Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62f872d8be | ||
|
|
41c6964aae | ||
|
|
95d3a2517c | ||
|
|
31063477ed | ||
|
|
4f05b95bd0 |
No files matched your search
@@ -1,125 +0,0 @@
|
|||||||
FROM ubuntu:22.04
|
|
||||||
|
|
||||||
ARG DEBIAN_FRONTEND=noninteractive
|
|
||||||
|
|
||||||
# Note installations that take a while (more than 10s)
|
|
||||||
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get install -y ca-certificates
|
|
||||||
RUN apt-get install -y ccache
|
|
||||||
RUN apt-get install -y curl
|
|
||||||
RUN apt-get install -y fish
|
|
||||||
RUN apt-get install -y gdb
|
|
||||||
# git-all: ~250s
|
|
||||||
RUN apt-get install -y git-all
|
|
||||||
RUN apt-get install -y gnupg
|
|
||||||
# liblttng-ust-dev: ~30s
|
|
||||||
RUN apt-get install -y liblttng-ust-dev
|
|
||||||
RUN apt-get install -y libssh2-1-dev
|
|
||||||
RUN apt-get install -y locales
|
|
||||||
RUN apt-get install -y ninja-build
|
|
||||||
# openssh-server: ~40s
|
|
||||||
RUN apt-get install -y openssh-server
|
|
||||||
RUN apt-get install -y pkg-config
|
|
||||||
RUN apt-get install -y software-properties-common
|
|
||||||
RUN apt-get install -y sudo
|
|
||||||
RUN apt-get install -y tzdata
|
|
||||||
RUN apt-get install -y vim
|
|
||||||
RUN apt-get install -y zip
|
|
||||||
|
|
||||||
# Python: ~30s
|
|
||||||
RUN apt-get install -y \
|
|
||||||
python3-pip \
|
|
||||||
python3-venv \
|
|
||||||
&& python3 -m pip install --upgrade pip
|
|
||||||
|
|
||||||
# GCC
|
|
||||||
ARG gcc_major=13
|
|
||||||
RUN \
|
|
||||||
add-apt-repository ppa:ubuntu-toolchain-r/test \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y g++-$gcc_major
|
|
||||||
|
|
||||||
# Clang
|
|
||||||
ARG clang_major=17
|
|
||||||
RUN \
|
|
||||||
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
|
|
||||||
| gpg --dearmor - \
|
|
||||||
| tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
|
|
||||||
&& echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
|
|
||||||
| tee -a "/etc/apt/sources.list.d/llvm.list" \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y \
|
|
||||||
clang-$clang_major \
|
|
||||||
clang-format-$clang_major \
|
|
||||||
clangd-$clang_major
|
|
||||||
|
|
||||||
# CMake
|
|
||||||
RUN \
|
|
||||||
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
|
|
||||||
| gpg --dearmor - \
|
|
||||||
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
|
|
||||||
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
|
|
||||||
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y cmake
|
|
||||||
|
|
||||||
# Dotnet
|
|
||||||
RUN apt-get update && apt-get install -y dotnet7
|
|
||||||
|
|
||||||
# Node
|
|
||||||
ARG node_major=18
|
|
||||||
RUN mkdir -p /etc/apt/keyrings \
|
|
||||||
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
|
|
||||||
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
|
|
||||||
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
|
|
||||||
| tee /etc/apt/sources.list.d/nodesource.list \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y nodejs
|
|
||||||
|
|
||||||
# Emumba (deps)
|
|
||||||
RUN apt-get update && apt-get -y install rapidjson-dev
|
|
||||||
# Emumba
|
|
||||||
COPY emumba /emumba
|
|
||||||
RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
|
|
||||||
&& rm -r /emumba
|
|
||||||
|
|
||||||
# Pcap
|
|
||||||
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
|
||||||
|
|
||||||
# Databento (deps)
|
|
||||||
RUN apt-get update && apt-get -y install libzstd-dev
|
|
||||||
|
|
||||||
# Set up default executables (update-alternatives)
|
|
||||||
RUN \
|
|
||||||
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
|
|
||||||
|
|
||||||
# Add compiler symlinks for ccache
|
|
||||||
RUN \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/clang; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/clang++; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/gcc; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/g++
|
|
||||||
|
|
||||||
RUN locale-gen en_US.UTF-8
|
|
||||||
|
|
||||||
ARG docker_user="bts"
|
|
||||||
ARG docker_uid="1000"
|
|
||||||
ARG docker_pass="bts"
|
|
||||||
|
|
||||||
RUN \
|
|
||||||
groupadd -g "$docker_uid" "$docker_user" \
|
|
||||||
&& useradd \
|
|
||||||
-m \
|
|
||||||
-u "$docker_uid" \
|
|
||||||
-g "$docker_uid" \
|
|
||||||
-G sudo \
|
|
||||||
--password "$(openssl passwd -1 "$docker_pass")" \
|
|
||||||
"$docker_user"
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
{
|
|
||||||
"dockerFile": "Dockerfile",
|
|
||||||
// Creates local dir to mount to HOME in docker
|
|
||||||
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
|
|
||||||
"hostRequirements": {
|
|
||||||
"memory": "4gb",
|
|
||||||
"storage": "6gb"
|
|
||||||
},
|
|
||||||
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
|
|
||||||
"containerUser": "bts",
|
|
||||||
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
|
|
||||||
"workspaceFolder": "/home/bts/spark",
|
|
||||||
"updateRemoteUserUID": true,
|
|
||||||
"shutdownAction": "none",
|
|
||||||
// Note: If running multiple containers on the same host, each must use a
|
|
||||||
// unique set of ports. Usually we just add 1000 to each port as needed. For
|
|
||||||
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
|
||||||
"runArgs": [
|
|
||||||
"--shm-size",
|
|
||||||
"2gb",
|
|
||||||
"-p",
|
|
||||||
"8000:8000",
|
|
||||||
"-p",
|
|
||||||
"8001:8001",
|
|
||||||
"-p",
|
|
||||||
"8002:8002",
|
|
||||||
"-p",
|
|
||||||
"8003:8003",
|
|
||||||
"-p",
|
|
||||||
"12345:12345"
|
|
||||||
],
|
|
||||||
"mounts": [
|
|
||||||
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
|
|
||||||
],
|
|
||||||
"customizations": {
|
|
||||||
"vscode": {
|
|
||||||
"settings": {
|
|
||||||
"clangd.arguments": [
|
|
||||||
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"extensions": [
|
|
||||||
"eamodio.gitlens",
|
|
||||||
"llvm-vs-code-extensions.vscode-clangd",
|
|
||||||
"ms-vscode.cmake-tools",
|
|
||||||
"ms-vscode.cpptools"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@@ -1,4 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
|
|
||||||
sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
*.cmd text=auto eol=crlf
|
|
||||||
*.cs text=auto eol=crlf
|
|
||||||
*.sh text=auto eol=lf
|
|
||||||
+4
-6
@@ -8,9 +8,7 @@ cmake-build-debug/
|
|||||||
.idea/
|
.idea/
|
||||||
.dir-locals.el
|
.dir-locals.el
|
||||||
*.pyc
|
*.pyc
|
||||||
**.vscode/*
|
**.vscode
|
||||||
!.vscode/launch.json
|
|
||||||
!.vscode/tasks.json
|
|
||||||
.vs
|
.vs
|
||||||
obj/
|
obj/
|
||||||
*.csproj.user
|
*.csproj.user
|
||||||
@@ -49,13 +47,13 @@ trace_data/
|
|||||||
/src/.clang-format
|
/src/.clang-format
|
||||||
/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
|
/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
|
||||||
/src/server/sparkd/cmds/spartan/analysis/data
|
/src/server/sparkd/cmds/spartan/analysis/data
|
||||||
.tmp/
|
*.tmp/*
|
||||||
.venv*/
|
/.venv*/*
|
||||||
.cache/
|
.cache/
|
||||||
/src/Spark.Admin.WebApp/tsconfig.*.g.json
|
/src/Spark.Admin.WebApp/tsconfig.*.g.json
|
||||||
/src/Spark.Admin.WebApp/src/bin/
|
/src/Spark.Admin.WebApp/src/bin/
|
||||||
**/CMakeDeps.cmake
|
**/CMakeDeps.cmake
|
||||||
**/CMakeDeps_win.cmake
|
**/CMakeDeps_win.cmake
|
||||||
.devcontainer/
|
**/devcontainer.json
|
||||||
*.Generated.cs
|
*.Generated.cs
|
||||||
*.lst
|
*.lst
|
||||||
Vendored
-158
@@ -1,158 +0,0 @@
|
|||||||
{
|
|
||||||
"version": "0.2.0",
|
|
||||||
"configurations": [
|
|
||||||
{
|
|
||||||
"name": ".NET Core Attach",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "attach"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "DataStreamPublisher",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "Build DataStream (Debug)",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"program": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "(gdb) Launch",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": ["run"],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "DataStreamQueryAnalyzer",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
|
||||||
"env": {
|
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "PubSubServer",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "Build DataStream (Debug)",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"program": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Spark",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
|
||||||
"program": "Spark.exe",
|
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Spark (Parameterized)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "Spark.exe",
|
|
||||||
"args": [
|
|
||||||
"--sparkd",
|
|
||||||
"${input:sparkConfigServer}"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Sparkd",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "Build Sparkd (Debug | Clang)",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run"
|
|
||||||
],
|
|
||||||
"cwd": "build/sparkd-debug-clang/server",
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"inputs": [
|
|
||||||
{
|
|
||||||
"id": "dataStreamPublisherConfig",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Publisher Configuration",
|
|
||||||
"options": [
|
|
||||||
"MarketDataPublisher",
|
|
||||||
"SimpleEchoClient",
|
|
||||||
"SimpleTestDataPublisher",
|
|
||||||
"TestDataPublisher",
|
|
||||||
],
|
|
||||||
"default": "MarketDataPublisher",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "pubSubServerConfig",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "PubSub Configuration",
|
|
||||||
"options": [
|
|
||||||
"PubSubServer",
|
|
||||||
"SimpleEchoClient",
|
|
||||||
],
|
|
||||||
"default": "PubSubServer",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkConfigServer",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Config Server URL",
|
|
||||||
"default": "http://localhost:8000/config",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
Vendored
-828
@@ -1,828 +0,0 @@
|
|||||||
{
|
|
||||||
"version": "2.0.0",
|
|
||||||
"tasks": [
|
|
||||||
{
|
|
||||||
"label": "Build Api Server",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_api_server.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_api_server.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"${input:buildConfiguration}",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Api Server (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_api_server.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_api_server.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Debug",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Api Server (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_api_server.cmd",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_api_server.sh",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build DataStream",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_data_stream.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"${input:buildConfiguration}",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build DataStream (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_data_stream.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Debug",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build DataStream (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_data_stream.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Spark",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_spark.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"${input:buildConfiguration}",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Spark (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_spark.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Debug",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Spark (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_spark.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--preset",
|
|
||||||
"${input:sparkdCmakePreset}",
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Debug",
|
|
||||||
"--compiler",
|
|
||||||
"${input:sparkdCompiler}"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Debug | Gcc)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Debug",
|
|
||||||
"--compiler",
|
|
||||||
"gcc"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Debug | Clang)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Debug",
|
|
||||||
"--compiler",
|
|
||||||
"clang"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Release",
|
|
||||||
"--compiler",
|
|
||||||
"${input:sparkdCompiler}"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Release | Gcc)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Release",
|
|
||||||
"--compiler",
|
|
||||||
"gcc"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Release | Clang)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Release",
|
|
||||||
"--compiler",
|
|
||||||
"clang"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Edit DataStream Config",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "editor",
|
|
||||||
"args": [
|
|
||||||
"${env:HOME}/.local/share/DataStream/conf/spark.conf"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "vim",
|
|
||||||
"args": [
|
|
||||||
"${env:APPDATA}/bts/DataStream/conf/spark.conf"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"reveal": "always",
|
|
||||||
"focus": true,
|
|
||||||
"panel": "shared",
|
|
||||||
"showReuseMessage": false,
|
|
||||||
"close": true,
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "wrench"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Edit Spark Config",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "vim",
|
|
||||||
"args": [
|
|
||||||
"${env:APPDATA}/bts/conf/spark.conf"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"reveal": "always",
|
|
||||||
"focus": true,
|
|
||||||
"panel": "dedicated",
|
|
||||||
"showReuseMessage": false,
|
|
||||||
"close": true,
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "wrench"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamQueryAnalyzer",
|
|
||||||
"dependsOn": "Build DataStream",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamQueryAnalyzer (Debug)",
|
|
||||||
"dependsOn": "Build DataStream (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamQueryAnalyzer (Release)",
|
|
||||||
"dependsOn": "Build DataStream (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamPublisher",
|
|
||||||
"dependsOn": "Build DataStream",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamPublisher (Debug)",
|
|
||||||
"dependsOn": "Build DataStream (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamPublisher (Release)",
|
|
||||||
"dependsOn": "Build DataStream (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer",
|
|
||||||
"dependsOn": "Build DataStream",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer (Debug)",
|
|
||||||
"dependsOn": "Build DataStream (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer (Debug | Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer (Release)",
|
|
||||||
"dependsOn": "Build DataStream (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer (Release | Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark",
|
|
||||||
"dependsOn": "Build Spark",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.exe",
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark (Debug)",
|
|
||||||
"dependsOn": "Build Spark (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.exe",
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark (Release)",
|
|
||||||
"dependsOn": "Build Spark (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.exe",
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark Config Query",
|
|
||||||
"dependsOn": "Build Spark",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "Spark.Console",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.Console.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"config",
|
|
||||||
"--query",
|
|
||||||
"${input:sparkConfigQuery}",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Sparkd",
|
|
||||||
"dependsOn": "Build Sparkd",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run"
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Sparkd Config Query",
|
|
||||||
"dependsOn": "Build Sparkd",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"config",
|
|
||||||
"--query",
|
|
||||||
"${input:sparkdConfigQuery}",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/logging/show_last_run.cmd",
|
|
||||||
"args": [
|
|
||||||
"DataStream",
|
|
||||||
"DataStreamQueryAnalyzer.exe",
|
|
||||||
"'Starting DataStreamQueryAnalyzer'"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"reveal": "always",
|
|
||||||
"focus": true,
|
|
||||||
"panel": "shared",
|
|
||||||
"showReuseMessage": false,
|
|
||||||
"close": true,
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "wrench"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "View PubSubServer Logs (Last Run)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/logging/show_last_run.sh",
|
|
||||||
"args": [
|
|
||||||
"${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
|
|
||||||
"Starting PubSubServer"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/logging/show_last_run.cmd",
|
|
||||||
"args": [
|
|
||||||
"DataStream",
|
|
||||||
"PubSubServer.exe",
|
|
||||||
"Starting PubSubServer"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"reveal": "always",
|
|
||||||
"focus": true,
|
|
||||||
"panel": "shared",
|
|
||||||
"showReuseMessage": false,
|
|
||||||
"close": true,
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "wrench"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
"inputs": [
|
|
||||||
{
|
|
||||||
"id": "buildConfiguration",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Configuration",
|
|
||||||
"options": [
|
|
||||||
"Debug",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"default": "Debug",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdCmakePreset",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Configuration",
|
|
||||||
"options": [
|
|
||||||
"sparkd-debug-clang",
|
|
||||||
"sparkd-debug-gcc",
|
|
||||||
"sparkd-release-clang",
|
|
||||||
"sparkd-release-gcc",
|
|
||||||
],
|
|
||||||
"default": "sparkd-debug-clang",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdCompiler",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Configuration",
|
|
||||||
"options": [
|
|
||||||
"clang",
|
|
||||||
"gcc",
|
|
||||||
],
|
|
||||||
"default": "sparkd-debug-clang",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "dataStreamPublisherConfig",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Publisher Configuration",
|
|
||||||
"options": [
|
|
||||||
"MarketDataPublisher",
|
|
||||||
"SimpleEchoClient",
|
|
||||||
"SimpleTestDataPublisher",
|
|
||||||
"TestDataPublisher",
|
|
||||||
],
|
|
||||||
"default": "MarketDataPublisher",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "pubSubServerConfig",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "PubSub Configuration",
|
|
||||||
"options": [
|
|
||||||
"PubSubServer",
|
|
||||||
"SimpleEchoClient",
|
|
||||||
],
|
|
||||||
"default": "PubSubServer",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkConfigQuery",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Config Type(s)"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdConfigQuery",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Config Type(s)"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
+13
-14
@@ -14,17 +14,12 @@ set(
|
|||||||
CACHE
|
CACHE
|
||||||
STRING
|
STRING
|
||||||
"External dependencies root folder (should contain folders: lib, bin, include")
|
"External dependencies root folder (should contain folders: lib, bin, include")
|
||||||
|
|
||||||
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
|
|
||||||
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
|
|
||||||
# api server, etc.).
|
|
||||||
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
|
|
||||||
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
|
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
|
||||||
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
|
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
|
||||||
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
||||||
|
set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
|
||||||
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
||||||
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
||||||
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
|
|
||||||
if (WIN32)
|
if (WIN32)
|
||||||
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
|
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
|
||||||
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
|
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
|
||||||
@@ -36,13 +31,13 @@ endif()
|
|||||||
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
||||||
link_directories(${SPARK_LIB})
|
link_directories(${SPARK_LIB})
|
||||||
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
if(BUILD_API_SERVER)
|
||||||
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
add_compile_definitions(SPARK_ENABLE_API_SERVER)
|
||||||
if(DEFINED APP_DATA_DIR_OVERRIDE)
|
|
||||||
add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
|
|
||||||
endif()
|
endif()
|
||||||
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
|
add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
|
||||||
|
add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
|
||||||
# subdirs
|
# subdirs
|
||||||
add_subdirectory(src/lib)
|
add_subdirectory(src/lib)
|
||||||
else()
|
else()
|
||||||
@@ -52,7 +47,13 @@ else()
|
|||||||
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
||||||
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
||||||
|
|
||||||
if(BUILD_SPARKD)
|
if(BUILD_API_SERVER)
|
||||||
|
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
|
||||||
|
add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
|
||||||
|
elseif(BUILD_SPARKD)
|
||||||
|
add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
|
||||||
|
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
|
||||||
|
|
||||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||||
# read by their code). We override some below.
|
# read by their code). We override some below.
|
||||||
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
||||||
@@ -132,8 +133,6 @@ else()
|
|||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
|
|
||||||
|
|
||||||
COMMAND
|
COMMAND
|
||||||
ln ARGS -sf
|
ln ARGS -sf
|
||||||
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
||||||
|
|||||||
+62
-113
@@ -15,13 +15,11 @@
|
|||||||
"rhs": "Windows"
|
"rhs": "Windows"
|
||||||
},
|
},
|
||||||
"generator": "Visual Studio 17 2022",
|
"generator": "Visual Studio 17 2022",
|
||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build",
|
||||||
"environment": {
|
"environment": {
|
||||||
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
||||||
},
|
},
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR": "${dollar}{APPDATA}/bts",
|
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
|
|
||||||
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||||
@@ -32,12 +30,7 @@
|
|||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
|
||||||
"EXPORT_MD_CONSUMER": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "ON"
|
|
||||||
},
|
|
||||||
"SHARED_MEMORY_PREFIX": "spark"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -53,8 +46,6 @@
|
|||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"platform": "",
|
"platform": "",
|
||||||
"APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
|
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
|
|
||||||
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -71,8 +62,7 @@
|
|||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
},
|
}
|
||||||
"SHARED_MEMORY_PREFIX": "sparkd"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -105,72 +95,98 @@
|
|||||||
{
|
{
|
||||||
"name": "sparkd-debug-clang",
|
"name": "sparkd-debug-clang",
|
||||||
"inherits": "sparkd-debug",
|
"inherits": "sparkd-debug",
|
||||||
"displayName": "Sparkd (Debug | Clang)",
|
"displayName": "Debug (clang)",
|
||||||
"description": "Debug build compiled w/ clang",
|
"description": "Debug build compiled w/ clang",
|
||||||
|
"binaryDir": "${sourceDir}/build/clang/debug",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "clang",
|
"CMAKE_C_COMPILER": "clang",
|
||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-debug-gcc",
|
"name": "sparkd-debug-gcc",
|
||||||
"inherits": "sparkd-debug",
|
"inherits": "sparkd-debug",
|
||||||
"displayName": "Sparkd (Debug | Gcc)",
|
"displayName": "Debug (gcc)",
|
||||||
"description": "Debug build compiled w/ gcc",
|
"description": "Debug build compiled w/ gcc",
|
||||||
|
"binaryDir": "${sourceDir}/build/gcc/debug",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "gcc",
|
"CMAKE_C_COMPILER": "gcc",
|
||||||
"CMAKE_CXX_COMPILER": "g++",
|
"CMAKE_CXX_COMPILER": "g++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-clang",
|
"name": "sparkd-release-clang",
|
||||||
"inherits": "sparkd-release",
|
"inherits": "sparkd-release",
|
||||||
"displayName": "Sparkd (Release | Clang)",
|
"displayName": "Release (clang)",
|
||||||
"description": "Release build compiled w/ clang",
|
"description": "Release build compiled w/ clang",
|
||||||
|
"binaryDir": "${sourceDir}/build/clang/release",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "clang",
|
"CMAKE_C_COMPILER": "clang",
|
||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-gcc",
|
"name": "sparkd-release-gcc",
|
||||||
"inherits": "sparkd-release",
|
"inherits": "sparkd-release",
|
||||||
"displayName": "Sparkd (Release | Gcc)",
|
"displayName": "Release (gcc)",
|
||||||
"description": "Release build compiled w/ gcc",
|
"description": "Release build compiled w/ gcc",
|
||||||
|
"binaryDir": "${sourceDir}/build/gcc/release",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "gcc",
|
"CMAKE_C_COMPILER": "gcc",
|
||||||
"CMAKE_CXX_COMPILER": "g++",
|
"CMAKE_CXX_COMPILER": "g++",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"name": "spark-api-server-base",
|
||||||
|
"hidden": true,
|
||||||
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
|
"cacheVariables": {
|
||||||
|
"BUILD_API_SERVER": {
|
||||||
|
"type": "bool",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
|
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows",
|
"name": "spark-api-server-windows",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"displayName": "Spark Api Server (Windows)",
|
"displayName": "Spark Api Server",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
"BUILD_API_SERVER": {
|
||||||
"SHARED_MEMORY_PREFIX": "spark_api_server"
|
"type": "bool",
|
||||||
|
"value": "ON"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-linux",
|
"name": "spark-api-server-linux",
|
||||||
"inherits": "sparkd",
|
"inherits": "sparkd",
|
||||||
"displayName": "Spark Api Server (Linux)",
|
"displayName": "Spark Api Server",
|
||||||
"generator": "Ninja Multi-Config",
|
"generator": "Ninja Multi-Config",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
|
||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
},
|
},
|
||||||
|
"BUILD_API_SERVER": {
|
||||||
|
"type": "bool",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
"BUILD_SPARKD": {
|
"BUILD_SPARKD": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
@@ -181,11 +197,7 @@
|
|||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
|
||||||
"EXPORT_MD_CONSUMER": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "ON"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -208,62 +220,13 @@
|
|||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite-windows",
|
"name": "data-stream-suite",
|
||||||
"condition": {
|
|
||||||
"type": "equals",
|
|
||||||
"lhs": "${hostSystemName}",
|
|
||||||
"rhs": "Windows"
|
|
||||||
},
|
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"displayName": "Data Stream Suite (Windows)",
|
"displayName": "Data Stream Suite",
|
||||||
"description": "Data Stream utils ported over from Edge",
|
"description": "Data Stream utils ported over from Edge",
|
||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
|
||||||
"SHARED_MEMORY_PREFIX": "data_stream"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "data-stream-suite-linux",
|
|
||||||
"displayName": "Data Stream Suite (Linux)",
|
|
||||||
"description": "Data Stream utils ported over from Edge",
|
|
||||||
"condition": {
|
|
||||||
"type": "equals",
|
|
||||||
"lhs": "${hostSystemName}",
|
|
||||||
"rhs": "Linux"
|
|
||||||
},
|
|
||||||
"inherits": "sparkd",
|
|
||||||
"generator": "Ninja Multi-Config",
|
|
||||||
"cacheVariables": {
|
|
||||||
"APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
|
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
|
||||||
"BUILD_ADMIN_WEB_APP": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "OFF"
|
|
||||||
},
|
|
||||||
"BUILD_SDK": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "OFF"
|
|
||||||
},
|
|
||||||
"BUILD_SPARKD": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "OFF"
|
|
||||||
},
|
|
||||||
"CMAKE_C_COMPILER": "clang",
|
|
||||||
"CMAKE_CXX_COMPILER": "clang++",
|
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
|
||||||
"EXPORT_MD_CONSUMER": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "ON"
|
|
||||||
},
|
|
||||||
"SHARED_MEMORY_PREFIX": "data_stream"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
@@ -281,7 +244,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-debug",
|
"name": "spark-debug",
|
||||||
"displayName": "Spark (Debug)",
|
"displayName": "Debug",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"configurePreset": "spark",
|
"configurePreset": "spark",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
@@ -289,7 +252,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-release",
|
"name": "spark-release",
|
||||||
"displayName": "Spark (Release)",
|
"displayName": "Release",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"configurePreset": "spark",
|
"configurePreset": "spark",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
@@ -317,28 +280,28 @@
|
|||||||
{
|
{
|
||||||
"name": "sparkd-debug-clang",
|
"name": "sparkd-debug-clang",
|
||||||
"configurePreset": "sparkd-debug-clang",
|
"configurePreset": "sparkd-debug-clang",
|
||||||
"displayName": "Sparkd (Debug | Clang)",
|
"displayName": "Debug (clang)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-debug-gcc",
|
"name": "sparkd-debug-gcc",
|
||||||
"configurePreset": "sparkd-debug-gcc",
|
"configurePreset": "sparkd-debug-gcc",
|
||||||
"displayName": "Sparkd (Debug | Gcc)",
|
"displayName": "Debug (gcc)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-clang",
|
"name": "sparkd-release-clang",
|
||||||
"configurePreset": "sparkd-release-clang",
|
"configurePreset": "sparkd-release-clang",
|
||||||
"displayName": "Sparkd (Release | Clang)",
|
"displayName": "Release (clang)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-gcc",
|
"name": "sparkd-release-gcc",
|
||||||
"configurePreset": "sparkd-release-gcc",
|
"configurePreset": "sparkd-release-gcc",
|
||||||
"displayName": "Sparkd (Release | Gcc)",
|
"displayName": "Release (gcc)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -355,14 +318,14 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows-debug",
|
"name": "spark-api-server-windows-debug",
|
||||||
"displayName": "Spark Api Server (Windows | Debug)",
|
"displayName": "Spark Api Server (Debug)",
|
||||||
"inherits": "spark-api-server-windows",
|
"inherits": "spark-api-server-windows",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows-release",
|
"name": "spark-api-server-windows-release",
|
||||||
"displayName": "Spark Api Server (Windows | Release)",
|
"displayName": "Spark Api Server (Debug)",
|
||||||
"inherits": "spark-api-server-windows",
|
"inherits": "spark-api-server-windows",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
},
|
},
|
||||||
@@ -381,14 +344,14 @@
|
|||||||
{
|
{
|
||||||
"name": "spark-api-server-linux-debug",
|
"name": "spark-api-server-linux-debug",
|
||||||
"inherits": "spark-api-server-linux",
|
"inherits": "spark-api-server-linux",
|
||||||
"displayName": "Spark Api Server (Linux | Debug)",
|
"displayName": "Spark Api Server (Debug)",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-linux-release",
|
"name": "spark-api-server-linux-release",
|
||||||
"inherits": "spark-api-server-linux",
|
"inherits": "spark-api-server-linux",
|
||||||
"displayName": "Spark Api Server (Linux | Release)",
|
"displayName": "Spark Api Server (Release)",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -408,30 +371,16 @@
|
|||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite-windows-debug",
|
"name": "data-stream-suite-debug",
|
||||||
"displayName": "Data Stream Suite (Windows | Debug)",
|
"displayName": "DataStreamSuite (Debug)",
|
||||||
"configurePreset": "data-stream-suite-windows",
|
"configurePreset": "data-stream-suite",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite-windows-release",
|
"name": "data-stream-suite-release",
|
||||||
"displayName": "Data Stream Suite (Windows | Release)",
|
"displayName": "DataStreamSuite (Release)",
|
||||||
"configurePreset": "data-stream-suite-windows",
|
"configurePreset": "data-stream-suite",
|
||||||
"configuration": "Release"
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "data-stream-suite-linux-debug",
|
|
||||||
"displayName": "Data Stream Suite (Linux | Debug)",
|
|
||||||
"configurePreset": "data-stream-suite-linux",
|
|
||||||
"configuration": "Debug"
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "data-stream-suite-linux-release",
|
|
||||||
"displayName": "Data Stream Suite (Linux | Release)",
|
|
||||||
"configurePreset": "data-stream-suite-linux",
|
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
-1
Submodule docker/linux deleted from 56c61231f3.
@@ -48,15 +48,9 @@ RUN apt-get update && apt-get install -y \
|
|||||||
# cmake
|
# cmake
|
||||||
RUN \
|
RUN \
|
||||||
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
|
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
|
||||||
| gpg --dearmor - \
|
| gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
|
||||||
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
|
&& apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
|
||||||
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
|
|
||||||
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
|
|
||||||
&& apt-get update \
|
&& apt-get update \
|
||||||
&& rm /usr/share/keyrings/kitware-archive-keyring.gpg \
|
|
||||||
&& apt-get install -y kitware-archive-keyring \
|
|
||||||
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
|
|
||||||
| tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
|
|
||||||
&& apt-get install -y cmake
|
&& apt-get install -y cmake
|
||||||
|
|
||||||
# node
|
# node
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
{
|
{
|
||||||
"image": "bluetradingsystems/sparkd-dev:latest",
|
"image": "bluetradingsystems/sparkd-dev:latest",
|
||||||
// Creates local dir to mount to HOME in docker
|
"initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
|
||||||
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
|
|
||||||
"hostRequirements": {
|
"hostRequirements": {
|
||||||
"memory": "4gb",
|
"memory": "4gb",
|
||||||
"storage": "6gb"
|
"storage": "6gb"
|
||||||
@@ -28,6 +27,6 @@
|
|||||||
"8003:8003"
|
"8003:8003"
|
||||||
],
|
],
|
||||||
"mounts": [
|
"mounts": [
|
||||||
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
|
"source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
Executable
+3
@@ -0,0 +1,3 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"
|
||||||
Submodule docker/windows deleted from de7d40c09d.
+1
-1
Submodule externals updated: 8069cc893f...cae5fd20da.
@@ -0,0 +1,6 @@
|
|||||||
|
cd ~/spark/build/clang/debug/; cmake --build . --target install; rm -rf ~/spark/install/Debug/bin.zip; tar -zcvf ~/spark/install/Debug/bin.zip ~/spark/install/Debug/bin; scp ~/spark/install/Debug/bin.zip bts@10.1.10.31:~/13000/; cd ~/spark/
|
||||||
|
cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; docker container stop spark_13000; docker container start spark_13000
|
||||||
|
cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; cp -r sparkbin ~/.sparkd-dev/mounts/sparkd_build/;
|
||||||
|
ES SECURITYTYPE_FUTURE at VENUE_GLBX
|
||||||
|
./sparkd md dump --symbol ES --type future --venue glbx --platform mdp3onixsex
|
||||||
|
BT5u53r
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
<PackageReference Include="YaccLexTools" Version="1.2.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -1,22 +1,41 @@
|
|||||||
using System.Diagnostics;
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG.Utilities.General;
|
using System;
|
||||||
|
using System.Runtime;
|
||||||
|
|
||||||
/// <summary>
|
namespace BCG.Utilities.General
|
||||||
/// Summary description for Crono.
|
{
|
||||||
/// </summary>
|
/// <summary>
|
||||||
public class Crono {
|
/// Summary description for Crono.
|
||||||
readonly Stopwatch myTimer = new();
|
/// </summary>
|
||||||
|
public class Crono
|
||||||
|
{
|
||||||
|
public Crono()
|
||||||
|
{
|
||||||
|
myTimer = new HighPerformanceTimer();
|
||||||
|
}
|
||||||
|
|
||||||
public void start() =>
|
public void start()
|
||||||
myTimer.Start();
|
{myTimer.start();}
|
||||||
|
|
||||||
public void stop() =>
|
public void stop()
|
||||||
myTimer.Stop();
|
{myTimer.stop();}
|
||||||
|
|
||||||
public TimeSpan time =>
|
public TimeSpan time
|
||||||
myTimer.Elapsed;
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public double miliseconds
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{return myTimer.totalMilliseconds;}
|
||||||
|
}
|
||||||
|
|
||||||
|
private HighPerformanceTimer myTimer;
|
||||||
|
}
|
||||||
|
|
||||||
public double miliseconds =>
|
|
||||||
myTimer.Elapsed.TotalMilliseconds;
|
|
||||||
}
|
}
|
||||||
@@ -1,192 +1,192 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
public enum RegistryRoot { HKCU, HKLM }
|
public enum RegistryRoot { HKCU, HKLM }
|
||||||
|
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class BlueRegistry
|
public class BlueRegistry
|
||||||
{
|
{
|
||||||
public const string BlueSubKey = @"Software\Blue Capital Group\";
|
public const string BlueSubKey = @"Software\Blue Capital Group\";
|
||||||
public static readonly string ApplicationPrefix;
|
public static readonly string ApplicationPrefix;
|
||||||
protected string myFullPath;
|
protected string myFullPath;
|
||||||
|
|
||||||
public BlueRegistry(string subPath)
|
public BlueRegistry(string subPath)
|
||||||
{
|
{
|
||||||
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
|
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
public BlueRegistry(BlueRegistry parent, string subPath)
|
public BlueRegistry(BlueRegistry parent, string subPath)
|
||||||
{
|
{
|
||||||
myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
|
myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
static BlueRegistry()
|
static BlueRegistry()
|
||||||
{
|
{
|
||||||
string app = "Applications";
|
string app = "Applications";
|
||||||
|
|
||||||
Assembly ass = Assembly.GetEntryAssembly();
|
Assembly ass = Assembly.GetEntryAssembly();
|
||||||
if (ass != null)
|
if (ass != null)
|
||||||
{
|
{
|
||||||
app = ass.GetName().Name;
|
app = ass.GetName().Name;
|
||||||
|
|
||||||
object[] attrs = ass.GetCustomAttributes(
|
object[] attrs = ass.GetCustomAttributes(
|
||||||
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
|
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
|
||||||
|
|
||||||
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
|
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
|
||||||
{
|
{
|
||||||
app = ((AssemblyTitleAttribute)attrs[0]).Title;
|
app = ((AssemblyTitleAttribute)attrs[0]).Title;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
|
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
|
||||||
}
|
}
|
||||||
|
|
||||||
public object getValue(string name)
|
public object getValue(string name)
|
||||||
{
|
{
|
||||||
return getValue(RegistryRoot.HKCU, name);
|
return getValue(RegistryRoot.HKCU, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
public object getValue(string name, object defaultValue)
|
public object getValue(string name, object defaultValue)
|
||||||
{
|
{
|
||||||
return getValue(RegistryRoot.HKCU, name, defaultValue);
|
return getValue(RegistryRoot.HKCU, name, defaultValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setValue(string name, object value)
|
public void setValue(string name, object value)
|
||||||
{
|
{
|
||||||
setValue(RegistryRoot.HKCU, name, value);
|
setValue(RegistryRoot.HKCU, name, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteValue(string name)
|
public void deleteValue(string name)
|
||||||
{
|
{
|
||||||
deleteValue(RegistryRoot.HKCU, name);
|
deleteValue(RegistryRoot.HKCU, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
private RegistryKey myGetRegistryKey(RegistryRoot root)
|
private RegistryKey myGetRegistryKey(RegistryRoot root)
|
||||||
{
|
{
|
||||||
if (root == RegistryRoot.HKCU)
|
if (root == RegistryRoot.HKCU)
|
||||||
return Registry.CurrentUser.CreateSubKey(myFullPath);
|
return Registry.CurrentUser.CreateSubKey(myFullPath);
|
||||||
else
|
else
|
||||||
return Registry.LocalMachine.CreateSubKey(myFullPath);
|
return Registry.LocalMachine.CreateSubKey(myFullPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
#region Generic Methods
|
#region Generic Methods
|
||||||
public virtual void setValue(RegistryRoot root, string name, object value)
|
internal virtual void setValue(RegistryRoot root, string name, object value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
rk.SetValue(name, value);
|
rk.SetValue(name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual object getValue(RegistryRoot root, string name)
|
internal virtual object getValue(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return rk.GetValue(name);
|
return rk.GetValue(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual void deleteValue(RegistryRoot root, string name)
|
internal virtual void deleteValue(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
rk.DeleteValue(name);
|
rk.DeleteValue(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public virtual object getValue(RegistryRoot root, string name, object defaultValue)
|
internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return rk.GetValue(name, defaultValue);
|
return rk.GetValue(name, defaultValue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Double
|
#region Double
|
||||||
internal virtual void setDouble(RegistryRoot root, string name, double value)
|
internal virtual void setDouble(RegistryRoot root, string name, double value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
DoubleRegistryHandler.setValue(rk, name, value);
|
DoubleRegistryHandler.setValue(rk, name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual double getDouble(RegistryRoot root, string name)
|
internal virtual double getDouble(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return DoubleRegistryHandler.getValue(rk, name);
|
return DoubleRegistryHandler.getValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Enum
|
#region Enum
|
||||||
internal virtual void setEnum(RegistryRoot root, string name, Enum value)
|
internal virtual void setEnum(RegistryRoot root, string name, Enum value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
EnumRegistryHandler.setValue(rk, name, value);
|
EnumRegistryHandler.setValue(rk, name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
|
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return EnumRegistryHandler.getValue(rk, name, enumType);
|
return EnumRegistryHandler.getValue(rk, name, enumType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Array
|
#region Array
|
||||||
internal virtual void setArray(RegistryRoot root, string name, Array array)
|
internal virtual void setArray(RegistryRoot root, string name, Array array)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
ArrayRegistryHandler.setValue(rk, name, array);
|
ArrayRegistryHandler.setValue(rk, name, array);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
|
internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return ArrayRegistryHandler.getValue(rk, name, elementType);
|
return ArrayRegistryHandler.getValue(rk, name, elementType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region SortedList
|
#region SortedList
|
||||||
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
|
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
SortedListRegistryHandler.setValue(rk, name, slist);
|
SortedListRegistryHandler.setValue(rk, name, slist);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual SortedList getSortedList(RegistryRoot root, string name)
|
internal virtual SortedList getSortedList(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return SortedListRegistryHandler.getValue(rk, name);
|
return SortedListRegistryHandler.getValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual void deleteSortedList(RegistryRoot root, string name)
|
internal virtual void deleteSortedList(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
SortedListRegistryHandler.deleteValue(rk, name);
|
SortedListRegistryHandler.deleteValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,10 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net7.0-windows</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
<UseWindowsForms>true</UseWindowsForms>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
+166
-166
@@ -1,166 +1,166 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
using System.Drawing;
|
using System.Drawing;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
public class PersistWindowState
|
public class PersistWindowState
|
||||||
{
|
{
|
||||||
// event info that allows form to persist extra window state data
|
// event info that allows form to persist extra window state data
|
||||||
public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
|
public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
|
||||||
public event WindowStateHandler LoadState;
|
public event WindowStateHandler LoadState;
|
||||||
public event WindowStateHandler SaveState;
|
public event WindowStateHandler SaveState;
|
||||||
|
|
||||||
private Form myParent;
|
private Form myParent;
|
||||||
private BlueRegistry myRegistry;
|
private BlueRegistry myRegistry;
|
||||||
|
|
||||||
private int myNormalLeft;
|
private int myNormalLeft;
|
||||||
private int myNormalTop;
|
private int myNormalTop;
|
||||||
private int myNormalWidth;
|
private int myNormalWidth;
|
||||||
private int myNormalHeight;
|
private int myNormalHeight;
|
||||||
private bool myResizable = true;
|
private bool myResizable = true;
|
||||||
private FormWindowState myWindowState;
|
private FormWindowState myWindowState;
|
||||||
private bool myAllowSaveMinimized = false;
|
private bool myAllowSaveMinimized = false;
|
||||||
private bool mySavePosition = true;
|
private bool mySavePosition = true;
|
||||||
private bool mySaveState = true;
|
private bool mySaveState = true;
|
||||||
|
|
||||||
public PersistWindowState (bool savePosition)
|
public PersistWindowState (bool savePosition)
|
||||||
{
|
{
|
||||||
mySavePosition = savePosition;
|
mySavePosition = savePosition;
|
||||||
}
|
}
|
||||||
|
|
||||||
public PersistWindowState ()
|
public PersistWindowState ()
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool saveState
|
public bool saveState
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
set { mySaveState = value; }
|
set { mySaveState = value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public Form parent
|
public Form parent
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
get { return myParent; }
|
get { return myParent; }
|
||||||
set
|
set
|
||||||
{
|
{
|
||||||
myParent = value;
|
myParent = value;
|
||||||
myRegistry = new BlueRegistry (myParent.Text);
|
myRegistry = new BlueRegistry (myParent.Text);
|
||||||
|
|
||||||
// subscribe to parent form's events
|
// subscribe to parent form's events
|
||||||
myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
|
myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
|
||||||
myParent.Resize += new System.EventHandler(myOnResize);
|
myParent.Resize += new System.EventHandler(myOnResize);
|
||||||
myParent.Move += new System.EventHandler(myOnMove);
|
myParent.Move += new System.EventHandler(myOnMove);
|
||||||
myParent.Load += new System.EventHandler(myOnLoad);
|
myParent.Load += new System.EventHandler(myOnLoad);
|
||||||
|
|
||||||
// get initial width and height in case form is never resized
|
// get initial width and height in case form is never resized
|
||||||
myNormalWidth = myParent.Width;
|
myNormalWidth = myParent.Width;
|
||||||
myNormalHeight = myParent.Height;
|
myNormalHeight = myParent.Height;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public BlueRegistry blueRegistry
|
public BlueRegistry blueRegistry
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
get { return myRegistry; }
|
get { return myRegistry; }
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
set { myRegistry = value; }
|
set { myRegistry = value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool resizable
|
public bool resizable
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
set { myResizable = value; }
|
set { myResizable = value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool allowSaveMinimized
|
public bool allowSaveMinimized
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
set { myAllowSaveMinimized = value; }
|
set { myAllowSaveMinimized = value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myOnResize(object sender, System.EventArgs e)
|
private void myOnResize(object sender, System.EventArgs e)
|
||||||
{
|
{
|
||||||
// save width and height
|
// save width and height
|
||||||
if (myParent.WindowState == FormWindowState.Normal && myResizable)
|
if (myParent.WindowState == FormWindowState.Normal && myResizable)
|
||||||
{
|
{
|
||||||
myNormalWidth = myParent.Width;
|
myNormalWidth = myParent.Width;
|
||||||
myNormalHeight = myParent.Height;
|
myNormalHeight = myParent.Height;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myOnMove(object sender, System.EventArgs e)
|
private void myOnMove(object sender, System.EventArgs e)
|
||||||
{
|
{
|
||||||
// save position
|
// save position
|
||||||
if(myParent.WindowState == FormWindowState.Normal)
|
if(myParent.WindowState == FormWindowState.Normal)
|
||||||
{
|
{
|
||||||
myNormalLeft = myParent.Left;
|
myNormalLeft = myParent.Left;
|
||||||
myNormalTop = myParent.Top;
|
myNormalTop = myParent.Top;
|
||||||
}
|
}
|
||||||
// save state
|
// save state
|
||||||
myWindowState = myParent.WindowState;
|
myWindowState = myParent.WindowState;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
|
private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
|
||||||
{
|
{
|
||||||
// save position, size and state
|
// save position, size and state
|
||||||
if (mySavePosition)
|
if (mySavePosition)
|
||||||
{
|
{
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
|
myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
|
myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (myResizable)
|
if (myResizable)
|
||||||
{
|
{
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
|
myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
|
myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
|
||||||
}
|
}
|
||||||
|
|
||||||
// check if we are allowed to save the state as minimized (not normally)
|
// check if we are allowed to save the state as minimized (not normally)
|
||||||
if (!myAllowSaveMinimized)
|
if (!myAllowSaveMinimized)
|
||||||
{
|
{
|
||||||
if(myWindowState == FormWindowState.Minimized)
|
if(myWindowState == FormWindowState.Minimized)
|
||||||
myWindowState = FormWindowState.Normal;
|
myWindowState = FormWindowState.Normal;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mySaveState)
|
if (mySaveState)
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
|
myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
|
||||||
|
|
||||||
// fire SaveState event
|
// fire SaveState event
|
||||||
if (SaveState != null)
|
if (SaveState != null)
|
||||||
SaveState (this, myRegistry);
|
SaveState (this, myRegistry);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myOnLoad (object sender, System.EventArgs e)
|
private void myOnLoad (object sender, System.EventArgs e)
|
||||||
{
|
{
|
||||||
if (mySavePosition)
|
if (mySavePosition)
|
||||||
{
|
{
|
||||||
int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
|
int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
|
||||||
int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
|
int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
|
||||||
myParent.Location = new Point (left, top);
|
myParent.Location = new Point (left, top);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (myResizable)
|
if (myResizable)
|
||||||
{
|
{
|
||||||
int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
|
int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
|
||||||
int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
|
int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
|
||||||
myParent.Size = new Size (width, height);
|
myParent.Size = new Size (width, height);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mySaveState)
|
if (mySaveState)
|
||||||
{
|
{
|
||||||
FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState",
|
FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState",
|
||||||
(int) myParent.WindowState);
|
(int) myParent.WindowState);
|
||||||
myParent.WindowState = windowState;
|
myParent.WindowState = windowState;
|
||||||
}
|
}
|
||||||
|
|
||||||
// fire LoadState event
|
// fire LoadState event
|
||||||
if(LoadState != null)
|
if(LoadState != null)
|
||||||
LoadState (this, myRegistry);
|
LoadState (this, myRegistry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
|
||||||
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
|
|
||||||
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup Label="ColumnExpression">
|
|
||||||
<Scanner Include="ColumnExpressionTokens.lex" />
|
|
||||||
<Parser Include="ColumnExpressionGrammar.y" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
Whitespace-only changes.
@@ -1,62 +0,0 @@
|
|||||||
namespace Bts.ColumnExpressions;
|
|
||||||
|
|
||||||
public class ColumnExpression {
|
|
||||||
public ColumnExpression(string name, Type? type = null, object? value = null) {
|
|
||||||
Name = name;
|
|
||||||
Type = type
|
|
||||||
?? value?.GetType()
|
|
||||||
?? throw new ArgumentException(
|
|
||||||
"Type must be specified explicitly when value is null", nameof(type));
|
|
||||||
Value = value
|
|
||||||
?? DBNull.Value;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string Name { get; }
|
|
||||||
|
|
||||||
public Type Type { get; }
|
|
||||||
|
|
||||||
public object Value { get; }
|
|
||||||
|
|
||||||
public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
|
|
||||||
// Try to convert value to 'type' to verify they are compatible
|
|
||||||
// Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
|
|
||||||
// account for implicit conversions (e.g., int -> long), just inheritance.
|
|
||||||
if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
|
|
||||||
throw new ArgumentException(
|
|
||||||
$"Type '{type}' is not assignable from value type '{value?.GetType()}'");
|
|
||||||
}
|
|
||||||
return new(name, type ?? typeof(T), value);
|
|
||||||
}
|
|
||||||
|
|
||||||
internal static Type ParseType(string typeId) {
|
|
||||||
return Candidates().ExceptNull().FirstOrDefault() ??
|
|
||||||
throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
|
|
||||||
|
|
||||||
IEnumerable<Type?> Candidates() {
|
|
||||||
if (typeId.Length == 0) {
|
|
||||||
yield break;
|
|
||||||
}
|
|
||||||
yield return Type.GetType(typeId);
|
|
||||||
yield return Type.GetType($"System.{typeId}");
|
|
||||||
if (typeId.Length > 1) {
|
|
||||||
yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
|
|
||||||
}
|
|
||||||
yield return typeId.ToLower() switch {
|
|
||||||
"float64" or "double" or "real64" or "number" => typeof(double),
|
|
||||||
"float32" or "single" or "real32" or "real" or "float" => typeof(float),
|
|
||||||
"int16" or "short" or "integer16" or "signed16" => typeof(short),
|
|
||||||
"int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
|
|
||||||
"int64" or "long" or "integer64" or "signed64" => typeof(long),
|
|
||||||
"uint16" or "ushort" or "unsigned16" => typeof(ushort),
|
|
||||||
"uint32" or "uint" or "unsigned" => typeof(uint),
|
|
||||||
"uint64" or "ulong" or "unsigned64" => typeof(ulong),
|
|
||||||
"char" => typeof(char),
|
|
||||||
"byte" or "uint8" => typeof(byte),
|
|
||||||
"bool" or "boolean" => typeof(bool),
|
|
||||||
"string" or "text" => typeof(string),
|
|
||||||
"date" or "datetime" => typeof(DateTime),
|
|
||||||
_ => null
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
%namespace Bts.ColumnExpressions
|
|
||||||
%partial
|
|
||||||
%visibility internal
|
|
||||||
%tokentype ColumnExpressionToken
|
|
||||||
%parsertype ColumnExpressionParser
|
|
||||||
|
|
||||||
%union {
|
|
||||||
public int Integer;
|
|
||||||
public double Real;
|
|
||||||
public bool Boolean;
|
|
||||||
public string String;
|
|
||||||
public string Identifier;
|
|
||||||
public string TypeId;
|
|
||||||
public DateTime DateTime;
|
|
||||||
public Tuple<string, Type> ColumnDefinition;
|
|
||||||
public ColumnExpression ColumnExpression;
|
|
||||||
public ImmutableList<ColumnExpression> ColumnExpressions;
|
|
||||||
}
|
|
||||||
|
|
||||||
%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
|
|
||||||
%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
|
|
||||||
|
|
||||||
%nonassoc EQUALS
|
|
||||||
|
|
||||||
%type <ColumnDefinition> ColumnDefinition
|
|
||||||
%type <ColumnExpression> ColumnExpression
|
|
||||||
%type <ColumnExpressions> ColumnExpressions
|
|
||||||
|
|
||||||
%%
|
|
||||||
|
|
||||||
ColumnExpressions :
|
|
||||||
ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
|
|
||||||
| ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
|
|
||||||
;
|
|
||||||
|
|
||||||
ColumnExpression :
|
|
||||||
ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
|
|
||||||
| ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
|
|
||||||
| ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
|
|
||||||
| ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
|
|
||||||
| ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
|
|
||||||
| ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
|
|
||||||
;
|
|
||||||
|
|
||||||
ColumnDefinition :
|
|
||||||
IDENTIFIER { $$ = new($1.Identifier, null); }
|
|
||||||
| IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
|
|
||||||
| IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
|
|
||||||
;
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace Bts.ColumnExpressions;
|
|
||||||
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
public class ColumnExpressionInterpreter {
|
|
||||||
public static IList<ColumnExpression> Evaluate(Stream stream) {
|
|
||||||
var parser = new ColumnExpressionParser();
|
|
||||||
var res = parser.Parse(stream);
|
|
||||||
return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
|
|
||||||
res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IList<ColumnExpression> Evaluate(string str) {
|
|
||||||
using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
|
|
||||||
return Evaluate(ms);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace Bts.ColumnExpressions;
|
|
||||||
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
partial class ColumnExpressionParser {
|
|
||||||
public ColumnExpressionParser() : base(null) { }
|
|
||||||
|
|
||||||
public ValueType? Parse(string s) {
|
|
||||||
using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
|
|
||||||
return Parse(ms);
|
|
||||||
}
|
|
||||||
|
|
||||||
public ValueType? Parse(Stream stream) {
|
|
||||||
Scanner = new ColumnExpressionScanner(stream);
|
|
||||||
Parse();
|
|
||||||
return CurrentSemanticValue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
namespace Bts.ColumnExpressions;
|
|
||||||
|
|
||||||
partial class ColumnExpressionScanner {
|
|
||||||
public override void yyerror(string format, params object[] args) {
|
|
||||||
base.yyerror(format, args);
|
|
||||||
Console.WriteLine(format, args);
|
|
||||||
Console.WriteLine();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
%namespace Bts.ColumnExpressions
|
|
||||||
%scannertype ColumnExpressionScanner
|
|
||||||
%visibility internal
|
|
||||||
%tokentype ColumnExpressionToken
|
|
||||||
|
|
||||||
%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
|
|
||||||
|
|
||||||
/* Literal Constants */
|
|
||||||
Letter [a-zA-Z]
|
|
||||||
BooleanConstant true|false
|
|
||||||
StringConstant '([^\'\\]*|\\.)*'
|
|
||||||
DateTimeConstant ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
|
|
||||||
IntegerConstant -?[0-9]+
|
|
||||||
NullConstant 'null'
|
|
||||||
RealConstant -?[0-9]+\.[0-9]+
|
|
||||||
|
|
||||||
/* Relational Operators */
|
|
||||||
Equals =
|
|
||||||
OfType /
|
|
||||||
|
|
||||||
Identifier {Letter}({Letter}*[0-9]*_*)*
|
|
||||||
NamespacePrefix {Identifier}\.
|
|
||||||
TypeId {NamespacePrefix}*{Identifier}
|
|
||||||
|
|
||||||
/* Utility Tokens */
|
|
||||||
OpenParenthesis \(
|
|
||||||
CloseParenthesis \)
|
|
||||||
Comma ,
|
|
||||||
QuoteOpen \[
|
|
||||||
QuoteClose \]
|
|
||||||
|
|
||||||
/* Comments */
|
|
||||||
LineComment \/\/[^\n\r]*(\n|\r|\r\n)
|
|
||||||
|
|
||||||
%%
|
|
||||||
|
|
||||||
{Equals} {
|
|
||||||
return (int)ColumnExpressionToken.EQUALS;
|
|
||||||
}
|
|
||||||
|
|
||||||
{OfType} {
|
|
||||||
return (int)ColumnExpressionToken.OF_TYPE;
|
|
||||||
}
|
|
||||||
|
|
||||||
{StringConstant} {
|
|
||||||
yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
|
|
||||||
return (int)ColumnExpressionToken.STRING_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{BooleanConstant} {
|
|
||||||
yylval.Boolean = yytext.ToLower() switch {
|
|
||||||
"true" => true,
|
|
||||||
"false" => false,
|
|
||||||
_ => throw new Exception(
|
|
||||||
string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
|
|
||||||
};
|
|
||||||
return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{DateTimeConstant} {
|
|
||||||
yylval.DateTime = DateTime.Parse(yytext);
|
|
||||||
return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{IntegerConstant} {
|
|
||||||
yylval.Integer = int.Parse(yytext);
|
|
||||||
return (int)ColumnExpressionToken.INTEGER_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{RealConstant} {
|
|
||||||
yylval.Real = double.Parse(yytext);
|
|
||||||
return (int)ColumnExpressionToken.REAL_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{Identifier} {
|
|
||||||
yylval.Identifier = yytext;
|
|
||||||
return (int)ColumnExpressionToken.IDENTIFIER;
|
|
||||||
}
|
|
||||||
|
|
||||||
{TypeId} {
|
|
||||||
yylval.TypeId = yytext;
|
|
||||||
return (int)ColumnExpressionToken.TYPE_ID;
|
|
||||||
}
|
|
||||||
|
|
||||||
%%
|
|
||||||
@@ -1,720 +0,0 @@
|
|||||||
// ==============================================================
|
|
||||||
// <auto-generated>
|
|
||||||
// This code automatically produced from an embedded resource.
|
|
||||||
// Do not edit this file, or it will become incompatible with
|
|
||||||
// the specification from which it was generated.
|
|
||||||
// </auto-generated>
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Runtime.Serialization;
|
|
||||||
using System.Globalization;
|
|
||||||
|
|
||||||
namespace QUT.GplexBuffers
|
|
||||||
{
|
|
||||||
// Code copied from GPLEX embedded resource
|
|
||||||
[Serializable]
|
|
||||||
internal class BufferException : Exception
|
|
||||||
{
|
|
||||||
public BufferException() { }
|
|
||||||
public BufferException(string message) : base(message) { }
|
|
||||||
public BufferException(string message, Exception innerException)
|
|
||||||
: base(message, innerException) { }
|
|
||||||
protected BufferException(SerializationInfo info, StreamingContext context)
|
|
||||||
: base(info, context) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
internal abstract class ScanBuff
|
|
||||||
{
|
|
||||||
private string fileNm;
|
|
||||||
|
|
||||||
public const int EndOfFile = -1;
|
|
||||||
public const int UnicodeReplacementChar = 0xFFFD;
|
|
||||||
|
|
||||||
public bool IsFile { get { return (fileNm != null); } }
|
|
||||||
public string FileName { get { return fileNm; } set { fileNm = value; } }
|
|
||||||
|
|
||||||
public abstract int Pos { get; set; }
|
|
||||||
public abstract int Read();
|
|
||||||
public virtual void Mark() { }
|
|
||||||
|
|
||||||
public abstract string GetString(int begin, int limit);
|
|
||||||
|
|
||||||
public static ScanBuff GetBuffer(string source)
|
|
||||||
{
|
|
||||||
return new StringBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ScanBuff GetBuffer(IList<string> source)
|
|
||||||
{
|
|
||||||
return new LineBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!NOFILES)
|
|
||||||
public static ScanBuff GetBuffer(Stream source)
|
|
||||||
{
|
|
||||||
return new BuildBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
return new BuildBuffer(source, fallbackCodePage);
|
|
||||||
}
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endif // !NOFILES
|
|
||||||
}
|
|
||||||
|
|
||||||
#region Buffer classes
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// ===== Definitions for various ScanBuff derived classes ====
|
|
||||||
// ==============================================================
|
|
||||||
// =============== String input ================
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class reads characters from a single string as
|
|
||||||
/// required, for example, by Visual Studio language services
|
|
||||||
/// </summary>
|
|
||||||
sealed class StringBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
string str; // input buffer
|
|
||||||
int bPos; // current position in buffer
|
|
||||||
int sLen;
|
|
||||||
|
|
||||||
public StringBuffer(string source)
|
|
||||||
{
|
|
||||||
this.str = source;
|
|
||||||
this.sLen = source.Length;
|
|
||||||
this.FileName = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
if (bPos < sLen) return str[bPos++];
|
|
||||||
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
|
|
||||||
else { bPos++; return EndOfFile; } // two strikes and you're out!
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
// "limit" can be greater than sLen with the BABEL
|
|
||||||
// option set. Read returns a "virtual" EOL if
|
|
||||||
// an attempt is made to read past the end of the
|
|
||||||
// string buffer. Without the guard any attempt
|
|
||||||
// to fetch yytext for a token that includes the
|
|
||||||
// EOL will throw an index exception.
|
|
||||||
if (limit > sLen) limit = sLen;
|
|
||||||
if (limit <= begin) return "";
|
|
||||||
else return str.Substring(begin, limit - begin);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return bPos; }
|
|
||||||
set { bPos = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() { return "StringBuffer"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// The LineBuff class contributed by Nigel Horspool,
|
|
||||||
// nigelh@cs.uvic.cs
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
sealed class LineBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
IList<string> line; // list of source lines from a file
|
|
||||||
int numLines; // number of strings in line list
|
|
||||||
string curLine; // current line in that list
|
|
||||||
int cLine; // index of current line in the list
|
|
||||||
int curLen; // length of current line
|
|
||||||
int curLineStart; // position of line start in whole file
|
|
||||||
int curLineEnd; // position of line end in whole file
|
|
||||||
int maxPos; // max position ever visited in whole file
|
|
||||||
int cPos; // ordinal number of code in source
|
|
||||||
|
|
||||||
// Constructed from a list of strings, one per source line.
|
|
||||||
// The lines have had trailing '\n' characters removed.
|
|
||||||
public LineBuffer(IList<string> lineList)
|
|
||||||
{
|
|
||||||
line = lineList;
|
|
||||||
numLines = line.Count;
|
|
||||||
cPos = curLineStart = 0;
|
|
||||||
curLine = (numLines > 0 ? line[0] : "");
|
|
||||||
maxPos = curLineEnd = curLen = curLine.Length;
|
|
||||||
cLine = 1;
|
|
||||||
FileName = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
if (cPos < curLineEnd)
|
|
||||||
return curLine[cPos++ - curLineStart];
|
|
||||||
if (cPos++ == curLineEnd)
|
|
||||||
return '\n';
|
|
||||||
if (cLine >= numLines)
|
|
||||||
return EndOfFile;
|
|
||||||
curLine = line[cLine];
|
|
||||||
curLen = curLine.Length;
|
|
||||||
curLineStart = curLineEnd + 1;
|
|
||||||
curLineEnd = curLineStart + curLen;
|
|
||||||
if (curLineEnd > maxPos)
|
|
||||||
maxPos = curLineEnd;
|
|
||||||
cLine++;
|
|
||||||
return curLen > 0 ? curLine[0] : '\n';
|
|
||||||
}
|
|
||||||
|
|
||||||
// To speed up searches for the line containing a position
|
|
||||||
private int cachedPosition;
|
|
||||||
private int cachedIxdex;
|
|
||||||
private int cachedLineStart;
|
|
||||||
|
|
||||||
// Given a position pos within the entire source, the results are
|
|
||||||
// ix -- the index of the containing line
|
|
||||||
// lstart -- the position of the first character on that line
|
|
||||||
private void findIndex(int pos, out int ix, out int lstart)
|
|
||||||
{
|
|
||||||
if (pos >= cachedPosition)
|
|
||||||
{
|
|
||||||
ix = cachedIxdex; lstart = cachedLineStart;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ix = lstart = 0;
|
|
||||||
}
|
|
||||||
while (ix < numLines)
|
|
||||||
{
|
|
||||||
int len = line[ix].Length + 1;
|
|
||||||
if (pos < lstart + len) break;
|
|
||||||
lstart += len;
|
|
||||||
ix++;
|
|
||||||
}
|
|
||||||
cachedPosition = pos;
|
|
||||||
cachedIxdex = ix;
|
|
||||||
cachedLineStart = lstart;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
if (begin >= maxPos || limit <= begin) return "";
|
|
||||||
int endIx, begIx, endLineStart, begLineStart;
|
|
||||||
findIndex(begin, out begIx, out begLineStart);
|
|
||||||
int begCol = begin - begLineStart;
|
|
||||||
findIndex(limit, out endIx, out endLineStart);
|
|
||||||
int endCol = limit - endLineStart;
|
|
||||||
string s = line[begIx];
|
|
||||||
if (begIx == endIx)
|
|
||||||
{
|
|
||||||
// the usual case, substring all on one line
|
|
||||||
return (endCol <= s.Length) ?
|
|
||||||
s.Substring(begCol, endCol - begCol)
|
|
||||||
: s.Substring(begCol) + "\n";
|
|
||||||
}
|
|
||||||
// the string spans multiple lines, yuk!
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
if (begCol < s.Length)
|
|
||||||
sb.Append(s.Substring(begCol));
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
sb.Append("\n");
|
|
||||||
s = line[++begIx];
|
|
||||||
if (begIx >= endIx) break;
|
|
||||||
sb.Append(s);
|
|
||||||
}
|
|
||||||
if (endCol <= s.Length)
|
|
||||||
{
|
|
||||||
sb.Append(s.Substring(0, endCol));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
sb.Append(s);
|
|
||||||
sb.Append("\n");
|
|
||||||
}
|
|
||||||
return sb.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return cPos; }
|
|
||||||
set
|
|
||||||
{
|
|
||||||
cPos = value;
|
|
||||||
findIndex(cPos, out cLine, out curLineStart);
|
|
||||||
// cLine should be the *next* line after curLine.
|
|
||||||
curLine = (cLine < numLines ? line[cLine++] : "");
|
|
||||||
curLineEnd = curLineStart + curLine.Length;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() { return "LineBuffer"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!NOFILES)
|
|
||||||
// ==============================================================
|
|
||||||
// ===== class BuildBuff : for unicode text files ========
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
class BuildBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
// Double buffer for char stream.
|
|
||||||
class BufferElement
|
|
||||||
{
|
|
||||||
StringBuilder bldr = new StringBuilder();
|
|
||||||
StringBuilder next = new StringBuilder();
|
|
||||||
int minIx;
|
|
||||||
int maxIx;
|
|
||||||
int brkIx;
|
|
||||||
bool appendToNext;
|
|
||||||
|
|
||||||
internal BufferElement() { }
|
|
||||||
|
|
||||||
internal int MaxIndex { get { return maxIx; } }
|
|
||||||
// internal int MinIndex { get { return minIx; } }
|
|
||||||
|
|
||||||
internal char this[int index]
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
if (index < minIx || index >= maxIx)
|
|
||||||
throw new BufferException("Index was outside data buffer");
|
|
||||||
else if (index < brkIx)
|
|
||||||
return bldr[index - minIx];
|
|
||||||
else
|
|
||||||
return next[index - brkIx];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Append(char[] block, int count)
|
|
||||||
{
|
|
||||||
maxIx += count;
|
|
||||||
if (appendToNext)
|
|
||||||
this.next.Append(block, 0, count);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
this.bldr.Append(block, 0, count);
|
|
||||||
brkIx = maxIx;
|
|
||||||
appendToNext = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal string GetString(int start, int limit)
|
|
||||||
{
|
|
||||||
if (limit <= start)
|
|
||||||
return "";
|
|
||||||
if (start >= minIx && limit <= maxIx)
|
|
||||||
if (limit < brkIx) // String entirely in bldr builder
|
|
||||||
return bldr.ToString(start - minIx, limit - start);
|
|
||||||
else if (start >= brkIx) // String entirely in next builder
|
|
||||||
return next.ToString(start - brkIx, limit - start);
|
|
||||||
else // Must do a string-concatenation
|
|
||||||
return
|
|
||||||
bldr.ToString(start - minIx, brkIx - start) +
|
|
||||||
next.ToString(0, limit - brkIx);
|
|
||||||
else
|
|
||||||
throw new BufferException("String was outside data buffer");
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Mark(int limit)
|
|
||||||
{
|
|
||||||
if (limit > brkIx + 16) // Rotate blocks
|
|
||||||
{
|
|
||||||
StringBuilder temp = bldr;
|
|
||||||
bldr = next;
|
|
||||||
next = temp;
|
|
||||||
next.Length = 0;
|
|
||||||
minIx = brkIx;
|
|
||||||
brkIx = maxIx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
BufferElement data = new BufferElement();
|
|
||||||
|
|
||||||
int bPos; // Postion index in the StringBuilder
|
|
||||||
BlockReader NextBlk; // Delegate that serves char-arrays;
|
|
||||||
|
|
||||||
private string EncodingName
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
StreamReader rdr = NextBlk.Target as StreamReader;
|
|
||||||
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public BuildBuffer(Stream stream)
|
|
||||||
{
|
|
||||||
FileStream fStrm = (stream as FileStream);
|
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
|
||||||
NextBlk = BlockReaderFactory.Raw(stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public BuildBuffer(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
FileStream fStrm = (stream as FileStream);
|
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
|
||||||
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Marks a conservative lower bound for the buffer,
|
|
||||||
/// allowing space to be reclaimed. If an application
|
|
||||||
/// needs to call GetString at arbitrary past locations
|
|
||||||
/// in the input stream, Mark() is not called.
|
|
||||||
/// </summary>
|
|
||||||
public override void Mark() { data.Mark(bPos - 2); }
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return bPos; }
|
|
||||||
set { bPos = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Read returns the ordinal number of the next char, or
|
|
||||||
/// EOF (-1) for an end of stream. Note that the next
|
|
||||||
/// code point may require *two* calls of Read().
|
|
||||||
/// </summary>
|
|
||||||
/// <returns></returns>
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
//
|
|
||||||
// Characters at positions
|
|
||||||
// [data.offset, data.offset + data.bldr.Length)
|
|
||||||
// are available in data.bldr.
|
|
||||||
//
|
|
||||||
if (bPos < data.MaxIndex)
|
|
||||||
{
|
|
||||||
// ch0 cannot be EOF
|
|
||||||
return (int)data[bPos++];
|
|
||||||
}
|
|
||||||
else // Read from underlying stream
|
|
||||||
{
|
|
||||||
// Experimental code, blocks of page size
|
|
||||||
char[] chrs = new char[4096];
|
|
||||||
int count = NextBlk(chrs, 0, 4096);
|
|
||||||
if (count == 0)
|
|
||||||
return EndOfFile;
|
|
||||||
else
|
|
||||||
{
|
|
||||||
data.Append(chrs, count);
|
|
||||||
return (int)data[bPos++];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
return data.GetString(begin, limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString()
|
|
||||||
{
|
|
||||||
return "StringBuilder buffer, encoding: " + this.EncodingName;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// =============== End ScanBuff-derived classes ==================
|
|
||||||
|
|
||||||
internal delegate int BlockReader(char[] block, int index, int number);
|
|
||||||
|
|
||||||
// A delegate factory, serving up a delegate that
|
|
||||||
// reads a block of characters from the underlying
|
|
||||||
// encoded stream, via a StreamReader object.
|
|
||||||
//
|
|
||||||
internal static class BlockReaderFactory
|
|
||||||
{
|
|
||||||
public static BlockReader Raw(Stream stream)
|
|
||||||
{
|
|
||||||
return delegate(char[] block, int index, int number)
|
|
||||||
{
|
|
||||||
byte[] b = new byte[number];
|
|
||||||
int count = stream.Read(b, 0, number);
|
|
||||||
int i = 0;
|
|
||||||
int j = index;
|
|
||||||
for (; i < count; i++, j++)
|
|
||||||
block[j] = (char)b[i];
|
|
||||||
return count;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static BlockReader Get(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
Encoding encoding;
|
|
||||||
int preamble = Preamble(stream);
|
|
||||||
|
|
||||||
if (preamble != 0) // There is a valid BOM here!
|
|
||||||
encoding = Encoding.GetEncoding(preamble);
|
|
||||||
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
|
|
||||||
return Raw(stream);
|
|
||||||
else if (fallbackCodePage != -2) // Anything but "guess"
|
|
||||||
encoding = Encoding.GetEncoding(fallbackCodePage);
|
|
||||||
else // This is the "guess" option
|
|
||||||
{
|
|
||||||
int guess = new Guesser(stream).GuessCodePage();
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (guess == -1) // ==> this is a 7-bit file
|
|
||||||
encoding = Encoding.ASCII;
|
|
||||||
else if (guess == 65001)
|
|
||||||
encoding = Encoding.UTF8;
|
|
||||||
else // ==> use the machine default
|
|
||||||
encoding = Encoding.Default;
|
|
||||||
}
|
|
||||||
StreamReader reader = new StreamReader(stream, encoding);
|
|
||||||
return reader.Read;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int Preamble(Stream stream)
|
|
||||||
{
|
|
||||||
int b0 = stream.ReadByte();
|
|
||||||
int b1 = stream.ReadByte();
|
|
||||||
|
|
||||||
if (b0 == 0xfe && b1 == 0xff)
|
|
||||||
return 1201; // UTF16BE
|
|
||||||
if (b0 == 0xff && b1 == 0xfe)
|
|
||||||
return 1200; // UTF16LE
|
|
||||||
|
|
||||||
int b2 = stream.ReadByte();
|
|
||||||
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
|
||||||
return 65001; // UTF8
|
|
||||||
//
|
|
||||||
// There is no unicode preamble, so we
|
|
||||||
// return denoter for the machine default.
|
|
||||||
//
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
}
|
|
||||||
#endif // !NOFILES
|
|
||||||
#endregion Buffer classes
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// ============ class CodePageHandling =============
|
|
||||||
// ==============================================================
|
|
||||||
#if (!NOFILES)
|
|
||||||
public static class CodePageHandling
|
|
||||||
{
|
|
||||||
public static int GetCodePage(string option)
|
|
||||||
{
|
|
||||||
string command = option.ToUpperInvariant();
|
|
||||||
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
|
|
||||||
command = command.Substring(9);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (command.Equals("RAW"))
|
|
||||||
return -1;
|
|
||||||
else if (command.Equals("GUESS"))
|
|
||||||
return -2;
|
|
||||||
else if (command.Equals("DEFAULT"))
|
|
||||||
return 0;
|
|
||||||
else if (char.IsDigit(command[0]))
|
|
||||||
return int.Parse(command, CultureInfo.InvariantCulture);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Encoding enc = Encoding.GetEncoding(command);
|
|
||||||
return enc.CodePage;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (FormatException)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine(
|
|
||||||
"Invalid format \"{0}\", using machine default", option);
|
|
||||||
}
|
|
||||||
catch (ArgumentException)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine(
|
|
||||||
"Unknown code page \"{0}\", using machine default", option);
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#region guesser
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
// ==============================================================
|
|
||||||
// ============ Encoding Guesser =============
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class provides a simple finite state automaton that
|
|
||||||
/// scans the file looking for (1) valid UTF-8 byte patterns,
|
|
||||||
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
|
|
||||||
/// The method then guesses whether it is UTF-8 or maybe some
|
|
||||||
/// local machine default encoding. This works well for the
|
|
||||||
/// various Latin encodings.
|
|
||||||
/// </summary>
|
|
||||||
internal class Guesser
|
|
||||||
{
|
|
||||||
ScanBuff buffer;
|
|
||||||
|
|
||||||
public int GuessCodePage() { return Scan(); }
|
|
||||||
|
|
||||||
const int maxAccept = 10;
|
|
||||||
const int initial = 0;
|
|
||||||
const int eofNum = 0;
|
|
||||||
const int goStart = -1;
|
|
||||||
const int INITIAL = 0;
|
|
||||||
const int EndToken = 0;
|
|
||||||
|
|
||||||
#region user code
|
|
||||||
/*
|
|
||||||
* Reads the bytes of a file to determine if it is
|
|
||||||
* UTF-8 or a single-byte code page file.
|
|
||||||
*/
|
|
||||||
public long utfX;
|
|
||||||
public long uppr;
|
|
||||||
#endregion user code
|
|
||||||
|
|
||||||
int state;
|
|
||||||
int currentStart = startState[0];
|
|
||||||
int code;
|
|
||||||
|
|
||||||
#region ScannerTables
|
|
||||||
static int[] startState = new int[] { 11, 0 };
|
|
||||||
|
|
||||||
#region CharacterMap
|
|
||||||
static sbyte[] map = new sbyte[256] {
|
|
||||||
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
||||||
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
static sbyte[][] nextState = new sbyte[][] {
|
|
||||||
new sbyte[] {0, 0, 0, 0, 0, 0},
|
|
||||||
new sbyte[] {-1, -1, 10, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, -1, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 8, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 5, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 6, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 7, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, -1, 9, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, 1, 2, 3, 4, 2}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
|
|
||||||
// Reason for suppression: cannot have self-reference in array initializer.
|
|
||||||
static Guesser()
|
|
||||||
{
|
|
||||||
nextState[7] = nextState[2];
|
|
||||||
nextState[9] = nextState[2];
|
|
||||||
nextState[10] = nextState[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
int NextState()
|
|
||||||
{
|
|
||||||
if (code == ScanBuff.EndOfFile)
|
|
||||||
return eofNum;
|
|
||||||
else
|
|
||||||
return nextState[state][map[code]];
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
public Guesser(System.IO.Stream file) { SetSource(file); }
|
|
||||||
|
|
||||||
public void SetSource(System.IO.Stream source)
|
|
||||||
{
|
|
||||||
this.buffer = new BuildBuffer(source);
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
|
|
||||||
int Scan()
|
|
||||||
{
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
int next;
|
|
||||||
state = currentStart;
|
|
||||||
while ((next = NextState()) == goStart)
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
while ((next = NextState()) > eofNum)
|
|
||||||
{
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
if (state <= maxAccept)
|
|
||||||
{
|
|
||||||
#region ActionSwitch
|
|
||||||
#pragma warning disable 162
|
|
||||||
switch (state)
|
|
||||||
{
|
|
||||||
case eofNum:
|
|
||||||
switch (currentStart)
|
|
||||||
{
|
|
||||||
case 11:
|
|
||||||
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
|
|
||||||
else if (uppr * 10 > utfX) return 0; /* default code page */
|
|
||||||
else return 65001; /* UTF-8 encoding */
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return EndToken;
|
|
||||||
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
|
|
||||||
case 2: // Recognized '{Upper128}', Shortest string "\x80"
|
|
||||||
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
|
|
||||||
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
|
|
||||||
uppr++;
|
|
||||||
break;
|
|
||||||
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
|
|
||||||
uppr += 3;
|
|
||||||
break;
|
|
||||||
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
|
|
||||||
utfX += 3;
|
|
||||||
break;
|
|
||||||
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
|
|
||||||
utfX += 2;
|
|
||||||
break;
|
|
||||||
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
|
|
||||||
utfX++;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
#pragma warning restore 162
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // end class Guesser
|
|
||||||
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endregion
|
|
||||||
#endif // !NOFILES
|
|
||||||
|
|
||||||
// End of code copied from embedded resource
|
|
||||||
}
|
|
||||||
@@ -1,884 +0,0 @@
|
|||||||
// ==============================================================
|
|
||||||
// <auto-generated>
|
|
||||||
// This code automatically produced from an embedded resource.
|
|
||||||
// Do not edit this file, or it will become incompatible with
|
|
||||||
// the specification from which it was generated.
|
|
||||||
// </auto-generated>
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
// Gardens Point Parser Generator
|
|
||||||
// Copyright (c) Wayne Kelly, QUT 2005-2010
|
|
||||||
// (see accompanying GPPGcopyright.rtf)
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Text;
|
|
||||||
using System.Globalization;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Runtime.Serialization;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
|
||||||
namespace QUT.Gppg {
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract class for GPPG shift-reduce parsers.
|
|
||||||
/// Parsers generated by GPPG derive from this base
|
|
||||||
/// class, overriding the abstract Initialize() and
|
|
||||||
/// DoAction() methods.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TValue">Semantic value type</typeparam>
|
|
||||||
/// <typeparam name="TSpan">Location type</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public abstract class ShiftReduceParser<TValue, TSpan>
|
|
||||||
#else
|
|
||||||
internal abstract class ShiftReduceParser<TValue, TSpan>
|
|
||||||
#endif
|
|
||||||
where TSpan : IMerge<TSpan>, new() {
|
|
||||||
private AbstractScanner<TValue, TSpan> scanner;
|
|
||||||
/// <summary>
|
|
||||||
/// The abstract scanner for this parser.
|
|
||||||
/// </summary>
|
|
||||||
protected AbstractScanner<TValue, TSpan> Scanner {
|
|
||||||
get { return scanner; }
|
|
||||||
set { scanner = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Constructor for base class
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="scanner">Scanner instance for this parser</param>
|
|
||||||
protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
|
|
||||||
this.scanner = scanner;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// TECHNICAL EXPLANATION.
|
|
||||||
// Why the next two fields are not exposed via properties.
|
|
||||||
// ==============================================================
|
|
||||||
// These fields are of the generic parameter types, and are
|
|
||||||
// frequently instantiated as struct types in derived classes.
|
|
||||||
// Semantic actions are defined in the derived classes and refer
|
|
||||||
// to instance fields of these structs. In such cases the code
|
|
||||||
// "get_CurrentSemanticValue().myField = blah;" will fail since
|
|
||||||
// the getter pushes the value of the field, not the reference.
|
|
||||||
// So, in the presence of properties, gppg would need to encode
|
|
||||||
// such field accesses as ...
|
|
||||||
// "tmp = get_CurrentSemanticValue(); // Fetch value
|
|
||||||
// tmp.myField = blah; // update
|
|
||||||
// set_CurrentSemanticValue(tmp); " // Write update back.
|
|
||||||
// There is no issue if TValue is restricted to be a ref type.
|
|
||||||
// The same explanation applies to scanner.yylval.
|
|
||||||
// ==============================================================
|
|
||||||
/// <summary>
|
|
||||||
/// The current value of the "$$" symbolic variable in the parser
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
protected TValue CurrentSemanticValue;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The current value of the "@$" symbolic variable in the parser
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
protected TSpan CurrentLocationSpan;
|
|
||||||
protected int NextToken;
|
|
||||||
|
|
||||||
private TSpan LastSpan;
|
|
||||||
private State FsaState;
|
|
||||||
private bool recovering;
|
|
||||||
private int tokensSinceLastError;
|
|
||||||
|
|
||||||
private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
|
|
||||||
private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
|
|
||||||
private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The stack of semantic value (YYSTYPE) values.
|
|
||||||
/// </summary>
|
|
||||||
protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The stack of location value (YYLTYPE) varlues.
|
|
||||||
/// </summary>
|
|
||||||
protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
|
|
||||||
|
|
||||||
private int errorToken;
|
|
||||||
private int endOfFileToken;
|
|
||||||
private string[] nonTerminals;
|
|
||||||
private State[] states;
|
|
||||||
private Rule[] rules;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes
|
|
||||||
/// to insert the rule list into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="rules">The array of Rule objects</param>
|
|
||||||
protected void InitRules( Rule[] rules ) { this.rules = rules; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes to
|
|
||||||
/// insert the states table into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="states">The pre-initialized states table</param>
|
|
||||||
protected void InitStates( State[] states ) { this.states = states; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// OBSOLETE FOR VERSION 1.4.0
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="size"></param>
|
|
||||||
protected void InitStateTable( int size ) { states = new State[size]; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes
|
|
||||||
/// to insert the special value for the error and EOF tokens.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="err">The error state ordinal</param>
|
|
||||||
/// <param name="end">The EOF stat ordinal</param>
|
|
||||||
protected void InitSpecialTokens( int err, int end ) {
|
|
||||||
errorToken = err;
|
|
||||||
endOfFileToken = end;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes to
|
|
||||||
/// insert the non-terminal symbol names into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="names">Non-terminal symbol names</param>
|
|
||||||
protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
|
|
||||||
|
|
||||||
#region YYAbort, YYAccept etcetera.
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class AcceptException : Exception {
|
|
||||||
internal AcceptException() { }
|
|
||||||
protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class AbortException : Exception {
|
|
||||||
internal AbortException() { }
|
|
||||||
protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class ErrorException : Exception {
|
|
||||||
internal ErrorException() { }
|
|
||||||
protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
// The following methods are only called from within
|
|
||||||
// a semantic action. The thrown exceptions can never
|
|
||||||
// propagate outside the ShiftReduceParser class in
|
|
||||||
// which they are nested.
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning "true"
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYAccept() { throw new AcceptException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning "false"
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYAbort() { throw new AbortException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning
|
|
||||||
/// "false" if error recovery fails.
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYError() { throw new ErrorException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Check if parser in error recovery state.
|
|
||||||
/// </summary>
|
|
||||||
protected bool YYRecovering { get { return recovering; } }
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract base method. ShiftReduceParser calls this
|
|
||||||
/// to initialize the base class data structures. Concrete
|
|
||||||
/// parser classes must override this method.
|
|
||||||
/// </summary>
|
|
||||||
protected abstract void Initialize();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Main entry point of the Shift-Reduce Parser.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if parse succeeds, else false for
|
|
||||||
/// unrecoverable errors</returns>
|
|
||||||
public bool Parse() {
|
|
||||||
Initialize(); // allow derived classes to instantiate rules, states and nonTerminals
|
|
||||||
|
|
||||||
NextToken = 0;
|
|
||||||
FsaState = states[0];
|
|
||||||
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
valueStack.Push( CurrentSemanticValue );
|
|
||||||
LocationStack.Push( CurrentLocationSpan );
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
|
|
||||||
DisplayStack();
|
|
||||||
#endif
|
|
||||||
int action = FsaState.defaultAction;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable != null) {
|
|
||||||
if (NextToken == 0) {
|
|
||||||
// We save the last token span, so that the location span
|
|
||||||
// of production right hand sides that begin or end with a
|
|
||||||
// nullable production will be correct.
|
|
||||||
LastSpan = scanner.yylloc;
|
|
||||||
NextToken = scanner.yylex();
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
else
|
|
||||||
Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
|
|
||||||
#endif
|
|
||||||
if (FsaState.ParserTable.ContainsKey( NextToken ))
|
|
||||||
action = FsaState.ParserTable[NextToken];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (action > 0) // shift
|
|
||||||
{
|
|
||||||
Shift( action );
|
|
||||||
}
|
|
||||||
else if (action < 0) // reduce
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
Reduce( -action );
|
|
||||||
if (action == -1) // accept
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (AbortException) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
catch (AcceptException) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (ErrorException) {
|
|
||||||
if (!ErrorRecovery())
|
|
||||||
return false;
|
|
||||||
else
|
|
||||||
throw; // Rethrow x, preserving information.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (action == 0 && !ErrorRecovery()) // error
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Shift( int stateIndex ) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
FsaState = states[stateIndex];
|
|
||||||
|
|
||||||
valueStack.Push( scanner.yylval );
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
LocationStack.Push( scanner.yylloc );
|
|
||||||
|
|
||||||
if (recovering) {
|
|
||||||
if (NextToken != errorToken)
|
|
||||||
tokensSinceLastError++;
|
|
||||||
|
|
||||||
if (tokensSinceLastError > 5)
|
|
||||||
recovering = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (NextToken != endOfFileToken)
|
|
||||||
NextToken = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Reduce( int ruleNumber ) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
DisplayRule(ruleNumber);
|
|
||||||
#endif
|
|
||||||
Rule rule = rules[ruleNumber];
|
|
||||||
int rhLen = rule.RightHandSide.Length;
|
|
||||||
//
|
|
||||||
// Default actions for unit productions.
|
|
||||||
//
|
|
||||||
if (rhLen == 1) {
|
|
||||||
CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1;
|
|
||||||
CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
|
|
||||||
}
|
|
||||||
else if (rhLen == 0) {
|
|
||||||
// Create a new blank value.
|
|
||||||
// Explicit semantic action may mutate this value
|
|
||||||
CurrentSemanticValue = default( TValue );
|
|
||||||
// The location span for an empty production will start with the
|
|
||||||
// beginning of the next lexeme, and end with the finish of the
|
|
||||||
// previous lexeme. This gives the correct behaviour when this
|
|
||||||
// nonsense value is used in later Merge operations.
|
|
||||||
CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
|
|
||||||
scanner.yylloc.Merge( LastSpan ) :
|
|
||||||
default( TSpan ));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
// Default action: $$ = $1;
|
|
||||||
CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
|
|
||||||
// Default action "@$ = @1.Merge(@N)" for location info.
|
|
||||||
TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
|
|
||||||
TSpan atN = LocationStack[LocationStack.Depth - 1];
|
|
||||||
CurrentLocationSpan =
|
|
||||||
((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
|
|
||||||
}
|
|
||||||
|
|
||||||
DoAction( ruleNumber );
|
|
||||||
|
|
||||||
for (int i = 0; i < rule.RightHandSide.Length; i++) {
|
|
||||||
StateStack.Pop();
|
|
||||||
valueStack.Pop();
|
|
||||||
LocationStack.Pop();
|
|
||||||
}
|
|
||||||
FsaState = StateStack.TopElement();
|
|
||||||
|
|
||||||
if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
|
|
||||||
FsaState = states[FsaState.Goto[rule.LeftHandSide]];
|
|
||||||
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
valueStack.Push( CurrentSemanticValue );
|
|
||||||
LocationStack.Push( CurrentLocationSpan );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Execute the selected action from array.
|
|
||||||
/// Must be overriden in derived classes.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actionNumber">Index of the action to perform</param>
|
|
||||||
protected abstract void DoAction( int actionNumber );
|
|
||||||
|
|
||||||
private bool ErrorRecovery() {
|
|
||||||
bool discard;
|
|
||||||
|
|
||||||
if (!recovering) // if not recovering from previous error
|
|
||||||
ReportError();
|
|
||||||
|
|
||||||
if (!FindErrorRecoveryState())
|
|
||||||
return false;
|
|
||||||
//
|
|
||||||
// The interim fix for the "looping in error recovery"
|
|
||||||
// artifact involved moving the setting of the recovering
|
|
||||||
// bool until after invalid tokens have been discarded.
|
|
||||||
//
|
|
||||||
ShiftErrorToken();
|
|
||||||
discard = DiscardInvalidTokens();
|
|
||||||
recovering = true;
|
|
||||||
tokensSinceLastError = 0;
|
|
||||||
return discard;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ReportError() {
|
|
||||||
StringBuilder errorMsg = new StringBuilder();
|
|
||||||
errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
|
|
||||||
|
|
||||||
if (FsaState.ParserTable.Count < 7) {
|
|
||||||
bool first = true;
|
|
||||||
foreach (int terminal in FsaState.ParserTable.Keys) {
|
|
||||||
if (first)
|
|
||||||
errorMsg.Append( ", expecting " );
|
|
||||||
else
|
|
||||||
errorMsg.Append( ", or " );
|
|
||||||
|
|
||||||
errorMsg.Append( TerminalToString( terminal ) );
|
|
||||||
first = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
scanner.yyerror( errorMsg.ToString() );
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ShiftErrorToken() {
|
|
||||||
int old_next = NextToken;
|
|
||||||
NextToken = errorToken;
|
|
||||||
|
|
||||||
Shift( FsaState.ParserTable[NextToken] );
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
|
|
||||||
#endif
|
|
||||||
NextToken = old_next;
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool FindErrorRecoveryState() {
|
|
||||||
while (true) // pop states until one found that accepts error token
|
|
||||||
{
|
|
||||||
if (FsaState.ParserTable != null &&
|
|
||||||
FsaState.ParserTable.ContainsKey( errorToken ) &&
|
|
||||||
FsaState.ParserTable[errorToken] > 0) // shift
|
|
||||||
return true;
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
|
|
||||||
#endif
|
|
||||||
StateStack.Pop();
|
|
||||||
valueStack.Pop();
|
|
||||||
LocationStack.Pop();
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
DisplayStack();
|
|
||||||
#endif
|
|
||||||
if (StateStack.IsEmpty()) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
|
|
||||||
#endif
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
FsaState = StateStack.TopElement();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool DiscardInvalidTokens() {
|
|
||||||
|
|
||||||
int action = FsaState.defaultAction;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable != null) {
|
|
||||||
// Discard tokens until find one that works ...
|
|
||||||
while (true) {
|
|
||||||
if (NextToken == 0) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.Write("Reading a token: ");
|
|
||||||
#endif
|
|
||||||
NextToken = scanner.yylex();
|
|
||||||
}
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
if (NextToken == endOfFileToken)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable.ContainsKey( NextToken ))
|
|
||||||
action = FsaState.ParserTable[NextToken];
|
|
||||||
|
|
||||||
if (action != 0)
|
|
||||||
return true;
|
|
||||||
else {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
NextToken = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (recovering && tokensSinceLastError == 0) {
|
|
||||||
//
|
|
||||||
// Boolean recovering is not set until after the first
|
|
||||||
// error token has been shifted. Thus if we get back
|
|
||||||
// here with recovering set and no tokens read we are
|
|
||||||
// looping on the same error recovery action. This
|
|
||||||
// happens if current_state.ParserTable is null because
|
|
||||||
// the state has an LR(0) reduction, but not all
|
|
||||||
// lookahead tokens are valid. This only occurs for
|
|
||||||
// error productions that *end* on "error".
|
|
||||||
//
|
|
||||||
// This action discards tokens one at a time until
|
|
||||||
// the looping stops. Another attack would be to always
|
|
||||||
// use the LALR(1) table if a production ends on "error"
|
|
||||||
//
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
if (NextToken == endOfFileToken)
|
|
||||||
return false;
|
|
||||||
NextToken = 0;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Traditional YACC method. Discards the next input token.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
protected void yyclearin() { NextToken = 0; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Tradional YACC method. Clear the "recovering" flag.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
protected void yyerrok() { recovering = false; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// OBSOLETE FOR VERSION 1.4.0
|
|
||||||
/// Method used by derived types to insert new
|
|
||||||
/// state instances in the "states" array.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="stateNumber">index of the state</param>
|
|
||||||
/// <param name="state">data for the state</param>
|
|
||||||
protected void AddState( int stateNumber, State state ) {
|
|
||||||
states[stateNumber] = state;
|
|
||||||
state.number = stateNumber;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayStack() {
|
|
||||||
Console.Error.Write( "State stack is now:" );
|
|
||||||
for (int i = 0; i < StateStack.Depth; i++)
|
|
||||||
Console.Error.Write( " {0}", StateStack[i].number );
|
|
||||||
Console.Error.WriteLine();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayRule( int ruleNumber ) {
|
|
||||||
Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
|
|
||||||
DisplayProduction( rules[ruleNumber] );
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayProduction( Rule rule ) {
|
|
||||||
if (rule.RightHandSide.Length == 0)
|
|
||||||
Console.Error.Write( "/* empty */ " );
|
|
||||||
else
|
|
||||||
foreach (int symbol in rule.RightHandSide)
|
|
||||||
Console.Error.Write( "{0} ", SymbolToString( symbol ) );
|
|
||||||
|
|
||||||
Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract state class naming terminal symbols.
|
|
||||||
/// This is overridden by derived classes with the
|
|
||||||
/// name (or alias) to be used in error messages.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="terminal">The terminal ordinal</param>
|
|
||||||
/// <returns></returns>
|
|
||||||
protected abstract string TerminalToString( int terminal );
|
|
||||||
|
|
||||||
private string SymbolToString( int symbol ) {
|
|
||||||
if (symbol < 0)
|
|
||||||
return nonTerminals[-symbol - 1];
|
|
||||||
else
|
|
||||||
return TerminalToString( symbol );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Return text representation of argument character
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The character to convert</param>
|
|
||||||
/// <returns>String representation of the character</returns>
|
|
||||||
protected static string CharToString( char input ) {
|
|
||||||
switch (input) {
|
|
||||||
case '\a': return @"'\a'";
|
|
||||||
case '\b': return @"'\b'";
|
|
||||||
case '\f': return @"'\f'";
|
|
||||||
case '\n': return @"'\n'";
|
|
||||||
case '\r': return @"'\r'";
|
|
||||||
case '\t': return @"'\t'";
|
|
||||||
case '\v': return @"'\v'";
|
|
||||||
case '\0': return @"'\0'";
|
|
||||||
default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Classes implementing this interface must supply a
|
|
||||||
/// method that merges two location objects to return
|
|
||||||
/// a new object of the same type.
|
|
||||||
/// GPPG-generated parsers have the default location
|
|
||||||
/// action equivalent to "@$ = @1.Merge(@N);" where N
|
|
||||||
/// is the right-hand-side length of the production.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TSpan">The Location type</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public interface IMerge<TSpan>
|
|
||||||
#else
|
|
||||||
internal interface IMerge<TSpan>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Interface method that creates a location object from
|
|
||||||
/// the current and last object. Typically used to create
|
|
||||||
/// a location object extending from the start of the @1
|
|
||||||
/// object to the end of the @N object.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="last">The lexically last object to merge</param>
|
|
||||||
/// <returns>The merged location object</returns>
|
|
||||||
TSpan Merge( TSpan last );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This is the default class that carries location
|
|
||||||
/// information from the scanner to the parser.
|
|
||||||
/// If you don't declare "%YYLTYPE Foo" the parser
|
|
||||||
/// will expect to deal with this type.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class LexLocation : IMerge<LexLocation>
|
|
||||||
#else
|
|
||||||
[SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
|
|
||||||
internal class LexLocation : IMerge<LexLocation>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
private int startLine; // start line
|
|
||||||
private int startColumn; // start column
|
|
||||||
private int endLine; // end line
|
|
||||||
private int endColumn; // end column
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The line at which the text span starts.
|
|
||||||
/// </summary>
|
|
||||||
public int StartLine { get { return startLine; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The column at which the text span starts.
|
|
||||||
/// </summary>
|
|
||||||
public int StartColumn { get { return startColumn; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The line on which the text span ends.
|
|
||||||
/// </summary>
|
|
||||||
public int EndLine { get { return endLine; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The column of the first character
|
|
||||||
/// beyond the end of the text span.
|
|
||||||
/// </summary>
|
|
||||||
public int EndColumn { get { return endColumn; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Default no-arg constructor.
|
|
||||||
/// </summary>
|
|
||||||
public LexLocation() { }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Constructor for text-span with given start and end.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="sl">start line</param>
|
|
||||||
/// <param name="sc">start column</param>
|
|
||||||
/// <param name="el">end line </param>
|
|
||||||
/// <param name="ec">end column</param>
|
|
||||||
public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Create a text location which spans from the
|
|
||||||
/// start of "this" to the end of the argument "last"
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="last">The last location in the result span</param>
|
|
||||||
/// <returns>The merged span</returns>
|
|
||||||
public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract scanner class that GPPG expects its scanners to
|
|
||||||
/// extend.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
|
|
||||||
/// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public abstract class AbstractScanner<TValue, TSpan>
|
|
||||||
#else
|
|
||||||
internal abstract class AbstractScanner<TValue, TSpan>
|
|
||||||
#endif
|
|
||||||
where TSpan : IMerge<TSpan> {
|
|
||||||
/// <summary>
|
|
||||||
/// Lexical value optionally set by the scanner. The value
|
|
||||||
/// is of the %YYSTYPE type declared in the parser spec.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
// A field must be declared for this value of parametric type,
|
|
||||||
// since it may be instantiated by a value struct. If it were
|
|
||||||
// implemented as a property, machine generated code in derived
|
|
||||||
// types would not be able to select on the returned value.
|
|
||||||
#pragma warning disable 649
|
|
||||||
public TValue yylval; // Lexical value: set by scanner
|
|
||||||
#pragma warning restore 649
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Current scanner location property. The value is of the
|
|
||||||
/// type declared by %YYLTYPE in the parser specification.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public virtual TSpan yylloc {
|
|
||||||
get { return default( TSpan ); } // Empty implementation allowing
|
|
||||||
set { /* skip */ } // yylloc to be ignored entirely.
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Main call point for LEX-like scanners. Returns an int
|
|
||||||
/// corresponding to the token recognized by the scanner.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>An int corresponding to the token</returns>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public abstract int yylex();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Traditional error reporting provided by LEX-like scanners
|
|
||||||
/// to their YACC-like clients.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="format">Message format string</param>
|
|
||||||
/// <param name="args">Optional array of args</param>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public virtual void yyerror( string format, params object[] args ) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Encapsulated state for the parser.
|
|
||||||
/// Opaque to users, visible to the tool-generated code.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class State
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The number of states in the automaton.
|
|
||||||
/// </summary>
|
|
||||||
public int number;
|
|
||||||
#else
|
|
||||||
internal class State {
|
|
||||||
/// <summary>
|
|
||||||
/// The index of this state in the states array.
|
|
||||||
/// </summary>
|
|
||||||
internal int number;
|
|
||||||
#endif
|
|
||||||
internal Dictionary<int, int> ParserTable; // Terminal -> ParseAction
|
|
||||||
internal Dictionary<int, int> Goto; // NonTerminal -> State;
|
|
||||||
internal int defaultAction; // = 0; // ParseAction
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// State transition data for this state. Pairs of elements of the
|
|
||||||
/// goto array associate symbol ordinals with next state indices.
|
|
||||||
/// The actions array is passed to another constructor.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actions">The action list</param>c
|
|
||||||
/// <param name="goToList">Next state data</param>
|
|
||||||
public State( int[] actions, int[] goToList )
|
|
||||||
: this( actions ) {
|
|
||||||
Goto = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < goToList.Length; i += 2)
|
|
||||||
Goto.Add( goToList[i], goToList[i + 1] );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Action data for this state. Pairs of elements of the
|
|
||||||
/// action array associate action ordinals with each of
|
|
||||||
/// those symbols that have actions in the current state.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actions">The action array</param>
|
|
||||||
public State( int[] actions ) {
|
|
||||||
ParserTable = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < actions.Length; i += 2)
|
|
||||||
ParserTable.Add( actions[i], actions[i + 1] );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set the default action for this state.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="defaultAction">Ordinal of the default action</param>
|
|
||||||
public State( int defaultAction ) {
|
|
||||||
this.defaultAction = defaultAction;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set the default action and the state transition table.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="defaultAction">The default action</param>
|
|
||||||
/// <param name="goToList">Transitions from this state</param>
|
|
||||||
public State( int defaultAction, int[] goToList )
|
|
||||||
: this( defaultAction ) {
|
|
||||||
Goto = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < goToList.Length; i += 2)
|
|
||||||
Goto.Add( goToList[i], goToList[i + 1] );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rule representation at runtime.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class Rule
|
|
||||||
#else
|
|
||||||
internal class Rule
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
internal int LeftHandSide; // symbol
|
|
||||||
internal int[] RightHandSide; // symbols
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rule constructor. This holds the ordinal of
|
|
||||||
/// the left hand side symbol, and the list of
|
|
||||||
/// right hand side symbols, in lexical order.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="left">The LHS non-terminal</param>
|
|
||||||
/// <param name="right">The RHS symbols, in lexical order</param>
|
|
||||||
public Rule( int left, int[] right ) {
|
|
||||||
this.LeftHandSide = left;
|
|
||||||
this.RightHandSide = right;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Stack utility for the shift-reduce parser.
|
|
||||||
/// GPPG parsers have three instances:
|
|
||||||
/// (1) The parser state stack, T = QUT.Gppg.State,
|
|
||||||
/// (2) The semantic value stack, T = TValue,
|
|
||||||
/// (3) The location stack, T = TSpan.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="T"></typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class PushdownPrefixState<T>
|
|
||||||
#else
|
|
||||||
internal class PushdownPrefixState<T>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
// Note that we cannot use the BCL Stack<T> class
|
|
||||||
// here as derived types need to index into stacks.
|
|
||||||
//
|
|
||||||
private T[] array = new T[8];
|
|
||||||
private int tos = 0;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Indexer for values of the stack below the top.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="index">index of the element, starting from the bottom</param>
|
|
||||||
/// <returns>the selected element</returns>
|
|
||||||
public T this[int index] { get { return array[index]; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The current depth of the stack.
|
|
||||||
/// </summary>
|
|
||||||
public int Depth { get { return tos; } }
|
|
||||||
|
|
||||||
internal void Push( T value ) {
|
|
||||||
if (tos >= array.Length) {
|
|
||||||
T[] newarray = new T[array.Length * 2];
|
|
||||||
System.Array.Copy( array, newarray, tos );
|
|
||||||
array = newarray;
|
|
||||||
}
|
|
||||||
array[tos++] = value;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal T Pop() {
|
|
||||||
T rslt = array[--tos];
|
|
||||||
array[tos] = default( T );
|
|
||||||
return rslt;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal T TopElement() { return array[tos - 1]; }
|
|
||||||
|
|
||||||
internal bool IsEmpty() { return tos == 0; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
global using System;
|
|
||||||
global using System.Linq;
|
|
||||||
global using System.Collections.Generic;
|
|
||||||
global using System.Collections.Immutable;
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
|
||||||
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
|
|
||||||
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup Label="ColumnSortExpression">
|
|
||||||
<Scanner Include="ColumnSortExpressionTokens.lex" />
|
|
||||||
<Parser Include="ColumnSortExpressionGrammar.y" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
namespace Bts.ColumnSortExpressions;
|
|
||||||
|
|
||||||
public class ColumnSortExpression {
|
|
||||||
public ColumnSortExpression(string name, bool isAscending = true) {
|
|
||||||
Name = name;
|
|
||||||
IsAscending = isAscending;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string Name { get; }
|
|
||||||
public bool IsAscending { get; }
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
%namespace Bts.ColumnSortExpressions
|
|
||||||
%partial
|
|
||||||
%visibility internal
|
|
||||||
%tokentype ColumnSortExpressionToken
|
|
||||||
%parsertype ColumnSortExpressionParser
|
|
||||||
|
|
||||||
%union {
|
|
||||||
public string Identifier;
|
|
||||||
public ColumnSortExpression Column;
|
|
||||||
public ImmutableList<ColumnSortExpression> Columns;
|
|
||||||
}
|
|
||||||
|
|
||||||
%token ASCENDING, DESCENDING, IDENTIFIER
|
|
||||||
|
|
||||||
%type <Column> Column
|
|
||||||
%type <Columns> Columns
|
|
||||||
|
|
||||||
%%
|
|
||||||
|
|
||||||
Columns :
|
|
||||||
Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
|
|
||||||
| Columns Column { $$ = $1.Add($2); }
|
|
||||||
;
|
|
||||||
|
|
||||||
Column :
|
|
||||||
IDENTIFIER { $$ = new($1.Identifier, true); }
|
|
||||||
| ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
|
|
||||||
| DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
|
|
||||||
;
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace Bts.ColumnSortExpressions;
|
|
||||||
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
public class ColumnSortExpressionInterpreter {
|
|
||||||
public static IList<ColumnSortExpression> Evaluate(Stream stream) {
|
|
||||||
var parser = new ColumnSortExpressionParser();
|
|
||||||
var res = parser.Parse(stream);
|
|
||||||
return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
|
|
||||||
res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IList<ColumnSortExpression> Evaluate(string str) {
|
|
||||||
using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
|
|
||||||
return Evaluate(ms);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace Bts.ColumnSortExpressions;
|
|
||||||
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
partial class ColumnSortExpressionParser {
|
|
||||||
public ColumnSortExpressionParser() : base(null) { }
|
|
||||||
|
|
||||||
public ValueType? Parse(string s) {
|
|
||||||
using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
|
|
||||||
return Parse(ms);
|
|
||||||
}
|
|
||||||
|
|
||||||
public ValueType? Parse(Stream stream) {
|
|
||||||
Scanner = new ColumnSortExpressionScanner(stream);
|
|
||||||
Parse();
|
|
||||||
return CurrentSemanticValue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
namespace Bts.ColumnSortExpressions;
|
|
||||||
|
|
||||||
partial class ColumnSortExpressionScanner {
|
|
||||||
public override void yyerror(string format, params object[] args) {
|
|
||||||
base.yyerror(format, args);
|
|
||||||
Console.WriteLine(format, args);
|
|
||||||
Console.WriteLine();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
%namespace Bts.ColumnSortExpressions
|
|
||||||
%scannertype ColumnSortExpressionScanner
|
|
||||||
%visibility internal
|
|
||||||
%tokentype ColumnSortExpressionToken
|
|
||||||
|
|
||||||
%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
|
|
||||||
|
|
||||||
/* Literal Constants */
|
|
||||||
Letter [a-zA-Z]
|
|
||||||
|
|
||||||
Identifier {Letter}({Letter}*[0-9]*_*)*
|
|
||||||
Ascending +
|
|
||||||
Descending -
|
|
||||||
|
|
||||||
%%
|
|
||||||
|
|
||||||
{Ascending} {
|
|
||||||
return (int)ColumnSortExpressionToken.ASCENDING;
|
|
||||||
}
|
|
||||||
|
|
||||||
{Descending} {
|
|
||||||
return (int)ColumnSortExpressionToken.DESCENDING;
|
|
||||||
}
|
|
||||||
|
|
||||||
{Identifier} {
|
|
||||||
yylval.Identifier = yytext;
|
|
||||||
return (int)ColumnSortExpressionToken.IDENTIFIER;
|
|
||||||
}
|
|
||||||
|
|
||||||
%%
|
|
||||||
@@ -1,720 +0,0 @@
|
|||||||
// ==============================================================
|
|
||||||
// <auto-generated>
|
|
||||||
// This code automatically produced from an embedded resource.
|
|
||||||
// Do not edit this file, or it will become incompatible with
|
|
||||||
// the specification from which it was generated.
|
|
||||||
// </auto-generated>
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Runtime.Serialization;
|
|
||||||
using System.Globalization;
|
|
||||||
|
|
||||||
namespace QUT.GplexBuffers
|
|
||||||
{
|
|
||||||
// Code copied from GPLEX embedded resource
|
|
||||||
[Serializable]
|
|
||||||
internal class BufferException : Exception
|
|
||||||
{
|
|
||||||
public BufferException() { }
|
|
||||||
public BufferException(string message) : base(message) { }
|
|
||||||
public BufferException(string message, Exception innerException)
|
|
||||||
: base(message, innerException) { }
|
|
||||||
protected BufferException(SerializationInfo info, StreamingContext context)
|
|
||||||
: base(info, context) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
internal abstract class ScanBuff
|
|
||||||
{
|
|
||||||
private string fileNm;
|
|
||||||
|
|
||||||
public const int EndOfFile = -1;
|
|
||||||
public const int UnicodeReplacementChar = 0xFFFD;
|
|
||||||
|
|
||||||
public bool IsFile { get { return (fileNm != null); } }
|
|
||||||
public string FileName { get { return fileNm; } set { fileNm = value; } }
|
|
||||||
|
|
||||||
public abstract int Pos { get; set; }
|
|
||||||
public abstract int Read();
|
|
||||||
public virtual void Mark() { }
|
|
||||||
|
|
||||||
public abstract string GetString(int begin, int limit);
|
|
||||||
|
|
||||||
public static ScanBuff GetBuffer(string source)
|
|
||||||
{
|
|
||||||
return new StringBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ScanBuff GetBuffer(IList<string> source)
|
|
||||||
{
|
|
||||||
return new LineBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!NOFILES)
|
|
||||||
public static ScanBuff GetBuffer(Stream source)
|
|
||||||
{
|
|
||||||
return new BuildBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
return new BuildBuffer(source, fallbackCodePage);
|
|
||||||
}
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endif // !NOFILES
|
|
||||||
}
|
|
||||||
|
|
||||||
#region Buffer classes
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// ===== Definitions for various ScanBuff derived classes ====
|
|
||||||
// ==============================================================
|
|
||||||
// =============== String input ================
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class reads characters from a single string as
|
|
||||||
/// required, for example, by Visual Studio language services
|
|
||||||
/// </summary>
|
|
||||||
sealed class StringBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
string str; // input buffer
|
|
||||||
int bPos; // current position in buffer
|
|
||||||
int sLen;
|
|
||||||
|
|
||||||
public StringBuffer(string source)
|
|
||||||
{
|
|
||||||
this.str = source;
|
|
||||||
this.sLen = source.Length;
|
|
||||||
this.FileName = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
if (bPos < sLen) return str[bPos++];
|
|
||||||
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
|
|
||||||
else { bPos++; return EndOfFile; } // two strikes and you're out!
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
// "limit" can be greater than sLen with the BABEL
|
|
||||||
// option set. Read returns a "virtual" EOL if
|
|
||||||
// an attempt is made to read past the end of the
|
|
||||||
// string buffer. Without the guard any attempt
|
|
||||||
// to fetch yytext for a token that includes the
|
|
||||||
// EOL will throw an index exception.
|
|
||||||
if (limit > sLen) limit = sLen;
|
|
||||||
if (limit <= begin) return "";
|
|
||||||
else return str.Substring(begin, limit - begin);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return bPos; }
|
|
||||||
set { bPos = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() { return "StringBuffer"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// The LineBuff class contributed by Nigel Horspool,
|
|
||||||
// nigelh@cs.uvic.cs
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
sealed class LineBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
IList<string> line; // list of source lines from a file
|
|
||||||
int numLines; // number of strings in line list
|
|
||||||
string curLine; // current line in that list
|
|
||||||
int cLine; // index of current line in the list
|
|
||||||
int curLen; // length of current line
|
|
||||||
int curLineStart; // position of line start in whole file
|
|
||||||
int curLineEnd; // position of line end in whole file
|
|
||||||
int maxPos; // max position ever visited in whole file
|
|
||||||
int cPos; // ordinal number of code in source
|
|
||||||
|
|
||||||
// Constructed from a list of strings, one per source line.
|
|
||||||
// The lines have had trailing '\n' characters removed.
|
|
||||||
public LineBuffer(IList<string> lineList)
|
|
||||||
{
|
|
||||||
line = lineList;
|
|
||||||
numLines = line.Count;
|
|
||||||
cPos = curLineStart = 0;
|
|
||||||
curLine = (numLines > 0 ? line[0] : "");
|
|
||||||
maxPos = curLineEnd = curLen = curLine.Length;
|
|
||||||
cLine = 1;
|
|
||||||
FileName = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
if (cPos < curLineEnd)
|
|
||||||
return curLine[cPos++ - curLineStart];
|
|
||||||
if (cPos++ == curLineEnd)
|
|
||||||
return '\n';
|
|
||||||
if (cLine >= numLines)
|
|
||||||
return EndOfFile;
|
|
||||||
curLine = line[cLine];
|
|
||||||
curLen = curLine.Length;
|
|
||||||
curLineStart = curLineEnd + 1;
|
|
||||||
curLineEnd = curLineStart + curLen;
|
|
||||||
if (curLineEnd > maxPos)
|
|
||||||
maxPos = curLineEnd;
|
|
||||||
cLine++;
|
|
||||||
return curLen > 0 ? curLine[0] : '\n';
|
|
||||||
}
|
|
||||||
|
|
||||||
// To speed up searches for the line containing a position
|
|
||||||
private int cachedPosition;
|
|
||||||
private int cachedIxdex;
|
|
||||||
private int cachedLineStart;
|
|
||||||
|
|
||||||
// Given a position pos within the entire source, the results are
|
|
||||||
// ix -- the index of the containing line
|
|
||||||
// lstart -- the position of the first character on that line
|
|
||||||
private void findIndex(int pos, out int ix, out int lstart)
|
|
||||||
{
|
|
||||||
if (pos >= cachedPosition)
|
|
||||||
{
|
|
||||||
ix = cachedIxdex; lstart = cachedLineStart;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ix = lstart = 0;
|
|
||||||
}
|
|
||||||
while (ix < numLines)
|
|
||||||
{
|
|
||||||
int len = line[ix].Length + 1;
|
|
||||||
if (pos < lstart + len) break;
|
|
||||||
lstart += len;
|
|
||||||
ix++;
|
|
||||||
}
|
|
||||||
cachedPosition = pos;
|
|
||||||
cachedIxdex = ix;
|
|
||||||
cachedLineStart = lstart;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
if (begin >= maxPos || limit <= begin) return "";
|
|
||||||
int endIx, begIx, endLineStart, begLineStart;
|
|
||||||
findIndex(begin, out begIx, out begLineStart);
|
|
||||||
int begCol = begin - begLineStart;
|
|
||||||
findIndex(limit, out endIx, out endLineStart);
|
|
||||||
int endCol = limit - endLineStart;
|
|
||||||
string s = line[begIx];
|
|
||||||
if (begIx == endIx)
|
|
||||||
{
|
|
||||||
// the usual case, substring all on one line
|
|
||||||
return (endCol <= s.Length) ?
|
|
||||||
s.Substring(begCol, endCol - begCol)
|
|
||||||
: s.Substring(begCol) + "\n";
|
|
||||||
}
|
|
||||||
// the string spans multiple lines, yuk!
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
if (begCol < s.Length)
|
|
||||||
sb.Append(s.Substring(begCol));
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
sb.Append("\n");
|
|
||||||
s = line[++begIx];
|
|
||||||
if (begIx >= endIx) break;
|
|
||||||
sb.Append(s);
|
|
||||||
}
|
|
||||||
if (endCol <= s.Length)
|
|
||||||
{
|
|
||||||
sb.Append(s.Substring(0, endCol));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
sb.Append(s);
|
|
||||||
sb.Append("\n");
|
|
||||||
}
|
|
||||||
return sb.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return cPos; }
|
|
||||||
set
|
|
||||||
{
|
|
||||||
cPos = value;
|
|
||||||
findIndex(cPos, out cLine, out curLineStart);
|
|
||||||
// cLine should be the *next* line after curLine.
|
|
||||||
curLine = (cLine < numLines ? line[cLine++] : "");
|
|
||||||
curLineEnd = curLineStart + curLine.Length;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() { return "LineBuffer"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!NOFILES)
|
|
||||||
// ==============================================================
|
|
||||||
// ===== class BuildBuff : for unicode text files ========
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
class BuildBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
// Double buffer for char stream.
|
|
||||||
class BufferElement
|
|
||||||
{
|
|
||||||
StringBuilder bldr = new StringBuilder();
|
|
||||||
StringBuilder next = new StringBuilder();
|
|
||||||
int minIx;
|
|
||||||
int maxIx;
|
|
||||||
int brkIx;
|
|
||||||
bool appendToNext;
|
|
||||||
|
|
||||||
internal BufferElement() { }
|
|
||||||
|
|
||||||
internal int MaxIndex { get { return maxIx; } }
|
|
||||||
// internal int MinIndex { get { return minIx; } }
|
|
||||||
|
|
||||||
internal char this[int index]
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
if (index < minIx || index >= maxIx)
|
|
||||||
throw new BufferException("Index was outside data buffer");
|
|
||||||
else if (index < brkIx)
|
|
||||||
return bldr[index - minIx];
|
|
||||||
else
|
|
||||||
return next[index - brkIx];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Append(char[] block, int count)
|
|
||||||
{
|
|
||||||
maxIx += count;
|
|
||||||
if (appendToNext)
|
|
||||||
this.next.Append(block, 0, count);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
this.bldr.Append(block, 0, count);
|
|
||||||
brkIx = maxIx;
|
|
||||||
appendToNext = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal string GetString(int start, int limit)
|
|
||||||
{
|
|
||||||
if (limit <= start)
|
|
||||||
return "";
|
|
||||||
if (start >= minIx && limit <= maxIx)
|
|
||||||
if (limit < brkIx) // String entirely in bldr builder
|
|
||||||
return bldr.ToString(start - minIx, limit - start);
|
|
||||||
else if (start >= brkIx) // String entirely in next builder
|
|
||||||
return next.ToString(start - brkIx, limit - start);
|
|
||||||
else // Must do a string-concatenation
|
|
||||||
return
|
|
||||||
bldr.ToString(start - minIx, brkIx - start) +
|
|
||||||
next.ToString(0, limit - brkIx);
|
|
||||||
else
|
|
||||||
throw new BufferException("String was outside data buffer");
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Mark(int limit)
|
|
||||||
{
|
|
||||||
if (limit > brkIx + 16) // Rotate blocks
|
|
||||||
{
|
|
||||||
StringBuilder temp = bldr;
|
|
||||||
bldr = next;
|
|
||||||
next = temp;
|
|
||||||
next.Length = 0;
|
|
||||||
minIx = brkIx;
|
|
||||||
brkIx = maxIx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
BufferElement data = new BufferElement();
|
|
||||||
|
|
||||||
int bPos; // Postion index in the StringBuilder
|
|
||||||
BlockReader NextBlk; // Delegate that serves char-arrays;
|
|
||||||
|
|
||||||
private string EncodingName
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
StreamReader rdr = NextBlk.Target as StreamReader;
|
|
||||||
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public BuildBuffer(Stream stream)
|
|
||||||
{
|
|
||||||
FileStream fStrm = (stream as FileStream);
|
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
|
||||||
NextBlk = BlockReaderFactory.Raw(stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public BuildBuffer(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
FileStream fStrm = (stream as FileStream);
|
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
|
||||||
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Marks a conservative lower bound for the buffer,
|
|
||||||
/// allowing space to be reclaimed. If an application
|
|
||||||
/// needs to call GetString at arbitrary past locations
|
|
||||||
/// in the input stream, Mark() is not called.
|
|
||||||
/// </summary>
|
|
||||||
public override void Mark() { data.Mark(bPos - 2); }
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return bPos; }
|
|
||||||
set { bPos = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Read returns the ordinal number of the next char, or
|
|
||||||
/// EOF (-1) for an end of stream. Note that the next
|
|
||||||
/// code point may require *two* calls of Read().
|
|
||||||
/// </summary>
|
|
||||||
/// <returns></returns>
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
//
|
|
||||||
// Characters at positions
|
|
||||||
// [data.offset, data.offset + data.bldr.Length)
|
|
||||||
// are available in data.bldr.
|
|
||||||
//
|
|
||||||
if (bPos < data.MaxIndex)
|
|
||||||
{
|
|
||||||
// ch0 cannot be EOF
|
|
||||||
return (int)data[bPos++];
|
|
||||||
}
|
|
||||||
else // Read from underlying stream
|
|
||||||
{
|
|
||||||
// Experimental code, blocks of page size
|
|
||||||
char[] chrs = new char[4096];
|
|
||||||
int count = NextBlk(chrs, 0, 4096);
|
|
||||||
if (count == 0)
|
|
||||||
return EndOfFile;
|
|
||||||
else
|
|
||||||
{
|
|
||||||
data.Append(chrs, count);
|
|
||||||
return (int)data[bPos++];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
return data.GetString(begin, limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString()
|
|
||||||
{
|
|
||||||
return "StringBuilder buffer, encoding: " + this.EncodingName;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// =============== End ScanBuff-derived classes ==================
|
|
||||||
|
|
||||||
internal delegate int BlockReader(char[] block, int index, int number);
|
|
||||||
|
|
||||||
// A delegate factory, serving up a delegate that
|
|
||||||
// reads a block of characters from the underlying
|
|
||||||
// encoded stream, via a StreamReader object.
|
|
||||||
//
|
|
||||||
internal static class BlockReaderFactory
|
|
||||||
{
|
|
||||||
public static BlockReader Raw(Stream stream)
|
|
||||||
{
|
|
||||||
return delegate(char[] block, int index, int number)
|
|
||||||
{
|
|
||||||
byte[] b = new byte[number];
|
|
||||||
int count = stream.Read(b, 0, number);
|
|
||||||
int i = 0;
|
|
||||||
int j = index;
|
|
||||||
for (; i < count; i++, j++)
|
|
||||||
block[j] = (char)b[i];
|
|
||||||
return count;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static BlockReader Get(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
Encoding encoding;
|
|
||||||
int preamble = Preamble(stream);
|
|
||||||
|
|
||||||
if (preamble != 0) // There is a valid BOM here!
|
|
||||||
encoding = Encoding.GetEncoding(preamble);
|
|
||||||
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
|
|
||||||
return Raw(stream);
|
|
||||||
else if (fallbackCodePage != -2) // Anything but "guess"
|
|
||||||
encoding = Encoding.GetEncoding(fallbackCodePage);
|
|
||||||
else // This is the "guess" option
|
|
||||||
{
|
|
||||||
int guess = new Guesser(stream).GuessCodePage();
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (guess == -1) // ==> this is a 7-bit file
|
|
||||||
encoding = Encoding.ASCII;
|
|
||||||
else if (guess == 65001)
|
|
||||||
encoding = Encoding.UTF8;
|
|
||||||
else // ==> use the machine default
|
|
||||||
encoding = Encoding.Default;
|
|
||||||
}
|
|
||||||
StreamReader reader = new StreamReader(stream, encoding);
|
|
||||||
return reader.Read;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int Preamble(Stream stream)
|
|
||||||
{
|
|
||||||
int b0 = stream.ReadByte();
|
|
||||||
int b1 = stream.ReadByte();
|
|
||||||
|
|
||||||
if (b0 == 0xfe && b1 == 0xff)
|
|
||||||
return 1201; // UTF16BE
|
|
||||||
if (b0 == 0xff && b1 == 0xfe)
|
|
||||||
return 1200; // UTF16LE
|
|
||||||
|
|
||||||
int b2 = stream.ReadByte();
|
|
||||||
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
|
||||||
return 65001; // UTF8
|
|
||||||
//
|
|
||||||
// There is no unicode preamble, so we
|
|
||||||
// return denoter for the machine default.
|
|
||||||
//
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
}
|
|
||||||
#endif // !NOFILES
|
|
||||||
#endregion Buffer classes
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// ============ class CodePageHandling =============
|
|
||||||
// ==============================================================
|
|
||||||
#if (!NOFILES)
|
|
||||||
public static class CodePageHandling
|
|
||||||
{
|
|
||||||
public static int GetCodePage(string option)
|
|
||||||
{
|
|
||||||
string command = option.ToUpperInvariant();
|
|
||||||
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
|
|
||||||
command = command.Substring(9);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (command.Equals("RAW"))
|
|
||||||
return -1;
|
|
||||||
else if (command.Equals("GUESS"))
|
|
||||||
return -2;
|
|
||||||
else if (command.Equals("DEFAULT"))
|
|
||||||
return 0;
|
|
||||||
else if (char.IsDigit(command[0]))
|
|
||||||
return int.Parse(command, CultureInfo.InvariantCulture);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Encoding enc = Encoding.GetEncoding(command);
|
|
||||||
return enc.CodePage;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (FormatException)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine(
|
|
||||||
"Invalid format \"{0}\", using machine default", option);
|
|
||||||
}
|
|
||||||
catch (ArgumentException)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine(
|
|
||||||
"Unknown code page \"{0}\", using machine default", option);
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#region guesser
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
// ==============================================================
|
|
||||||
// ============ Encoding Guesser =============
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class provides a simple finite state automaton that
|
|
||||||
/// scans the file looking for (1) valid UTF-8 byte patterns,
|
|
||||||
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
|
|
||||||
/// The method then guesses whether it is UTF-8 or maybe some
|
|
||||||
/// local machine default encoding. This works well for the
|
|
||||||
/// various Latin encodings.
|
|
||||||
/// </summary>
|
|
||||||
internal class Guesser
|
|
||||||
{
|
|
||||||
ScanBuff buffer;
|
|
||||||
|
|
||||||
public int GuessCodePage() { return Scan(); }
|
|
||||||
|
|
||||||
const int maxAccept = 10;
|
|
||||||
const int initial = 0;
|
|
||||||
const int eofNum = 0;
|
|
||||||
const int goStart = -1;
|
|
||||||
const int INITIAL = 0;
|
|
||||||
const int EndToken = 0;
|
|
||||||
|
|
||||||
#region user code
|
|
||||||
/*
|
|
||||||
* Reads the bytes of a file to determine if it is
|
|
||||||
* UTF-8 or a single-byte code page file.
|
|
||||||
*/
|
|
||||||
public long utfX;
|
|
||||||
public long uppr;
|
|
||||||
#endregion user code
|
|
||||||
|
|
||||||
int state;
|
|
||||||
int currentStart = startState[0];
|
|
||||||
int code;
|
|
||||||
|
|
||||||
#region ScannerTables
|
|
||||||
static int[] startState = new int[] { 11, 0 };
|
|
||||||
|
|
||||||
#region CharacterMap
|
|
||||||
static sbyte[] map = new sbyte[256] {
|
|
||||||
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
||||||
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
static sbyte[][] nextState = new sbyte[][] {
|
|
||||||
new sbyte[] {0, 0, 0, 0, 0, 0},
|
|
||||||
new sbyte[] {-1, -1, 10, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, -1, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 8, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 5, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 6, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 7, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, -1, 9, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, 1, 2, 3, 4, 2}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
|
|
||||||
// Reason for suppression: cannot have self-reference in array initializer.
|
|
||||||
static Guesser()
|
|
||||||
{
|
|
||||||
nextState[7] = nextState[2];
|
|
||||||
nextState[9] = nextState[2];
|
|
||||||
nextState[10] = nextState[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
int NextState()
|
|
||||||
{
|
|
||||||
if (code == ScanBuff.EndOfFile)
|
|
||||||
return eofNum;
|
|
||||||
else
|
|
||||||
return nextState[state][map[code]];
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
public Guesser(System.IO.Stream file) { SetSource(file); }
|
|
||||||
|
|
||||||
public void SetSource(System.IO.Stream source)
|
|
||||||
{
|
|
||||||
this.buffer = new BuildBuffer(source);
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
|
|
||||||
int Scan()
|
|
||||||
{
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
int next;
|
|
||||||
state = currentStart;
|
|
||||||
while ((next = NextState()) == goStart)
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
while ((next = NextState()) > eofNum)
|
|
||||||
{
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
if (state <= maxAccept)
|
|
||||||
{
|
|
||||||
#region ActionSwitch
|
|
||||||
#pragma warning disable 162
|
|
||||||
switch (state)
|
|
||||||
{
|
|
||||||
case eofNum:
|
|
||||||
switch (currentStart)
|
|
||||||
{
|
|
||||||
case 11:
|
|
||||||
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
|
|
||||||
else if (uppr * 10 > utfX) return 0; /* default code page */
|
|
||||||
else return 65001; /* UTF-8 encoding */
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return EndToken;
|
|
||||||
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
|
|
||||||
case 2: // Recognized '{Upper128}', Shortest string "\x80"
|
|
||||||
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
|
|
||||||
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
|
|
||||||
uppr++;
|
|
||||||
break;
|
|
||||||
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
|
|
||||||
uppr += 3;
|
|
||||||
break;
|
|
||||||
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
|
|
||||||
utfX += 3;
|
|
||||||
break;
|
|
||||||
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
|
|
||||||
utfX += 2;
|
|
||||||
break;
|
|
||||||
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
|
|
||||||
utfX++;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
#pragma warning restore 162
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // end class Guesser
|
|
||||||
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endregion
|
|
||||||
#endif // !NOFILES
|
|
||||||
|
|
||||||
// End of code copied from embedded resource
|
|
||||||
}
|
|
||||||
@@ -1,884 +0,0 @@
|
|||||||
// ==============================================================
|
|
||||||
// <auto-generated>
|
|
||||||
// This code automatically produced from an embedded resource.
|
|
||||||
// Do not edit this file, or it will become incompatible with
|
|
||||||
// the specification from which it was generated.
|
|
||||||
// </auto-generated>
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
// Gardens Point Parser Generator
|
|
||||||
// Copyright (c) Wayne Kelly, QUT 2005-2010
|
|
||||||
// (see accompanying GPPGcopyright.rtf)
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Text;
|
|
||||||
using System.Globalization;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Runtime.Serialization;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
|
||||||
namespace QUT.Gppg {
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract class for GPPG shift-reduce parsers.
|
|
||||||
/// Parsers generated by GPPG derive from this base
|
|
||||||
/// class, overriding the abstract Initialize() and
|
|
||||||
/// DoAction() methods.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TValue">Semantic value type</typeparam>
|
|
||||||
/// <typeparam name="TSpan">Location type</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public abstract class ShiftReduceParser<TValue, TSpan>
|
|
||||||
#else
|
|
||||||
internal abstract class ShiftReduceParser<TValue, TSpan>
|
|
||||||
#endif
|
|
||||||
where TSpan : IMerge<TSpan>, new() {
|
|
||||||
private AbstractScanner<TValue, TSpan> scanner;
|
|
||||||
/// <summary>
|
|
||||||
/// The abstract scanner for this parser.
|
|
||||||
/// </summary>
|
|
||||||
protected AbstractScanner<TValue, TSpan> Scanner {
|
|
||||||
get { return scanner; }
|
|
||||||
set { scanner = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Constructor for base class
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="scanner">Scanner instance for this parser</param>
|
|
||||||
protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
|
|
||||||
this.scanner = scanner;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// TECHNICAL EXPLANATION.
|
|
||||||
// Why the next two fields are not exposed via properties.
|
|
||||||
// ==============================================================
|
|
||||||
// These fields are of the generic parameter types, and are
|
|
||||||
// frequently instantiated as struct types in derived classes.
|
|
||||||
// Semantic actions are defined in the derived classes and refer
|
|
||||||
// to instance fields of these structs. In such cases the code
|
|
||||||
// "get_CurrentSemanticValue().myField = blah;" will fail since
|
|
||||||
// the getter pushes the value of the field, not the reference.
|
|
||||||
// So, in the presence of properties, gppg would need to encode
|
|
||||||
// such field accesses as ...
|
|
||||||
// "tmp = get_CurrentSemanticValue(); // Fetch value
|
|
||||||
// tmp.myField = blah; // update
|
|
||||||
// set_CurrentSemanticValue(tmp); " // Write update back.
|
|
||||||
// There is no issue if TValue is restricted to be a ref type.
|
|
||||||
// The same explanation applies to scanner.yylval.
|
|
||||||
// ==============================================================
|
|
||||||
/// <summary>
|
|
||||||
/// The current value of the "$$" symbolic variable in the parser
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
protected TValue CurrentSemanticValue;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The current value of the "@$" symbolic variable in the parser
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
protected TSpan CurrentLocationSpan;
|
|
||||||
protected int NextToken;
|
|
||||||
|
|
||||||
private TSpan LastSpan;
|
|
||||||
private State FsaState;
|
|
||||||
private bool recovering;
|
|
||||||
private int tokensSinceLastError;
|
|
||||||
|
|
||||||
private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
|
|
||||||
private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
|
|
||||||
private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The stack of semantic value (YYSTYPE) values.
|
|
||||||
/// </summary>
|
|
||||||
protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The stack of location value (YYLTYPE) varlues.
|
|
||||||
/// </summary>
|
|
||||||
protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
|
|
||||||
|
|
||||||
private int errorToken;
|
|
||||||
private int endOfFileToken;
|
|
||||||
private string[] nonTerminals;
|
|
||||||
private State[] states;
|
|
||||||
private Rule[] rules;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes
|
|
||||||
/// to insert the rule list into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="rules">The array of Rule objects</param>
|
|
||||||
protected void InitRules( Rule[] rules ) { this.rules = rules; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes to
|
|
||||||
/// insert the states table into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="states">The pre-initialized states table</param>
|
|
||||||
protected void InitStates( State[] states ) { this.states = states; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// OBSOLETE FOR VERSION 1.4.0
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="size"></param>
|
|
||||||
protected void InitStateTable( int size ) { states = new State[size]; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes
|
|
||||||
/// to insert the special value for the error and EOF tokens.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="err">The error state ordinal</param>
|
|
||||||
/// <param name="end">The EOF stat ordinal</param>
|
|
||||||
protected void InitSpecialTokens( int err, int end ) {
|
|
||||||
errorToken = err;
|
|
||||||
endOfFileToken = end;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes to
|
|
||||||
/// insert the non-terminal symbol names into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="names">Non-terminal symbol names</param>
|
|
||||||
protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
|
|
||||||
|
|
||||||
#region YYAbort, YYAccept etcetera.
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class AcceptException : Exception {
|
|
||||||
internal AcceptException() { }
|
|
||||||
protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class AbortException : Exception {
|
|
||||||
internal AbortException() { }
|
|
||||||
protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class ErrorException : Exception {
|
|
||||||
internal ErrorException() { }
|
|
||||||
protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
// The following methods are only called from within
|
|
||||||
// a semantic action. The thrown exceptions can never
|
|
||||||
// propagate outside the ShiftReduceParser class in
|
|
||||||
// which they are nested.
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning "true"
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYAccept() { throw new AcceptException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning "false"
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYAbort() { throw new AbortException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning
|
|
||||||
/// "false" if error recovery fails.
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYError() { throw new ErrorException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Check if parser in error recovery state.
|
|
||||||
/// </summary>
|
|
||||||
protected bool YYRecovering { get { return recovering; } }
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract base method. ShiftReduceParser calls this
|
|
||||||
/// to initialize the base class data structures. Concrete
|
|
||||||
/// parser classes must override this method.
|
|
||||||
/// </summary>
|
|
||||||
protected abstract void Initialize();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Main entry point of the Shift-Reduce Parser.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if parse succeeds, else false for
|
|
||||||
/// unrecoverable errors</returns>
|
|
||||||
public bool Parse() {
|
|
||||||
Initialize(); // allow derived classes to instantiate rules, states and nonTerminals
|
|
||||||
|
|
||||||
NextToken = 0;
|
|
||||||
FsaState = states[0];
|
|
||||||
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
valueStack.Push( CurrentSemanticValue );
|
|
||||||
LocationStack.Push( CurrentLocationSpan );
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
|
|
||||||
DisplayStack();
|
|
||||||
#endif
|
|
||||||
int action = FsaState.defaultAction;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable != null) {
|
|
||||||
if (NextToken == 0) {
|
|
||||||
// We save the last token span, so that the location span
|
|
||||||
// of production right hand sides that begin or end with a
|
|
||||||
// nullable production will be correct.
|
|
||||||
LastSpan = scanner.yylloc;
|
|
||||||
NextToken = scanner.yylex();
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
else
|
|
||||||
Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
|
|
||||||
#endif
|
|
||||||
if (FsaState.ParserTable.ContainsKey( NextToken ))
|
|
||||||
action = FsaState.ParserTable[NextToken];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (action > 0) // shift
|
|
||||||
{
|
|
||||||
Shift( action );
|
|
||||||
}
|
|
||||||
else if (action < 0) // reduce
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
Reduce( -action );
|
|
||||||
if (action == -1) // accept
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (AbortException) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
catch (AcceptException) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (ErrorException) {
|
|
||||||
if (!ErrorRecovery())
|
|
||||||
return false;
|
|
||||||
else
|
|
||||||
throw; // Rethrow x, preserving information.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (action == 0 && !ErrorRecovery()) // error
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Shift( int stateIndex ) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
FsaState = states[stateIndex];
|
|
||||||
|
|
||||||
valueStack.Push( scanner.yylval );
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
LocationStack.Push( scanner.yylloc );
|
|
||||||
|
|
||||||
if (recovering) {
|
|
||||||
if (NextToken != errorToken)
|
|
||||||
tokensSinceLastError++;
|
|
||||||
|
|
||||||
if (tokensSinceLastError > 5)
|
|
||||||
recovering = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (NextToken != endOfFileToken)
|
|
||||||
NextToken = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Reduce( int ruleNumber ) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
DisplayRule(ruleNumber);
|
|
||||||
#endif
|
|
||||||
Rule rule = rules[ruleNumber];
|
|
||||||
int rhLen = rule.RightHandSide.Length;
|
|
||||||
//
|
|
||||||
// Default actions for unit productions.
|
|
||||||
//
|
|
||||||
if (rhLen == 1) {
|
|
||||||
CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1;
|
|
||||||
CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
|
|
||||||
}
|
|
||||||
else if (rhLen == 0) {
|
|
||||||
// Create a new blank value.
|
|
||||||
// Explicit semantic action may mutate this value
|
|
||||||
CurrentSemanticValue = default( TValue );
|
|
||||||
// The location span for an empty production will start with the
|
|
||||||
// beginning of the next lexeme, and end with the finish of the
|
|
||||||
// previous lexeme. This gives the correct behaviour when this
|
|
||||||
// nonsense value is used in later Merge operations.
|
|
||||||
CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
|
|
||||||
scanner.yylloc.Merge( LastSpan ) :
|
|
||||||
default( TSpan ));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
// Default action: $$ = $1;
|
|
||||||
CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
|
|
||||||
// Default action "@$ = @1.Merge(@N)" for location info.
|
|
||||||
TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
|
|
||||||
TSpan atN = LocationStack[LocationStack.Depth - 1];
|
|
||||||
CurrentLocationSpan =
|
|
||||||
((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
|
|
||||||
}
|
|
||||||
|
|
||||||
DoAction( ruleNumber );
|
|
||||||
|
|
||||||
for (int i = 0; i < rule.RightHandSide.Length; i++) {
|
|
||||||
StateStack.Pop();
|
|
||||||
valueStack.Pop();
|
|
||||||
LocationStack.Pop();
|
|
||||||
}
|
|
||||||
FsaState = StateStack.TopElement();
|
|
||||||
|
|
||||||
if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
|
|
||||||
FsaState = states[FsaState.Goto[rule.LeftHandSide]];
|
|
||||||
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
valueStack.Push( CurrentSemanticValue );
|
|
||||||
LocationStack.Push( CurrentLocationSpan );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Execute the selected action from array.
|
|
||||||
/// Must be overriden in derived classes.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actionNumber">Index of the action to perform</param>
|
|
||||||
protected abstract void DoAction( int actionNumber );
|
|
||||||
|
|
||||||
private bool ErrorRecovery() {
|
|
||||||
bool discard;
|
|
||||||
|
|
||||||
if (!recovering) // if not recovering from previous error
|
|
||||||
ReportError();
|
|
||||||
|
|
||||||
if (!FindErrorRecoveryState())
|
|
||||||
return false;
|
|
||||||
//
|
|
||||||
// The interim fix for the "looping in error recovery"
|
|
||||||
// artifact involved moving the setting of the recovering
|
|
||||||
// bool until after invalid tokens have been discarded.
|
|
||||||
//
|
|
||||||
ShiftErrorToken();
|
|
||||||
discard = DiscardInvalidTokens();
|
|
||||||
recovering = true;
|
|
||||||
tokensSinceLastError = 0;
|
|
||||||
return discard;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ReportError() {
|
|
||||||
StringBuilder errorMsg = new StringBuilder();
|
|
||||||
errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
|
|
||||||
|
|
||||||
if (FsaState.ParserTable.Count < 7) {
|
|
||||||
bool first = true;
|
|
||||||
foreach (int terminal in FsaState.ParserTable.Keys) {
|
|
||||||
if (first)
|
|
||||||
errorMsg.Append( ", expecting " );
|
|
||||||
else
|
|
||||||
errorMsg.Append( ", or " );
|
|
||||||
|
|
||||||
errorMsg.Append( TerminalToString( terminal ) );
|
|
||||||
first = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
scanner.yyerror( errorMsg.ToString() );
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ShiftErrorToken() {
|
|
||||||
int old_next = NextToken;
|
|
||||||
NextToken = errorToken;
|
|
||||||
|
|
||||||
Shift( FsaState.ParserTable[NextToken] );
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
|
|
||||||
#endif
|
|
||||||
NextToken = old_next;
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool FindErrorRecoveryState() {
|
|
||||||
while (true) // pop states until one found that accepts error token
|
|
||||||
{
|
|
||||||
if (FsaState.ParserTable != null &&
|
|
||||||
FsaState.ParserTable.ContainsKey( errorToken ) &&
|
|
||||||
FsaState.ParserTable[errorToken] > 0) // shift
|
|
||||||
return true;
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
|
|
||||||
#endif
|
|
||||||
StateStack.Pop();
|
|
||||||
valueStack.Pop();
|
|
||||||
LocationStack.Pop();
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
DisplayStack();
|
|
||||||
#endif
|
|
||||||
if (StateStack.IsEmpty()) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
|
|
||||||
#endif
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
FsaState = StateStack.TopElement();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool DiscardInvalidTokens() {
|
|
||||||
|
|
||||||
int action = FsaState.defaultAction;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable != null) {
|
|
||||||
// Discard tokens until find one that works ...
|
|
||||||
while (true) {
|
|
||||||
if (NextToken == 0) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.Write("Reading a token: ");
|
|
||||||
#endif
|
|
||||||
NextToken = scanner.yylex();
|
|
||||||
}
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
if (NextToken == endOfFileToken)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable.ContainsKey( NextToken ))
|
|
||||||
action = FsaState.ParserTable[NextToken];
|
|
||||||
|
|
||||||
if (action != 0)
|
|
||||||
return true;
|
|
||||||
else {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
NextToken = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (recovering && tokensSinceLastError == 0) {
|
|
||||||
//
|
|
||||||
// Boolean recovering is not set until after the first
|
|
||||||
// error token has been shifted. Thus if we get back
|
|
||||||
// here with recovering set and no tokens read we are
|
|
||||||
// looping on the same error recovery action. This
|
|
||||||
// happens if current_state.ParserTable is null because
|
|
||||||
// the state has an LR(0) reduction, but not all
|
|
||||||
// lookahead tokens are valid. This only occurs for
|
|
||||||
// error productions that *end* on "error".
|
|
||||||
//
|
|
||||||
// This action discards tokens one at a time until
|
|
||||||
// the looping stops. Another attack would be to always
|
|
||||||
// use the LALR(1) table if a production ends on "error"
|
|
||||||
//
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
if (NextToken == endOfFileToken)
|
|
||||||
return false;
|
|
||||||
NextToken = 0;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Traditional YACC method. Discards the next input token.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
protected void yyclearin() { NextToken = 0; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Tradional YACC method. Clear the "recovering" flag.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
protected void yyerrok() { recovering = false; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// OBSOLETE FOR VERSION 1.4.0
|
|
||||||
/// Method used by derived types to insert new
|
|
||||||
/// state instances in the "states" array.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="stateNumber">index of the state</param>
|
|
||||||
/// <param name="state">data for the state</param>
|
|
||||||
protected void AddState( int stateNumber, State state ) {
|
|
||||||
states[stateNumber] = state;
|
|
||||||
state.number = stateNumber;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayStack() {
|
|
||||||
Console.Error.Write( "State stack is now:" );
|
|
||||||
for (int i = 0; i < StateStack.Depth; i++)
|
|
||||||
Console.Error.Write( " {0}", StateStack[i].number );
|
|
||||||
Console.Error.WriteLine();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayRule( int ruleNumber ) {
|
|
||||||
Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
|
|
||||||
DisplayProduction( rules[ruleNumber] );
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayProduction( Rule rule ) {
|
|
||||||
if (rule.RightHandSide.Length == 0)
|
|
||||||
Console.Error.Write( "/* empty */ " );
|
|
||||||
else
|
|
||||||
foreach (int symbol in rule.RightHandSide)
|
|
||||||
Console.Error.Write( "{0} ", SymbolToString( symbol ) );
|
|
||||||
|
|
||||||
Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract state class naming terminal symbols.
|
|
||||||
/// This is overridden by derived classes with the
|
|
||||||
/// name (or alias) to be used in error messages.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="terminal">The terminal ordinal</param>
|
|
||||||
/// <returns></returns>
|
|
||||||
protected abstract string TerminalToString( int terminal );
|
|
||||||
|
|
||||||
private string SymbolToString( int symbol ) {
|
|
||||||
if (symbol < 0)
|
|
||||||
return nonTerminals[-symbol - 1];
|
|
||||||
else
|
|
||||||
return TerminalToString( symbol );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Return text representation of argument character
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The character to convert</param>
|
|
||||||
/// <returns>String representation of the character</returns>
|
|
||||||
protected static string CharToString( char input ) {
|
|
||||||
switch (input) {
|
|
||||||
case '\a': return @"'\a'";
|
|
||||||
case '\b': return @"'\b'";
|
|
||||||
case '\f': return @"'\f'";
|
|
||||||
case '\n': return @"'\n'";
|
|
||||||
case '\r': return @"'\r'";
|
|
||||||
case '\t': return @"'\t'";
|
|
||||||
case '\v': return @"'\v'";
|
|
||||||
case '\0': return @"'\0'";
|
|
||||||
default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Classes implementing this interface must supply a
|
|
||||||
/// method that merges two location objects to return
|
|
||||||
/// a new object of the same type.
|
|
||||||
/// GPPG-generated parsers have the default location
|
|
||||||
/// action equivalent to "@$ = @1.Merge(@N);" where N
|
|
||||||
/// is the right-hand-side length of the production.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TSpan">The Location type</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public interface IMerge<TSpan>
|
|
||||||
#else
|
|
||||||
internal interface IMerge<TSpan>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Interface method that creates a location object from
|
|
||||||
/// the current and last object. Typically used to create
|
|
||||||
/// a location object extending from the start of the @1
|
|
||||||
/// object to the end of the @N object.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="last">The lexically last object to merge</param>
|
|
||||||
/// <returns>The merged location object</returns>
|
|
||||||
TSpan Merge( TSpan last );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This is the default class that carries location
|
|
||||||
/// information from the scanner to the parser.
|
|
||||||
/// If you don't declare "%YYLTYPE Foo" the parser
|
|
||||||
/// will expect to deal with this type.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class LexLocation : IMerge<LexLocation>
|
|
||||||
#else
|
|
||||||
[SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
|
|
||||||
internal class LexLocation : IMerge<LexLocation>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
private int startLine; // start line
|
|
||||||
private int startColumn; // start column
|
|
||||||
private int endLine; // end line
|
|
||||||
private int endColumn; // end column
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The line at which the text span starts.
|
|
||||||
/// </summary>
|
|
||||||
public int StartLine { get { return startLine; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The column at which the text span starts.
|
|
||||||
/// </summary>
|
|
||||||
public int StartColumn { get { return startColumn; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The line on which the text span ends.
|
|
||||||
/// </summary>
|
|
||||||
public int EndLine { get { return endLine; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The column of the first character
|
|
||||||
/// beyond the end of the text span.
|
|
||||||
/// </summary>
|
|
||||||
public int EndColumn { get { return endColumn; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Default no-arg constructor.
|
|
||||||
/// </summary>
|
|
||||||
public LexLocation() { }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Constructor for text-span with given start and end.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="sl">start line</param>
|
|
||||||
/// <param name="sc">start column</param>
|
|
||||||
/// <param name="el">end line </param>
|
|
||||||
/// <param name="ec">end column</param>
|
|
||||||
public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Create a text location which spans from the
|
|
||||||
/// start of "this" to the end of the argument "last"
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="last">The last location in the result span</param>
|
|
||||||
/// <returns>The merged span</returns>
|
|
||||||
public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract scanner class that GPPG expects its scanners to
|
|
||||||
/// extend.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
|
|
||||||
/// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public abstract class AbstractScanner<TValue, TSpan>
|
|
||||||
#else
|
|
||||||
internal abstract class AbstractScanner<TValue, TSpan>
|
|
||||||
#endif
|
|
||||||
where TSpan : IMerge<TSpan> {
|
|
||||||
/// <summary>
|
|
||||||
/// Lexical value optionally set by the scanner. The value
|
|
||||||
/// is of the %YYSTYPE type declared in the parser spec.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
// A field must be declared for this value of parametric type,
|
|
||||||
// since it may be instantiated by a value struct. If it were
|
|
||||||
// implemented as a property, machine generated code in derived
|
|
||||||
// types would not be able to select on the returned value.
|
|
||||||
#pragma warning disable 649
|
|
||||||
public TValue yylval; // Lexical value: set by scanner
|
|
||||||
#pragma warning restore 649
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Current scanner location property. The value is of the
|
|
||||||
/// type declared by %YYLTYPE in the parser specification.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public virtual TSpan yylloc {
|
|
||||||
get { return default( TSpan ); } // Empty implementation allowing
|
|
||||||
set { /* skip */ } // yylloc to be ignored entirely.
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Main call point for LEX-like scanners. Returns an int
|
|
||||||
/// corresponding to the token recognized by the scanner.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>An int corresponding to the token</returns>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public abstract int yylex();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Traditional error reporting provided by LEX-like scanners
|
|
||||||
/// to their YACC-like clients.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="format">Message format string</param>
|
|
||||||
/// <param name="args">Optional array of args</param>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public virtual void yyerror( string format, params object[] args ) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Encapsulated state for the parser.
|
|
||||||
/// Opaque to users, visible to the tool-generated code.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class State
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The number of states in the automaton.
|
|
||||||
/// </summary>
|
|
||||||
public int number;
|
|
||||||
#else
|
|
||||||
internal class State {
|
|
||||||
/// <summary>
|
|
||||||
/// The index of this state in the states array.
|
|
||||||
/// </summary>
|
|
||||||
internal int number;
|
|
||||||
#endif
|
|
||||||
internal Dictionary<int, int> ParserTable; // Terminal -> ParseAction
|
|
||||||
internal Dictionary<int, int> Goto; // NonTerminal -> State;
|
|
||||||
internal int defaultAction; // = 0; // ParseAction
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// State transition data for this state. Pairs of elements of the
|
|
||||||
/// goto array associate symbol ordinals with next state indices.
|
|
||||||
/// The actions array is passed to another constructor.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actions">The action list</param>c
|
|
||||||
/// <param name="goToList">Next state data</param>
|
|
||||||
public State( int[] actions, int[] goToList )
|
|
||||||
: this( actions ) {
|
|
||||||
Goto = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < goToList.Length; i += 2)
|
|
||||||
Goto.Add( goToList[i], goToList[i + 1] );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Action data for this state. Pairs of elements of the
|
|
||||||
/// action array associate action ordinals with each of
|
|
||||||
/// those symbols that have actions in the current state.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actions">The action array</param>
|
|
||||||
public State( int[] actions ) {
|
|
||||||
ParserTable = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < actions.Length; i += 2)
|
|
||||||
ParserTable.Add( actions[i], actions[i + 1] );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set the default action for this state.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="defaultAction">Ordinal of the default action</param>
|
|
||||||
public State( int defaultAction ) {
|
|
||||||
this.defaultAction = defaultAction;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set the default action and the state transition table.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="defaultAction">The default action</param>
|
|
||||||
/// <param name="goToList">Transitions from this state</param>
|
|
||||||
public State( int defaultAction, int[] goToList )
|
|
||||||
: this( defaultAction ) {
|
|
||||||
Goto = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < goToList.Length; i += 2)
|
|
||||||
Goto.Add( goToList[i], goToList[i + 1] );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rule representation at runtime.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class Rule
|
|
||||||
#else
|
|
||||||
internal class Rule
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
internal int LeftHandSide; // symbol
|
|
||||||
internal int[] RightHandSide; // symbols
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rule constructor. This holds the ordinal of
|
|
||||||
/// the left hand side symbol, and the list of
|
|
||||||
/// right hand side symbols, in lexical order.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="left">The LHS non-terminal</param>
|
|
||||||
/// <param name="right">The RHS symbols, in lexical order</param>
|
|
||||||
public Rule( int left, int[] right ) {
|
|
||||||
this.LeftHandSide = left;
|
|
||||||
this.RightHandSide = right;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Stack utility for the shift-reduce parser.
|
|
||||||
/// GPPG parsers have three instances:
|
|
||||||
/// (1) The parser state stack, T = QUT.Gppg.State,
|
|
||||||
/// (2) The semantic value stack, T = TValue,
|
|
||||||
/// (3) The location stack, T = TSpan.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="T"></typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class PushdownPrefixState<T>
|
|
||||||
#else
|
|
||||||
internal class PushdownPrefixState<T>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
// Note that we cannot use the BCL Stack<T> class
|
|
||||||
// here as derived types need to index into stacks.
|
|
||||||
//
|
|
||||||
private T[] array = new T[8];
|
|
||||||
private int tos = 0;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Indexer for values of the stack below the top.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="index">index of the element, starting from the bottom</param>
|
|
||||||
/// <returns>the selected element</returns>
|
|
||||||
public T this[int index] { get { return array[index]; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The current depth of the stack.
|
|
||||||
/// </summary>
|
|
||||||
public int Depth { get { return tos; } }
|
|
||||||
|
|
||||||
internal void Push( T value ) {
|
|
||||||
if (tos >= array.Length) {
|
|
||||||
T[] newarray = new T[array.Length * 2];
|
|
||||||
System.Array.Copy( array, newarray, tos );
|
|
||||||
array = newarray;
|
|
||||||
}
|
|
||||||
array[tos++] = value;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal T Pop() {
|
|
||||||
T rslt = array[--tos];
|
|
||||||
array[tos] = default( T );
|
|
||||||
return rslt;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal T TopElement() { return array[tos - 1]; }
|
|
||||||
|
|
||||||
internal bool IsEmpty() { return tos == 0; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
global using System;
|
|
||||||
global using System.Linq;
|
|
||||||
global using System.Collections.Generic;
|
|
||||||
global using System.Collections.Immutable;
|
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
<PackageReference Include="YaccLexTools" Version="1.2.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -9,17 +9,9 @@
|
|||||||
<ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
|
<ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
|
||||||
<ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
|
<ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
|
||||||
<ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
|
<ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
|
||||||
<ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
|
|
||||||
<ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
|
|
||||||
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
||||||
<ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
|
<ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
|
||||||
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
|
|
||||||
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
|
||||||
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Reactive.Linq;
|
|
||||||
|
|
||||||
public static class DataStreamExtensions {
|
|
||||||
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
|
||||||
source as ViewStream ?? new ObservableViewStream(source);
|
|
||||||
|
|
||||||
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
|
||||||
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
|
||||||
|
|
||||||
IDisposable Subscribe(IObserver<DataPacket> observer) {
|
|
||||||
source.onChange += OnChange;
|
|
||||||
return Disposable.Create(() => source.onChange -= OnChange);
|
|
||||||
|
|
||||||
void OnChange(object? newValue) =>
|
|
||||||
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
|
|
||||||
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
|
|
||||||
public class PandasScriptStream : ViewStream {
|
|
||||||
public PandasScriptStream(ViewStream vs, string file)
|
|
||||||
: this(vs, file, null) { }
|
|
||||||
|
|
||||||
public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
|
|
||||||
: this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
|
|
||||||
mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
|
|
||||||
|
|
||||||
string readFile() {
|
|
||||||
try {
|
|
||||||
return File.ReadAllText(file);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Log.Gen.error($"PandasScriptStream error reading file: {ex}");
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
using BCG.Extensions;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Runtime.CompilerServices;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
public class PandasStream : ViewStream, IWorker {
|
|
||||||
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
|
||||||
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
|
||||||
readonly string pythonExe;
|
|
||||||
readonly ViewStreamDatabase vsd;
|
|
||||||
readonly CSVSaverStream inputCsvStream;
|
|
||||||
readonly Lazy<string> pythonCode;
|
|
||||||
IDictionary<int, IBlueRow>? lastRowMap = null;
|
|
||||||
int inputVersion = 0;
|
|
||||||
int outputVersion = 0;
|
|
||||||
DateTime lastFireTime = DateTime.MinValue;
|
|
||||||
TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
|
|
||||||
IDisposable? dispose;
|
|
||||||
|
|
||||||
public PandasStream(ViewStream vs, string dataFrameExpression)
|
|
||||||
: this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
|
|
||||||
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
|
|
||||||
this.pythonExe = pythonExe;
|
|
||||||
this.vsd = vsd;
|
|
||||||
// This should fire an update when it writes to the output file
|
|
||||||
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
|
|
||||||
pythonCode = new(getPythonCode);
|
|
||||||
|
|
||||||
string getPythonCode() {
|
|
||||||
var imports =
|
|
||||||
@"import numpy as np
|
|
||||||
import pandas as pd
|
|
||||||
import sys
|
|
||||||
";
|
|
||||||
var bd = new StringBuilder(imports);
|
|
||||||
bd.AppendLine($"df=pd.read_csv('{inputFile}')");
|
|
||||||
bd.AppendLine(dataFrameExpression);
|
|
||||||
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
|
|
||||||
return bd.ToString();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
vsd.addWorker(this);
|
|
||||||
inputCsvStream.onChange += onInputChanged;
|
|
||||||
this.dispose = Disposable.Create(dispose);
|
|
||||||
|
|
||||||
void onInputChanged(object newValue) =>
|
|
||||||
inputVersion++;
|
|
||||||
|
|
||||||
void dispose() {
|
|
||||||
inputCsvStream.onChange -= onInputChanged;
|
|
||||||
vsd.removeWorker(this);
|
|
||||||
deleteFile(inputFile);
|
|
||||||
deleteFile(outputFile);
|
|
||||||
}
|
|
||||||
|
|
||||||
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
|
|
||||||
if (!File.Exists(file)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
File.Delete(file);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose(ref dispose);
|
|
||||||
|
|
||||||
static void executePythonCode(string pythonCode, string? pythonExe = null) {
|
|
||||||
pythonExe ??= "python";
|
|
||||||
var start = new ProcessStartInfo {
|
|
||||||
FileName = pythonExe,
|
|
||||||
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
|
|
||||||
UseShellExecute = false,
|
|
||||||
RedirectStandardOutput = true
|
|
||||||
};
|
|
||||||
using var process = Process.Start(start)
|
|
||||||
?? throw new InvalidOperationException("Failed to launch python");
|
|
||||||
process.WaitForExit();
|
|
||||||
}
|
|
||||||
|
|
||||||
void IWorker.doWork() {
|
|
||||||
if (outputVersion == inputVersion) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
var now = DateTime.Now;
|
|
||||||
var elapsedSinceLastFire = now - lastFireTime;
|
|
||||||
if (elapsedSinceLastFire < fireThrottle) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
outputVersion = inputVersion;
|
|
||||||
executePythonCode(pythonCode.Value, pythonExe);
|
|
||||||
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
|
|
||||||
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
|
|
||||||
var comp = BlueRowEqualityComparer.Instance;
|
|
||||||
var deletedRows = new BlueTable(schema);
|
|
||||||
var addedRows = new BlueTable(schema);
|
|
||||||
if (lastRowMap is not null) {
|
|
||||||
deletedRows.SetItems(lastRowMap.Where(oldRow =>
|
|
||||||
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
|
|
||||||
addedRows.SetItems(rowMap.Where(newRow =>
|
|
||||||
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
|
|
||||||
}
|
|
||||||
lastRowMap = rowMap;
|
|
||||||
lastFireTime = DateTime.Now;
|
|
||||||
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
|
|
||||||
class State {
|
|
||||||
public int n_sends = 0;
|
|
||||||
public int n_recvs = 0;
|
|
||||||
}
|
|
||||||
private readonly string _ip;
|
|
||||||
private readonly int _port;
|
|
||||||
private readonly ViewStreamDatabase _vsd;
|
|
||||||
State _state = new State();
|
|
||||||
private readonly int _msg_size;
|
|
||||||
|
|
||||||
private IBlueTableSchema _schema;
|
|
||||||
private TimeSpan _send_period;
|
|
||||||
private ITCPConnection _connection = null;
|
|
||||||
private byte[] _buff;
|
|
||||||
|
|
||||||
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
|
|
||||||
this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
|
|
||||||
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
|
|
||||||
_ip = ip;
|
|
||||||
_port = port;
|
|
||||||
_msg_size = msg_size;
|
|
||||||
_vsd = vsd;
|
|
||||||
_schema = BlueDataUtility.form_schema("TCPClient",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("n_sends", TypeCode.UInt32),
|
|
||||||
Tuple.Create("n_recvs", TypeCode.UInt32));
|
|
||||||
_send_period = TimeSpan.FromSeconds(send_period_seconds);
|
|
||||||
_buff = new byte[msg_size];
|
|
||||||
for (int i = 0; i < _buff.Length; i++) {
|
|
||||||
_buff[i] = (byte)i;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
|
||||||
var server_rci = new RemoteConnectionInfo(_ip, _port);
|
|
||||||
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
MasterWorker.getInstance(false).removeWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork() {
|
|
||||||
if (_connection != null) {
|
|
||||||
_connection.send(_buff);
|
|
||||||
_state.n_sends++;
|
|
||||||
}
|
|
||||||
IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
|
|
||||||
BlueTable rows = new BlueTable(r.schema);
|
|
||||||
rows.addRow(r);
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void onConnection(ITCPConnection connection) {
|
|
||||||
_connection = connection;
|
|
||||||
_connection.register(this);
|
|
||||||
}
|
|
||||||
public bool isDisposed { get; }
|
|
||||||
|
|
||||||
public void onData(RemoteMessage msg) {
|
|
||||||
bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
|
|
||||||
_state.n_recvs++;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
file_path = r"C:\Temp\test2.txt" # r prefix for raw string to avoid escape characters
|
|
||||||
with open(file_path, "w") as f:
|
|
||||||
f.write("Hello, world!\n")
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
|
|
||||||
readonly IObservable<DataPacket> source;
|
|
||||||
IDisposable? subscription;
|
|
||||||
|
|
||||||
public ObservableViewStream(IObservable<DataPacket> source) =>
|
|
||||||
this.source = source;
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
if (subscription is not null) {
|
|
||||||
subscription.Dispose();
|
|
||||||
subscription = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
Log.isnull(subscription);
|
|
||||||
subscription = source.Subscribe(myFireEvent);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
Log.notnull(subscription);
|
|
||||||
subscription.Dispose();
|
|
||||||
subscription = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
|
|
||||||
source.Subscribe(observer);
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
using BCG;
|
||||||
|
|
||||||
|
namespace BCG
|
||||||
|
{
|
||||||
|
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
||||||
|
[DataStreamUserRespresentation("add_to_cloud")]
|
||||||
|
public class AddToCloudStream : ViewStream
|
||||||
|
{
|
||||||
|
private readonly string _name, _stream_exp;
|
||||||
|
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
|
||||||
|
public AddToCloudStream(string name, params string[] lines)
|
||||||
|
: this(name, 0, lines)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public AddToCloudStream(string name, int persist, params string[] lines)
|
||||||
|
{
|
||||||
|
_name = name;
|
||||||
|
_stream_exp = getExpression(lines);
|
||||||
|
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
|
||||||
|
mySetRealSource(new TableStream("cloud_streams"));
|
||||||
|
}
|
||||||
|
|
||||||
|
static string getExpression(params string[] lines) {
|
||||||
|
for (int i = 0; i < lines.Length; i++) {
|
||||||
|
// unescape quotes
|
||||||
|
lines[i] = lines[i].Replace("\\\"", "\"");
|
||||||
|
}
|
||||||
|
return string.Join("\n", lines);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,22 +1,31 @@
|
|||||||
|
using BCG;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using mbts.config;
|
||||||
|
using bts.utils;
|
||||||
|
using System.Threading.Tasks;
|
||||||
using Bts.DataStreamExpressions;
|
using Bts.DataStreamExpressions;
|
||||||
|
using System.Collections.Generic;
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
using bts;
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
|
{
|
||||||
// listens to the requests posted on "cloud_streams" request table, and
|
// listens to the requests posted on "cloud_streams" request table, and
|
||||||
// if unfulfilled starts a ViewPubsliher
|
// if unfulfilled starts a ViewPubsliher
|
||||||
public class RemoteQueryManager : IWorker {
|
public class CloudStreamManager : IWorker
|
||||||
public class StreamPubUnit {
|
{
|
||||||
|
public class StreamPubUnit
|
||||||
|
{
|
||||||
public string name;
|
public string name;
|
||||||
public string stream_expression;
|
public string stream_expression;
|
||||||
public ViewStream stream;
|
public ViewStream stream;
|
||||||
public bool persist;
|
public bool persist;
|
||||||
public ViewPublisher view_publisher;
|
public ViewPublisher view_publisher;
|
||||||
|
|
||||||
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
|
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
|
||||||
|
{
|
||||||
name = name_;
|
name = name_;
|
||||||
stream_expression = stream_expression_;
|
stream_expression = stream_expression_;
|
||||||
stream = stream_;
|
stream = stream_;
|
||||||
@@ -24,7 +33,7 @@ namespace BCG {
|
|||||||
view_publisher = view_publisher_;
|
view_publisher = view_publisher_;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
static ViewStream? build_stream(string stream_expression)
|
static ViewStream build_stream(string stream_expression)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@@ -32,7 +41,9 @@ namespace BCG {
|
|||||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||||
return rv;
|
return rv;
|
||||||
|
|
||||||
} catch (Exception ex) {
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -41,37 +52,34 @@ namespace BCG {
|
|||||||
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
||||||
private ViewStream _stream = new TableStream("cloud_streams");
|
private ViewStream _stream = new TableStream("cloud_streams");
|
||||||
private PubTableBroker _broker = new PubTableBroker();
|
private PubTableBroker _broker = new PubTableBroker();
|
||||||
private ManagedSubscriber _sub = ManagedSubscriber.instance;
|
|
||||||
|
|
||||||
|
public CloudStreamManager()
|
||||||
public RemoteQueryManager() {
|
{
|
||||||
_stream.onChange += _on_str;
|
_stream.onChange += _on_str;
|
||||||
_sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
|
|
||||||
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
|
|
||||||
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
private void _on_str(object obj)
|
private void _on_str(object obj)
|
||||||
{
|
{
|
||||||
if (obj is not DataPacket dp) {
|
DataPacket dp = obj as DataPacket;
|
||||||
return;
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in dp.newRows)
|
foreach (IBlueRow r in dp.newRows)
|
||||||
{
|
{
|
||||||
string name = r.get<string>("name");
|
string name = r.get<string>("name");
|
||||||
bool persist = r.get<bool>("persist");
|
bool persist = r.get<bool>("persist");
|
||||||
string stream_exp = r.get<string>("stream_exp");
|
string stream_exp = r.get<string>("stream_exp");
|
||||||
if (!_stream_pub_map.ContainsKey(name)) {
|
if (!_stream_pub_map.ContainsKey(name))
|
||||||
_spawn_remote_stream(name, stream_exp, persist);
|
{
|
||||||
|
_assign(name, stream_exp, persist);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
|
private void _assign(string name, string stream_expression, bool persist)
|
||||||
|
{
|
||||||
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
||||||
ViewStream stream = build_stream(stream_expression);
|
ViewStream stream = build_stream(stream_expression);
|
||||||
_expose_to_ws(name, stream);
|
_expose_to_ws(name, stream);
|
||||||
if (persist) {
|
if (persist)
|
||||||
|
{
|
||||||
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
||||||
}
|
}
|
||||||
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
||||||
@@ -79,12 +87,14 @@ namespace BCG {
|
|||||||
_stream_pub_map.Add(name, u);
|
_stream_pub_map.Add(name, u);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _expose_to_ws(string name, ViewStream stream) {
|
private void _expose_to_ws(string name, ViewStream stream)
|
||||||
|
{
|
||||||
// TODO: john
|
// TODO: john
|
||||||
}
|
}
|
||||||
|
|
||||||
public void doWork() {
|
public void doWork()
|
||||||
|
{
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
|
||||||
[DataStreamUserRespresentation("remoteQuery")]
|
|
||||||
public class RemoteQueryStream : ViewStream {
|
|
||||||
private readonly string _name, _stream_exp;
|
|
||||||
private PublisherCore _publisher_core = new PublisherCore();
|
|
||||||
private ViewStream _source_str;
|
|
||||||
|
|
||||||
public RemoteQueryStream(params string[] lines) {
|
|
||||||
_stream_exp = get_expression(lines);
|
|
||||||
_name = Guid.NewGuid().ToString();
|
|
||||||
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
|
|
||||||
//_name.Replace("-", "_");
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
|
|
||||||
byte[] rem_req_bytes = rem_req.serialize();
|
|
||||||
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
|
|
||||||
_source_str = new TableStream(_name);
|
|
||||||
_source_str.onChange += new ChangeDelegate(_handle_data);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
//maybe there should be a way to turn remote requests off ?
|
|
||||||
_source_str.onChange -= new ChangeDelegate(_handle_data);
|
|
||||||
}
|
|
||||||
|
|
||||||
static string get_expression(params string[] lines) {
|
|
||||||
for (int i = 0; i < lines.Length; i++) {
|
|
||||||
// unescape quotes
|
|
||||||
lines[i] = lines[i].Replace("\\\"", "\"");
|
|
||||||
}
|
|
||||||
return string.Join("\n", lines);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _handle_data(object o) {
|
|
||||||
myFireEvent(o);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
namespace bts {
|
|
||||||
using System.IO;
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System.IO;
|
|
||||||
using System.Threading.Tasks.Dataflow;
|
|
||||||
|
|
||||||
public class RemoteRequest {
|
|
||||||
public readonly string stream_expression;
|
|
||||||
public readonly string published_table_name;
|
|
||||||
public RemoteRequest(string stream_expression_, string published_table_name_) {
|
|
||||||
stream_expression = stream_expression_;
|
|
||||||
published_table_name = published_table_name_;
|
|
||||||
}
|
|
||||||
public byte[] serialize() {
|
|
||||||
var str = new MemoryStream();
|
|
||||||
var bw = new BinaryWriter(str);
|
|
||||||
bw.write_string_cpp(stream_expression);
|
|
||||||
bw.write_string_cpp(published_table_name);
|
|
||||||
return str.ToArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
|
|
||||||
BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
|
|
||||||
string stream_expression_ = br.read_string_cpp();
|
|
||||||
string published_table_name_ = br.read_string_cpp();
|
|
||||||
return new RemoteRequest(stream_expression_, published_table_name_);
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,54 +1,59 @@
|
|||||||
// using BCG;
|
using BCG;
|
||||||
|
|
||||||
// using bts;
|
using System.Linq;
|
||||||
|
|
||||||
// using System.Linq;
|
namespace BCG
|
||||||
|
{
|
||||||
|
public class StreamOffloadRequestor
|
||||||
|
{
|
||||||
|
public class OffloadOp
|
||||||
|
{
|
||||||
|
string name;
|
||||||
|
string stream_exp;
|
||||||
|
}
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
private IBlueRowCollection _rows;
|
||||||
|
private PubTableBroker _broker = new PubTableBroker();
|
||||||
|
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
||||||
|
public StreamOffloadRequestor()
|
||||||
|
{
|
||||||
|
_schema = new BlueTableSchema("cloud_streams",
|
||||||
|
new ColumnInfo("id", 0, typeof(int)),
|
||||||
|
new ColumnInfo("name", 1, typeof(string)),
|
||||||
|
new ColumnInfo("persist", 2, typeof(bool)),
|
||||||
|
new ColumnInfo("stream_exp", 3, typeof(string)));
|
||||||
|
_existing_offloads_vs.onChange += _on_str;
|
||||||
|
}
|
||||||
|
private void _on_str(object obj)
|
||||||
|
{
|
||||||
|
DataPacket dp = obj as DataPacket;
|
||||||
|
_rows = dp.newRows;
|
||||||
|
// this is needed so the broker knows the state of the table
|
||||||
|
_broker.createTable(_rows);
|
||||||
|
}
|
||||||
|
public void add_to_cloud(string name, string stream_exp, bool persist)
|
||||||
|
{
|
||||||
|
// check if it exists
|
||||||
|
if (_rows == null)
|
||||||
|
{
|
||||||
|
BlueTable table = new BlueTable(_schema);
|
||||||
|
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
||||||
|
table.addRow(nr);
|
||||||
|
_broker.createTable(table);
|
||||||
|
// create table
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (exists(name))
|
||||||
|
return;
|
||||||
|
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
||||||
|
_broker.addRow(nr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// namespace BCG {
|
bool exists(string name)
|
||||||
// // used on the client
|
{
|
||||||
// public class StreamOffloadRequestor {
|
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
||||||
// public class OffloadOp {
|
}
|
||||||
// string name;
|
}
|
||||||
// string stream_exp;
|
}
|
||||||
// }
|
|
||||||
// private IBlueTableSchema _schema;
|
|
||||||
// private IBlueRowCollection _rows;
|
|
||||||
// private PubTableBroker _broker = new PubTableBroker();
|
|
||||||
// ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
|
||||||
// public StreamOffloadRequestor() {
|
|
||||||
// _schema = new BlueTableSchema("cloud_streams",
|
|
||||||
// new ColumnInfo("id", 0, typeof(int)),
|
|
||||||
// new ColumnInfo("name", 1, typeof(string)),
|
|
||||||
// new ColumnInfo("persist", 2, typeof(bool)),
|
|
||||||
// new ColumnInfo("stream_exp", 3, typeof(string)));
|
|
||||||
// _existing_offloads_vs.onChange += _on_str;
|
|
||||||
// }
|
|
||||||
// private void _on_str(object obj) {
|
|
||||||
// DataPacket dp = obj as DataPacket;
|
|
||||||
// _rows = dp.newRows;
|
|
||||||
// // this is needed so the broker knows the state of the table
|
|
||||||
// _broker.create_table(_rows);
|
|
||||||
// }
|
|
||||||
// public void add_to_cloud(string name, string stream_exp, bool persist) {
|
|
||||||
// // check if it exists
|
|
||||||
// if (_rows == null) {
|
|
||||||
// BlueTable table = new BlueTable(_schema);
|
|
||||||
// IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
|
||||||
// table.addRow(nr);
|
|
||||||
// _broker.create_table(table);
|
|
||||||
// // create table
|
|
||||||
// } else {
|
|
||||||
// if (exists(name))
|
|
||||||
// return;
|
|
||||||
// IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
|
||||||
// // TODO: use make_batch_changes instead
|
|
||||||
// Log.assert(false, "Fix addRow");
|
|
||||||
// //_broker.addRow(nr);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// bool exists(string name) {
|
|
||||||
// return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,14 +1,17 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
internal class AddColumnsRow : IBlueRow {
|
{
|
||||||
|
internal class AddColumnsRow : IBlueRow
|
||||||
|
{
|
||||||
private IBlueTableSchema mySchema;
|
private IBlueTableSchema mySchema;
|
||||||
private IBlueRow myRow;
|
private IBlueRow myRow;
|
||||||
//private object[] myCachedValues;
|
//private object[] myCachedValues;
|
||||||
|
|
||||||
#region Constructors
|
#region Constructors
|
||||||
public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
|
public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
|
||||||
|
{
|
||||||
mySchema = schema;
|
mySchema = schema;
|
||||||
myRow = row;
|
myRow = row;
|
||||||
//myCachedValues = new object[schema.columnNames.Length];
|
//myCachedValues = new object[schema.columnNames.Length];
|
||||||
@@ -16,78 +19,85 @@ namespace BCG {
|
|||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IBlueRow members
|
#region IBlueRow members
|
||||||
public IBlueTableSchema schema { get { return mySchema; } }
|
public IBlueTableSchema schema{get{return mySchema;}}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region ICollection members
|
#region ICollection members
|
||||||
public int Count { get { return mySchema.columnNames.Length; } }
|
public int Count {get{return mySchema.columnNames.Length;}}
|
||||||
|
|
||||||
public bool IsSynchronized { get { return false; } }
|
public bool IsSynchronized {get{return false;}}
|
||||||
|
|
||||||
public object SyncRoot { get { return this; } }
|
public object SyncRoot {get{return this;}}
|
||||||
|
|
||||||
public void CopyTo(Array array, int index) {
|
public void CopyTo (Array array, int index)
|
||||||
|
{
|
||||||
int i = index;
|
int i = index;
|
||||||
foreach (object o in this as IEnumerable)
|
foreach (object o in this as IEnumerable)
|
||||||
array.SetValue(o, i++);
|
array.SetValue (o, i++);
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IDictionary members
|
#region IDictionary members
|
||||||
public bool IsFixedSize { get { return true; } }
|
public bool IsFixedSize {get{return true;}}
|
||||||
|
|
||||||
public bool IsReadOnly { get { return true; } }
|
public bool IsReadOnly {get{return true;}}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The key can either be the string corrsponding to the column name or
|
/// The key can either be the string corrsponding to the column name or
|
||||||
/// the order of the column
|
/// the order of the column
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public object this[object key] {
|
public object this [object key]
|
||||||
get {
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
if (key is string)
|
if (key is string)
|
||||||
return this[(string)key];
|
return this[(string)key];
|
||||||
return this[(int)key];
|
return this[(int)key];
|
||||||
}
|
}
|
||||||
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
set {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
}
|
}
|
||||||
|
|
||||||
public object this[int index] {
|
public object this[int index]
|
||||||
get {
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
//if (myCachedValues[index] == null)
|
//if (myCachedValues[index] == null)
|
||||||
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
|
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
|
||||||
//return myCachedValues[index];
|
//return myCachedValues[index];
|
||||||
return mySchema[index].isConstant ? DBNull.Value : myRow[index];
|
return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
|
public object this[string colName] {get{return this[mySchema[colName].colNo];}}
|
||||||
|
|
||||||
public ICollection Keys { get { return mySchema.columnNames; } }
|
public ICollection Keys{get{return mySchema.columnNames;}}
|
||||||
|
|
||||||
public ICollection Values {
|
public ICollection Values
|
||||||
get {
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
object[] retVal = new object[mySchema.columnNames.Length];
|
object[] retVal = new object[mySchema.columnNames.Length];
|
||||||
CopyTo(retVal, 0);
|
CopyTo (retVal, 0);
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
|
||||||
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
|
||||||
public bool Contains(object key) { return mySchema[(string)key] != null; }
|
public bool Contains (object key){return mySchema[(string)key] != null;}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
||||||
/// iterate through the values, typecast this into an IEnumerable
|
/// iterate through the values, typecast this into an IEnumerable
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
|
||||||
|
|
||||||
|
IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
|
||||||
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,34 +1,13 @@
|
|||||||
using System;
|
using System;
|
||||||
|
using System.Data;
|
||||||
|
using System.Collections;
|
||||||
|
using BCG.Utilities;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
namespace BCG
|
||||||
|
{
|
||||||
|
|
||||||
namespace BCG {
|
public class AddColumnsStream : MapStream
|
||||||
public class AddColumnsStream : ViewStream {
|
{
|
||||||
public static Tuple<string, Type, object> parse_exp(string exp) {
|
public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef)) {}
|
||||||
string[] toks = exp.Split(':');
|
|
||||||
string col_name = toks[0];
|
|
||||||
string col_value = toks[1];
|
|
||||||
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
|
|
||||||
return Tuple.Create(col_name, tup.Item1, tup.Item2);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Tuple<string, Type, object> _new_col;
|
|
||||||
private IBlueTableSchema _out_schema;
|
|
||||||
|
|
||||||
public AddColumnsStream(ViewStream vs, string col_def) {
|
|
||||||
_new_col = parse_exp(col_def);
|
|
||||||
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
|
|
||||||
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
|
|
||||||
if (_out_schema == null)
|
|
||||||
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
|
|
||||||
return _out_schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
private IBlueRow _map(IBlueRow r) {
|
|
||||||
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
|
|
||||||
rv.setValue(_new_col.Item1, _new_col.Item3);
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
namespace bts {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
|
|
||||||
public class AddConstantColumnRow {
|
|
||||||
AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +1,73 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections;
|
||||||
using System.Diagnostics.CodeAnalysis;
|
// using BCG.Utilities;
|
||||||
using System.Linq;
|
// using BCG.Evaluator.AddColumnsExpression;
|
||||||
|
|
||||||
using Bts.ColumnExpressions;
|
// namespace BCG
|
||||||
|
// {
|
||||||
namespace BCG;
|
// /// <summary>
|
||||||
|
// /// Provides methods to add columns to the schema
|
||||||
/// <summary>
|
// /// </summary>
|
||||||
/// Provides methods to add columns to the schema
|
// public class ColumnAdder : IStreamMapper
|
||||||
/// </summary>
|
// {
|
||||||
/// <param name="columnsExpression">
|
// protected IDictionary myNewColumns;
|
||||||
/// String represting the columns to add to the output table
|
// protected IBlueTableSchema mySchema;
|
||||||
/// </param>
|
|
||||||
public class ColumnAdder : IStreamMapper {
|
// static syntax myParser =
|
||||||
// Store tuples of (column, index) to preserve order
|
// new syntax();
|
||||||
protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
|
|
||||||
protected IBlueTableSchema? mySchema;
|
// /// <summary>
|
||||||
|
// /// string represting the columns to be added to the output table.
|
||||||
public ColumnAdder(string columnsExpression) =>
|
// /// </summary>
|
||||||
myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
|
// /// <param name="colAddExpression"></param>
|
||||||
throw new Exception("Invalid columnn expression: " + columnsExpression))
|
// public ColumnAdder (string columnDef)
|
||||||
.Select((c, i) => (c, i))
|
// {
|
||||||
.ToDictionary(c => c.c.Name, c => (c.c, c.i));
|
// AddColumnsExpression ace;
|
||||||
|
|
||||||
[MemberNotNull(nameof(mySchema))]
|
// if (DataStreamLoadFileConfig.instance.useStaticParser)
|
||||||
public virtual IBlueRow map(IBlueRow row) {
|
// {
|
||||||
// Only rebuild schema if necessary (i.e., if it's null)
|
// lock (myParser)
|
||||||
mySchema ??= buildSchema(row.schema, myNewColumns);
|
// {
|
||||||
return new AddColumnsRow(mySchema, row);
|
// ace =
|
||||||
}
|
// (AddColumnsExpression)myParser.Parse(columnDef);
|
||||||
|
// }
|
||||||
[MemberNotNull(nameof(mySchema))]
|
// }
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
|
// else
|
||||||
// Always rebuild schema (even if it's already non-null)
|
// {
|
||||||
mySchema = buildSchema(inputSchema, myNewColumns);
|
// syntax parser = new syntax();
|
||||||
|
// ace =
|
||||||
[MemberNotNull(nameof(mySchema))]
|
// (AddColumnsExpression)parser.Parse(columnDef);
|
||||||
static BlueTableSchema buildSchema(
|
// }
|
||||||
IBlueTableSchema schema,
|
|
||||||
IDictionary<string, (ColumnExpression column, int index)> newColumns
|
// if (ace == null)
|
||||||
) {
|
// throw new Exception ("Invalid columnn expression: " + columnDef);
|
||||||
ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
|
// myNewColumns = ace.columns;
|
||||||
schema.columnInfo.CopyTo(cis, 0);
|
// }
|
||||||
foreach (var (col, index) in newColumns.Values) {
|
|
||||||
var schemaIdx = schema.columnInfo.Length + index;
|
// public virtual IBlueRow map (IBlueRow row)
|
||||||
cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
|
// {
|
||||||
}
|
// if (mySchema == null)
|
||||||
return new BlueTableSchema(cis, schema.tableName);
|
// myConstructSchema (row.schema);
|
||||||
}
|
// return new AddColumnsRow (mySchema, row);
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
// public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
|
||||||
|
// {
|
||||||
|
// myConstructSchema (inputSchema);
|
||||||
|
// return mySchema;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void myConstructSchema (IBlueTableSchema schema)
|
||||||
|
// {
|
||||||
|
// ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
|
||||||
|
// int ciIndex = 0;
|
||||||
|
// foreach (ColumnInfo ci in schema.columnInfo)
|
||||||
|
// cis[ciIndex++] = ci;
|
||||||
|
// foreach (DictionaryEntry de in myNewColumns)
|
||||||
|
// cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++,
|
||||||
|
// ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
|
||||||
|
// mySchema = new BlueTableSchema (cis, schema.tableName);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,24 +1,23 @@
|
|||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Provides a stream that with distinct rows with respect to the columns
|
/// Provides a stream that with distinct rows with respect to the columns
|
||||||
/// the user specifies in the argument
|
/// the user specifies in the argument
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[DataStreamUserRespresentation("distinct")]
|
[DataStreamUserRespresentation ("distinct")]
|
||||||
public class DistinctRowStream : ViewStream {
|
public class DistinctRowStream : ViewStream
|
||||||
public DistinctRowStream(ViewStream stream, params string[] cols) {
|
{
|
||||||
var g_by_eval = new DistinctRowEvaluator(cols);
|
public DistinctRowStream (ViewStream stream, params string[] cols)
|
||||||
ViewStream r = new GroupByStream(stream, g_by_eval);
|
{
|
||||||
var proj_exp = String.Empty;
|
ArrayList groupByCols = new ArrayList ();
|
||||||
foreach (var c in cols)
|
foreach (string col in cols)
|
||||||
proj_exp += $"{c} ";
|
foreach (string c in col.Split (' '))
|
||||||
proj_exp = proj_exp.Trim();
|
groupByCols.Add (c);
|
||||||
r = new NewProjectStream(r, proj_exp);
|
mySetRealSource (new GroupByStream (stream,
|
||||||
mySetRealSource(r);
|
new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
// using System;
|
||||||
|
// using System.Collections.Generic;
|
||||||
|
// using System.Text;
|
||||||
|
// using System.IO;
|
||||||
|
|
||||||
|
// using System.Runtime.Serialization.Formatters.Binary;
|
||||||
|
|
||||||
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// public class DumpLoaderStream : ViewStream, ISideEffectStream
|
||||||
|
// {
|
||||||
|
// //wtf
|
||||||
|
// //public class Exception : System.Exception
|
||||||
|
// //{public Exception(string msg) : base(msg) { }}
|
||||||
|
|
||||||
|
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
|
||||||
|
// {
|
||||||
|
// mySetRealSource(
|
||||||
|
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
||||||
|
// ViewStreamDatabase.instance));
|
||||||
|
// }
|
||||||
|
|
||||||
|
// public DumpLoaderStream(string fileName)
|
||||||
|
// {
|
||||||
|
// if (fileName == null)
|
||||||
|
// throw new LoaderStream.Exception("File name cannot be null");
|
||||||
|
|
||||||
|
// for (int i = 0; i < 5; i++)
|
||||||
|
// try
|
||||||
|
// {
|
||||||
|
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
||||||
|
// {
|
||||||
|
// IClientDeserializer deser = new ClientUniversalDeserializer();
|
||||||
|
// byte[] the_ops;
|
||||||
|
// Array the_rows;
|
||||||
|
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
|
||||||
|
|
||||||
|
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
|
||||||
|
// }
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// catch (IOException)
|
||||||
|
// {
|
||||||
|
// System.Threading.Thread.Sleep(50);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,25 +1,48 @@
|
|||||||
using BCG;
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Text;
|
||||||
|
using System.IO;
|
||||||
|
|
||||||
namespace bts {
|
using System.Runtime.Serialization.Formatters.Binary;
|
||||||
// This is the other side of the SaverStream, its "loads" the stream and fires once
|
|
||||||
|
namespace BCG
|
||||||
|
{
|
||||||
[DataStreamUserRespresentation("loader")]
|
[DataStreamUserRespresentation("loader")]
|
||||||
public class LoaderStream : ViewStream {
|
public class LoaderStream : ViewStream, ISideEffectStream
|
||||||
|
{
|
||||||
|
public class Exception : System.Exception
|
||||||
|
{public Exception(string msg) : base(msg) { }}
|
||||||
|
|
||||||
private string _filename;
|
public LoaderStream(string fileName, int secondsBetweenPolls)
|
||||||
private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
|
{
|
||||||
|
mySetRealSource(
|
||||||
|
new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
||||||
public LoaderStream(string filename) {
|
ViewStreamDatabase.instance));
|
||||||
_filename = filename;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
public LoaderStream(string fileName)
|
||||||
byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
|
{
|
||||||
StreamIOMessage msg = _deser.deserialize(ser_msg);
|
if (fileName == null)
|
||||||
Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
|
throw new LoaderStream.Exception("File name cannot be null");
|
||||||
IBlueRowCollection rows = msg.create_message.rows;
|
|
||||||
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
|
for (int i = 0; i < 5; i++)
|
||||||
myFireEvent(dp);
|
try
|
||||||
|
{
|
||||||
|
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
||||||
|
{
|
||||||
|
BinaryFormatter bf = new BinaryFormatter();
|
||||||
|
#pragma warning disable SYSLIB0011
|
||||||
|
BlueTable bt = (BlueTable)bf.Deserialize(fs);
|
||||||
|
#pragma warning restore SYSLIB0011
|
||||||
|
mySetRealSource(new BlueRowCollectionStaticStream(bt));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
catch (IOException)
|
||||||
|
{
|
||||||
|
System.Threading.Thread.Sleep(50);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,11 +5,9 @@ using System.Text;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using BCG;
|
|
||||||
|
|
||||||
namespace bts {
|
namespace BCG {
|
||||||
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
|
// Writes all the udpates to a stream in a file.
|
||||||
// which just writes one update, as a create)
|
|
||||||
// File naming:
|
// File naming:
|
||||||
// token_epoch_yyyyMMdd_HHmmss
|
// token_epoch_yyyyMMdd_HHmmss
|
||||||
// File location: directory(passed)\token\
|
// File location: directory(passed)\token\
|
||||||
@@ -17,16 +15,11 @@ namespace bts {
|
|||||||
// data:
|
// data:
|
||||||
// epochtime, size, serialized data
|
// epochtime, size, serialized data
|
||||||
[DataStreamUserRespresentation("pers")]
|
[DataStreamUserRespresentation("pers")]
|
||||||
// Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
|
|
||||||
// which maps the name of the directory where the streams are saved. The directory (3rd arg) is the
|
|
||||||
// parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
|
|
||||||
// file is created. The file names are token_epoch_datetime
|
|
||||||
// the first entry in a file is the complete state
|
|
||||||
public class PersistorStream : ViewStream {
|
public class PersistorStream : ViewStream {
|
||||||
private ViewStream _source_stream;
|
private ViewStream _source_stream;
|
||||||
private string _token;
|
private string _token;
|
||||||
private FileStream _out_fs;
|
private FileStream _out_fs;
|
||||||
private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
|
private IBlueDataSerializer _serializer;
|
||||||
private string _file_name = String.Empty;
|
private string _file_name = String.Empty;
|
||||||
private long _rollover_size;
|
private long _rollover_size;
|
||||||
private readonly string _directory_path;
|
private readonly string _directory_path;
|
||||||
@@ -95,19 +88,20 @@ namespace bts {
|
|||||||
void _on_change(object new_value) {
|
void _on_change(object new_value) {
|
||||||
DataPacket dp = new_value as DataPacket;
|
DataPacket dp = new_value as DataPacket;
|
||||||
byte[] serialized_update = null;
|
byte[] serialized_update = null;
|
||||||
bool first_write = _serializer == null || _n_updates_in_current_file == 0;
|
bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
|
||||||
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
_serializer = _serializer ?? new UniversalSerializer(dp.schema);
|
||||||
if (first_write) {
|
if (_write_create) {
|
||||||
// first file always starts with create
|
// new files always starts with create
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
_serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
|
||||||
} else
|
} else
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
_serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
|
||||||
|
|
||||||
BinaryWriter bw = new BinaryWriter(_out_fs);
|
BinaryWriter bw = new BinaryWriter(_out_fs);
|
||||||
long utc_epoch = epoch_now();
|
long utc_epoch = epoch_now();
|
||||||
|
|
||||||
//time, length of data, data
|
//time, length of data, data
|
||||||
//bw.Write(utc_epoch);
|
bw.Write(utc_epoch);
|
||||||
//bw.Write(serialized_update.Length);
|
bw.Write(serialized_update.Length);
|
||||||
bw.Write(serialized_update);
|
bw.Write(serialized_update);
|
||||||
_out_fs.Flush();
|
_out_fs.Flush();
|
||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
|
|||||||
@@ -0,0 +1,190 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.IO;
|
||||||
|
using System.Runtime.Serialization.Formatters.Binary;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using bts.collections;
|
||||||
|
using System.Reflection.Metadata;
|
||||||
|
|
||||||
|
|
||||||
|
namespace BCG {
|
||||||
|
|
||||||
|
public class PlaybackStream : ViewStream, IWorker {
|
||||||
|
private readonly PlaybackType _type;
|
||||||
|
private BlueTable _current_table;
|
||||||
|
private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||||
|
// these are the packets already downloaded from the current file
|
||||||
|
private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
|
||||||
|
//private IEnumerable<FileStream> _file_streams = new List<FileStream>();
|
||||||
|
private IEnumerable<string> _file_names;
|
||||||
|
private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
|
||||||
|
private DateTime _playback_t0, _playback_t1;
|
||||||
|
private bool _all_files_read = false;
|
||||||
|
SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
|
||||||
|
|
||||||
|
public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
|
||||||
|
_type = PlaybackType.Streaming;
|
||||||
|
_playback_t0 = PersistorStream.parse_datetime(datetime_start);
|
||||||
|
_playback_t1 = PersistorStream.parse_datetime(datetime_end);
|
||||||
|
string token_directory_name = $"{directory_name}\\{token}";
|
||||||
|
bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
|
||||||
|
List<string> files_in_range = new List<string>();
|
||||||
|
if (Directory.Exists(token_directory_name)) {
|
||||||
|
_file_names =
|
||||||
|
Directory.GetFiles(token_directory_name).Where(f => {
|
||||||
|
DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
|
||||||
|
return file_time > _playback_t0 && file_time < _playback_t1;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
|
||||||
|
_snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
|
||||||
|
PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
|
||||||
|
if (snapshot_packet != null) {
|
||||||
|
foreach (PlaybackPacket p in _snapshot_data.packets) {
|
||||||
|
byte[] ops;
|
||||||
|
Array row_array;
|
||||||
|
_deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
||||||
|
for (int i = 0; i < ops.Length; i++) {
|
||||||
|
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
|
byte op = ops[i];
|
||||||
|
if (op == StreamOpCode.create) {
|
||||||
|
IBlueRowCollection full_table = rows_args[0];
|
||||||
|
_current_table = new BlueTable(full_table);
|
||||||
|
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
|
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
|
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
} else {
|
||||||
|
bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
|
}
|
||||||
|
if (p.idx == snapshot_packet.idx) {
|
||||||
|
myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void _load_packets_from_current_file() {
|
||||||
|
int curr_idx = _current_fs.Item1;
|
||||||
|
FileStream curr_fs = _current_fs.Item2;
|
||||||
|
_packets = new List<PlaybackPacket>();
|
||||||
|
_packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool current_file_all_replayed {
|
||||||
|
get {
|
||||||
|
bool rv = true;
|
||||||
|
_packets.ForEach(p => rv = rv && p.replayed);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
var fs = _current_fs.Item2;
|
||||||
|
if (fs != null) {
|
||||||
|
fs.Close();
|
||||||
|
}
|
||||||
|
ViewStreamDatabase.instance.removeWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
// public event PlaybackStreamProgress onProgress;
|
||||||
|
// public event Action onFileExhausted;
|
||||||
|
private PlaybackPacket _locate_packet_to_replay() {
|
||||||
|
foreach (var p in _packets) {
|
||||||
|
// returns the first non replayed packet
|
||||||
|
if (!p.replayed)
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
|
||||||
|
myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
|
||||||
|
if (onProgress != null)
|
||||||
|
onProgress(myFileReader.Position, myFileReader.Length);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#region IWorker Members
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
if (_all_files_read)
|
||||||
|
return;
|
||||||
|
if (current_file_all_replayed) {
|
||||||
|
_load_next_file();
|
||||||
|
}
|
||||||
|
PlaybackPacket curr_packet = _locate_packet_to_replay();
|
||||||
|
if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
|
||||||
|
byte[] ops;
|
||||||
|
Array row_array;
|
||||||
|
// what a stupid interface!
|
||||||
|
_deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
||||||
|
_fire_packet(ops, row_array);
|
||||||
|
curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
||||||
|
} else {
|
||||||
|
PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
|
||||||
|
// this is the time we need to fire (replay) this packet
|
||||||
|
long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
|
||||||
|
long now_epoch = PersistorStream.epoch_now();
|
||||||
|
// fire when we are past the
|
||||||
|
if (now_epoch >= time_to_fire) {
|
||||||
|
byte[] ops;
|
||||||
|
Array row_array;
|
||||||
|
_deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
||||||
|
_fire_packet(ops, row_array);
|
||||||
|
curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _fire_packet(byte[] ops, Array row_array) {
|
||||||
|
for (int i = 0; i < ops.Length; i++) {
|
||||||
|
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
|
byte op = ops[i];
|
||||||
|
if (op == StreamOpCode.create) {
|
||||||
|
IBlueRowCollection full_table = rows_args[0];
|
||||||
|
_current_table = new BlueTable(full_table);
|
||||||
|
//_current_table.addRows(full_table);
|
||||||
|
myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
||||||
|
(myLastValue as DataPacket).newRows));
|
||||||
|
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
|
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
|
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
myFireEvent(new DataPacket(_current_table, adds, dels));
|
||||||
|
} else {
|
||||||
|
bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _load_next_file() {
|
||||||
|
var new_idx = _current_fs.Item1 + 1;
|
||||||
|
if (new_idx >= _file_names.Count()) {
|
||||||
|
_all_files_read = true;
|
||||||
|
bts.Log.Gen.info("All files read");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
var prev_fs = _current_fs.Item2;
|
||||||
|
if (prev_fs != null)
|
||||||
|
prev_fs.Close();
|
||||||
|
var new_filename = _file_names.ElementAt(new_idx);
|
||||||
|
bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
|
||||||
|
var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
|
||||||
|
_current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
|
||||||
|
_load_packets_from_current_file();
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.IO;
|
||||||
|
|
||||||
|
namespace BCG
|
||||||
|
{
|
||||||
|
public class RecorderStream : ViewStream
|
||||||
|
{
|
||||||
|
private ViewStream mySource;
|
||||||
|
private DateTime myStartTime;
|
||||||
|
private FileStream myOutFile;
|
||||||
|
private string myFileName;
|
||||||
|
private IBlueDataSerializer mySerializer;
|
||||||
|
|
||||||
|
public RecorderStream(ViewStream src, string recordFile)
|
||||||
|
{
|
||||||
|
mySource = src;
|
||||||
|
myFileName = recordFile;
|
||||||
|
|
||||||
|
myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber()
|
||||||
|
{
|
||||||
|
myStartTime = DateTime.Now;
|
||||||
|
mySource.onChange += new ChangeDelegate(mySource_onChange);
|
||||||
|
myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber()
|
||||||
|
{
|
||||||
|
mySource.onChange -= mySource_onChange;
|
||||||
|
myOutFile.Close();
|
||||||
|
}
|
||||||
|
|
||||||
|
void mySource_onChange(object newValue)
|
||||||
|
{
|
||||||
|
DataPacket dp = (DataPacket)newValue;
|
||||||
|
|
||||||
|
BlueTable toWrite = new BlueTable(dp.schema);
|
||||||
|
|
||||||
|
if (mySerializer == null)
|
||||||
|
{
|
||||||
|
mySerializer = new UniversalSerializer(dp.schema);
|
||||||
|
}
|
||||||
|
foreach (IBlueRow r in dp.newRows)
|
||||||
|
toWrite.addRow(r);
|
||||||
|
|
||||||
|
byte[] result;
|
||||||
|
|
||||||
|
if (myLastValue == null)
|
||||||
|
mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
|
||||||
|
else
|
||||||
|
mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
|
||||||
|
|
||||||
|
BinaryWriter bw = new BinaryWriter(myOutFile);
|
||||||
|
//write down what time this happened
|
||||||
|
bw.Write((DateTime.Now - myStartTime).TotalSeconds);
|
||||||
|
bw.Write(result.Length);
|
||||||
|
bw.Write(result);
|
||||||
|
myOutFile.Flush();
|
||||||
|
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,37 +1,64 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections.Generic;
|
||||||
using System.Text;
|
// using System.Text;
|
||||||
using System.IO;
|
// using System.IO;
|
||||||
using BCG;
|
|
||||||
|
|
||||||
namespace bts {
|
// using System.Runtime.Serialization.Formatters.Binary;
|
||||||
// this saves the updates as a Create on the same file, each update calls -save, which
|
|
||||||
// reqrites the whole data to the packet. The other side of this stream is the Loader
|
|
||||||
[DataStreamUserRespresentation("saver")]
|
|
||||||
public class SaverStream : ViewStream {
|
|
||||||
private ViewStream _stream;
|
|
||||||
private string _filename;
|
|
||||||
private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
|
|
||||||
|
|
||||||
public SaverStream(ViewStream vs, string filename) {
|
// namespace BCG
|
||||||
_stream = vs;
|
// {
|
||||||
_filename = filename;
|
// [DataStreamUserRespresentation("saver")]
|
||||||
}
|
// public class SaverStream : ViewStream, ISideEffectStream
|
||||||
|
// {
|
||||||
|
// private ViewStream myStream;
|
||||||
|
// private string myFileName;
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
|
// public class Exception : System.Exception
|
||||||
|
// {
|
||||||
|
// public Exception(string msg) : base(msg) { }
|
||||||
|
// }
|
||||||
|
|
||||||
void _handle_changes(object dp_obj) {
|
// public SaverStream(ViewStream vs, string fileName)
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
// {
|
||||||
_save(dp);
|
// if (fileName == null)
|
||||||
myFireEvent(dp_obj);
|
// throw new SaverStream.Exception("File name cannot be null");
|
||||||
}
|
// myStream = vs;
|
||||||
|
// myFileName = fileName;
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
|
// }
|
||||||
|
|
||||||
protected void _save(DataPacket dp) {
|
// protected override void myOnFirstSubscriber()
|
||||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
// { myStream.onChange += new ChangeDelegate(myHandleChanges); }
|
||||||
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
|
||||||
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
// void myHandleChanges(object newValue)
|
||||||
}
|
// {
|
||||||
}
|
// DataPacket dp = newValue as DataPacket;
|
||||||
}
|
// mySave(dp);
|
||||||
|
// myFireEvent(newValue);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnLastSubscriber()
|
||||||
|
// { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
|
||||||
|
|
||||||
|
// protected void mySave(DataPacket dp)
|
||||||
|
// {
|
||||||
|
// for (int i = 0; i < 5 ; i++)
|
||||||
|
// try
|
||||||
|
// {
|
||||||
|
// using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
|
||||||
|
// {
|
||||||
|
// BinaryFormatter bf = new BinaryFormatter();
|
||||||
|
// BlueTable t = new BlueTable(dp.schema);
|
||||||
|
// foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
|
||||||
|
// t.addRow(new BlueRow(r));
|
||||||
|
// bf.Serialize(fs, t);
|
||||||
|
// }
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// catch (IOException)
|
||||||
|
// {
|
||||||
|
// System.Threading.Thread.Sleep(5);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,129 +1,115 @@
|
|||||||
// using System;
|
using System;
|
||||||
// using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
// using System.Linq;
|
using System.Linq;
|
||||||
// using System.Text;
|
using System.Text;
|
||||||
// using System.IO;
|
using System.IO;
|
||||||
// using System.Runtime.Serialization.Formatters.Binary;
|
using System.Runtime.Serialization.Formatters.Binary;
|
||||||
// using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
// using bts.collections;
|
using bts.collections;
|
||||||
// using System.Reflection.Metadata;
|
using System.Reflection.Metadata;
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public class SnapshotStream : ViewStream {
|
public class SnapshotStream : ViewStream {
|
||||||
public enum SpecificSnapshot {
|
|
||||||
FirstOfDay,
|
|
||||||
LastOfDay
|
|
||||||
};
|
|
||||||
// private BlueTable _current_table;
|
|
||||||
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
|
||||||
// public DateTime _snapshot_time;
|
|
||||||
// public string _snapshot_file;
|
|
||||||
// public IEnumerable<PlaybackPacket> _packets;
|
|
||||||
// public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
|
|
||||||
// DateTime dt;
|
|
||||||
// var parsed = DateTime.TryParse(snapshot_date, out dt);
|
|
||||||
// dt = dt.Date;
|
|
||||||
// bts.Log.assert(parsed);
|
|
||||||
// SpecificSnapshot specific_snapshot_enum;
|
|
||||||
// parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
|
|
||||||
// bts.Log.assert(parsed);
|
|
||||||
// string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
|
|
||||||
// ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
|
|
||||||
// mySetRealSource(rs);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public SnapshotStream(string snapshot_dt, string directory_name, string token) {
|
private BlueTable _current_table;
|
||||||
// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
|
private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||||
// string token_directory_name = $"{directory_name}\\{token}";
|
public DateTime _snapshot_time;
|
||||||
// _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
|
public string _snapshot_file;
|
||||||
// }
|
public IEnumerable<PlaybackPacket> _packets;
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
|
||||||
// FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
|
|
||||||
// _packets = StreamPersistorUtils.load_packets_from_file(fs);
|
|
||||||
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
|
|
||||||
// if (snapshot_packet != null) {
|
|
||||||
// foreach (PlaybackPacket p in _packets) {
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
|
||||||
// byte op = ops[i];
|
|
||||||
// if (op == StreamOpCode.create) {
|
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
// if (p.idx == snapshot_packet.idx) {
|
|
||||||
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _absorb_packet(byte[] ops, Array row_array) {
|
public SnapshotStream(string snapshot_dt, string directory_name, string token) {
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
_snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
string token_directory_name = $"{directory_name}\\{token}";
|
||||||
// byte op = ops[i];
|
_snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
|
||||||
// if (op == StreamOpCode.create) {
|
}
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Operator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
// }
|
FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
|
||||||
|
_packets = StreamPersistorUtils.load_packets_from_file(fs);
|
||||||
|
PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
|
||||||
|
if (snapshot_packet != null) {
|
||||||
|
foreach (PlaybackPacket p in _packets) {
|
||||||
|
byte[] ops;
|
||||||
|
Array row_array;
|
||||||
|
_deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
||||||
|
for (int i = 0; i < ops.Length; i++) {
|
||||||
|
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
|
byte op = ops[i];
|
||||||
|
if (op == StreamOpCode.create) {
|
||||||
|
IBlueRowCollection full_table = rows_args[0];
|
||||||
|
_current_table = new BlueTable(full_table);
|
||||||
|
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
|
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
|
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
} else {
|
||||||
|
bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
|
}
|
||||||
|
if (p.idx == snapshot_packet.idx) {
|
||||||
|
myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// // public event SnapshotStream.Progress onProgress;
|
private void _absorb_packet(byte[] ops, Array row_array) {
|
||||||
// // public event Action onFileExhausted;
|
for (int i = 0; i < ops.Length; i++) {
|
||||||
// private PlaybackPacket _locate_packet_to_replay() {
|
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
// foreach (var p in _packets) {
|
byte op = ops[i];
|
||||||
// // returns the first non replayed packet
|
if (op == StreamOpCode.create) {
|
||||||
// if (!p.replayed)
|
IBlueRowCollection full_table = rows_args[0];
|
||||||
// return p;
|
_current_table = new BlueTable(full_table);
|
||||||
// }
|
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
// return null;
|
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
// }
|
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
} else {
|
||||||
|
bts.Log.assert(false, $"Operator: {op} is not expected");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// private void _fire_packet(byte[] ops, Array row_array) {
|
protected override void myOnLastSubscriber() {
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
}
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
|
||||||
// byte op = ops[i];
|
|
||||||
// if (op == StreamOpCode.create) {
|
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// //_current_table.addRows(full_table);
|
|
||||||
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
|
||||||
// (myLastValue as DataPacket).newRows));
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
|
|
||||||
// }
|
// public event SnapshotStream.Progress onProgress;
|
||||||
// }
|
// public event Action onFileExhausted;
|
||||||
|
private PlaybackPacket _locate_packet_to_replay() {
|
||||||
|
foreach (var p in _packets) {
|
||||||
|
// returns the first non replayed packet
|
||||||
|
if (!p.replayed)
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _fire_packet(byte[] ops, Array row_array) {
|
||||||
|
for (int i = 0; i < ops.Length; i++) {
|
||||||
|
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
|
byte op = ops[i];
|
||||||
|
if (op == StreamOpCode.create) {
|
||||||
|
IBlueRowCollection full_table = rows_args[0];
|
||||||
|
_current_table = new BlueTable(full_table);
|
||||||
|
//_current_table.addRows(full_table);
|
||||||
|
myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
||||||
|
(myLastValue as DataPacket).newRows));
|
||||||
|
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
|
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
|
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
myFireEvent(new DataPacket(_current_table, adds, dels));
|
||||||
|
} else {
|
||||||
|
bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,7 +3,6 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using bts;
|
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
@@ -38,20 +37,6 @@ namespace BCG {
|
|||||||
public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
|
public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
|
||||||
}
|
}
|
||||||
public static class StreamPersistorUtils {
|
public static class StreamPersistorUtils {
|
||||||
public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
|
|
||||||
DateTime max_dt = DateTime.MinValue;
|
|
||||||
string rv = String.Empty;
|
|
||||||
string[] file_names = Directory.GetFiles(directory_name);
|
|
||||||
for (int i = 0; i < file_names.Count() - 1; i++) {
|
|
||||||
string fn = file_names[i];
|
|
||||||
var dt = parse_datetime_from_filename(fn);
|
|
||||||
if (dt > max_dt) {
|
|
||||||
max_dt = dt;
|
|
||||||
rv = fn;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
|
public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
|
||||||
string[] file_names = Directory.GetFiles(directory_name);
|
string[] file_names = Directory.GetFiles(directory_name);
|
||||||
for (int i = 0; i < file_names.Count() - 1; i++) {
|
for (int i = 0; i < file_names.Count() - 1; i++) {
|
||||||
|
|||||||
@@ -1,10 +1,16 @@
|
|||||||
|
using BCG;
|
||||||
|
using System;
|
||||||
|
using System.Data;
|
||||||
|
using System.Collections;
|
||||||
|
using BCG.Utilities;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
|
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
|
|
||||||
namespace BCG.StreamWorld.Addition
|
namespace BCG.StreamWorld.Addition
|
||||||
{
|
{
|
||||||
[DataStreamUserRespresentation (false)]
|
[DataStreamUserRespresentation (false)]
|
||||||
@@ -145,6 +151,7 @@ namespace BCG
|
|||||||
private string myFilterString;
|
private string myFilterString;
|
||||||
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
||||||
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
|
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
|
||||||
|
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||||
|
|
||||||
#region constructors
|
#region constructors
|
||||||
public delegate bool OnFilter(IBlueRow r);
|
public delegate bool OnFilter(IBlueRow r);
|
||||||
@@ -219,6 +226,7 @@ namespace BCG
|
|||||||
|
|
||||||
private void myHandleCallbacks(object newValue)
|
private void myHandleCallbacks(object newValue)
|
||||||
{
|
{
|
||||||
|
_timer.start();
|
||||||
DataPacket retVal;
|
DataPacket retVal;
|
||||||
DataPacket arrivedPacket = newValue as DataPacket;
|
DataPacket arrivedPacket = newValue as DataPacket;
|
||||||
myLastReceivedRows = arrivedPacket.newRows;
|
myLastReceivedRows = arrivedPacket.newRows;
|
||||||
@@ -247,6 +255,7 @@ namespace BCG
|
|||||||
if (myFilter.filter(r))
|
if (myFilter.filter(r))
|
||||||
myLastFiredRows.addRow(r);
|
myLastFiredRows.addRow(r);
|
||||||
}
|
}
|
||||||
|
_timer.stop();
|
||||||
myFireEvent(retVal);
|
myFireEvent(retVal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,66 +1,37 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Text.RegularExpressions;
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
|
{
|
||||||
|
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
|
||||||
public static TimeSpan parse_duration(string input) {
|
{
|
||||||
int days = 0, hours = 0, minutes = 0, seconds = 0;
|
|
||||||
|
|
||||||
// Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
|
|
||||||
Regex regex = new Regex(@"(\d+)([dhms])");
|
|
||||||
MatchCollection matches = regex.Matches(input);
|
|
||||||
|
|
||||||
foreach (Match match in matches) {
|
|
||||||
int value = int.Parse(match.Groups[1].Value);
|
|
||||||
string unit = match.Groups[2].Value;
|
|
||||||
|
|
||||||
switch (unit) {
|
|
||||||
case "d":
|
|
||||||
days = value;
|
|
||||||
break;
|
|
||||||
case "h":
|
|
||||||
hours = value;
|
|
||||||
break;
|
|
||||||
case "m":
|
|
||||||
minutes = value;
|
|
||||||
break;
|
|
||||||
case "s":
|
|
||||||
seconds = value;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return new TimeSpan(days, hours, minutes, seconds);
|
|
||||||
}
|
|
||||||
|
|
||||||
#region constructors
|
#region constructors
|
||||||
|
|
||||||
public TimeBasedFilter(string col_name, string row_time_to_live)
|
|
||||||
: this(col_name, parse_duration(row_time_to_live)) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
|
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
|
||||||
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
|
: this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
|
||||||
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
|
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
|
||||||
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
|
: this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
|
||||||
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
||||||
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
|
: this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
|
||||||
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
|
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
|
||||||
myVsd = vsd;
|
{
|
||||||
_time_col = rowTimeColumnName;
|
myVsd = vsd;
|
||||||
_ttl = rowTimeToLive;
|
myRowTimeColumnName = rowTimeColumnName;
|
||||||
myFilterFrequency = filterFrequency;
|
myRowTimeToLive = rowTimeToLive;
|
||||||
myNextFilter = DateTime.MinValue;
|
myFilterFrequency = filterFrequency;
|
||||||
|
myNextFilter = DateTime.MinValue;
|
||||||
|
|
||||||
// add worker
|
// add worker
|
||||||
myVsd.addWorker(this);
|
myVsd.addWorker(this);
|
||||||
@@ -70,25 +41,32 @@ namespace BCG {
|
|||||||
public event FilterChangeDelegate onFilterChange;
|
public event FilterChangeDelegate onFilterChange;
|
||||||
|
|
||||||
#region IStreamFilter
|
#region IStreamFilter
|
||||||
public bool filter(IBlueRow row) {
|
public bool filter(IBlueRow row)
|
||||||
DateTime now = DateTime.Now;
|
{
|
||||||
|
DateTime now = DateTime.Now;
|
||||||
|
DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
|
||||||
|
|
||||||
DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
|
if (now > timeStamp)
|
||||||
if (now > row_time) {
|
{
|
||||||
TimeSpan span = now - row_time;
|
TimeSpan span = now - timeStamp;
|
||||||
if (span >= _ttl) {
|
|
||||||
|
if (span >= myRowTimeToLive)
|
||||||
|
{
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return (true);
|
return (true);
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IWorker
|
#region IWorker
|
||||||
public void doWork() {
|
public void doWork()
|
||||||
|
{
|
||||||
DateTime now = DateTime.Now;
|
DateTime now = DateTime.Now;
|
||||||
|
|
||||||
if (now > myNextFilter) {
|
if (now > myNextFilter)
|
||||||
|
{
|
||||||
// fire event
|
// fire event
|
||||||
onFilterChange();
|
onFilterChange();
|
||||||
|
|
||||||
@@ -98,7 +76,8 @@ namespace BCG {
|
|||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IDisposable
|
#region IDisposable
|
||||||
public void Dispose() {
|
public void Dispose()
|
||||||
|
{
|
||||||
// remove worker
|
// remove worker
|
||||||
myVsd.removeWorker(this);
|
myVsd.removeWorker(this);
|
||||||
}
|
}
|
||||||
@@ -106,20 +85,10 @@ namespace BCG {
|
|||||||
|
|
||||||
#region state
|
#region state
|
||||||
private ViewStreamDatabase myVsd;
|
private ViewStreamDatabase myVsd;
|
||||||
private string _time_col;
|
private string myRowTimeColumnName;
|
||||||
private TimeSpan _ttl;
|
private TimeSpan myRowTimeToLive;
|
||||||
private TimeSpan myFilterFrequency;
|
private TimeSpan myFilterFrequency;
|
||||||
private DateTime myNextFilter;
|
private DateTime myNextFilter;
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
[DataStreamUserRespresentation("timefilter")]
|
|
||||||
public class TimeBasedFilterStream : ViewStream {
|
|
||||||
public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
|
|
||||||
:
|
|
||||||
this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
|
|
||||||
public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
|
|
||||||
TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
|
|
||||||
mySetRealSource(new FilterStream(base_stream, filter));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -4,14 +4,18 @@ using BCG.Utilities;
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
public class GroupByStream : MethodDrivenStream {
|
{
|
||||||
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
public class GroupByStream : MethodDrivenStream
|
||||||
|
{
|
||||||
|
private class MyFuncGroupByEvaluator : IGroupByEvaluator
|
||||||
|
{
|
||||||
private string[] _groupby_cols;
|
private string[] _groupby_cols;
|
||||||
private Func<DataPacket, IBlueRow> _groupby_func;
|
private Func<DataPacket, IBlueRow> _groupby_func;
|
||||||
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
|
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
||||||
|
{
|
||||||
_groupby_cols = groupby_cols.ToArray();
|
_groupby_cols = groupby_cols.ToArray();
|
||||||
_groupby_func = groupby_func;
|
_groupby_func = groupby_func;
|
||||||
}
|
}
|
||||||
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
|
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
|
||||||
|
|
||||||
@@ -25,7 +29,7 @@ namespace BCG {
|
|||||||
/// Serves as a quick way to access all rows belonging to the same group
|
/// Serves as a quick way to access all rows belonging to the same group
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private bool myIdGuard = true;
|
private bool myIdGuard = true;
|
||||||
private RowDictionaryTable myBookKeepingTable;
|
private RowDictionaryTable myBookKeepingTable ;
|
||||||
private IGroupByEvaluator myGroupByEvaluator;
|
private IGroupByEvaluator myGroupByEvaluator;
|
||||||
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
|
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
|
||||||
//can map a schema to another
|
//can map a schema to another
|
||||||
@@ -33,98 +37,112 @@ namespace BCG {
|
|||||||
private int myIdGenerator = 0;
|
private int myIdGenerator = 0;
|
||||||
|
|
||||||
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
||||||
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
|
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
|
||||||
|
{ }
|
||||||
|
|
||||||
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
|
public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
|
||||||
: this(stream, groupByEvaluator, true) { }
|
: this (stream, groupByEvaluator, true)
|
||||||
|
{}
|
||||||
|
|
||||||
public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
|
public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
|
||||||
: base(stream) {
|
: base (stream)
|
||||||
|
{
|
||||||
myGroupByEvaluator = groupbyEvaluator;
|
myGroupByEvaluator = groupbyEvaluator;
|
||||||
myIdGuard = idGuard;
|
myIdGuard = idGuard;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleFirstCall() {
|
protected override void myHandleFirstCall ()
|
||||||
|
{
|
||||||
mySchema = myLastArrivedPacket.schema;
|
mySchema = myLastArrivedPacket.schema;
|
||||||
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
|
myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
|
||||||
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
|
myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows,
|
||||||
myGroupByEvaluator.groupByColumns);
|
myGroupByEvaluator.groupByColumns);
|
||||||
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
|
GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
|
||||||
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
|
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
|
||||||
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
|
{
|
||||||
rowGroup, new BlueTable(mySchema)));
|
IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
|
||||||
|
rowGroup, new BlueTable (mySchema)));
|
||||||
if (newRow != null)
|
if (newRow != null)
|
||||||
temp.addRow(newRow, myGetGroupString(rowGroup));
|
temp.addRow (newRow, myGetGroupString (rowGroup));
|
||||||
}
|
}
|
||||||
myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
|
myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adding need not to anything as deletion already takes care
|
/// Adding need not to anything as deletion already takes care
|
||||||
/// of addition
|
/// of addition
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected override void myHandleIncrementalAdd() { }
|
protected override void myHandleIncrementalAdd() {}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Both deletes and adds are handled here to make the intermediate results not
|
/// Both deletes and adds are handled here to make the intermediate results not
|
||||||
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream
|
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream
|
||||||
/// because of the nature of it)
|
/// because of the nature of it)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected override void myHandleIncrementalDelete() {
|
protected override void myHandleIncrementalDelete()
|
||||||
|
{
|
||||||
myFireSwitch = false;
|
myFireSwitch = false;
|
||||||
RowDictionaryTable delGroups, addGroups;
|
RowDictionaryTable delGroups, addGroups;
|
||||||
//form groups for deleted and added rows
|
//form groups for deleted and added rows
|
||||||
myInitIncrementalChangeData(out delGroups, out addGroups);
|
myInitIncrementalChangeData (out delGroups, out addGroups);
|
||||||
ArrayList handledAddGroups = new ArrayList();
|
ArrayList handledAddGroups = new ArrayList ();
|
||||||
string groupRepString;
|
string groupRepString;
|
||||||
|
|
||||||
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
|
foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
|
||||||
|
{
|
||||||
// BlueEventLog.logAssert (delGroup.Count > 0);
|
// BlueEventLog.logAssert (delGroup.Count > 0);
|
||||||
groupRepString = myGetGroupString(delGroup);
|
groupRepString = myGetGroupString (delGroup);
|
||||||
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
|
IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
|
||||||
new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
|
new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
|
||||||
handledAddGroups.Add(addGroup);
|
handledAddGroups.Add (addGroup);
|
||||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
|
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
|
||||||
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
|
new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
|
||||||
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
|
IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
|
||||||
myFireSwitch |= myMergeRow(newRow, groupRepString);
|
myFireSwitch |= myMergeRow (newRow, groupRepString);
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
|
foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
|
||||||
|
{
|
||||||
// BlueEventLog.logAssert (addGroup.Count > 0);
|
// BlueEventLog.logAssert (addGroup.Count > 0);
|
||||||
if (!handledAddGroups.Contains(addGroup)) {
|
if (!handledAddGroups.Contains (addGroup))
|
||||||
|
{
|
||||||
//this is the case when there are rows added to a group and none
|
//this is the case when there are rows added to a group and none
|
||||||
//have been removed, hence it 'escapes' from being dealt with in
|
//have been removed, hence it 'escapes' from being dealt with in
|
||||||
//the loop above
|
//the loop above
|
||||||
|
|
||||||
groupRepString = myGetGroupString(addGroup);
|
groupRepString = myGetGroupString (addGroup);
|
||||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
|
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
|
||||||
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
|
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
|
||||||
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
|
IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup,
|
||||||
new BlueTable(aggGroup.schema)));
|
new BlueTable (aggGroup.schema)));
|
||||||
myFireSwitch |= myMergeRow(newRow, groupRepString);
|
myFireSwitch |= myMergeRow (newRow, groupRepString);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool myMergeRow(IBlueRow newRow, string groupRepString) {
|
private bool myMergeRow (IBlueRow newRow, string groupRepString)
|
||||||
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
|
{
|
||||||
|
IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
|
||||||
//BlueEventLog.logAssert (newRow != null || existingRow != null);
|
//BlueEventLog.logAssert (newRow != null || existingRow != null);
|
||||||
if (existingRow == null && newRow == null)
|
if (existingRow == null && newRow == null)
|
||||||
return false;
|
return false;
|
||||||
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
|
else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
|
||||||
return false;
|
return false;
|
||||||
else if (existingRow == null) {
|
else if (existingRow == null)
|
||||||
|
{
|
||||||
//row added for the first time
|
//row added for the first time
|
||||||
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
|
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
|
||||||
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
|
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
|
||||||
return true;
|
return true;
|
||||||
} else {
|
}
|
||||||
((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
|
else
|
||||||
((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
|
{
|
||||||
if (newRow != null) {
|
((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
|
||||||
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
|
((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
|
||||||
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
|
if (newRow != null)
|
||||||
|
{
|
||||||
|
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
|
||||||
|
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -136,68 +154,77 @@ namespace BCG {
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="delGroups"></param>
|
/// <param name="delGroups"></param>
|
||||||
/// <param name="addGroups"></param>
|
/// <param name="addGroups"></param>
|
||||||
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
|
private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
|
||||||
|
{
|
||||||
delGroups =
|
delGroups =
|
||||||
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
|
new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
|
||||||
addGroups =
|
addGroups =
|
||||||
new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
|
new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
|
||||||
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
|
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
|
||||||
myBookKeepingTable.removeRow(deletedRow);
|
myBookKeepingTable.removeRow (deletedRow);
|
||||||
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
|
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
|
||||||
myBookKeepingTable.addRow(addedRow);
|
myBookKeepingTable.addRow (addedRow);
|
||||||
myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
|
myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
|
||||||
myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
|
myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnTableClear() {
|
protected override void myOnTableClear ()
|
||||||
|
{
|
||||||
if (myBookKeepingTable.Count == 0)
|
if (myBookKeepingTable.Count == 0)
|
||||||
return; //ie there arent any rows, so no need to optimize the 'cleansing' process
|
return; //ie there arent any rows, so no need to optimize the 'cleansing' process
|
||||||
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
|
myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
|
||||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
||||||
(myLastFiredPacket.newRows as GroupByOutputTable).clear();
|
(myLastFiredPacket.newRows as GroupByOutputTable).clear ();
|
||||||
myBookKeepingTable.clear();
|
myBookKeepingTable.clear ();
|
||||||
myGroupByEvaluator.clear();
|
myGroupByEvaluator.clear ();
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRow myEvaluate(DataPacket dp) {
|
private IBlueRow myEvaluate (DataPacket dp)
|
||||||
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
|
{
|
||||||
return evalRow == null ? null :
|
IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
|
||||||
|
return evalRow == null ? null :
|
||||||
myIdGuard ?
|
myIdGuard ?
|
||||||
BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
|
BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
|
||||||
}
|
}
|
||||||
|
|
||||||
private string myGetGroupString(IBlueRowCollection group) {
|
private string myGetGroupString (IBlueRowCollection group)
|
||||||
return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
|
{
|
||||||
|
return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber ()
|
||||||
|
{
|
||||||
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
|
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
|
||||||
//gets resubscribed, we start the process of building data again, and treat the callback as the first
|
//gets resubscribed, we start the process of building data again, and treat the callback as the first
|
||||||
//callback.
|
//callback.
|
||||||
myInit();
|
myInit ();
|
||||||
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myInit() {
|
protected override void myInit ()
|
||||||
base.myInit();
|
{
|
||||||
if (myBookKeepingTable != null) {
|
base.myInit ();
|
||||||
|
if (myBookKeepingTable != null)
|
||||||
|
{
|
||||||
//to avoid calling init before the first callback
|
//to avoid calling init before the first callback
|
||||||
myBookKeepingTable.clear();
|
myBookKeepingTable.clear ();
|
||||||
myGroupByEvaluator.clear();
|
myGroupByEvaluator.clear ();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static string getRepString(DataPacket dp, string[] colNames) {
|
public static string getRepString (DataPacket dp, string[] colNames)
|
||||||
return BlueDataUtility.rowToString(getRepRow(dp), colNames);
|
{
|
||||||
|
return BlueDataUtility.rowToString (getRepRow (dp), colNames);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static IBlueRow getRepRow(DataPacket dp) {
|
public static IBlueRow getRepRow (DataPacket dp)
|
||||||
|
{
|
||||||
if (dp.newRows.Count > 0)
|
if (dp.newRows.Count > 0)
|
||||||
return BlueDataUtility.extractRow(dp.newRows, 0);
|
return BlueDataUtility.extractRow (dp.newRows, 0);
|
||||||
else if (dp.addedRows.Count > 0)
|
else if (dp.addedRows.Count > 0)
|
||||||
return BlueDataUtility.extractRow(dp.addedRows, 0);
|
return BlueDataUtility.extractRow (dp.addedRows, 0);
|
||||||
else if (dp.deletedRows.Count > 0)
|
else if (dp.deletedRows.Count > 0)
|
||||||
return BlueDataUtility.extractRow(dp.deletedRows, 0);
|
return BlueDataUtility.extractRow (dp.deletedRows, 0);
|
||||||
// BlueEventLog.logAssert (false, "Illegal Data Packet");
|
// BlueEventLog.logAssert (false, "Illegal Data Packet");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -210,21 +237,22 @@ namespace BCG {
|
|||||||
/// <param name="newValue"></param>
|
/// <param name="newValue"></param>
|
||||||
/// <param name="isAdd"></param>
|
/// <param name="isAdd"></param>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
|
public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
|
||||||
if (lastValue == null || lastValue.Equals(DBNull.Value))
|
{
|
||||||
|
if (lastValue == null || lastValue.Equals (DBNull.Value))
|
||||||
return newValue;
|
return newValue;
|
||||||
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
|
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
|
||||||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
|
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
|
||||||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
|
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
|
||||||
Object retVal = null;
|
Object retVal = null;
|
||||||
if (newValue.GetType().Equals(typeof(int)))
|
if (newValue.GetType().Equals (typeof(int)))
|
||||||
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
|
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
|
||||||
else if (newValue.GetType().Equals(typeof(Double)))
|
else if (newValue.GetType().Equals (typeof(Double)))
|
||||||
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
|
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
|
||||||
else if (lastValue.GetType().Equals(typeof(Decimal)))
|
else if (lastValue.GetType().Equals (typeof(Decimal)))
|
||||||
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
|
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
|
||||||
else { }
|
else{}
|
||||||
// BlueEventLog.logAssert (false, "Illegal type");
|
// BlueEventLog.logAssert (false, "Illegal type");
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,42 +1,47 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections;
|
// using System.Collections;
|
||||||
|
// using BCG.Utilities;
|
||||||
|
|
||||||
using BCG.Utilities;
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// /// <summary>
|
||||||
|
// /// Computes the max for a given column
|
||||||
|
// /// </summary>
|
||||||
|
// public class MaxGroupByEvaluator : GroupByEvaluator
|
||||||
|
// {
|
||||||
|
|
||||||
namespace BCG {
|
// public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
|
||||||
/// <summary>
|
// {}
|
||||||
/// Computes the max for a given column
|
|
||||||
/// </summary>
|
|
||||||
public class MaxGroupByEvaluator : GroupByEvaluator {
|
|
||||||
|
|
||||||
public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
|
// public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
|
||||||
|
// : base (colMapEpression, outputCol, groupByCols) {}
|
||||||
|
|
||||||
public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
// public override IBlueRow evaluate (DataPacket dp)
|
||||||
: base(colMapEpression, outputCol, groupByCols) { }
|
// {
|
||||||
|
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
|
||||||
public override IBlueRow evaluate(DataPacket dp) {
|
// myOutputTable;
|
||||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
|
// string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
|
||||||
myOutputTable;
|
// IBlueRow currMaxRow = myOutputTable.getRow (hashString);
|
||||||
string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
// IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
|
||||||
IBlueRow currMaxRow = myOutputTable.getRow(hashString);
|
// if (maxRow != null)
|
||||||
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
|
// {
|
||||||
if (maxRow != null) {
|
// BlueEventLog.logAssert (currMaxRow != null);
|
||||||
bts.Log.assert(currMaxRow != null);
|
// BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
|
||||||
bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
|
// if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
|
||||||
if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
|
// {
|
||||||
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
|
// maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
|
||||||
if (maxRow == null)
|
// if (maxRow == null)
|
||||||
myOutputTable.removeRow(hashString);
|
// myOutputTable.removeRow (hashString);
|
||||||
else
|
// else
|
||||||
myOutputTable.addRow(maxRow, hashString);
|
// myOutputTable.addRow (maxRow, hashString);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
|
// maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
|
||||||
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
|
// // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
|
||||||
// otherwise if maxRow > currMaxRow then its replaced
|
// // otherwise if maxRow > currMaxRow then its replaced
|
||||||
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
|
// if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
|
||||||
myOutputTable.addRow(maxRow, hashString);
|
// myOutputTable.addRow (maxRow, hashString);
|
||||||
return myProject(myOutputTable.getRow(hashString));
|
// return myProject (myOutputTable.getRow (hashString));
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -1,15 +1,20 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
public class MaxStream : ViewStream {
|
// {
|
||||||
public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
|
// public class MaxStream : ViewStream
|
||||||
}
|
// {
|
||||||
public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
|
// public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// {
|
||||||
new MaxGroupByEvaluator(outputCol, inputCols)));
|
// }
|
||||||
}
|
// public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
|
||||||
|
// {
|
||||||
|
// mySetRealSource (new GroupByStream (stream,
|
||||||
|
// new MaxGroupByEvaluator (outputCol, inputCols)));
|
||||||
|
// }
|
||||||
|
|
||||||
public MaxStream(ViewStream stream, string outputCol)
|
// public MaxStream (ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
|
||||||
}
|
// {}
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
@@ -1,36 +1,40 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
// using BCG.Utilities;
|
||||||
|
|
||||||
using BCG.Utilities;
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// public class MinGroupByEvaluator : MaxGroupByEvaluator
|
||||||
|
// {
|
||||||
|
// public MinGroupByEvaluator (string outputCol, string[] groupByCols)
|
||||||
|
// : base (outputCol, groupByCols) {}
|
||||||
|
|
||||||
namespace BCG {
|
// public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
|
||||||
public class MinGroupByEvaluator : MaxGroupByEvaluator {
|
// base (colMapEpression, outputCol, groupByCols) {}
|
||||||
public MinGroupByEvaluator(string outputCol, string[] groupByCols)
|
|
||||||
: base(outputCol, groupByCols) { }
|
|
||||||
|
|
||||||
public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
|
// public override IBlueRow evaluate (DataPacket dp)
|
||||||
base(colMapEpression, outputCol, groupByCols) { }
|
// {
|
||||||
|
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
|
||||||
|
// string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
|
||||||
|
// IBlueRow currMinRow = myOutputTable.getRow (stringRep);
|
||||||
|
// IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
|
||||||
|
// if (minRow != null)
|
||||||
|
// {
|
||||||
|
// // BlueEventLog.logAssert (currMinRow != null);
|
||||||
|
// // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
|
||||||
|
// if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
|
||||||
|
// {
|
||||||
|
// minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
|
||||||
|
// if (minRow == null)
|
||||||
|
// myOutputTable.removeRow (stringRep);
|
||||||
|
// else
|
||||||
|
// myOutputTable.addRow (minRow, stringRep);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
|
||||||
|
// if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
|
||||||
|
// myOutputTable.addRow (minRow, stringRep);
|
||||||
|
// return myOutputTable.getRow (stringRep);
|
||||||
|
// }
|
||||||
|
|
||||||
public override IBlueRow evaluate(DataPacket dp) {
|
// }
|
||||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
// }
|
||||||
string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
|
||||||
IBlueRow currMinRow = myOutputTable.getRow(stringRep);
|
|
||||||
IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
|
|
||||||
if (minRow != null) {
|
|
||||||
// BlueEventLog.logAssert (currMinRow != null);
|
|
||||||
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
|
|
||||||
if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
|
|
||||||
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
|
|
||||||
if (minRow == null)
|
|
||||||
myOutputTable.removeRow(stringRep);
|
|
||||||
else
|
|
||||||
myOutputTable.addRow(minRow, stringRep);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
|
|
||||||
if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
|
|
||||||
myOutputTable.addRow(minRow, stringRep);
|
|
||||||
return myOutputTable.getRow(stringRep);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +1,17 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
public class MinStream : ViewStream {
|
// {
|
||||||
public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
|
// public class MinStream : ViewStream
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// {
|
||||||
new MinGroupByEvaluator(outputCol, inputCol)));
|
// public MinStream (ViewStream stream, string outputCol, string[] inputCol)
|
||||||
}
|
// {
|
||||||
|
// mySetRealSource (new GroupByStream (stream,
|
||||||
|
// new MinGroupByEvaluator (outputCol, inputCol)));
|
||||||
|
// }
|
||||||
|
|
||||||
public MinStream(ViewStream stream, string outputCol)
|
// public MinStream (ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
|
||||||
}
|
// {}
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
+14
-18
@@ -1,8 +1,10 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
using bts;
|
using System.Threading.Tasks;
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -12,7 +14,7 @@ namespace BCG
|
|||||||
private INonIncGroupByEvaluator _groupby_func;
|
private INonIncGroupByEvaluator _groupby_func;
|
||||||
private BlueRowHasher _row_hasher;
|
private BlueRowHasher _row_hasher;
|
||||||
private string[] _groupby_cols;
|
private string[] _groupby_cols;
|
||||||
private Stopwatch _timer = new();
|
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
|
|
||||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
||||||
@@ -31,27 +33,21 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
DataPacket dp = nv as DataPacket;
|
DataPacket dp = nv as DataPacket;
|
||||||
///grouping together
|
///grouping together
|
||||||
_timer.Restart();
|
_timer.start();
|
||||||
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
||||||
_timer.Stop();
|
_timer.stop();
|
||||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||||
if (groupElapsedMs >= 50) {
|
|
||||||
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
|
||||||
}
|
|
||||||
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
|
||||||
/// reducing
|
/// reducing
|
||||||
_timer.Restart();
|
_timer.start();
|
||||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||||
_timer.Stop();
|
_timer.stop();
|
||||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (applyElapsedMs >= 50)
|
if (_timer.lastIntervalTime > 50)
|
||||||
{
|
{
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
Log.DataFlow.info(
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
|
||||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
|
||||||
}
|
}
|
||||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,82 +1,94 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
/// <summary>
|
// {
|
||||||
/// Computes the sum for a given column
|
// /// <summary>
|
||||||
/// </summary>
|
// /// Computes the sum for a given column
|
||||||
public class SumGroupByEvaluator : GroupByEvaluator {
|
// /// </summary>
|
||||||
public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
|
// public class SumGroupByEvaluator : GroupByEvaluator
|
||||||
|
// {
|
||||||
|
// public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
|
||||||
|
// {}
|
||||||
|
|
||||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
// public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
|
||||||
: base(colMapEpression, outputCol, groupByCols) { }
|
// : base (colMapEpression, outputCol, groupByCols) {}
|
||||||
|
|
||||||
public override IBlueRow evaluate(DataPacket dp) {
|
// public override IBlueRow evaluate (DataPacket dp)
|
||||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
// {
|
||||||
|
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
|
||||||
IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
|
|
||||||
return sumRow;
|
// IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
|
||||||
}
|
// return sumRow;
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
|
// public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
|
||||||
private string[] myOutputCols, myGroupByCols;
|
// {
|
||||||
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
|
// private string[] myOutputCols, myGroupByCols;
|
||||||
myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
|
||||||
myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
// {
|
||||||
}
|
// myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
||||||
|
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
||||||
|
// }
|
||||||
|
|
||||||
public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
|
// public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
|
||||||
myOutputCols = output_columns;
|
// {
|
||||||
myGroupByCols = groupby_columns;
|
// myOutputCols = output_columns;
|
||||||
}
|
// myGroupByCols = groupby_columns;
|
||||||
|
// }
|
||||||
|
|
||||||
#region IGroupByEvaluator Members
|
// #region IGroupByEvaluator Members
|
||||||
|
|
||||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
|
// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
|
||||||
|
|
||||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
// public string[] groupByColumns { get { return myGroupByCols; } }
|
||||||
|
|
||||||
public void clear() {/*stateless*/}
|
// public void clear() {/*stateless*/}
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
|
// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
|
||||||
|
|
||||||
#endregion
|
// #endregion
|
||||||
}
|
// }
|
||||||
|
|
||||||
public class SumGroupByEvaluator<T> : GroupByEvaluator {
|
// public class SumGroupByEvaluator<T> : GroupByEvaluator
|
||||||
public SumGroupByEvaluator(string outputCol, string[] groupByCols)
|
// {
|
||||||
: base(outputCol, groupByCols) { }
|
// public SumGroupByEvaluator(string outputCol, string[] groupByCols)
|
||||||
|
// : base(outputCol, groupByCols)
|
||||||
|
// { }
|
||||||
|
|
||||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
// public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||||
: base(colMapEpression, outputCol, groupByCols) { }
|
// : base(colMapEpression, outputCol, groupByCols) { }
|
||||||
|
|
||||||
public override IBlueRow evaluate(DataPacket dp) {
|
// public override IBlueRow evaluate(DataPacket dp)
|
||||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
// {
|
||||||
|
// myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||||
|
|
||||||
IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
|
// IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
|
||||||
return sumRow;
|
// return sumRow;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
|
// public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
|
||||||
private string[] myOutputCols, myGroupByCols;
|
// {
|
||||||
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
|
// private string[] myOutputCols, myGroupByCols;
|
||||||
myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
|
||||||
myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
// {
|
||||||
}
|
// myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
||||||
|
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
||||||
|
// }
|
||||||
|
|
||||||
#region IGroupByEvaluator Members
|
// #region IGroupByEvaluator Members
|
||||||
|
|
||||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
|
// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
|
||||||
|
|
||||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
// public string[] groupByColumns { get { return myGroupByCols; } }
|
||||||
|
|
||||||
public void clear() {/*stateless*/}
|
// public void clear() {/*stateless*/}
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
|
// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
|
||||||
|
|
||||||
#endregion
|
// #endregion
|
||||||
}
|
// }
|
||||||
|
|
||||||
}
|
// }
|
||||||
@@ -1,89 +1,107 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections.Generic;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
public class NullMapper : IStreamMapper {
|
// {
|
||||||
private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
|
// public class NullMapper : IStreamMapper
|
||||||
|
// {
|
||||||
|
// private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
|
||||||
|
|
||||||
public NullMapper(string[] valid_cols) {
|
// public NullMapper(string[] valid_cols)
|
||||||
foreach (string c in valid_cols)
|
// {
|
||||||
_valid_cols[c] = true;
|
// foreach (string c in valid_cols)
|
||||||
_valid_cols["id"] = true;
|
// _valid_cols[c] = true;
|
||||||
}
|
// _valid_cols["id"] = true;
|
||||||
|
// }
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
|
// public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row) {
|
// public IBlueRow map(IBlueRow row)
|
||||||
MutableBlueRow out_row = new MutableBlueRow(row);
|
// {
|
||||||
foreach (string c in row.schema.columnNames)
|
// MutableBlueRow out_row = new MutableBlueRow(row);
|
||||||
if (!_valid_cols.ContainsKey(c))
|
// foreach (string c in row.schema.columnNames)
|
||||||
out_row.setValue(c, DBNull.Value);
|
// if (!_valid_cols.ContainsKey(c))
|
||||||
return out_row;
|
// out_row.setValue(c, DBNull.Value);
|
||||||
}
|
// return out_row;
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
public class SumStream : ViewStream {
|
// public class SumStream : ViewStream
|
||||||
public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
|
// {
|
||||||
string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
// public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
|
||||||
string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
// {
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
||||||
outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
|
// string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
||||||
new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
// mySetRealSource(new GroupByStream(stream,
|
||||||
}
|
// outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
|
||||||
|
// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
||||||
|
// }
|
||||||
|
|
||||||
public SumStream(ViewStream stream, string[] output_columns) {
|
// public SumStream(ViewStream stream, string[] output_columns)
|
||||||
ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
|
// {
|
||||||
rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
|
// ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
|
||||||
rs = new ProjectStream(rs, "!fake_col");
|
// rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
|
||||||
mySetRealSource(rs);
|
// rs = new ProjectStream(rs, "!fake_col");
|
||||||
}
|
// mySetRealSource(rs);
|
||||||
|
// }
|
||||||
|
|
||||||
public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
|
// public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
|
||||||
string[] output_columns = space_seperated_output_columns.Split(' ');
|
// {
|
||||||
string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
|
// string[] output_columns = space_seperated_output_columns.Split(' ');
|
||||||
mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
|
// string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
|
||||||
}
|
// mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
|
||||||
|
// }
|
||||||
|
|
||||||
public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
|
// public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
|
||||||
ViewStream rs = null;
|
// {
|
||||||
groupby_columns = groupby_columns ?? new string[] { };
|
// ViewStream rs = null;
|
||||||
if (groupby_columns.Length == 0) {
|
// groupby_columns = groupby_columns ?? new string[] {};
|
||||||
// rs = new AddColumnsStream(stream, "fake_col=0");
|
// if (groupby_columns.Length == 0)
|
||||||
// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
|
// {
|
||||||
// rs = new ProjectStream(rs, "!fake_col");
|
// rs = new AddColumnsStream(stream, "fake_col=0");
|
||||||
} else {
|
// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
|
||||||
rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
|
// rs = new ProjectStream(rs, "!fake_col");
|
||||||
}
|
// }
|
||||||
if (force_null_into_other_columns) {
|
// else
|
||||||
List<string> valid_cols = new List<string>(output_columns);
|
// {
|
||||||
valid_cols.AddRange(groupby_columns);
|
// rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
|
||||||
rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
|
// }
|
||||||
}
|
// if (force_null_into_other_columns)
|
||||||
mySetRealSource(rs);
|
// {
|
||||||
}
|
// List<string> valid_cols = new List<string>(output_columns);
|
||||||
|
// valid_cols.AddRange(groupby_columns);
|
||||||
|
// rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
|
||||||
|
// }
|
||||||
|
// mySetRealSource(rs);
|
||||||
|
// }
|
||||||
|
|
||||||
public SumStream(ViewStream stream, string outputCol)
|
// public SumStream (ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
|
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
|
||||||
}
|
// {
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
public class IntSumStream : ViewStream {
|
// public class IntSumStream : ViewStream
|
||||||
public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
|
// {
|
||||||
string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
// public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
|
||||||
string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
// {
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
||||||
outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
|
// string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
||||||
new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
// mySetRealSource(new GroupByStream(stream,
|
||||||
}
|
// outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
|
||||||
|
// new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
||||||
|
// }
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
public class SumStream<T> : ViewStream {
|
// public class SumStream<T> : ViewStream
|
||||||
public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
|
// {
|
||||||
string[] inputCols = spaceSeparatedInputCols.Split(' ');
|
// public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// {
|
||||||
new SumGroupByEvaluator<T>(outputCol, inputCols)));
|
// string[] inputCols = spaceSeparatedInputCols.Split(' ');
|
||||||
}
|
// mySetRealSource(new GroupByStream(stream,
|
||||||
}
|
// new SumGroupByEvaluator<T>(outputCol, inputCols)));
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
using ZstdSharp.Unsafe;
|
|
||||||
|
|
||||||
namespace bts {
|
|
||||||
public class RowDiffStream : ViewStream {
|
|
||||||
private ViewStream _vs;
|
|
||||||
private readonly string _order_col, _col_1, _col_2;
|
|
||||||
private readonly int _diff;
|
|
||||||
private IBlueTableSchema _out_schema;
|
|
||||||
private readonly string _diff_col_name;
|
|
||||||
public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
|
|
||||||
_vs = vs;
|
|
||||||
_order_col = order_col;
|
|
||||||
_col_1 = col_1;
|
|
||||||
_col_2 = col_2;
|
|
||||||
_diff = diff;
|
|
||||||
_diff_col_name = diff_col_name;
|
|
||||||
}
|
|
||||||
|
|
||||||
private string _diff_col_name_perc {
|
|
||||||
get {
|
|
||||||
return $"{_diff_col_name}_percentage";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_vs.onChange += _handle_vs;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_vs.onChange -= _handle_vs;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _handle_vs(object dp_obj) {
|
|
||||||
var dp = dp_obj as DataPacket;
|
|
||||||
IBlueRowCollection nrows = dp.newRows;
|
|
||||||
_out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
|
|
||||||
_out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
|
|
||||||
List<IBlueRow> row_list = new List<IBlueRow>();
|
|
||||||
foreach (IBlueRow r in nrows)
|
|
||||||
row_list.Add(r);
|
|
||||||
//OrderBy sorts in ascedning order
|
|
||||||
var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
|
|
||||||
var sorted_row_array = sorted_rows.ToArray();
|
|
||||||
// reverse it to make it descending
|
|
||||||
sorted_rows.Reverse();
|
|
||||||
//sort rows so we can do the diff
|
|
||||||
OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
|
|
||||||
int i = 0;
|
|
||||||
foreach (IBlueRow r in sorted_rows) {
|
|
||||||
MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
|
|
||||||
//So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
|
|
||||||
// we assume x - y, and name variables like that
|
|
||||||
double x = r.get<double>(_col_1);
|
|
||||||
int y_row_idx = i + _diff;
|
|
||||||
double diff = System.Double.NaN;
|
|
||||||
double perc = System.Double.NaN;
|
|
||||||
if (y_row_idx < sorted_rows.Count()) {
|
|
||||||
var y = sorted_rows[y_row_idx].get<double>(_col_2);
|
|
||||||
diff = x - y;
|
|
||||||
perc = 100.0 * diff / y;
|
|
||||||
}
|
|
||||||
nr.setValue(_diff_col_name, diff);
|
|
||||||
nr.setValue(_diff_col_name_perc, perc);
|
|
||||||
rv.addRow(nr);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -27,8 +27,7 @@ namespace BCG
|
|||||||
string prependString = isLeft ? "left" : "right";
|
string prependString = isLeft ? "left" : "right";
|
||||||
return String.Concat (prependString, colName).GetHashCode ();
|
return String.Concat (prependString, colName).GetHashCode ();
|
||||||
}
|
}
|
||||||
public override bool Equals (object? obj) =>
|
public override bool Equals (object obj) {return GetHashCode () == obj.GetHashCode ();}
|
||||||
GetHashCode() == obj?.GetHashCode();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
public class JoinStream : ViewStream
|
public class JoinStream : ViewStream
|
||||||
{
|
{
|
||||||
protected InternalJoinStream? myIjs;
|
protected InternalJoinStream myIjs;
|
||||||
protected ProjectStream? myPs;
|
protected ProjectStream myPs;
|
||||||
protected ColumnMap? myLeftCm, myRightCm;
|
protected ColumnMap myLeftCm, myRightCm;
|
||||||
protected bool myIsProject = true;
|
protected bool myIsProject = true;
|
||||||
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
||||||
|
|
||||||
|
|||||||
+13
-11
@@ -1,10 +1,10 @@
|
|||||||
using bts;
|
using System;
|
||||||
|
|
||||||
using mbts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -40,7 +40,8 @@ namespace BCG
|
|||||||
|
|
||||||
public void _on_stream_cb(object dp_obj)
|
public void _on_stream_cb(object dp_obj)
|
||||||
{
|
{
|
||||||
var timer = Stopwatch.StartNew();
|
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
|
timer.start();
|
||||||
DataPacket dp = (DataPacket)dp_obj;
|
DataPacket dp = (DataPacket)dp_obj;
|
||||||
if (_schema == null)
|
if (_schema == null)
|
||||||
{
|
{
|
||||||
@@ -66,12 +67,13 @@ namespace BCG
|
|||||||
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
timer.Stop();
|
timer.stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (timer.ElapsedMilliseconds >= 50) {
|
if (timer.lastIntervalTime > 50)
|
||||||
|
{
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
|
string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
||||||
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||||
}
|
}
|
||||||
if (on_new_data != null)
|
if (on_new_data != null)
|
||||||
on_new_data();
|
on_new_data();
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
using bts;
|
using System;
|
||||||
|
|
||||||
using mbts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -114,7 +114,8 @@ namespace BCG
|
|||||||
if (_is_ready())
|
if (_is_ready())
|
||||||
{
|
{
|
||||||
_construct_schema();
|
_construct_schema();
|
||||||
var timer = Stopwatch.StartNew();
|
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
|
timer.start();
|
||||||
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
||||||
List<object[]> output_rows = new List<object[]>();
|
List<object[]> output_rows = new List<object[]>();
|
||||||
|
|
||||||
@@ -137,12 +138,13 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
||||||
timer.Stop();
|
timer.stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (timer.ElapsedMilliseconds >= 250) {
|
if (timer.lastIntervalTime >= 250)
|
||||||
|
{
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
||||||
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||||
}
|
}
|
||||||
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
@@ -152,12 +154,15 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
{
|
{
|
||||||
|
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
IBlueRowCollection output_rows;
|
IBlueRowCollection output_rows;
|
||||||
var timer = Stopwatch.StartNew();
|
timer.start();
|
||||||
output_rows = _output_mapper.map_rows(unmapped_rows);
|
output_rows = _output_mapper.map_rows(unmapped_rows);
|
||||||
timer.Stop();
|
timer.stop();
|
||||||
if (timer.ElapsedMilliseconds >= 50) {
|
if (timer.lastIntervalTime >= 50)
|
||||||
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
{
|
||||||
|
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
|
||||||
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||||
}
|
}
|
||||||
return output_rows;
|
return output_rows;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Text;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using BCG.Utilities;
|
||||||
using mbts;
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
using Spark.Core.Collections;
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
@@ -222,8 +226,7 @@ namespace BCG
|
|||||||
|
|
||||||
#region IDictionary Members
|
#region IDictionary Members
|
||||||
|
|
||||||
public void Add(object key, object? value) =>
|
public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
|
||||||
throw new NotSupportedException("IBlueRow is readonly");
|
|
||||||
|
|
||||||
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
|
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
|
||||||
|
|
||||||
@@ -253,9 +256,10 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public object? this[object key] {
|
public object this[object key]
|
||||||
get => key is string str ? this[str] : this[(int)key];
|
{
|
||||||
set => throw new NotSupportedException("IBlueRow is readonly");
|
get { return key is string ? this[key.ToString()] : this[(int)key]; }
|
||||||
|
set { throw new NotSupportedException("IBlueRow is readonly"); }
|
||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
@@ -290,6 +294,7 @@ namespace BCG
|
|||||||
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
||||||
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
||||||
private bool myManageUnmappedLeftRows = false;
|
private bool myManageUnmappedLeftRows = false;
|
||||||
|
protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
|
||||||
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
||||||
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
||||||
protected JoinProjectHelper myJoinProjectHelper;
|
protected JoinProjectHelper myJoinProjectHelper;
|
||||||
@@ -356,6 +361,7 @@ namespace BCG
|
|||||||
|
|
||||||
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
||||||
{
|
{
|
||||||
|
myTimer.start();
|
||||||
myInitSchema(dp.schema, isLeft);
|
myInitSchema(dp.schema, isLeft);
|
||||||
BlueTable dels = null, adds = null;
|
BlueTable dels = null, adds = null;
|
||||||
if (myState != null)
|
if (myState != null)
|
||||||
@@ -365,6 +371,7 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
||||||
myAddRows(dp.addedRows, isLeft, myState, adds);
|
myAddRows(dp.addedRows, isLeft, myState, adds);
|
||||||
|
myTimer.stop();
|
||||||
if (myState != null)
|
if (myState != null)
|
||||||
{
|
{
|
||||||
myFireEvent(new DataPacket(myState, adds, dels));
|
myFireEvent(new DataPacket(myState, adds, dels));
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
namespace BCG {
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public class KeyValueToRelationalStream : ViewStream {
|
|
||||||
private readonly ViewStream _kv_str;
|
|
||||||
private readonly string _doc_id_col, _key_col, _val_col;
|
|
||||||
public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
|
|
||||||
_kv_str = kv_str;
|
|
||||||
_doc_id_col = doc_id_col;
|
|
||||||
_key_col = key_col;
|
|
||||||
_val_col = val_col;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_kv_str.onChange += _on_str_update;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_kv_str.onChange -= _on_str_update;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _on_str_update(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
var rows = dp.newRows;
|
|
||||||
var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -91,8 +91,7 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Add (object key, object? value) =>
|
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
throw new NotSupportedException ("The IBlueRow is readonly");
|
|
||||||
|
|
||||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
|
||||||
|
|||||||
@@ -1,64 +1,51 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
public class LiteralStream : ViewStream {
|
{
|
||||||
|
public class LiteralStream : ViewStream
|
||||||
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
{
|
||||||
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
|
public LiteralStream(string tableName, string schema, string value)
|
||||||
string[] pairs = key_value_pairs.Split(' ');
|
{
|
||||||
foreach (string p in pairs) {
|
string [] columns = schema.Split(' ');
|
||||||
string[] kvp = p.Split(':');
|
|
||||||
string col_name = kvp[0];
|
|
||||||
string val = kvp[1];
|
|
||||||
var tup = BlueDataUtility.guess_type(val);
|
|
||||||
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
|
|
||||||
rv.Add(Tuple.Create(ci, tup.Item2));
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public LiteralStream(string tableName, string schema, string value) {
|
|
||||||
string[] columns = schema.Split(' ');
|
|
||||||
|
|
||||||
|
|
||||||
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
|
ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
|
||||||
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
ci[0] = new ColumnInfo("id",0,typeof(int));
|
||||||
int n = 1;
|
int n = 1;
|
||||||
foreach (string c in columns) {
|
foreach (string c in columns)
|
||||||
string[] columnAndType = c.Split('/');
|
{
|
||||||
|
string [] columnAndType = c.Split('/');
|
||||||
if (columnAndType.Length == 1)
|
if (columnAndType.Length == 1)
|
||||||
ci[n] = new ColumnInfo(c, n, typeof(double));
|
ci[n] = new ColumnInfo(c,n,typeof(double));
|
||||||
else
|
else
|
||||||
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
|
||||||
n++;
|
n++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
|
BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
|
||||||
string[] rows = value.Split(';');
|
string [] rows = value.Split(';');
|
||||||
n = 0;
|
n = 0;
|
||||||
foreach (string r in rows)
|
foreach (string r in rows)
|
||||||
try {
|
try
|
||||||
if (r == string.Empty)
|
{
|
||||||
|
if(r == string.Empty)
|
||||||
continue;
|
continue;
|
||||||
t.addRow(
|
t.addRow(
|
||||||
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
|
BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));
|
||||||
} catch (Exception e) {
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
||||||
}
|
}
|
||||||
|
|
||||||
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
||||||
}
|
}
|
||||||
|
|
||||||
public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
public LiteralStream(string schema, string value) : this ("literal",schema, value)
|
||||||
|
{}
|
||||||
// public LiteralStream(string value) {
|
|
||||||
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,24 +1,24 @@
|
|||||||
// using System;
|
using System;
|
||||||
|
|
||||||
// namespace BCG
|
namespace BCG
|
||||||
// {
|
{
|
||||||
// public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
|
public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
|
||||||
// {
|
{
|
||||||
// IBlueTableSchema mySchema;
|
IBlueTableSchema mySchema;
|
||||||
|
|
||||||
// public IBlueRow map (IBlueRow row)
|
public IBlueRow map (IBlueRow row)
|
||||||
// {
|
{
|
||||||
// if (mySchema == null)
|
if (mySchema == null)
|
||||||
// {
|
{
|
||||||
// ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
|
ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
|
||||||
// int i = 0;
|
int i = 0;
|
||||||
// foreach (ColumnInfo ci in row.schema.columnInfo)
|
foreach (ColumnInfo ci in row.schema.columnInfo)
|
||||||
// cis[i++] = ci.colType == null ?
|
cis[i++] = ci.colType == null ?
|
||||||
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
|
new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
|
||||||
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
|
new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
|
||||||
// mySchema = new BlueTableSchema (row.schema.tableName, cis);
|
mySchema = new BlueTableSchema (row.schema.tableName, cis);
|
||||||
// }
|
}
|
||||||
// return new BlueRow (mySchema, row);
|
return new BlueRow (mySchema, row);
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
@@ -1,10 +1,15 @@
|
|||||||
|
using System;
|
||||||
|
using System.Data;
|
||||||
|
using System.Collections;
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
|
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
|
|
||||||
using System;
|
namespace BCG
|
||||||
|
{
|
||||||
namespace BCG {
|
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -16,45 +21,24 @@ namespace BCG {
|
|||||||
[DataStreamUserRespresentation (false)]
|
[DataStreamUserRespresentation (false)]
|
||||||
public class MapStream : MethodDrivenStream
|
public class MapStream : MethodDrivenStream
|
||||||
{
|
{
|
||||||
|
private HighPerformanceTimer t = new HighPerformanceTimer ();
|
||||||
protected IStreamMapper myMapper;
|
protected IStreamMapper myMapper;
|
||||||
protected IChangeableStreamMapper myChangeableMapper = null;
|
protected IChangeableStreamMapper myChangeableMapper = null;
|
||||||
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
||||||
|
|
||||||
public class FuncWrapper : IStreamMapper {
|
public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
|
||||||
private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func;
|
{
|
||||||
Func<IBlueRow, IBlueRow> _row_map_func;
|
|
||||||
|
|
||||||
public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
|
|
||||||
Func<IBlueRow, IBlueRow> row_map_func) {
|
|
||||||
_schema_map_func = schema_map_func;
|
|
||||||
_row_map_func = row_map_func;
|
|
||||||
}
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
|
|
||||||
return _schema_map_func(input_schema);
|
|
||||||
}
|
|
||||||
public IBlueRow map(IBlueRow row) {
|
|
||||||
return _row_map_func(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
|
|
||||||
if (myUseNewMap)
|
if (myUseNewMap)
|
||||||
mySetRealSource(new SuperMapStream(vs, mapper));
|
mySetRealSource (new SuperMapStream (vs, mapper));
|
||||||
else
|
else
|
||||||
myMapper = mapper;
|
myMapper = mapper;
|
||||||
}
|
}
|
||||||
|
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
|
||||||
public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
|
{
|
||||||
Func<IBlueRow, IBlueRow> row_map_func)
|
|
||||||
:
|
|
||||||
this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
|
|
||||||
if (myUseNewMap)
|
if (myUseNewMap)
|
||||||
mySetRealSource(new SuperMapStream(vs, mapper));
|
mySetRealSource(new SuperMapStream(vs, mapper));
|
||||||
else {
|
else
|
||||||
|
{
|
||||||
mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
|
mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
|
||||||
myChangeableMapper = mapper;
|
myChangeableMapper = mapper;
|
||||||
}
|
}
|
||||||
@@ -63,68 +47,77 @@ namespace BCG {
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the value of the mapper. Setting the value will result in re
|
/// Gets or sets the value of the mapper. Setting the value will result in re
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private void myMapperChanged() {
|
private void myMapperChanged ()
|
||||||
|
{
|
||||||
if (myLastArrivedPacket != null)
|
if (myLastArrivedPacket != null)
|
||||||
myChangeHandler(new DataPacket(myLastArrivedPacket.newRows,
|
myChangeHandler (new DataPacket (myLastArrivedPacket.newRows,
|
||||||
myLastArrivedPacket.newRows,
|
myLastArrivedPacket.newRows,
|
||||||
myLastArrivedPacket.newRows));
|
myLastArrivedPacket.newRows));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
protected override void myHandleFirstCall() {
|
protected override void myHandleFirstCall ()
|
||||||
|
{
|
||||||
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
|
|
||||||
myLastFiredPacket = new DataPacket();
|
IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
|
||||||
myLastFiredPacket.newRows = new BlueTable(outputSchema);
|
myLastFiredPacket = new DataPacket ();
|
||||||
myLastFiredPacket.deletedRows = new BlueTable(outputSchema);
|
myLastFiredPacket.newRows = new BlueTable (outputSchema);
|
||||||
|
myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.newRows)
|
foreach (IBlueRow row in myLastArrivedPacket.newRows)
|
||||||
((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row));
|
((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
|
||||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
|
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleIncrementalAdd() {
|
protected override void myHandleIncrementalAdd()
|
||||||
|
{
|
||||||
//t.start ();
|
//t.start ();
|
||||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
|
foreach (IBlueRow row in myLastArrivedPacket.addedRows)
|
||||||
IBlueRow newRow = myMapper.map(row);
|
{
|
||||||
(myLastFiredPacket.newRows as BlueTable).addRow(newRow);
|
IBlueRow newRow = myMapper.map (row);
|
||||||
(myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
|
(myLastFiredPacket.newRows as BlueTable).addRow (newRow);
|
||||||
|
(myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
|
||||||
}
|
}
|
||||||
//t.stop ();
|
//t.stop ();
|
||||||
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
||||||
//System.Diagnostics.Trace.WriteLine (" ");
|
//System.Diagnostics.Trace.WriteLine (" ");
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleIncrementalDelete() {
|
protected override void myHandleIncrementalDelete()
|
||||||
|
{
|
||||||
//t.start ();
|
//t.start ();
|
||||||
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
|
foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
|
||||||
|
{
|
||||||
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
|
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
|
||||||
(myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
|
(myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
|
||||||
(myLastFiredPacket.newRows as BlueTable).deleteRow(row);
|
(myLastFiredPacket.newRows as BlueTable).deleteRow (row);
|
||||||
}
|
}
|
||||||
//t.stop ();
|
//t.stop ();
|
||||||
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber()
|
||||||
protected override void myOnFirstSubscriber() {
|
{
|
||||||
myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
|
myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber ()
|
||||||
myInit();
|
{
|
||||||
|
myInit ();
|
||||||
if (myStream != null)
|
if (myStream != null)
|
||||||
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnTableClear() {
|
protected override void myOnTableClear ()
|
||||||
|
{
|
||||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||||
myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
||||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|||||||
@@ -1,13 +1,12 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
using mbts;
|
using mbts;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// SuperMap gives the user the ability
|
/// SuperMap gives the user the ability
|
||||||
/// to map one row to one or more rows
|
/// to map one row to one or more rows
|
||||||
@@ -18,11 +17,13 @@ namespace BCG {
|
|||||||
/// can chage itself and raise an event causing
|
/// can chage itself and raise an event causing
|
||||||
/// a remapping to happen
|
/// a remapping to happen
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class SuperMapStream : ViewStream {
|
public class SuperMapStream : ViewStream
|
||||||
|
{
|
||||||
#region super mapper that wraps around stream mapper
|
#region super mapper that wraps around stream mapper
|
||||||
private class MyMappingWrapper : ISuperStreamMapper {
|
private class MyMappingWrapper : ISuperStreamMapper
|
||||||
|
{
|
||||||
private IStreamMapper myMapper;
|
private IStreamMapper myMapper;
|
||||||
|
|
||||||
public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
|
public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
|
||||||
|
|
||||||
#region ISuperStreamMapper Members
|
#region ISuperStreamMapper Members
|
||||||
@@ -31,7 +32,8 @@ namespace BCG {
|
|||||||
|
|
||||||
public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
|
public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
|
||||||
|
|
||||||
public event MapperChangeDelegate onMapperChange {
|
public event MapperChangeDelegate onMapperChange
|
||||||
|
{
|
||||||
add { }
|
add { }
|
||||||
remove { }
|
remove { }
|
||||||
}
|
}
|
||||||
@@ -40,18 +42,21 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
private class MyFuncWrapper : ISuperStreamMapper {
|
private class MyFuncWrapper : ISuperStreamMapper
|
||||||
|
{
|
||||||
private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
|
private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
|
||||||
private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
|
private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
|
||||||
public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
||||||
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) {
|
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
||||||
|
{
|
||||||
_get_schema = get_schema;
|
_get_schema = get_schema;
|
||||||
_map_rows = map_rows;
|
_map_rows = map_rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
|
||||||
|
|
||||||
public IBlueRow[] map(IBlueRow r) {
|
public IBlueRow[] map(IBlueRow r)
|
||||||
|
{
|
||||||
IEnumerable<IBlueRow> rows = _map_rows(r);
|
IEnumerable<IBlueRow> rows = _map_rows(r);
|
||||||
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
||||||
}
|
}
|
||||||
@@ -71,15 +76,19 @@ namespace BCG {
|
|||||||
|
|
||||||
public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
||||||
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
||||||
: this(vs, new MyFuncWrapper(get_schema, map_rows)) { }
|
: this(vs, new MyFuncWrapper(get_schema, map_rows))
|
||||||
|
{ }
|
||||||
|
|
||||||
public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
||||||
: this(vs, s => out_schema, map_rows) { }
|
: this(vs, s => out_schema, map_rows)
|
||||||
|
{ }
|
||||||
|
|
||||||
public SuperMapStream(ViewStream vs, IStreamMapper mapper)
|
public SuperMapStream(ViewStream vs, IStreamMapper mapper)
|
||||||
: this(vs, new MyMappingWrapper(mapper)) { }
|
: this(vs, new MyMappingWrapper(mapper))
|
||||||
|
{ }
|
||||||
|
|
||||||
public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) {
|
public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
|
||||||
|
{
|
||||||
myStream = vs;
|
myStream = vs;
|
||||||
myMapper = mapper;
|
myMapper = mapper;
|
||||||
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
|
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
|
||||||
@@ -87,7 +96,8 @@ namespace BCG {
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
|
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber()
|
||||||
|
{
|
||||||
myStream.onChange -= new ChangeDelegate(myHandleData);
|
myStream.onChange -= new ChangeDelegate(myHandleData);
|
||||||
myRows = null;
|
myRows = null;
|
||||||
myBaseRows = null;
|
myBaseRows = null;
|
||||||
@@ -95,7 +105,8 @@ namespace BCG {
|
|||||||
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
|
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleData(object newValue) {
|
private void myHandleData(object newValue)
|
||||||
|
{
|
||||||
DataPacket idp = newValue as DataPacket;
|
DataPacket idp = newValue as DataPacket;
|
||||||
//HighPerformanceTimer timer = new HighPerformanceTimer();
|
//HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
//timer.start();
|
//timer.start();
|
||||||
@@ -106,19 +117,24 @@ namespace BCG {
|
|||||||
BlueTable dels = new BlueTable(myRows.schema),
|
BlueTable dels = new BlueTable(myRows.schema),
|
||||||
adds = new BlueTable(myRows.schema);
|
adds = new BlueTable(myRows.schema);
|
||||||
|
|
||||||
foreach (IBlueRow r in dp.deletedRows) {
|
foreach (IBlueRow r in dp.deletedRows)
|
||||||
|
{
|
||||||
List<IBlueRow> mappedRows = myMap[(int)r["id"]];
|
List<IBlueRow> mappedRows = myMap[(int)r["id"]];
|
||||||
foreach (IBlueRow mr in mappedRows) {
|
foreach (IBlueRow mr in mappedRows)
|
||||||
|
{
|
||||||
dels.addRow(mr);
|
dels.addRow(mr);
|
||||||
myRows.deleteRow(mr);
|
myRows.deleteRow(mr);
|
||||||
}
|
}
|
||||||
myMap.Remove((int)r["id"]);
|
myMap.Remove((int)r["id"]);
|
||||||
}
|
}
|
||||||
foreach (IBlueRow r in dp.addedRows) {
|
foreach (IBlueRow r in dp.addedRows)
|
||||||
|
{
|
||||||
IBlueRow[] mappedRows = myMapper.map(r);
|
IBlueRow[] mappedRows = myMapper.map(r);
|
||||||
foreach (IBlueRow mr in mappedRows) {
|
foreach (IBlueRow mr in mappedRows)
|
||||||
|
{
|
||||||
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
||||||
if (!myMap.ContainsKey((int)r["id"])) {
|
if (!myMap.ContainsKey((int)r["id"]))
|
||||||
|
{
|
||||||
myMap.Add((int)r["id"], new List<IBlueRow>());
|
myMap.Add((int)r["id"], new List<IBlueRow>());
|
||||||
}
|
}
|
||||||
myMap[(int)r["id"]].Add(idChangedMapRow);
|
myMap[(int)r["id"]].Add(idChangedMapRow);
|
||||||
@@ -129,12 +145,13 @@ namespace BCG {
|
|||||||
//timer.stop();
|
//timer.stop();
|
||||||
// if (timer.lastIntervalTime > 20)
|
// if (timer.lastIntervalTime > 20)
|
||||||
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
|
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
|
||||||
myFireEvent(new DataPacket(myRows, adds, dels));
|
// myFireEvent(new DataPacket(myRows, adds, dels));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleMapChange() {
|
private void myHandleMapChange()
|
||||||
|
{
|
||||||
if (myBaseRows != null)
|
if (myBaseRows != null)
|
||||||
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
|
myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using MongoDB.Bson;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Runtime.Intrinsics.X86;
|
|
||||||
|
|
||||||
|
|
||||||
namespace bts {
|
|
||||||
// listens to the underlying stream, and writes each seriialized packet in a
|
|
||||||
// seperate file
|
|
||||||
[DataStreamUserRespresentation("dp_recorder")]
|
|
||||||
public class DataPacketRecorderStream : ViewStream {
|
|
||||||
private ViewStream _base_stream;
|
|
||||||
private readonly string _folder;
|
|
||||||
StreamIOSerializationController _serializer = new StreamIOSerializationController();
|
|
||||||
private UInt32 _n_updates = 0;
|
|
||||||
public DataPacketRecorderStream(ViewStream base_stream, string folder) {
|
|
||||||
_base_stream = base_stream;
|
|
||||||
_folder = folder;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_base_stream.onChange += _handle_update;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_base_stream.onChange -= _handle_update;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _handle_update(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
byte[] ser_data = null;
|
|
||||||
if (_n_updates == 0) {
|
|
||||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
|
||||||
ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
|
|
||||||
} else {
|
|
||||||
StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
|
|
||||||
ser_data = _serializer.serialize(new StreamIOMessage(mra));
|
|
||||||
}
|
|
||||||
Log.assert(ser_data != null);
|
|
||||||
_n_updates++;
|
|
||||||
string file_name = $"{_folder}\\{_n_updates}.bin";
|
|
||||||
_write_to_file(ser_data, file_name);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void _write_to_file(byte[] data, string file_name) {
|
|
||||||
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
|
|
||||||
// Write the byte array to the file
|
|
||||||
fs.Write(data, 0, data.Length);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
[DataStreamUserRespresentation("dp_replay")]
|
|
||||||
public class DataPacketReplayStream : ViewStream, IWorker {
|
|
||||||
private ViewStreamDatabase _vsd;
|
|
||||||
private readonly string _folder_path;
|
|
||||||
private readonly TimeSpan _fire_period;
|
|
||||||
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
|
|
||||||
private int _file_ptr = 0;
|
|
||||||
private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
|
|
||||||
private BlueTable _state = null;
|
|
||||||
|
|
||||||
public DataPacketReplayStream(string folder_path, string fire_period)
|
|
||||||
:
|
|
||||||
this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
|
|
||||||
_vsd = vsd;
|
|
||||||
_folder_path = folder_path;
|
|
||||||
_fire_period = fire_period;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
MasterWorker.getInstance(false).addWorker(this, _fire_period);
|
|
||||||
_load_all_files();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
MasterWorker.getInstance(false).removeWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork() {
|
|
||||||
if (_file_ptr >= _file_data.Count())
|
|
||||||
// we can set _file_ptr to 0 and keep going as well
|
|
||||||
return;
|
|
||||||
var curr_d = _file_data[_file_ptr++];
|
|
||||||
var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
|
|
||||||
StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
|
|
||||||
Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
|
|
||||||
switch (msg.op) {
|
|
||||||
case StreamIOMessage.StreamOp.CREATE: {
|
|
||||||
_state = new BlueTable(msg.create_message.rows);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
|
|
||||||
var mra = msg.multiple_remove_add_message;
|
|
||||||
foreach (IBlueRow r in mra.removed_rows)
|
|
||||||
_state.deleteRow(r);
|
|
||||||
foreach (IBlueRow r in mra.added_rows)
|
|
||||||
_state.addRow(r);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _load_all_files() {
|
|
||||||
Func<string, string> extract_file_name = (string full_path) => {
|
|
||||||
string fn = full_path.Split("\\").Last();
|
|
||||||
fn = fn.Split(".").First();
|
|
||||||
return fn;
|
|
||||||
|
|
||||||
};
|
|
||||||
string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
|
|
||||||
|
|
||||||
// Iterate over the .bin files and perform an action
|
|
||||||
foreach (string file in bin_files) {
|
|
||||||
byte[] file_data = File.ReadAllBytes(file);
|
|
||||||
_file_data.Add(Tuple.Create(file, file_data));
|
|
||||||
}
|
|
||||||
_file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
|
|
||||||
string fn1 = extract_file_name(t1.Item1);
|
|
||||||
string fn2 = extract_file_name(t2.Item1);
|
|
||||||
int fn_i1 = Int32.Parse(fn1);
|
|
||||||
int fn_i2 = Int32.Parse(fn2);
|
|
||||||
if (fn_i1 < fn_i2)
|
|
||||||
return -1;
|
|
||||||
else if (fn_i1 == fn_i2)
|
|
||||||
return 0;
|
|
||||||
else if (fn_i1 > fn_i2)
|
|
||||||
return 1;
|
|
||||||
return 0;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,227 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
//using MongoDB.Driver;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
namespace bts
|
|
||||||
{
|
|
||||||
public class PubSubTestingStream : ViewStream, IWorker
|
|
||||||
{
|
|
||||||
public class PayloadMessage
|
|
||||||
{
|
|
||||||
public int msg_id = 0;
|
|
||||||
public string guid;
|
|
||||||
public Int64 travel_epoch_ms = 0;
|
|
||||||
public byte[] payload = null;
|
|
||||||
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
|
|
||||||
{
|
|
||||||
msg_id = msg_id_args;
|
|
||||||
travel_epoch_ms = send_epoch_args;
|
|
||||||
payload = new byte[payload_size];
|
|
||||||
for (int i = 0; i < payload_size; i++)
|
|
||||||
{
|
|
||||||
payload[i] = payload_byte;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
private PayloadMessage() { }
|
|
||||||
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
|
|
||||||
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
public byte[] serialize()
|
|
||||||
{
|
|
||||||
var str = new MemoryStream();
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
bw.Write(msg_id);
|
|
||||||
bw.Write(guid);
|
|
||||||
bw.Write(travel_epoch_ms);
|
|
||||||
bw.Write(payload.Length);
|
|
||||||
bw.Write(payload);
|
|
||||||
str.Position = 0;
|
|
||||||
var bytes = str.ToArray();
|
|
||||||
return bytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
|
||||||
{
|
|
||||||
var str = new MemoryStream(ser_bytes);
|
|
||||||
BinaryReader br = new BinaryReader(str);
|
|
||||||
int msg_id = br.ReadInt32();
|
|
||||||
string guid = br.ReadString();
|
|
||||||
Int64 epoch = br.ReadInt64();
|
|
||||||
int len = br.ReadInt32();
|
|
||||||
var payload = br.ReadBytes(len);
|
|
||||||
return new PayloadMessage()
|
|
||||||
{
|
|
||||||
msg_id = msg_id,
|
|
||||||
guid = guid,
|
|
||||||
travel_epoch_ms = epoch,
|
|
||||||
payload = payload
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
//public void
|
|
||||||
private readonly int _msg_size;
|
|
||||||
private readonly TimeSpan _publish_period;
|
|
||||||
private DateTime _last_publish_time = DateTime.MinValue;
|
|
||||||
private PublisherCore _pub_core = new PublisherCore();
|
|
||||||
private BlueTable _table;
|
|
||||||
private int _n_pubs = 0;
|
|
||||||
private readonly int _max_pubs;
|
|
||||||
private int _n_updates = 0;
|
|
||||||
private bool _continue_sub_thread = true;
|
|
||||||
private ViewStreamDatabase _vsd;
|
|
||||||
private bool _fire_dirty = false;
|
|
||||||
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
|
||||||
private readonly int _payload_sz;
|
|
||||||
private const byte _payload_byte = (byte)'A';
|
|
||||||
private readonly string _tag_name;
|
|
||||||
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
|
||||||
{
|
|
||||||
_vsd = vsd;
|
|
||||||
_payload_sz = payload_size;
|
|
||||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("n_pubs", TypeCode.Int32),
|
|
||||||
Tuple.Create("n_recvs", TypeCode.Int32),
|
|
||||||
Tuple.Create("last_sent_id", TypeCode.Int32),
|
|
||||||
Tuple.Create("last_recv_id", TypeCode.Int32),
|
|
||||||
Tuple.Create("last_sent_guid", TypeCode.String),
|
|
||||||
Tuple.Create("last_recv_guid", TypeCode.String),
|
|
||||||
Tuple.Create("match", TypeCode.Boolean),
|
|
||||||
Tuple.Create("echo_time", TypeCode.Int32));
|
|
||||||
_table = new BlueTable(schema);
|
|
||||||
_publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
|
|
||||||
_msg_size = payload_size;
|
|
||||||
_max_pubs = max_pubs;
|
|
||||||
_tag_name = tag_name;
|
|
||||||
}
|
|
||||||
|
|
||||||
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
|
||||||
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
|
||||||
{
|
|
||||||
_vsd.addWorker(this);
|
|
||||||
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
|
|
||||||
th.IsBackground = true;
|
|
||||||
th.Start();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
|
||||||
{
|
|
||||||
_vsd.removeWorker(this);
|
|
||||||
_continue_sub_thread = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
void _log_updates(IEnumerable<PubSubServerMessage> updates)
|
|
||||||
{
|
|
||||||
var idx = 0;
|
|
||||||
foreach (var msg in updates)
|
|
||||||
{
|
|
||||||
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
|
|
||||||
idx++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _subscribe_thread_func()
|
|
||||||
{
|
|
||||||
// subscription is done in another thread
|
|
||||||
SubscriberCore sub_core = new SubscriberCore();
|
|
||||||
sub_core.subscribe(_tag_name);
|
|
||||||
while (_continue_sub_thread)
|
|
||||||
{
|
|
||||||
// TODO recv_id
|
|
||||||
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
|
||||||
if (updates == null || updates.Count() == 0)
|
|
||||||
continue;
|
|
||||||
_log_updates(updates);
|
|
||||||
_n_updates += updates.Count();
|
|
||||||
if (updates.Count() > 0)
|
|
||||||
{
|
|
||||||
var server_msg = updates.Last();
|
|
||||||
if (server_msg.data != null && server_msg.data.Length > 0)
|
|
||||||
{
|
|
||||||
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
|
||||||
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// why are we getting empty or null data ? (server is sending data regardless?)
|
|
||||||
}
|
|
||||||
_fire_dirty = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _publish_next_packet()
|
|
||||||
{
|
|
||||||
if (_last_sent_msg == null)
|
|
||||||
{
|
|
||||||
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
|
|
||||||
{
|
|
||||||
guid = Guid.NewGuid().ToString()
|
|
||||||
};
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_last_sent_msg.msg_id++;
|
|
||||||
_last_sent_msg.guid = Guid.NewGuid().ToString();
|
|
||||||
_last_sent_msg.travel_epoch_ms = epoch_now();
|
|
||||||
}
|
|
||||||
var ser_bytes = _last_sent_msg.serialize();
|
|
||||||
_n_pubs++;
|
|
||||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
|
||||||
_fire_dirty = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _fire_dp()
|
|
||||||
{
|
|
||||||
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
|
|
||||||
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
|
|
||||||
object echo_time = DBNull.Value;
|
|
||||||
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
|
|
||||||
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
|
|
||||||
if (_last_sent_msg != null && _last_recvd_msg != null)
|
|
||||||
{
|
|
||||||
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
|
|
||||||
}
|
|
||||||
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
|
|
||||||
IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
|
|
||||||
last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
|
|
||||||
BlueTable table = new BlueTable(_table.schema);
|
|
||||||
table.addRow(nr);
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork()
|
|
||||||
{
|
|
||||||
if (DateTime.Now - _last_publish_time > _publish_period)
|
|
||||||
if (_n_pubs <= _max_pubs)
|
|
||||||
{
|
|
||||||
_publish_next_packet();
|
|
||||||
_last_publish_time = DateTime.Now;
|
|
||||||
}
|
|
||||||
// publish
|
|
||||||
// TODO: send id
|
|
||||||
if (_fire_dirty)
|
|
||||||
{
|
|
||||||
_fire_dirty = false;
|
|
||||||
_fire_dp();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static long epoch_now()
|
|
||||||
{
|
|
||||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,223 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Runtime.CompilerServices;
|
|
||||||
using bts.api;
|
|
||||||
|
|
||||||
namespace bts
|
|
||||||
{
|
|
||||||
[DataStreamUserRespresentation("pss_echo")]
|
|
||||||
public class PubsubEchoTestStream : ViewStream, IWorker
|
|
||||||
{
|
|
||||||
public class PayloadMessage
|
|
||||||
{
|
|
||||||
public readonly string msg_id;
|
|
||||||
public readonly Int64 timestamp = 0;
|
|
||||||
public readonly byte[] payload = null;
|
|
||||||
|
|
||||||
public static byte[] create_data(int payload_size, byte payload_byte)
|
|
||||||
{
|
|
||||||
var rv = new byte[payload_size];
|
|
||||||
for (int i = 0; i < payload_size; i++)
|
|
||||||
{
|
|
||||||
rv[i] = payload_byte;
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
|
|
||||||
// {
|
|
||||||
// guid = guid_;
|
|
||||||
// timestamp = timestamp_;
|
|
||||||
// payload = new byte[payload_size];
|
|
||||||
// for (int i = 0; i < payload_size; i++)
|
|
||||||
// {
|
|
||||||
// payload[i] = payload_byte;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
|
|
||||||
{
|
|
||||||
msg_id = guid_;
|
|
||||||
timestamp = timestamp_;
|
|
||||||
payload = payload_;
|
|
||||||
}
|
|
||||||
|
|
||||||
private PayloadMessage() { }
|
|
||||||
public byte[] serialize()
|
|
||||||
{
|
|
||||||
var str = new MemoryStream();
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
bw.Write(msg_id);
|
|
||||||
bw.Write(timestamp);
|
|
||||||
bw.Write(payload.Length);
|
|
||||||
bw.Write(payload);
|
|
||||||
str.Position = 0;
|
|
||||||
var bytes = str.ToArray();
|
|
||||||
return bytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
|
||||||
{
|
|
||||||
var str = new MemoryStream(ser_bytes);
|
|
||||||
BinaryReader br = new BinaryReader(str);
|
|
||||||
string guid = br.ReadString();
|
|
||||||
Int64 timestamp_ = br.ReadInt64();
|
|
||||||
int len = br.ReadInt32();
|
|
||||||
var payload = br.ReadBytes(len);
|
|
||||||
return new PayloadMessage(guid, timestamp_, payload);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
//public void
|
|
||||||
private readonly int _msg_size;
|
|
||||||
private readonly TimeSpan _publish_period;
|
|
||||||
private DateTime _last_publish_time = DateTime.MinValue;
|
|
||||||
private PublisherCore _pub_core;
|
|
||||||
private ISubscriberCore _sub_core;
|
|
||||||
private BlueTable _table;
|
|
||||||
private int _n_pubs = 0;
|
|
||||||
private readonly int _max_pubs;
|
|
||||||
private int _n_updates = 0;
|
|
||||||
private ViewStreamDatabase _vsd;
|
|
||||||
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
|
||||||
private const byte _payload_byte = (byte)'A';
|
|
||||||
private readonly string _tag_name;
|
|
||||||
private readonly int _payload_sz;
|
|
||||||
private readonly IPubSubRouter _router;
|
|
||||||
private enum IOState
|
|
||||||
{
|
|
||||||
SEND = 0,
|
|
||||||
RECV = 1
|
|
||||||
};
|
|
||||||
IOState _state = IOState.SEND;
|
|
||||||
|
|
||||||
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
|
||||||
{
|
|
||||||
var route = new PubSubRoute(server_url, ".*");
|
|
||||||
_router = new PubSubRouter(route);
|
|
||||||
_vsd = vsd;
|
|
||||||
_payload_sz = payload_size;
|
|
||||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("sent_id", TypeCode.String),
|
|
||||||
Tuple.Create("sent_time", TypeCode.Int64),
|
|
||||||
Tuple.Create("recv_id", TypeCode.String),
|
|
||||||
Tuple.Create("recv_time", TypeCode.Int64),
|
|
||||||
Tuple.Create("match", TypeCode.Boolean),
|
|
||||||
Tuple.Create("echo_time", TypeCode.Int32));
|
|
||||||
_table = new BlueTable(schema);
|
|
||||||
_publish_period = publish_period;
|
|
||||||
_tag_name = tag_name;
|
|
||||||
}
|
|
||||||
|
|
||||||
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
|
||||||
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
|
||||||
{
|
|
||||||
_pub_core = new PublisherCore(_router);
|
|
||||||
_sub_core = new SubscriberCore(_router);
|
|
||||||
_sub_core.subscribe(_tag_name);
|
|
||||||
_vsd.addWorker(this);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
|
||||||
{
|
|
||||||
_vsd.removeWorker(this);
|
|
||||||
_sub_core.unsubscribe(_tag_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _fire_dp_after_send()
|
|
||||||
{
|
|
||||||
// as soon as the data is published, its reflected in the stream state
|
|
||||||
Log.assert(_last_sent_msg != null);
|
|
||||||
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
|
||||||
DBNull.Value, DBNull.Value);
|
|
||||||
_table.addRow(r);
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _fire_dp_after_recv()
|
|
||||||
{
|
|
||||||
Log.assert(_last_recvd_msg != null);
|
|
||||||
// update the last publish row
|
|
||||||
int id = _table.Count - 1;
|
|
||||||
IBlueRow last_row = _table[id];
|
|
||||||
_table.deleteRow(last_row);
|
|
||||||
var recv_ts = epoch_now();
|
|
||||||
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
|
|
||||||
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
|
|
||||||
_table.addRow(nr);
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
|
||||||
|
|
||||||
private string _get_next_id()
|
|
||||||
{
|
|
||||||
return _table.Count.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _publish_next_packet()
|
|
||||||
{
|
|
||||||
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
|
||||||
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
|
||||||
var ser_bytes = _last_sent_msg.serialize();
|
|
||||||
_n_pubs++;
|
|
||||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
|
||||||
_last_publish_time = DateTime.Now;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork()
|
|
||||||
{
|
|
||||||
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
|
|
||||||
{
|
|
||||||
foreach (var u in updates)
|
|
||||||
{
|
|
||||||
var recv_msg = PayloadMessage.deserialize(u.data);
|
|
||||||
if (recv_msg.msg_id == _last_sent_msg.msg_id)
|
|
||||||
{
|
|
||||||
return recv_msg;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
switch (_state)
|
|
||||||
{
|
|
||||||
case IOState.SEND:
|
|
||||||
if (DateTime.Now - _last_publish_time >= _publish_period)
|
|
||||||
{
|
|
||||||
_publish_next_packet();
|
|
||||||
_last_publish_time = DateTime.Now;
|
|
||||||
_fire_dp_after_send();
|
|
||||||
_state = IOState.RECV;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case IOState.RECV:
|
|
||||||
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
|
||||||
updates = updates.Where(u => u.tag == _tag_name);
|
|
||||||
var recv_msg = find_packet(updates);
|
|
||||||
if (recv_msg != null)
|
|
||||||
{
|
|
||||||
_last_recvd_msg = recv_msg;
|
|
||||||
_fire_dp_after_recv();
|
|
||||||
_state = IOState.SEND;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static long epoch_now()
|
|
||||||
{
|
|
||||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
namespace bts {
|
|
||||||
using BCG;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class SimpleTCPEchoStream : ViewStream, IWorker {
|
|
||||||
private string _ip;
|
|
||||||
private int _port;
|
|
||||||
private SimpleTCPClient _tcp_client;
|
|
||||||
private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("event", TypeCode.String));
|
|
||||||
private BlueTable _table;
|
|
||||||
private SyncQueue<string> _action_queue = new SyncQueue<string>();
|
|
||||||
|
|
||||||
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
|
|
||||||
: this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
|
|
||||||
|
|
||||||
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
|
|
||||||
_ip = ip;
|
|
||||||
_port = port;
|
|
||||||
_table = new BlueTable(_schema);
|
|
||||||
_tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
ViewStreamDatabase.instance.addWorker(this);
|
|
||||||
_tcp_client.register_listener((msg) => { _on_msg(msg); });
|
|
||||||
_tcp_client.start();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _on_msg(string msg) {
|
|
||||||
_action_queue.enqueue(msg);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork() {
|
|
||||||
if (_action_queue.getCount() == 0)
|
|
||||||
return;
|
|
||||||
BlueTable added_rows = new BlueTable(_schema);
|
|
||||||
var sz = _action_queue.getCount();
|
|
||||||
for (int i = 0; i < sz; i++) {
|
|
||||||
string msg = _action_queue.dequeue();
|
|
||||||
if (msg == null)
|
|
||||||
break;
|
|
||||||
IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
|
|
||||||
_table.addRow(nr);
|
|
||||||
added_rows.addRow(nr);
|
|
||||||
}
|
|
||||||
if (added_rows.Count == 0)
|
|
||||||
return;
|
|
||||||
DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
using System.Net.Sockets;
|
|
||||||
using System.Net;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using MongoDB.Bson;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
|
|
||||||
namespace bts {
|
|
||||||
// this class, a little specialized, takes the first snapshot thats fired
|
|
||||||
// from the base stream sorts them by a given column, and fires one row at a time
|
|
||||||
public class SnapshotReplayStream : ViewStream, IWorker {
|
|
||||||
|
|
||||||
private ViewStreamDatabase _vsd;
|
|
||||||
private ViewStream _snapshot_stream;
|
|
||||||
private readonly string _sort_col;
|
|
||||||
private readonly int _buffer_size;
|
|
||||||
private TimeSpan _fire_period;
|
|
||||||
private int _in_row_ptr = 0;
|
|
||||||
private List<IBlueRow> _in_rows = new List<IBlueRow>();
|
|
||||||
private CircularBufferTable _out_table;
|
|
||||||
|
|
||||||
|
|
||||||
public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period)
|
|
||||||
: this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { }
|
|
||||||
|
|
||||||
public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) {
|
|
||||||
_vsd = vsd;
|
|
||||||
_snapshot_stream = base_str;
|
|
||||||
_sort_col = sort_col;
|
|
||||||
_buffer_size = buffer_size;
|
|
||||||
_fire_period = fire_period;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_snapshot_stream.onChange += _handle_stream_update;
|
|
||||||
MasterWorker.getInstance(false).addWorker(this, _fire_period);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
base.myOnLastSubscriber();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _handle_stream_update(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
if (_in_rows.Count > 0)
|
|
||||||
return;
|
|
||||||
_out_table = new CircularBufferTable(dp.schema, _buffer_size);
|
|
||||||
// _in_rows is filled up only once
|
|
||||||
foreach (IBlueRow r in dp.newRows) {
|
|
||||||
_in_rows.Add(r);
|
|
||||||
}
|
|
||||||
_in_rows.Sort((IBlueRow r1, IBlueRow r2) => {
|
|
||||||
return BlueDataUtility.compare_rows(r1, r2, _sort_col);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork() {
|
|
||||||
if (_in_row_ptr >= _in_rows.Count) {
|
|
||||||
// we can loop around or not here
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (_in_rows.Count > 0) {
|
|
||||||
// fire row
|
|
||||||
IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
|
|
||||||
_out_table.add_row(curr_row);
|
|
||||||
DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
|
|
||||||
myLastValue == null ? null : myLastValue as DataPacket, false);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,809 +0,0 @@
|
|||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Reflection;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
|
|
||||||
/// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
|
|
||||||
/// </summary>
|
|
||||||
[DataStreamUserRespresentation("tests")]
|
|
||||||
public class TesStream : ViewStream {
|
|
||||||
static byte[] create_byte_array() {
|
|
||||||
Random random = new Random();
|
|
||||||
var sz = random.Next(1, 1000);// % 1000;
|
|
||||||
|
|
||||||
// Generate a random byte
|
|
||||||
byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
|
|
||||||
byte[] rv = new byte[sz];
|
|
||||||
for (int i = 0; i < sz; i++) {
|
|
||||||
rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
private IBlueTableSchema _schema;
|
|
||||||
class TestResult {
|
|
||||||
public string test_name = MethodBase.GetCurrentMethod()?.Name
|
|
||||||
?? throw new InvalidOperationException("Failed to get test name from method");
|
|
||||||
public bool test_result;
|
|
||||||
}
|
|
||||||
private List<Func<TestResult>> _tests = new
|
|
||||||
List<Func<TestResult>>();
|
|
||||||
|
|
||||||
public TesStream() {
|
|
||||||
_schema = BlueDataUtility.form_schema("Tests",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("test", TypeCode.String),
|
|
||||||
Tuple.Create("result", TypeCode.Boolean),
|
|
||||||
Tuple.Create("exception", TypeCode.String)
|
|
||||||
);
|
|
||||||
// register tests here
|
|
||||||
_tests.Add(new Func<TestResult>(_run_sub_msg_test));
|
|
||||||
_tests.Add(new Func<TestResult>(_run_pub_msg_test));
|
|
||||||
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
|
|
||||||
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
|
|
||||||
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
|
|
||||||
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
|
|
||||||
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
|
|
||||||
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
|
|
||||||
//_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
|
|
||||||
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
|
|
||||||
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
|
|
||||||
// _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
|
|
||||||
// _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
|
|
||||||
//_tests.Add(new Func<TestResult>(_cpp_data_test));
|
|
||||||
//_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
|
|
||||||
_tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
BlueTable rows = new BlueTable(_schema);
|
|
||||||
// u the tests here
|
|
||||||
foreach (var test in _tests) {
|
|
||||||
string ex_msg = String.Empty;
|
|
||||||
TestResult tr = new TestResult() {
|
|
||||||
test_name = test.Method.Name,
|
|
||||||
test_result = false
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
tr = test();
|
|
||||||
} catch (Exception ex) {
|
|
||||||
ex_msg = ex.Message + "\n" + ex.StackTrace;
|
|
||||||
}
|
|
||||||
IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
|
|
||||||
rows.addRow(r);
|
|
||||||
}
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _run_sub_msg_test() {
|
|
||||||
bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
|
|
||||||
connection = Guid.NewGuid().ToString(),
|
|
||||||
subscribing_app = Guid.NewGuid().ToString(),
|
|
||||||
tags = new string[] { "one", "two", "three" }
|
|
||||||
};
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
sub_msg.serialize_into_str(str);
|
|
||||||
str.Position = 0;
|
|
||||||
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
|
|
||||||
sub_msg_.consruct_deserialize(str);
|
|
||||||
bool res = sub_msg.Equals(sub_msg_);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "SubscribeMsg",
|
|
||||||
test_result = res
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _run_pub_msg_test() {
|
|
||||||
var pub_msg = new bts.PublishMsg() {
|
|
||||||
connection = Guid.NewGuid().ToString(),
|
|
||||||
publishing_app = Guid.NewGuid().ToString(),
|
|
||||||
tag = Guid.NewGuid().ToString(),
|
|
||||||
tag_type = "DS_",
|
|
||||||
is_compressed = false,
|
|
||||||
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
|
||||||
data_to_publish = create_byte_array(),
|
|
||||||
};
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
pub_msg.serialize_into_str(str);
|
|
||||||
str.Position = 0;
|
|
||||||
var pub_msg_ = new bts.PublishMsg();
|
|
||||||
pub_msg_.consruct_deserialize(str);
|
|
||||||
bool res = pub_msg.Equals(pub_msg_);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "PublishMsg",
|
|
||||||
test_result = res
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _run_pubsub_server_msg_test() {
|
|
||||||
var msg = new bts.PubSubServerMessage() {
|
|
||||||
tag = Guid.NewGuid().ToString(),
|
|
||||||
tag_type = Guid.NewGuid().ToString(),
|
|
||||||
data = create_byte_array()
|
|
||||||
};
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
msg.serialize_into_str(str);
|
|
||||||
str.Position = 0;
|
|
||||||
var msg_ = new bts.PubSubServerMessage();
|
|
||||||
msg_.consruct_deserialize(str);
|
|
||||||
bool res = msg.Equals(msg_);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "PubSubServerMessage",
|
|
||||||
test_result = res
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _run_pubsub_client_msg_test() {
|
|
||||||
PubSubClientMessage msg = new PubSubClientMessage() {
|
|
||||||
msg_type = PubSubMessageType.SUBSCRIBE,
|
|
||||||
sub_msg = new bts.SubscribeMsg() {
|
|
||||||
connection = Guid.NewGuid().ToString(),
|
|
||||||
subscribing_app = Guid.NewGuid().ToString(),
|
|
||||||
tags = new string[] { "one", "two", "three" }
|
|
||||||
}
|
|
||||||
};
|
|
||||||
var str = new MemoryStream();
|
|
||||||
msg.serialize_into_str(str);
|
|
||||||
str.Position = 0;
|
|
||||||
var msg_ = new PubSubClientMessage();
|
|
||||||
msg_.consruct_deserialize(str);
|
|
||||||
var res = msg.Equals(msg_);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "PubSubClientMessage",
|
|
||||||
test_result = res
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _run_columninfo_serialization_test() {
|
|
||||||
Func<ColumnInfo, bool> test_ci = (ci) => {
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
StreamSerializationUtils.serialize_column_info(ci, ref bw);
|
|
||||||
str.Position = 0;
|
|
||||||
//var ci_txt = BlueDataUtility.to_string(ci);
|
|
||||||
//var ci_ser =
|
|
||||||
//bts.Log.Gen.info("ci: {}")
|
|
||||||
BinaryReader br = new BinaryReader(str);
|
|
||||||
var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
|
|
||||||
return BlueDataUtility.compute_equality(ci, ci_prime);
|
|
||||||
};
|
|
||||||
int i = 0;
|
|
||||||
foreach (Type t in BlueDataUtility.valid_column_types()) {
|
|
||||||
ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
|
|
||||||
i++, t);
|
|
||||||
var res = test_ci(ci);
|
|
||||||
if (!res) {
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "ColumnInfoSerialization",
|
|
||||||
test_result = false
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "ColumnInfoSerialization",
|
|
||||||
test_result = true
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private TestResult _run_schema_serialization_test() {
|
|
||||||
var table_name = "test_table";
|
|
||||||
var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
|
|
||||||
IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
StreamSerializationUtils.serialize_schema(schema, ref bw);
|
|
||||||
_write_to_file(str, @"c:\Temp\tests\schema.bin");
|
|
||||||
var hex_txt = StreamSerializationUtils.to_hex(str);
|
|
||||||
bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
|
|
||||||
|
|
||||||
str.Position = 0;
|
|
||||||
BinaryReader br = new BinaryReader(str);
|
|
||||||
var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
|
|
||||||
var res = BlueDataUtility.compute_equality(schema, schema_prime);
|
|
||||||
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "SchemaSerialization",
|
|
||||||
test_result = res
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private TestResult _run_datetime_seriliaztion_test() {
|
|
||||||
DateTime dt = DateTime.Now;
|
|
||||||
DateTimeOffset dt_off = new DateTimeOffset(dt);
|
|
||||||
long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
|
|
||||||
DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
|
|
||||||
DateTime dt_p = dt_off_pr.DateTime;
|
|
||||||
var passes = dt.Equals(dt_p);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "_run_datetime_seriliaztion_test",
|
|
||||||
test_result = false
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _run_full_stream_serialization_test() {
|
|
||||||
// create message
|
|
||||||
var schema = create_schema_with_all_types();
|
|
||||||
BlueTable cr_rows = new BlueTable(schema);
|
|
||||||
// 100 rows
|
|
||||||
int n_rows = 100;
|
|
||||||
for (int i = 0; i < n_rows; i++) {
|
|
||||||
var nr = generate_random_row(schema, i);
|
|
||||||
cr_rows.addRow(nr);
|
|
||||||
}
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
|
||||||
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
|
|
||||||
UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
|
|
||||||
ser.serialize(stream_io_msg, ref bw);
|
|
||||||
str.Position = 0;
|
|
||||||
BinaryReader br = new System.IO.BinaryReader(str);
|
|
||||||
UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
|
|
||||||
var stream_io_msg_prime = deser.deserialize(ref br);
|
|
||||||
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
|
|
||||||
var n_dels = 50;
|
|
||||||
BlueTable del_rows = new BlueTable(cr_rows.schema);
|
|
||||||
foreach (IBlueRow r in cr_rows) {
|
|
||||||
del_rows.addRow(r);
|
|
||||||
if (del_rows.Count == n_dels)
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
IBlueRowCollection added_rows = del_rows;
|
|
||||||
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
|
||||||
str = new MemoryStream();
|
|
||||||
bw = new BinaryWriter(str);
|
|
||||||
stream_io_msg = new bts.StreamIOMessage(mra_msg);
|
|
||||||
ser.serialize(stream_io_msg, ref bw);
|
|
||||||
bw.BaseStream.Position = 0;
|
|
||||||
br = new BinaryReader(str);
|
|
||||||
//deser = new UniversalStreamIODeserializer();
|
|
||||||
stream_io_msg_prime = deser.deserialize(ref br);
|
|
||||||
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "UniversalSerialization",
|
|
||||||
test_result = passes
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _run_full_stream_serialization_with_controller_test() {
|
|
||||||
// create message
|
|
||||||
var schema = create_schema_with_all_types();
|
|
||||||
BlueTable cr_rows = new BlueTable(schema);
|
|
||||||
// 100 rows
|
|
||||||
int n_rows = 100;
|
|
||||||
for (int i = 0; i < n_rows; i++) {
|
|
||||||
var nr = generate_random_row(schema, i);
|
|
||||||
cr_rows.addRow(nr);
|
|
||||||
}
|
|
||||||
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
|
||||||
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
|
|
||||||
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
|
|
||||||
var ser_bytes = ser.serialize(stream_io_msg);
|
|
||||||
IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
|
|
||||||
var stream_io_msg_prime = deser.deserialize(ser_bytes);
|
|
||||||
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
|
|
||||||
//mult remove adds
|
|
||||||
// remove all rows and add 50 more
|
|
||||||
|
|
||||||
BlueTable del_rows = new BlueTable(cr_rows);
|
|
||||||
BlueTable added_rows = new BlueTable(cr_rows.schema);
|
|
||||||
var n_adds = 50;
|
|
||||||
for (int i = 0; i < n_adds; i++) {
|
|
||||||
var nr = generate_random_row(schema, i);
|
|
||||||
added_rows.addRow(nr);
|
|
||||||
}
|
|
||||||
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
|
||||||
stream_io_msg = new bts.StreamIOMessage(mra_msg);
|
|
||||||
ser_bytes = ser.serialize(stream_io_msg);
|
|
||||||
var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
|
|
||||||
Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
|
|
||||||
stream_io_msg_prime = deser.deserialize(ser_bytes);
|
|
||||||
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "UniversalSerializationController",
|
|
||||||
test_result = passes
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _row_serialization_tests_no_dbnulls() {
|
|
||||||
// IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
|
||||||
// new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
|
||||||
// new Tuple<string, TypeCode>("i1", TypeCode.Int32),
|
|
||||||
// new Tuple<string, TypeCode>("d1", TypeCode.Double),
|
|
||||||
// new Tuple<string, TypeCode>("string_1", TypeCode.String));
|
|
||||||
// bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
||||||
// IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
|
|
||||||
var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
|
|
||||||
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
IBlueRow r = generate_random_row(schema, 0, false);
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
row_ser.serialize_row(r, 0, ref bw);
|
|
||||||
_write_to_file(str, @"C:\Temp\tests\row.bin");
|
|
||||||
str.Position = 0;
|
|
||||||
byte[] raw_byte_array = str.ToArray();
|
|
||||||
var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
|
|
||||||
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
|
|
||||||
BinaryReader br = new BinaryReader(str);
|
|
||||||
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
|
||||||
IBlueRow r_prime = des.deserialize_row(ref br);
|
|
||||||
string row_str = r_prime.to_string();
|
|
||||||
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
|
|
||||||
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "UniversalRowSerialization_NonNull",
|
|
||||||
test_result = passes
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _deserialize_pub_msg_test() {
|
|
||||||
var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
|
|
||||||
PubSubClientMessage ps_client_msg = new PubSubClientMessage();
|
|
||||||
ps_client_msg.consruct_deserialize(pub_msg_ser);
|
|
||||||
bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "_deserialize_pub_msg_test",
|
|
||||||
test_result = passes
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _cpp_data_test() {
|
|
||||||
//create and write schema
|
|
||||||
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
|
||||||
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
|
||||||
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
|
|
||||||
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
|
|
||||||
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
|
|
||||||
new Tuple<string, TypeCode>("string_1", TypeCode.String)
|
|
||||||
);
|
|
||||||
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
||||||
UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
|
|
||||||
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
|
||||||
StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
|
||||||
|
|
||||||
IBlueTableSchema schema_p = null;
|
|
||||||
var schema_file = @"c:\Temp\tests\schema.bin";
|
|
||||||
var row_file = @"C:\Temp\tests\row.bin";
|
|
||||||
var create_op_file = @"C:\Temp\tests\create_op.bin";
|
|
||||||
var mra_op_file = @"C:\Temp\tests\mra_op.bin";
|
|
||||||
IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
|
|
||||||
|
|
||||||
//serialize schema, write it to a file, read it back, deserialize it and match schemas
|
|
||||||
byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
|
|
||||||
_write_to_file(ser_schema_bytes, schema_file);
|
|
||||||
|
|
||||||
byte[] schema_file_bytes = _read_from_file(schema_file);
|
|
||||||
schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
|
|
||||||
bool passes = BlueDataUtility.compute_equality(schema, schema_p);
|
|
||||||
|
|
||||||
//serialize row, write it to a file, read it back, deserialize it and match schemas
|
|
||||||
byte[] ser_row_bytes = row_ser.serialize_row(row);
|
|
||||||
_write_to_file(ser_row_bytes, row_file);
|
|
||||||
|
|
||||||
byte[] row_file_bytes = _read_from_file(row_file);
|
|
||||||
var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
|
|
||||||
Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
|
|
||||||
|
|
||||||
IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
|
|
||||||
passes = passes && BlueDataUtility.compute_equality(row, row_p);
|
|
||||||
|
|
||||||
{
|
|
||||||
// serialize a create operation
|
|
||||||
int n_rows = 100;
|
|
||||||
BlueTable table = new BlueTable(schema);
|
|
||||||
for (int i = 0; i < n_rows; i++) {
|
|
||||||
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
|
|
||||||
table.addRow(nr);
|
|
||||||
}
|
|
||||||
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
|
|
||||||
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
|
||||||
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
|
|
||||||
_write_to_file(create_ser_bytes, create_op_file);
|
|
||||||
|
|
||||||
// read it and make sure its right
|
|
||||||
byte[] create_op_file_bytes = _read_from_file(create_op_file);
|
|
||||||
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
|
||||||
var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
|
|
||||||
_write_to_file(create_ser_bytes, mra_op_file);
|
|
||||||
|
|
||||||
passes = passes && cr_msg.Equals(cr_msg_p);
|
|
||||||
|
|
||||||
}
|
|
||||||
//multiple remove adds
|
|
||||||
{
|
|
||||||
// remove all rows and add them again
|
|
||||||
int n_rows = 100;
|
|
||||||
BlueTable table = new BlueTable(schema);
|
|
||||||
for (int i = 0; i < n_rows; i++) {
|
|
||||||
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
|
|
||||||
table.addRow(nr);
|
|
||||||
}
|
|
||||||
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
|
|
||||||
|
|
||||||
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
|
||||||
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
|
|
||||||
_write_to_file(mra_ser_bytes, mra_op_file);
|
|
||||||
|
|
||||||
// reasd it and make sure it matches
|
|
||||||
byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
|
|
||||||
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
|
||||||
StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
|
|
||||||
passes = passes && mra_msg.Equals(mra_msg_p);
|
|
||||||
}
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "_cpp_data_test",
|
|
||||||
test_result = passes
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _write_stream_updates_to_files() {
|
|
||||||
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
|
||||||
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
|
||||||
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
|
|
||||||
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
|
|
||||||
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
|
|
||||||
new Tuple<string, TypeCode>("string_1", TypeCode.String)
|
|
||||||
);
|
|
||||||
var packets_path = @"C:\Temp\tests\stream_packets\";
|
|
||||||
Func<long> now_epochs = () => {
|
|
||||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
||||||
// Convert to Unix time in milliseconds
|
|
||||||
long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
|
|
||||||
return unix_time_milliseconds;
|
|
||||||
};
|
|
||||||
Func<int, bool, string> curr_file_path = (id, create) => {
|
|
||||||
//var epoch_ms = now_epochs();
|
|
||||||
var file_path = $"{packets_path}{id}.bin";
|
|
||||||
if (create)
|
|
||||||
file_path = $"{packets_path}create_{id}.bin";
|
|
||||||
return file_path;
|
|
||||||
};
|
|
||||||
// write 1000 stream updates
|
|
||||||
{
|
|
||||||
int init_sz = 100;
|
|
||||||
BlueTable table = new BlueTable(schema);
|
|
||||||
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
|
||||||
// create table of 100 rows
|
|
||||||
for (int i = 0; i < init_sz; i++) {
|
|
||||||
IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
|
|
||||||
table.addRow(nr);
|
|
||||||
}
|
|
||||||
// write create msg
|
|
||||||
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
|
|
||||||
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
|
|
||||||
var create_file_path = curr_file_path(table.Count, true);
|
|
||||||
_write_to_file(create_ser_bytes, create_file_path);
|
|
||||||
|
|
||||||
// add 1000 rows and write an io msg for each update and write it to a file
|
|
||||||
int n_updates = 1000;
|
|
||||||
int period_ms = 50;
|
|
||||||
|
|
||||||
// write a create
|
|
||||||
for (int i = 0; i < n_updates; i++) {
|
|
||||||
// addition
|
|
||||||
BlueTable adds = new BlueTable(schema);
|
|
||||||
int id = table.Count;
|
|
||||||
IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
|
|
||||||
adds.addRow(nr);
|
|
||||||
BlueTable dels = new BlueTable(schema);
|
|
||||||
table.addRow(nr);
|
|
||||||
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
|
|
||||||
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
|
|
||||||
var mra_file_path = curr_file_path(id, false);
|
|
||||||
_write_to_file(mra_ser_bytes, mra_file_path);
|
|
||||||
|
|
||||||
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
|
|
||||||
}
|
|
||||||
// now start writing increments and add 1 row at a time
|
|
||||||
}
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "_write_updates_as_files",
|
|
||||||
test_result = true
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private TestResult _row_serialization_tests_with_dbnulls() {
|
|
||||||
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
|
||||||
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
|
||||||
new Tuple<string, TypeCode>("i1", TypeCode.Int32),
|
|
||||||
new Tuple<string, TypeCode>("d1", TypeCode.Double),
|
|
||||||
new Tuple<string, TypeCode>("string_1", TypeCode.String));
|
|
||||||
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
||||||
IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
row_ser.serialize_row(r, 0, ref bw);
|
|
||||||
str.Position = 0;
|
|
||||||
BinaryReader br = new BinaryReader(str);
|
|
||||||
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
|
||||||
IBlueRow r_prime = des.deserialize_row(ref br);
|
|
||||||
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "UniversalRowSerialization_WithDBNulls",
|
|
||||||
test_result = passes
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private TestResult _row_serialization_tests_all_types() {
|
|
||||||
var schema = create_schema_with_all_types();
|
|
||||||
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
||||||
IBlueRow r = generate_random_row(schema, 0, false);
|
|
||||||
MemoryStream str = new MemoryStream();
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
row_ser.serialize_row(r, 0, ref bw);
|
|
||||||
_write_to_file(str, @"C:\Temp\tests\row.bin");
|
|
||||||
str.Position = 0;
|
|
||||||
BinaryReader br = new BinaryReader(str);
|
|
||||||
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
|
||||||
IBlueRow r_prime = des.deserialize_row(ref br);
|
|
||||||
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
|
||||||
return new TestResult {
|
|
||||||
test_name = "UniversalRowSerialization_AllTypes",
|
|
||||||
test_result = passes
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private static TestResult _run_pubsub_server_stream_tag_handler_test() {
|
|
||||||
// create message
|
|
||||||
var schema = create_schema_with_all_types();
|
|
||||||
var tag = "DS_" + schema.tableName;
|
|
||||||
BlueTable cr_rows = new BlueTable(schema);
|
|
||||||
// 100 rows
|
|
||||||
int n_rows = 100;
|
|
||||||
for (int i = 0; i < n_rows; i++) {
|
|
||||||
var nr = generate_random_row(schema, i);
|
|
||||||
cr_rows.addRow(nr);
|
|
||||||
}
|
|
||||||
|
|
||||||
PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
|
|
||||||
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
|
||||||
bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
|
|
||||||
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
|
|
||||||
byte[] msg_bytes = ser.serialize(str_io_msg);
|
|
||||||
handler.update(tag, msg_bytes);
|
|
||||||
|
|
||||||
//now get it back for the handler
|
|
||||||
byte[] content = handler.get_content(tag);
|
|
||||||
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
|
||||||
bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
|
|
||||||
// test here
|
|
||||||
var passes = str_io_msg.Equals(str_io_msg_p);
|
|
||||||
// delete all rows
|
|
||||||
BlueTable del_rows = new BlueTable(cr_rows);
|
|
||||||
cr_rows.clear();
|
|
||||||
// added 50 new rows
|
|
||||||
BlueTable added_rows = new BlueTable(del_rows.schema);
|
|
||||||
int new_state_sz = 50;
|
|
||||||
for (int i = 0; i < new_state_sz; i++) {
|
|
||||||
var nr = generate_random_row(schema, i);
|
|
||||||
added_rows.addRow(nr);
|
|
||||||
}
|
|
||||||
cr_rows.addRows(added_rows);
|
|
||||||
// the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
|
|
||||||
// 1. generate op
|
|
||||||
bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
|
||||||
str_io_msg = new bts.StreamIOMessage(mra);
|
|
||||||
// 2. serialize it into bytes
|
|
||||||
msg_bytes = ser.serialize(str_io_msg);
|
|
||||||
// 3. send it to handler
|
|
||||||
handler.update(tag, msg_bytes);
|
|
||||||
// 4. get the state from the handler
|
|
||||||
byte[] content_p = handler.get_content(tag);
|
|
||||||
// 5. deserialize the state into a stream op
|
|
||||||
str_io_msg_p = deser.deserialize(content_p);
|
|
||||||
// 6. check to make sure the state's rows are good
|
|
||||||
passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
|
|
||||||
BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
|
|
||||||
return new TestResult {
|
|
||||||
test_result = passes,
|
|
||||||
test_name = "run_pubsub_server_stream_tag_handler_test"
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static TestResult _run_single_tag_data_test_clean() {
|
|
||||||
// create message
|
|
||||||
var test_name = MethodBase.GetCurrentMethod().Name;
|
|
||||||
var schema = create_schema_with_all_types();
|
|
||||||
var tag = "DS_" + schema.tableName;
|
|
||||||
BlueTable table = new BlueTable(schema);
|
|
||||||
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
|
|
||||||
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
|
||||||
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
|
|
||||||
// create with 100 rows
|
|
||||||
int n_rows = 100;
|
|
||||||
bool is_succ = true;
|
|
||||||
for (int i = 0; i < n_rows; i++) {
|
|
||||||
var nr = generate_random_row(schema, i);
|
|
||||||
// perform the same action SingletagData, and verify the states match
|
|
||||||
PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
|
|
||||||
table.addRow(nr);
|
|
||||||
st_data.handle_publish(pub_msg);
|
|
||||||
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
|
||||||
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
|
||||||
}
|
|
||||||
// change all rows
|
|
||||||
for (int i = 0; i < n_rows; i++) {
|
|
||||||
var nr = generate_random_row(schema, i);
|
|
||||||
var dr = table[i];
|
|
||||||
table.deleteRow(dr);
|
|
||||||
table.addRow(nr);
|
|
||||||
PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
|
|
||||||
st_data.handle_publish(pub_msg);
|
|
||||||
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
|
||||||
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
|
||||||
}
|
|
||||||
// delete all rows
|
|
||||||
var aux_table = new BlueTable(table);
|
|
||||||
foreach (IBlueRow r in aux_table) {
|
|
||||||
table.deleteRow(r);
|
|
||||||
PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
|
|
||||||
st_data.handle_publish(pub_msg);
|
|
||||||
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
|
||||||
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
|
||||||
}
|
|
||||||
return new TestResult() {
|
|
||||||
test_result = is_succ,
|
|
||||||
test_name = test_name
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#region Helper functions
|
|
||||||
|
|
||||||
private static void _write_to_file(byte[] ser_bytes, string file_name) {
|
|
||||||
_write_to_file(new MemoryStream(ser_bytes), file_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void _write_to_file(MemoryStream str, string file_name) {
|
|
||||||
if (String.IsNullOrEmpty(file_name))
|
|
||||||
return;
|
|
||||||
byte[] str_bytes = str.ToArray();
|
|
||||||
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
|
|
||||||
// Write the byte array to the file
|
|
||||||
fs.Write(str_bytes, 0, str_bytes.Length);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static byte[] _read_from_file(string file_path) {
|
|
||||||
var rv = File.ReadAllBytes(file_path);
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
|
|
||||||
var content = st_data.get_current_content();
|
|
||||||
var msg_p = deser.deserialize(content);
|
|
||||||
Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
|
|
||||||
return msg_p.create_message.rows;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
|
|
||||||
ref IStreamIOMsgSerializer serializer) {
|
|
||||||
Func<IBlueTableSchema> get_schema = () => {
|
|
||||||
return (added_row != null ? added_row : removed_row).schema;
|
|
||||||
};
|
|
||||||
bts.StreamIOMessage str_io_msg = null;
|
|
||||||
if (is_create) {
|
|
||||||
Log.assert(added_row != null);
|
|
||||||
var aux_new = new BlueTable(added_row.schema);
|
|
||||||
aux_new.addRow(added_row);
|
|
||||||
var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
|
|
||||||
str_io_msg = new bts.StreamIOMessage(cr_msg);
|
|
||||||
} else {
|
|
||||||
var schema = get_schema();
|
|
||||||
BlueTable added_table = new BlueTable(schema);
|
|
||||||
if (added_row != null)
|
|
||||||
added_table.addRow(added_row);
|
|
||||||
BlueTable rem_table = new BlueTable(schema);
|
|
||||||
if (removed_row != null)
|
|
||||||
rem_table.addRow(removed_row);
|
|
||||||
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
|
|
||||||
new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
|
|
||||||
str_io_msg = new bts.StreamIOMessage(mra_msg);
|
|
||||||
}
|
|
||||||
byte[] msg_bytes = serializer.serialize(str_io_msg);
|
|
||||||
PublishMsg pub_msg = new PublishMsg() {
|
|
||||||
connection = "test_connection",
|
|
||||||
tag = $"DS_{get_schema()}",
|
|
||||||
tag_type = "DS",
|
|
||||||
data_to_publish = msg_bytes
|
|
||||||
};
|
|
||||||
return pub_msg;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
|
|
||||||
excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
|
|
||||||
var table_name = "test_table";
|
|
||||||
BlueTableSchema schema = new BlueTableSchema();
|
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
|
||||||
foreach (Type t in BlueDataUtility.valid_column_types()) {
|
|
||||||
if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
|
|
||||||
continue;
|
|
||||||
var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
|
|
||||||
ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
|
|
||||||
cis.Add(ci);
|
|
||||||
}
|
|
||||||
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
|
|
||||||
schema = new BlueTableSchema(table_name, cis.ToArray());
|
|
||||||
return schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
|
|
||||||
Random rnd = new Random();
|
|
||||||
object[] row_vals = new object[schema.columnInfo.Length];
|
|
||||||
foreach (ColumnInfo ci in schema.columnInfo) {
|
|
||||||
if (ci.colName == "id") {
|
|
||||||
row_vals[ci.colNo] = id;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
object v = get_random_object(ci.type_code());
|
|
||||||
if (include_dbnulls && rnd.NextSingle() % 4 == 0)
|
|
||||||
v = DBNull.Value;
|
|
||||||
row_vals[ci.colNo] = v;
|
|
||||||
}
|
|
||||||
return new BlueRow(schema, row_vals);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static object get_random_object(TypeCode type_code) {
|
|
||||||
Random random = new Random();
|
|
||||||
switch (type_code) {
|
|
||||||
case TypeCode.Boolean:
|
|
||||||
return random.Next(2) == 0; // Random boolean value (true or false)
|
|
||||||
case TypeCode.Byte:
|
|
||||||
return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
|
|
||||||
case TypeCode.Char:
|
|
||||||
return (char)random.Next(65, 65 + 27); // Random char value
|
|
||||||
case TypeCode.Decimal:
|
|
||||||
Decimal dec = 100.123M;
|
|
||||||
return dec;
|
|
||||||
case TypeCode.DateTime:
|
|
||||||
return DateTime.Now;
|
|
||||||
case TypeCode.Double:
|
|
||||||
return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
|
|
||||||
case TypeCode.Int16:
|
|
||||||
return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
|
|
||||||
case TypeCode.Int32:
|
|
||||||
return random.Next(int.MinValue, int.MaxValue); // Random int value
|
|
||||||
case TypeCode.Int64:
|
|
||||||
return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
|
|
||||||
case TypeCode.SByte:
|
|
||||||
return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
|
|
||||||
case TypeCode.Single:
|
|
||||||
return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
|
|
||||||
case TypeCode.String:
|
|
||||||
return Guid.NewGuid().ToString();
|
|
||||||
case TypeCode.UInt16:
|
|
||||||
return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
|
|
||||||
case TypeCode.UInt32:
|
|
||||||
return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
|
|
||||||
case TypeCode.UInt64:
|
|
||||||
byte[] ulongBuffer = new byte[8];
|
|
||||||
random.NextBytes(ulongBuffer);
|
|
||||||
return BitConverter.ToUInt64(ulongBuffer, 0);
|
|
||||||
default:
|
|
||||||
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
namespace BCG {
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class TickingTestStream : ViewStream {
|
|
||||||
public TickingTestStream(int ticking_period_msecs)
|
|
||||||
:
|
|
||||||
this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
|
|
||||||
|
|
||||||
public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
|
|
||||||
ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
|
|
||||||
vs = new MathStream(vs, "x=id*123 y=x*x");
|
|
||||||
mySetRealSource(vs);
|
|
||||||
//vs.onChange += _on_str_update;
|
|
||||||
}
|
|
||||||
|
|
||||||
// private void _on_str_update(object dp_obj) {
|
|
||||||
// DataPacket dp = dp_obj as DataPacket;
|
|
||||||
// IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
|
|
||||||
// BlueTable state = new BlueTable(out_schema);
|
|
||||||
// foreach (IBlueRow r in dp.newRows) {
|
|
||||||
// MutableBlueRow nr = new MutableBlueRow(out_schema, r);
|
|
||||||
// DateTime time = r.get_datetime("time");
|
|
||||||
// var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
|
||||||
// nr.setValue("time_txt", time_txt);
|
|
||||||
// state.addRow(nr);
|
|
||||||
// }
|
|
||||||
// var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
|
|
||||||
// myFireEvent(dp_out);
|
|
||||||
// }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,37 +3,45 @@ using Google.Protobuf.WellKnownTypes;
|
|||||||
using System;
|
using System;
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
|
{
|
||||||
// Keeps adding changing rows to the stream, hence preserving the history
|
// Keeps adding changing rows to the stream, hence preserving the history
|
||||||
// and creating a time series, for example this can be used to make bbo updates
|
// and creating a time series, for example this can be used to make bbo updates
|
||||||
// into a time series of prices/qty
|
// into a time series of prices/qty
|
||||||
public class TimeSeriesStream : ViewStream {
|
public class TimeSeriesStream : ViewStream
|
||||||
|
{
|
||||||
private BlueTable _table;
|
private BlueTable _table;
|
||||||
private ViewStream _stream;
|
private ViewStream _stream;
|
||||||
private int _id = 0;
|
private int _id = 0;
|
||||||
|
|
||||||
public TimeSeriesStream(ViewStream stream) {
|
public TimeSeriesStream(ViewStream stream)
|
||||||
|
{
|
||||||
_stream = stream;
|
_stream = stream;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber()
|
||||||
|
{
|
||||||
_stream.onChange += new ChangeDelegate(_onstream_change);
|
_stream.onChange += new ChangeDelegate(_onstream_change);
|
||||||
}
|
}
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber()
|
||||||
|
{
|
||||||
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void _onstream_change(object o) {
|
private void _onstream_change(object o)
|
||||||
|
{
|
||||||
DataPacket dp = o as DataPacket;
|
DataPacket dp = o as DataPacket;
|
||||||
DateTime now = DateTime.Now;
|
DateTime now = DateTime.Now;
|
||||||
if (_table == null) {
|
if (_table == null)
|
||||||
|
{
|
||||||
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
|
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
|
||||||
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
|
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
|
||||||
_table = new BlueTable(schema);
|
_table = new BlueTable(schema);
|
||||||
}
|
}
|
||||||
BlueTable added_rows = new BlueTable(_table.schema);
|
BlueTable added_rows = new BlueTable(_table.schema);
|
||||||
foreach (IBlueRow r in dp.addedRows) {
|
foreach (IBlueRow r in dp.addedRows)
|
||||||
|
{
|
||||||
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
|
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
|
||||||
ar.setValue("timestamp", now);
|
ar.setValue("timestamp", now);
|
||||||
ar.setValue("id", _id++);
|
ar.setValue("id", _id++);
|
||||||
@@ -45,6 +53,114 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public class TimeBucketStream : ViewStream, IStreamMapper
|
||||||
|
{
|
||||||
|
private ViewStream _src;
|
||||||
|
private readonly string _time_stamp_col;
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
private int _bucket_size_seconds;
|
||||||
|
|
||||||
|
public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
|
||||||
|
{
|
||||||
|
_time_stamp_col = time_stamp_col;
|
||||||
|
_bucket_size_seconds = bucket_size_seconds;
|
||||||
|
var vs = new MapStream(input, this);
|
||||||
|
mySetRealSource(vs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#region IStreamMapper Members
|
||||||
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
|
||||||
|
{
|
||||||
|
IBlueTableSchema rv = input_schema;
|
||||||
|
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
|
||||||
|
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
|
||||||
|
return rv;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region IDynamicStreamMapper Members
|
||||||
|
|
||||||
|
public IBlueRow map(IBlueRow row)
|
||||||
|
{
|
||||||
|
DateTime ts = row.get<DateTime>(_time_stamp_col);
|
||||||
|
TimeSpan time_of_day = (ts - ts.Date);
|
||||||
|
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
|
||||||
|
double buckets = seconds / _bucket_size_seconds;
|
||||||
|
int bucket_number = Convert.ToInt32(Math.Floor(buckets));
|
||||||
|
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
|
||||||
|
rv.setValue("bucket_number", bucket_number);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
|
||||||
|
// public class ThrottleStream : ViewStream, IWorker
|
||||||
|
// {
|
||||||
|
// private ViewStreamDatabase _vsd;
|
||||||
|
// private ViewStream _stream;
|
||||||
|
// private int _throttle_ms;
|
||||||
|
// private TimeSpan _min_fire_period;
|
||||||
|
// private DateTime _last_Fire_time = DateTime.MinValue;
|
||||||
|
// private DataPacket _unfired_dp;
|
||||||
|
// public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
|
||||||
|
// {
|
||||||
|
// _vsd = vsd;
|
||||||
|
// _stream = base_stream;
|
||||||
|
// _throttle_ms = throttle_ms;
|
||||||
|
// _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnFirstSubscriber()
|
||||||
|
// {
|
||||||
|
// _vsd.addWorker(this);
|
||||||
|
// _stream.onChange += new ChangeDelegate(_onstream_change);
|
||||||
|
// }
|
||||||
|
// protected override void myOnLastSubscriber()
|
||||||
|
// {
|
||||||
|
// _vsd.removeWorker(this);
|
||||||
|
// _stream.onChange -= new ChangeDelegate(_onstream_change);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void _onstream_change(object obj)
|
||||||
|
// {
|
||||||
|
// DataPacket dp = obj as DataPacket;
|
||||||
|
// if (DateTime.Now - _last_Fire_time >= _min_fire_period)
|
||||||
|
// {
|
||||||
|
// // ok to fire here
|
||||||
|
// }
|
||||||
|
// else
|
||||||
|
// {
|
||||||
|
// _last_cached_state = dp.newRows;
|
||||||
|
// }
|
||||||
|
// _possibly_fire(dp.newRows);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// public void doWork()
|
||||||
|
// {
|
||||||
|
// if (_unfired_dp != null)
|
||||||
|
// {
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void _possibly_fire()
|
||||||
|
// {
|
||||||
|
// if (_unfired_dp != null)
|
||||||
|
// {
|
||||||
|
// if (DateTime.Now - _last_fire_time > _min_fire_period)
|
||||||
|
// {
|
||||||
|
// DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
|
||||||
|
// _unfired_dp = null;
|
||||||
|
// _last_Fire_time = DateTime.Now;
|
||||||
|
// myFireEvent(dp);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,121 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Security.Cryptography.X509Certificates;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using HtmlAgilityPack;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
class Program {
|
|
||||||
static void Main() {
|
|
||||||
string url = "http://quotes.toscrape.com";
|
|
||||||
List<string> quotes = ScrapeQuotes(url);
|
|
||||||
|
|
||||||
Console.WriteLine("Quotes from http://quotes.toscrape.com:");
|
|
||||||
foreach (var quote in quotes) {
|
|
||||||
Console.WriteLine(quote);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static List<string> ScrapeQuotes(string url) {
|
|
||||||
List<string> quotes = new List<string>();
|
|
||||||
|
|
||||||
// Create a new HtmlWeb instance
|
|
||||||
HtmlWeb web = new HtmlWeb();
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Load the HTML document from the specified URL
|
|
||||||
HtmlDocument document = web.Load(url);
|
|
||||||
|
|
||||||
// Extract quotes from the HTML document
|
|
||||||
var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
|
|
||||||
if (quoteNodes != null) {
|
|
||||||
foreach (var quoteNode in quoteNodes) {
|
|
||||||
string quote = quoteNode.InnerText.Trim();
|
|
||||||
quotes.Add(quote);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Console.WriteLine($"Error: {ex.Message}");
|
|
||||||
}
|
|
||||||
|
|
||||||
return quotes;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public class WebScapperStream : ViewStream, IWorker {
|
|
||||||
private readonly string _url;
|
|
||||||
private readonly string _xpath;
|
|
||||||
private readonly ViewStreamDatabase _vsd;
|
|
||||||
private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
|
|
||||||
private readonly IBlueTableSchema _schema;
|
|
||||||
|
|
||||||
public WebScapperStream()
|
|
||||||
:
|
|
||||||
this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
|
|
||||||
public WebScapperStream(string url, string xpath)
|
|
||||||
: this(url, xpath, ViewStreamDatabase.instance) { }
|
|
||||||
public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
|
|
||||||
bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
|
|
||||||
_schema = BlueDataUtility.form_schema($"{url}_{xpath}",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("url", TypeCode.UInt32),
|
|
||||||
Tuple.Create("line", TypeCode.String));
|
|
||||||
_url = url;
|
|
||||||
_xpath = xpath;
|
|
||||||
_vsd = vsd;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
MasterWorker.getInstance(false).removeWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork() {
|
|
||||||
var lines = scrape_lines(_url, _xpath);
|
|
||||||
Func<string, int> get_id = (line) => {
|
|
||||||
int rv = 0;
|
|
||||||
if (!_line_id_map.TryGetValue(line, out rv))
|
|
||||||
_line_id_map.Add(line, _line_id_map.Count);
|
|
||||||
return rv;
|
|
||||||
};
|
|
||||||
BlueTable nrows = new BlueTable(_schema);
|
|
||||||
foreach (var l in lines) {
|
|
||||||
bts.Log.Gen.info($"line: {l}");
|
|
||||||
var id = get_id(l);
|
|
||||||
BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
|
|
||||||
nrows.addRow(nr);
|
|
||||||
}
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
static List<string> scrape_lines(string url, string xpath) {
|
|
||||||
List<string> quotes = new List<string>();
|
|
||||||
// Create a new HtmlWeb instance
|
|
||||||
HtmlWeb web = new HtmlWeb();
|
|
||||||
try {
|
|
||||||
// Load the HTML document from the specified URL
|
|
||||||
HtmlDocument document = web.Load(url);
|
|
||||||
// Extract quotes from the HTML document
|
|
||||||
var quote_nodes = document.DocumentNode.SelectNodes(xpath);
|
|
||||||
if (quote_nodes != null) {
|
|
||||||
foreach (var qn in quote_nodes) {
|
|
||||||
string quote = qn.InnerText.Trim();
|
|
||||||
quotes.Add(quote);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Console.WriteLine($"Error: {ex.Message}");
|
|
||||||
}
|
|
||||||
|
|
||||||
return quotes;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loaded 100 of 894 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user