Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
20617c6ee9 | ||
|
|
753deba2f1 | ||
|
|
f2b931fdc5 | ||
|
|
5162cd6711 | ||
|
|
80252502e7 | ||
|
|
ad3fe9e6c0 | ||
|
|
e2f5e6717d | ||
|
|
a4005d1e74 | ||
|
|
a1de0faac9 | ||
|
|
ee4e011611 | ||
|
|
4512e085f2 | ||
|
|
e8bfe89780 | ||
|
|
ad6f86ca98 | ||
|
|
a79126c43b | ||
|
|
b4d7bbcfcd | ||
|
|
d301d75305 | ||
|
|
65d85c07e9 | ||
|
|
7e558bc108 | ||
|
|
c545aeb1ea | ||
|
|
ad51d9f7f5 | ||
|
|
c7722e223b |
No files matched your search
@@ -1,125 +0,0 @@
|
|||||||
FROM ubuntu:22.04
|
|
||||||
|
|
||||||
ARG DEBIAN_FRONTEND=noninteractive
|
|
||||||
|
|
||||||
# Note installations that take a while (more than 10s)
|
|
||||||
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get install -y ca-certificates
|
|
||||||
RUN apt-get install -y ccache
|
|
||||||
RUN apt-get install -y curl
|
|
||||||
RUN apt-get install -y fish
|
|
||||||
RUN apt-get install -y gdb
|
|
||||||
# git-all: ~250s
|
|
||||||
RUN apt-get install -y git-all
|
|
||||||
RUN apt-get install -y gnupg
|
|
||||||
# liblttng-ust-dev: ~30s
|
|
||||||
RUN apt-get install -y liblttng-ust-dev
|
|
||||||
RUN apt-get install -y libssh2-1-dev
|
|
||||||
RUN apt-get install -y locales
|
|
||||||
RUN apt-get install -y ninja-build
|
|
||||||
# openssh-server: ~40s
|
|
||||||
RUN apt-get install -y openssh-server
|
|
||||||
RUN apt-get install -y pkg-config
|
|
||||||
RUN apt-get install -y software-properties-common
|
|
||||||
RUN apt-get install -y sudo
|
|
||||||
RUN apt-get install -y tzdata
|
|
||||||
RUN apt-get install -y vim
|
|
||||||
RUN apt-get install -y zip
|
|
||||||
|
|
||||||
# Python: ~30s
|
|
||||||
RUN apt-get install -y \
|
|
||||||
python3-pip \
|
|
||||||
python3-venv \
|
|
||||||
&& python3 -m pip install --upgrade pip
|
|
||||||
|
|
||||||
# GCC
|
|
||||||
ARG gcc_major=13
|
|
||||||
RUN \
|
|
||||||
add-apt-repository ppa:ubuntu-toolchain-r/test \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y g++-$gcc_major
|
|
||||||
|
|
||||||
# Clang
|
|
||||||
ARG clang_major=17
|
|
||||||
RUN \
|
|
||||||
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
|
|
||||||
| gpg --dearmor - \
|
|
||||||
| tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
|
|
||||||
&& echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
|
|
||||||
| tee -a "/etc/apt/sources.list.d/llvm.list" \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y \
|
|
||||||
clang-$clang_major \
|
|
||||||
clang-format-$clang_major \
|
|
||||||
clangd-$clang_major
|
|
||||||
|
|
||||||
# CMake
|
|
||||||
RUN \
|
|
||||||
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
|
|
||||||
| gpg --dearmor - \
|
|
||||||
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
|
|
||||||
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
|
|
||||||
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y cmake
|
|
||||||
|
|
||||||
# Dotnet
|
|
||||||
RUN apt-get update && apt-get install -y dotnet7
|
|
||||||
|
|
||||||
# Node
|
|
||||||
ARG node_major=18
|
|
||||||
RUN mkdir -p /etc/apt/keyrings \
|
|
||||||
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
|
|
||||||
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
|
|
||||||
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
|
|
||||||
| tee /etc/apt/sources.list.d/nodesource.list \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y nodejs
|
|
||||||
|
|
||||||
# Emumba (deps)
|
|
||||||
RUN apt-get update && apt-get -y install rapidjson-dev
|
|
||||||
# Emumba
|
|
||||||
COPY emumba /emumba
|
|
||||||
RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
|
|
||||||
&& rm -r /emumba
|
|
||||||
|
|
||||||
# Pcap
|
|
||||||
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
|
||||||
|
|
||||||
# Databento (deps)
|
|
||||||
RUN apt-get update && apt-get -y install libzstd-dev
|
|
||||||
|
|
||||||
# Set up default executables (update-alternatives)
|
|
||||||
RUN \
|
|
||||||
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
|
|
||||||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
|
|
||||||
|
|
||||||
# Add compiler symlinks for ccache
|
|
||||||
RUN \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/clang; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/clang++; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/gcc; \
|
|
||||||
ln -s /usr/bin/ccache /usr/local/bin/g++
|
|
||||||
|
|
||||||
RUN locale-gen en_US.UTF-8
|
|
||||||
|
|
||||||
ARG docker_user="bts"
|
|
||||||
ARG docker_uid="1000"
|
|
||||||
ARG docker_pass="bts"
|
|
||||||
|
|
||||||
RUN \
|
|
||||||
groupadd -g "$docker_uid" "$docker_user" \
|
|
||||||
&& useradd \
|
|
||||||
-m \
|
|
||||||
-u "$docker_uid" \
|
|
||||||
-g "$docker_uid" \
|
|
||||||
-G sudo \
|
|
||||||
--password "$(openssl passwd -1 "$docker_pass")" \
|
|
||||||
"$docker_user"
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
{
|
|
||||||
"dockerFile": "Dockerfile",
|
|
||||||
// Creates local dir to mount to HOME in docker
|
|
||||||
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
|
|
||||||
"hostRequirements": {
|
|
||||||
"memory": "4gb",
|
|
||||||
"storage": "6gb"
|
|
||||||
},
|
|
||||||
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
|
|
||||||
"containerUser": "bts",
|
|
||||||
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
|
|
||||||
"workspaceFolder": "/home/bts/spark",
|
|
||||||
"updateRemoteUserUID": true,
|
|
||||||
"shutdownAction": "none",
|
|
||||||
// Note: If running multiple containers on the same host, each must use a
|
|
||||||
// unique set of ports. Usually we just add 1000 to each port as needed. For
|
|
||||||
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
|
||||||
"runArgs": [
|
|
||||||
"--shm-size",
|
|
||||||
"2gb",
|
|
||||||
"-p",
|
|
||||||
"8000:8000",
|
|
||||||
"-p",
|
|
||||||
"8001:8001",
|
|
||||||
"-p",
|
|
||||||
"8002:8002",
|
|
||||||
"-p",
|
|
||||||
"8003:8003",
|
|
||||||
"-p",
|
|
||||||
"12345:12345"
|
|
||||||
],
|
|
||||||
"mounts": [
|
|
||||||
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
|
|
||||||
],
|
|
||||||
"customizations": {
|
|
||||||
"vscode": {
|
|
||||||
"settings": {
|
|
||||||
"clangd.arguments": [
|
|
||||||
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"extensions": [
|
|
||||||
"eamodio.gitlens",
|
|
||||||
"llvm-vs-code-extensions.vscode-clangd",
|
|
||||||
"ms-vscode.cmake-tools",
|
|
||||||
"ms-vscode.cpptools"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@@ -1,4 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
|
|
||||||
sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
*.cmd text=auto eol=crlf
|
|
||||||
*.cs text=auto eol=crlf
|
|
||||||
*.sh text=auto eol=lf
|
|
||||||
+2
-4
@@ -8,9 +8,7 @@ cmake-build-debug/
|
|||||||
.idea/
|
.idea/
|
||||||
.dir-locals.el
|
.dir-locals.el
|
||||||
*.pyc
|
*.pyc
|
||||||
**.vscode/*
|
**.vscode
|
||||||
!.vscode/launch.json
|
|
||||||
!.vscode/tasks.json
|
|
||||||
.vs
|
.vs
|
||||||
obj/
|
obj/
|
||||||
*.csproj.user
|
*.csproj.user
|
||||||
@@ -56,6 +54,6 @@ trace_data/
|
|||||||
/src/Spark.Admin.WebApp/src/bin/
|
/src/Spark.Admin.WebApp/src/bin/
|
||||||
**/CMakeDeps.cmake
|
**/CMakeDeps.cmake
|
||||||
**/CMakeDeps_win.cmake
|
**/CMakeDeps_win.cmake
|
||||||
.devcontainer/
|
**/.devcontainer
|
||||||
*.Generated.cs
|
*.Generated.cs
|
||||||
*.lst
|
*.lst
|
||||||
Vendored
-158
@@ -1,158 +0,0 @@
|
|||||||
{
|
|
||||||
"version": "0.2.0",
|
|
||||||
"configurations": [
|
|
||||||
{
|
|
||||||
"name": ".NET Core Attach",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "attach"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "DataStreamPublisher",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "Build DataStream (Debug)",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"program": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "(gdb) Launch",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": ["run"],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "DataStreamQueryAnalyzer",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
|
||||||
"env": {
|
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "PubSubServer",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "Build DataStream (Debug)",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"program": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Spark",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
|
||||||
"program": "Spark.exe",
|
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Spark (Parameterized)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "Spark.exe",
|
|
||||||
"args": [
|
|
||||||
"--sparkd",
|
|
||||||
"${input:sparkConfigServer}"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Sparkd",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "Build Sparkd (Debug | Clang)",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run"
|
|
||||||
],
|
|
||||||
"cwd": "build/sparkd-debug-clang/server",
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"inputs": [
|
|
||||||
{
|
|
||||||
"id": "dataStreamPublisherConfig",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Publisher Configuration",
|
|
||||||
"options": [
|
|
||||||
"MarketDataPublisher",
|
|
||||||
"SimpleEchoClient",
|
|
||||||
"SimpleTestDataPublisher",
|
|
||||||
"TestDataPublisher",
|
|
||||||
],
|
|
||||||
"default": "MarketDataPublisher",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "pubSubServerConfig",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "PubSub Configuration",
|
|
||||||
"options": [
|
|
||||||
"PubSubServer",
|
|
||||||
"SimpleEchoClient",
|
|
||||||
],
|
|
||||||
"default": "PubSubServer",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkConfigServer",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Config Server URL",
|
|
||||||
"default": "http://localhost:8000/config",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
Vendored
-828
@@ -1,828 +0,0 @@
|
|||||||
{
|
|
||||||
"version": "2.0.0",
|
|
||||||
"tasks": [
|
|
||||||
{
|
|
||||||
"label": "Build Api Server",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_api_server.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_api_server.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"${input:buildConfiguration}",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Api Server (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_api_server.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_api_server.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Debug",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Api Server (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_api_server.cmd",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_api_server.sh",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build DataStream",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_data_stream.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"${input:buildConfiguration}",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build DataStream (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_data_stream.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Debug",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build DataStream (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_data_stream.sh",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Spark",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_spark.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"${input:buildConfiguration}",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Spark (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_spark.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Debug",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Spark (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_spark.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--preset",
|
|
||||||
"${input:sparkdCmakePreset}",
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Debug",
|
|
||||||
"--compiler",
|
|
||||||
"${input:sparkdCompiler}"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Debug | Gcc)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Debug",
|
|
||||||
"--compiler",
|
|
||||||
"gcc"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Debug | Clang)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Debug",
|
|
||||||
"--compiler",
|
|
||||||
"clang"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Release",
|
|
||||||
"--compiler",
|
|
||||||
"${input:sparkdCompiler}"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Release | Gcc)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Release",
|
|
||||||
"--compiler",
|
|
||||||
"gcc"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build Sparkd (Release | Clang)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "shell",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/build_sparkd.sh",
|
|
||||||
"args": [
|
|
||||||
"--build-type",
|
|
||||||
"Release",
|
|
||||||
"--compiler",
|
|
||||||
"clang"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Edit DataStream Config",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "editor",
|
|
||||||
"args": [
|
|
||||||
"${env:HOME}/.local/share/DataStream/conf/spark.conf"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "vim",
|
|
||||||
"args": [
|
|
||||||
"${env:APPDATA}/bts/DataStream/conf/spark.conf"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"reveal": "always",
|
|
||||||
"focus": true,
|
|
||||||
"panel": "shared",
|
|
||||||
"showReuseMessage": false,
|
|
||||||
"close": true,
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "wrench"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Edit Spark Config",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "vim",
|
|
||||||
"args": [
|
|
||||||
"${env:APPDATA}/bts/conf/spark.conf"
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"reveal": "always",
|
|
||||||
"focus": true,
|
|
||||||
"panel": "dedicated",
|
|
||||||
"showReuseMessage": false,
|
|
||||||
"close": true,
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "wrench"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamQueryAnalyzer",
|
|
||||||
"dependsOn": "Build DataStream",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamQueryAnalyzer (Debug)",
|
|
||||||
"dependsOn": "Build DataStream (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamQueryAnalyzer (Release)",
|
|
||||||
"dependsOn": "Build DataStream (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamPublisher",
|
|
||||||
"dependsOn": "Build DataStream",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamPublisher (Debug)",
|
|
||||||
"dependsOn": "Build DataStream (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run DataStreamPublisher (Release)",
|
|
||||||
"dependsOn": "Build DataStream (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer",
|
|
||||||
"dependsOn": "Build DataStream",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer (Debug)",
|
|
||||||
"dependsOn": "Build DataStream (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer (Debug | Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer (Release)",
|
|
||||||
"dependsOn": "Build DataStream (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run PubSubServer (Release | Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark",
|
|
||||||
"dependsOn": "Build Spark",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.exe",
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark (Debug)",
|
|
||||||
"dependsOn": "Build Spark (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.exe",
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark (Release)",
|
|
||||||
"dependsOn": "Build Spark (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.exe",
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark Config Query",
|
|
||||||
"dependsOn": "Build Spark",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "Spark.Console",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.Console.exe",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"config",
|
|
||||||
"--query",
|
|
||||||
"${input:sparkConfigQuery}",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Sparkd",
|
|
||||||
"dependsOn": "Build Sparkd",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run"
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Sparkd Config Query",
|
|
||||||
"dependsOn": "Build Sparkd",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"config",
|
|
||||||
"--query",
|
|
||||||
"${input:sparkdConfigQuery}",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/logging/show_last_run.cmd",
|
|
||||||
"args": [
|
|
||||||
"DataStream",
|
|
||||||
"DataStreamQueryAnalyzer.exe",
|
|
||||||
"'Starting DataStreamQueryAnalyzer'"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"reveal": "always",
|
|
||||||
"focus": true,
|
|
||||||
"panel": "shared",
|
|
||||||
"showReuseMessage": false,
|
|
||||||
"close": true,
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "wrench"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "View PubSubServer Logs (Last Run)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "tools/logging/show_last_run.sh",
|
|
||||||
"args": [
|
|
||||||
"${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
|
|
||||||
"Starting PubSubServer"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/logging/show_last_run.cmd",
|
|
||||||
"args": [
|
|
||||||
"DataStream",
|
|
||||||
"PubSubServer.exe",
|
|
||||||
"Starting PubSubServer"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"reveal": "always",
|
|
||||||
"focus": true,
|
|
||||||
"panel": "shared",
|
|
||||||
"showReuseMessage": false,
|
|
||||||
"close": true,
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "wrench"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
"inputs": [
|
|
||||||
{
|
|
||||||
"id": "buildConfiguration",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Configuration",
|
|
||||||
"options": [
|
|
||||||
"Debug",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"default": "Debug",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdCmakePreset",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Configuration",
|
|
||||||
"options": [
|
|
||||||
"sparkd-debug-clang",
|
|
||||||
"sparkd-debug-gcc",
|
|
||||||
"sparkd-release-clang",
|
|
||||||
"sparkd-release-gcc",
|
|
||||||
],
|
|
||||||
"default": "sparkd-debug-clang",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdCompiler",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Configuration",
|
|
||||||
"options": [
|
|
||||||
"clang",
|
|
||||||
"gcc",
|
|
||||||
],
|
|
||||||
"default": "sparkd-debug-clang",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "dataStreamPublisherConfig",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Publisher Configuration",
|
|
||||||
"options": [
|
|
||||||
"MarketDataPublisher",
|
|
||||||
"SimpleEchoClient",
|
|
||||||
"SimpleTestDataPublisher",
|
|
||||||
"TestDataPublisher",
|
|
||||||
],
|
|
||||||
"default": "MarketDataPublisher",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "pubSubServerConfig",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "PubSub Configuration",
|
|
||||||
"options": [
|
|
||||||
"PubSubServer",
|
|
||||||
"SimpleEchoClient",
|
|
||||||
],
|
|
||||||
"default": "PubSubServer",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkConfigQuery",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Config Type(s)"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdConfigQuery",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Config Type(s)"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
+13
-14
@@ -14,17 +14,12 @@ set(
|
|||||||
CACHE
|
CACHE
|
||||||
STRING
|
STRING
|
||||||
"External dependencies root folder (should contain folders: lib, bin, include")
|
"External dependencies root folder (should contain folders: lib, bin, include")
|
||||||
|
|
||||||
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
|
|
||||||
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
|
|
||||||
# api server, etc.).
|
|
||||||
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
|
|
||||||
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
|
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
|
||||||
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
|
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
|
||||||
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
||||||
|
set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
|
||||||
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
||||||
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
||||||
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
|
|
||||||
if (WIN32)
|
if (WIN32)
|
||||||
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
|
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
|
||||||
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
|
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
|
||||||
@@ -36,13 +31,13 @@ endif()
|
|||||||
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
||||||
link_directories(${SPARK_LIB})
|
link_directories(${SPARK_LIB})
|
||||||
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
if(BUILD_API_SERVER)
|
||||||
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
add_compile_definitions(SPARK_ENABLE_API_SERVER)
|
||||||
if(DEFINED APP_DATA_DIR_OVERRIDE)
|
|
||||||
add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
|
|
||||||
endif()
|
endif()
|
||||||
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
|
add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
|
||||||
|
add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
|
||||||
# subdirs
|
# subdirs
|
||||||
add_subdirectory(src/lib)
|
add_subdirectory(src/lib)
|
||||||
else()
|
else()
|
||||||
@@ -52,7 +47,13 @@ else()
|
|||||||
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
||||||
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
||||||
|
|
||||||
if(BUILD_SPARKD)
|
if(BUILD_API_SERVER)
|
||||||
|
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
|
||||||
|
add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
|
||||||
|
elseif(BUILD_SPARKD)
|
||||||
|
add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
|
||||||
|
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
|
||||||
|
|
||||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||||
# read by their code). We override some below.
|
# read by their code). We override some below.
|
||||||
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
||||||
@@ -132,8 +133,6 @@ else()
|
|||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
|
|
||||||
|
|
||||||
COMMAND
|
COMMAND
|
||||||
ln ARGS -sf
|
ln ARGS -sf
|
||||||
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
||||||
|
|||||||
+62
-113
@@ -15,13 +15,11 @@
|
|||||||
"rhs": "Windows"
|
"rhs": "Windows"
|
||||||
},
|
},
|
||||||
"generator": "Visual Studio 17 2022",
|
"generator": "Visual Studio 17 2022",
|
||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build",
|
||||||
"environment": {
|
"environment": {
|
||||||
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
||||||
},
|
},
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR": "${dollar}{APPDATA}/bts",
|
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
|
|
||||||
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||||
@@ -32,12 +30,7 @@
|
|||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
|
||||||
"EXPORT_MD_CONSUMER": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "ON"
|
|
||||||
},
|
|
||||||
"SHARED_MEMORY_PREFIX": "spark"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -53,8 +46,6 @@
|
|||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"platform": "",
|
"platform": "",
|
||||||
"APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
|
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
|
|
||||||
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -71,8 +62,7 @@
|
|||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
},
|
}
|
||||||
"SHARED_MEMORY_PREFIX": "sparkd"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -105,72 +95,98 @@
|
|||||||
{
|
{
|
||||||
"name": "sparkd-debug-clang",
|
"name": "sparkd-debug-clang",
|
||||||
"inherits": "sparkd-debug",
|
"inherits": "sparkd-debug",
|
||||||
"displayName": "Sparkd (Debug | Clang)",
|
"displayName": "Debug (clang)",
|
||||||
"description": "Debug build compiled w/ clang",
|
"description": "Debug build compiled w/ clang",
|
||||||
|
"binaryDir": "${sourceDir}/build/clang/debug",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "clang",
|
"CMAKE_C_COMPILER": "clang",
|
||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-debug-gcc",
|
"name": "sparkd-debug-gcc",
|
||||||
"inherits": "sparkd-debug",
|
"inherits": "sparkd-debug",
|
||||||
"displayName": "Sparkd (Debug | Gcc)",
|
"displayName": "Debug (gcc)",
|
||||||
"description": "Debug build compiled w/ gcc",
|
"description": "Debug build compiled w/ gcc",
|
||||||
|
"binaryDir": "${sourceDir}/build/gcc/debug",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "gcc",
|
"CMAKE_C_COMPILER": "gcc",
|
||||||
"CMAKE_CXX_COMPILER": "g++",
|
"CMAKE_CXX_COMPILER": "g++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-clang",
|
"name": "sparkd-release-clang",
|
||||||
"inherits": "sparkd-release",
|
"inherits": "sparkd-release",
|
||||||
"displayName": "Sparkd (Release | Clang)",
|
"displayName": "Release (clang)",
|
||||||
"description": "Release build compiled w/ clang",
|
"description": "Release build compiled w/ clang",
|
||||||
|
"binaryDir": "${sourceDir}/build/clang/release",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "clang",
|
"CMAKE_C_COMPILER": "clang",
|
||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-gcc",
|
"name": "sparkd-release-gcc",
|
||||||
"inherits": "sparkd-release",
|
"inherits": "sparkd-release",
|
||||||
"displayName": "Sparkd (Release | Gcc)",
|
"displayName": "Release (gcc)",
|
||||||
"description": "Release build compiled w/ gcc",
|
"description": "Release build compiled w/ gcc",
|
||||||
|
"binaryDir": "${sourceDir}/build/gcc/release",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "gcc",
|
"CMAKE_C_COMPILER": "gcc",
|
||||||
"CMAKE_CXX_COMPILER": "g++",
|
"CMAKE_CXX_COMPILER": "g++",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"name": "spark-api-server-base",
|
||||||
|
"hidden": true,
|
||||||
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
|
"cacheVariables": {
|
||||||
|
"BUILD_API_SERVER": {
|
||||||
|
"type": "bool",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
|
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows",
|
"name": "spark-api-server-windows",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"displayName": "Spark Api Server (Windows)",
|
"displayName": "Spark Api Server",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
"BUILD_API_SERVER": {
|
||||||
"SHARED_MEMORY_PREFIX": "spark_api_server"
|
"type": "bool",
|
||||||
|
"value": "ON"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-linux",
|
"name": "spark-api-server-linux",
|
||||||
"inherits": "sparkd",
|
"inherits": "sparkd",
|
||||||
"displayName": "Spark Api Server (Linux)",
|
"displayName": "Spark Api Server",
|
||||||
"generator": "Ninja Multi-Config",
|
"generator": "Ninja Multi-Config",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
|
||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
},
|
},
|
||||||
|
"BUILD_API_SERVER": {
|
||||||
|
"type": "bool",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
"BUILD_SPARKD": {
|
"BUILD_SPARKD": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
@@ -181,11 +197,7 @@
|
|||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
|
||||||
"EXPORT_MD_CONSUMER": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "ON"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -208,62 +220,13 @@
|
|||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite-windows",
|
"name": "data-stream-suite",
|
||||||
"condition": {
|
|
||||||
"type": "equals",
|
|
||||||
"lhs": "${hostSystemName}",
|
|
||||||
"rhs": "Windows"
|
|
||||||
},
|
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"displayName": "Data Stream Suite (Windows)",
|
"displayName": "Data Stream Suite",
|
||||||
"description": "Data Stream utils ported over from Edge",
|
"description": "Data Stream utils ported over from Edge",
|
||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
|
||||||
"SHARED_MEMORY_PREFIX": "data_stream"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "data-stream-suite-linux",
|
|
||||||
"displayName": "Data Stream Suite (Linux)",
|
|
||||||
"description": "Data Stream utils ported over from Edge",
|
|
||||||
"condition": {
|
|
||||||
"type": "equals",
|
|
||||||
"lhs": "${hostSystemName}",
|
|
||||||
"rhs": "Linux"
|
|
||||||
},
|
|
||||||
"inherits": "sparkd",
|
|
||||||
"generator": "Ninja Multi-Config",
|
|
||||||
"cacheVariables": {
|
|
||||||
"APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
|
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
|
||||||
"BUILD_ADMIN_WEB_APP": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "OFF"
|
|
||||||
},
|
|
||||||
"BUILD_SDK": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "OFF"
|
|
||||||
},
|
|
||||||
"BUILD_SPARKD": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "OFF"
|
|
||||||
},
|
|
||||||
"CMAKE_C_COMPILER": "clang",
|
|
||||||
"CMAKE_CXX_COMPILER": "clang++",
|
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
|
||||||
"EXPORT_MD_CONSUMER": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "ON"
|
|
||||||
},
|
|
||||||
"SHARED_MEMORY_PREFIX": "data_stream"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
@@ -281,7 +244,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-debug",
|
"name": "spark-debug",
|
||||||
"displayName": "Spark (Debug)",
|
"displayName": "Debug",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"configurePreset": "spark",
|
"configurePreset": "spark",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
@@ -289,7 +252,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-release",
|
"name": "spark-release",
|
||||||
"displayName": "Spark (Release)",
|
"displayName": "Release",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"configurePreset": "spark",
|
"configurePreset": "spark",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
@@ -317,28 +280,28 @@
|
|||||||
{
|
{
|
||||||
"name": "sparkd-debug-clang",
|
"name": "sparkd-debug-clang",
|
||||||
"configurePreset": "sparkd-debug-clang",
|
"configurePreset": "sparkd-debug-clang",
|
||||||
"displayName": "Sparkd (Debug | Clang)",
|
"displayName": "Debug (clang)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-debug-gcc",
|
"name": "sparkd-debug-gcc",
|
||||||
"configurePreset": "sparkd-debug-gcc",
|
"configurePreset": "sparkd-debug-gcc",
|
||||||
"displayName": "Sparkd (Debug | Gcc)",
|
"displayName": "Debug (gcc)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-clang",
|
"name": "sparkd-release-clang",
|
||||||
"configurePreset": "sparkd-release-clang",
|
"configurePreset": "sparkd-release-clang",
|
||||||
"displayName": "Sparkd (Release | Clang)",
|
"displayName": "Release (clang)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-gcc",
|
"name": "sparkd-release-gcc",
|
||||||
"configurePreset": "sparkd-release-gcc",
|
"configurePreset": "sparkd-release-gcc",
|
||||||
"displayName": "Sparkd (Release | Gcc)",
|
"displayName": "Release (gcc)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -355,14 +318,14 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows-debug",
|
"name": "spark-api-server-windows-debug",
|
||||||
"displayName": "Spark Api Server (Windows | Debug)",
|
"displayName": "Spark Api Server (Debug)",
|
||||||
"inherits": "spark-api-server-windows",
|
"inherits": "spark-api-server-windows",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows-release",
|
"name": "spark-api-server-windows-release",
|
||||||
"displayName": "Spark Api Server (Windows | Release)",
|
"displayName": "Spark Api Server (Debug)",
|
||||||
"inherits": "spark-api-server-windows",
|
"inherits": "spark-api-server-windows",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
},
|
},
|
||||||
@@ -381,14 +344,14 @@
|
|||||||
{
|
{
|
||||||
"name": "spark-api-server-linux-debug",
|
"name": "spark-api-server-linux-debug",
|
||||||
"inherits": "spark-api-server-linux",
|
"inherits": "spark-api-server-linux",
|
||||||
"displayName": "Spark Api Server (Linux | Debug)",
|
"displayName": "Spark Api Server (Debug)",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-linux-release",
|
"name": "spark-api-server-linux-release",
|
||||||
"inherits": "spark-api-server-linux",
|
"inherits": "spark-api-server-linux",
|
||||||
"displayName": "Spark Api Server (Linux | Release)",
|
"displayName": "Spark Api Server (Release)",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -408,30 +371,16 @@
|
|||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite-windows-debug",
|
"name": "data-stream-suite-debug",
|
||||||
"displayName": "Data Stream Suite (Windows | Debug)",
|
"displayName": "DataStreamSuite (Debug)",
|
||||||
"configurePreset": "data-stream-suite-windows",
|
"configurePreset": "data-stream-suite",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite-windows-release",
|
"name": "data-stream-suite-release",
|
||||||
"displayName": "Data Stream Suite (Windows | Release)",
|
"displayName": "DataStreamSuite (Release)",
|
||||||
"configurePreset": "data-stream-suite-windows",
|
"configurePreset": "data-stream-suite",
|
||||||
"configuration": "Release"
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "data-stream-suite-linux-debug",
|
|
||||||
"displayName": "Data Stream Suite (Linux | Debug)",
|
|
||||||
"configurePreset": "data-stream-suite-linux",
|
|
||||||
"configuration": "Debug"
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "data-stream-suite-linux-release",
|
|
||||||
"displayName": "Data Stream Suite (Linux | Release)",
|
|
||||||
"configurePreset": "data-stream-suite-linux",
|
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
+1
-1
Submodule externals updated: 8069cc893f...cae5fd20da.
@@ -0,0 +1,45 @@
|
|||||||
|
/* global use, db */
|
||||||
|
// MongoDB Playground
|
||||||
|
// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
|
||||||
|
// Make sure you are connected to enable completions and to be able to run a playground.
|
||||||
|
// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
|
||||||
|
// The result of the last command run in a playground is shown on the results panel.
|
||||||
|
// By default the first 20 documents will be returned with a cursor.
|
||||||
|
// Use 'console.log()' to print to the debug output.
|
||||||
|
// For more documentation on playgrounds please refer to
|
||||||
|
// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
|
||||||
|
|
||||||
|
// Select the database to use.
|
||||||
|
use('mongodbVSCodePlaygroundDB');
|
||||||
|
|
||||||
|
// Insert a few documents into the sales collection.
|
||||||
|
db.getCollection('sales').insertMany([
|
||||||
|
{ 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
|
||||||
|
{ 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
|
||||||
|
{ 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
|
||||||
|
{ 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
|
||||||
|
{ 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
|
||||||
|
{ 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
|
||||||
|
{ 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
|
||||||
|
{ 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Run a find command to view items sold on April 4th, 2014.
|
||||||
|
const salesOnApril4th = db.getCollection('sales').find({
|
||||||
|
date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
|
||||||
|
}).count();
|
||||||
|
|
||||||
|
// Print a message to the output window.
|
||||||
|
console.log(`${salesOnApril4th} sales occurred in 2014.`);
|
||||||
|
|
||||||
|
// Here we run an aggregation and open a cursor to the results.
|
||||||
|
// Use '.toArray()' to exhaust the cursor to return the whole result set.
|
||||||
|
// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
|
||||||
|
db.getCollection('sales').aggregate([
|
||||||
|
// Find all of the sales that occurred in 2014.
|
||||||
|
{ $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
|
||||||
|
// Group the total sales for each product.
|
||||||
|
{ $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
|
||||||
|
]);
|
||||||
|
|
||||||
|
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
<PackageReference Include="YaccLexTools" Version="1.2.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -1,22 +1,41 @@
|
|||||||
using System.Diagnostics;
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG.Utilities.General;
|
using System;
|
||||||
|
using System.Runtime;
|
||||||
|
|
||||||
/// <summary>
|
namespace BCG.Utilities.General
|
||||||
/// Summary description for Crono.
|
{
|
||||||
/// </summary>
|
/// <summary>
|
||||||
public class Crono {
|
/// Summary description for Crono.
|
||||||
readonly Stopwatch myTimer = new();
|
/// </summary>
|
||||||
|
public class Crono
|
||||||
|
{
|
||||||
|
public Crono()
|
||||||
|
{
|
||||||
|
myTimer = new HighPerformanceTimer();
|
||||||
|
}
|
||||||
|
|
||||||
public void start() =>
|
public void start()
|
||||||
myTimer.Start();
|
{myTimer.start();}
|
||||||
|
|
||||||
public void stop() =>
|
public void stop()
|
||||||
myTimer.Stop();
|
{myTimer.stop();}
|
||||||
|
|
||||||
public TimeSpan time =>
|
public TimeSpan time
|
||||||
myTimer.Elapsed;
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public double miliseconds
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{return myTimer.totalMilliseconds;}
|
||||||
|
}
|
||||||
|
|
||||||
|
private HighPerformanceTimer myTimer;
|
||||||
|
}
|
||||||
|
|
||||||
public double miliseconds =>
|
|
||||||
myTimer.Elapsed.TotalMilliseconds;
|
|
||||||
}
|
}
|
||||||
@@ -1,192 +1,192 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
public enum RegistryRoot { HKCU, HKLM }
|
public enum RegistryRoot { HKCU, HKLM }
|
||||||
|
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class BlueRegistry
|
public class BlueRegistry
|
||||||
{
|
{
|
||||||
public const string BlueSubKey = @"Software\Blue Capital Group\";
|
public const string BlueSubKey = @"Software\Blue Capital Group\";
|
||||||
public static readonly string ApplicationPrefix;
|
public static readonly string ApplicationPrefix;
|
||||||
protected string myFullPath;
|
protected string myFullPath;
|
||||||
|
|
||||||
public BlueRegistry(string subPath)
|
public BlueRegistry(string subPath)
|
||||||
{
|
{
|
||||||
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
|
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
public BlueRegistry(BlueRegistry parent, string subPath)
|
public BlueRegistry(BlueRegistry parent, string subPath)
|
||||||
{
|
{
|
||||||
myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
|
myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
static BlueRegistry()
|
static BlueRegistry()
|
||||||
{
|
{
|
||||||
string app = "Applications";
|
string app = "Applications";
|
||||||
|
|
||||||
Assembly ass = Assembly.GetEntryAssembly();
|
Assembly ass = Assembly.GetEntryAssembly();
|
||||||
if (ass != null)
|
if (ass != null)
|
||||||
{
|
{
|
||||||
app = ass.GetName().Name;
|
app = ass.GetName().Name;
|
||||||
|
|
||||||
object[] attrs = ass.GetCustomAttributes(
|
object[] attrs = ass.GetCustomAttributes(
|
||||||
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
|
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
|
||||||
|
|
||||||
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
|
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
|
||||||
{
|
{
|
||||||
app = ((AssemblyTitleAttribute)attrs[0]).Title;
|
app = ((AssemblyTitleAttribute)attrs[0]).Title;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
|
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
|
||||||
}
|
}
|
||||||
|
|
||||||
public object getValue(string name)
|
public object getValue(string name)
|
||||||
{
|
{
|
||||||
return getValue(RegistryRoot.HKCU, name);
|
return getValue(RegistryRoot.HKCU, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
public object getValue(string name, object defaultValue)
|
public object getValue(string name, object defaultValue)
|
||||||
{
|
{
|
||||||
return getValue(RegistryRoot.HKCU, name, defaultValue);
|
return getValue(RegistryRoot.HKCU, name, defaultValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setValue(string name, object value)
|
public void setValue(string name, object value)
|
||||||
{
|
{
|
||||||
setValue(RegistryRoot.HKCU, name, value);
|
setValue(RegistryRoot.HKCU, name, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteValue(string name)
|
public void deleteValue(string name)
|
||||||
{
|
{
|
||||||
deleteValue(RegistryRoot.HKCU, name);
|
deleteValue(RegistryRoot.HKCU, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
private RegistryKey myGetRegistryKey(RegistryRoot root)
|
private RegistryKey myGetRegistryKey(RegistryRoot root)
|
||||||
{
|
{
|
||||||
if (root == RegistryRoot.HKCU)
|
if (root == RegistryRoot.HKCU)
|
||||||
return Registry.CurrentUser.CreateSubKey(myFullPath);
|
return Registry.CurrentUser.CreateSubKey(myFullPath);
|
||||||
else
|
else
|
||||||
return Registry.LocalMachine.CreateSubKey(myFullPath);
|
return Registry.LocalMachine.CreateSubKey(myFullPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
#region Generic Methods
|
#region Generic Methods
|
||||||
public virtual void setValue(RegistryRoot root, string name, object value)
|
internal virtual void setValue(RegistryRoot root, string name, object value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
rk.SetValue(name, value);
|
rk.SetValue(name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual object getValue(RegistryRoot root, string name)
|
internal virtual object getValue(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return rk.GetValue(name);
|
return rk.GetValue(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual void deleteValue(RegistryRoot root, string name)
|
internal virtual void deleteValue(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
rk.DeleteValue(name);
|
rk.DeleteValue(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public virtual object getValue(RegistryRoot root, string name, object defaultValue)
|
internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return rk.GetValue(name, defaultValue);
|
return rk.GetValue(name, defaultValue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Double
|
#region Double
|
||||||
internal virtual void setDouble(RegistryRoot root, string name, double value)
|
internal virtual void setDouble(RegistryRoot root, string name, double value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
DoubleRegistryHandler.setValue(rk, name, value);
|
DoubleRegistryHandler.setValue(rk, name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual double getDouble(RegistryRoot root, string name)
|
internal virtual double getDouble(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return DoubleRegistryHandler.getValue(rk, name);
|
return DoubleRegistryHandler.getValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Enum
|
#region Enum
|
||||||
internal virtual void setEnum(RegistryRoot root, string name, Enum value)
|
internal virtual void setEnum(RegistryRoot root, string name, Enum value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
EnumRegistryHandler.setValue(rk, name, value);
|
EnumRegistryHandler.setValue(rk, name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
|
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return EnumRegistryHandler.getValue(rk, name, enumType);
|
return EnumRegistryHandler.getValue(rk, name, enumType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Array
|
#region Array
|
||||||
internal virtual void setArray(RegistryRoot root, string name, Array array)
|
internal virtual void setArray(RegistryRoot root, string name, Array array)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
ArrayRegistryHandler.setValue(rk, name, array);
|
ArrayRegistryHandler.setValue(rk, name, array);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
|
internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return ArrayRegistryHandler.getValue(rk, name, elementType);
|
return ArrayRegistryHandler.getValue(rk, name, elementType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region SortedList
|
#region SortedList
|
||||||
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
|
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
SortedListRegistryHandler.setValue(rk, name, slist);
|
SortedListRegistryHandler.setValue(rk, name, slist);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual SortedList getSortedList(RegistryRoot root, string name)
|
internal virtual SortedList getSortedList(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return SortedListRegistryHandler.getValue(rk, name);
|
return SortedListRegistryHandler.getValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual void deleteSortedList(RegistryRoot root, string name)
|
internal virtual void deleteSortedList(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
SortedListRegistryHandler.deleteValue(rk, name);
|
SortedListRegistryHandler.deleteValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,10 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net7.0-windows</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
<UseWindowsForms>true</UseWindowsForms>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
+166
-166
@@ -1,166 +1,166 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
using System.Drawing;
|
using System.Drawing;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
public class PersistWindowState
|
public class PersistWindowState
|
||||||
{
|
{
|
||||||
// event info that allows form to persist extra window state data
|
// event info that allows form to persist extra window state data
|
||||||
public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
|
public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
|
||||||
public event WindowStateHandler LoadState;
|
public event WindowStateHandler LoadState;
|
||||||
public event WindowStateHandler SaveState;
|
public event WindowStateHandler SaveState;
|
||||||
|
|
||||||
private Form myParent;
|
private Form myParent;
|
||||||
private BlueRegistry myRegistry;
|
private BlueRegistry myRegistry;
|
||||||
|
|
||||||
private int myNormalLeft;
|
private int myNormalLeft;
|
||||||
private int myNormalTop;
|
private int myNormalTop;
|
||||||
private int myNormalWidth;
|
private int myNormalWidth;
|
||||||
private int myNormalHeight;
|
private int myNormalHeight;
|
||||||
private bool myResizable = true;
|
private bool myResizable = true;
|
||||||
private FormWindowState myWindowState;
|
private FormWindowState myWindowState;
|
||||||
private bool myAllowSaveMinimized = false;
|
private bool myAllowSaveMinimized = false;
|
||||||
private bool mySavePosition = true;
|
private bool mySavePosition = true;
|
||||||
private bool mySaveState = true;
|
private bool mySaveState = true;
|
||||||
|
|
||||||
public PersistWindowState (bool savePosition)
|
public PersistWindowState (bool savePosition)
|
||||||
{
|
{
|
||||||
mySavePosition = savePosition;
|
mySavePosition = savePosition;
|
||||||
}
|
}
|
||||||
|
|
||||||
public PersistWindowState ()
|
public PersistWindowState ()
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool saveState
|
public bool saveState
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
set { mySaveState = value; }
|
set { mySaveState = value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public Form parent
|
public Form parent
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
get { return myParent; }
|
get { return myParent; }
|
||||||
set
|
set
|
||||||
{
|
{
|
||||||
myParent = value;
|
myParent = value;
|
||||||
myRegistry = new BlueRegistry (myParent.Text);
|
myRegistry = new BlueRegistry (myParent.Text);
|
||||||
|
|
||||||
// subscribe to parent form's events
|
// subscribe to parent form's events
|
||||||
myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
|
myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
|
||||||
myParent.Resize += new System.EventHandler(myOnResize);
|
myParent.Resize += new System.EventHandler(myOnResize);
|
||||||
myParent.Move += new System.EventHandler(myOnMove);
|
myParent.Move += new System.EventHandler(myOnMove);
|
||||||
myParent.Load += new System.EventHandler(myOnLoad);
|
myParent.Load += new System.EventHandler(myOnLoad);
|
||||||
|
|
||||||
// get initial width and height in case form is never resized
|
// get initial width and height in case form is never resized
|
||||||
myNormalWidth = myParent.Width;
|
myNormalWidth = myParent.Width;
|
||||||
myNormalHeight = myParent.Height;
|
myNormalHeight = myParent.Height;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public BlueRegistry blueRegistry
|
public BlueRegistry blueRegistry
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
get { return myRegistry; }
|
get { return myRegistry; }
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
set { myRegistry = value; }
|
set { myRegistry = value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool resizable
|
public bool resizable
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
set { myResizable = value; }
|
set { myResizable = value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool allowSaveMinimized
|
public bool allowSaveMinimized
|
||||||
{
|
{
|
||||||
[DebuggerStepThrough]
|
[DebuggerStepThrough]
|
||||||
set { myAllowSaveMinimized = value; }
|
set { myAllowSaveMinimized = value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myOnResize(object sender, System.EventArgs e)
|
private void myOnResize(object sender, System.EventArgs e)
|
||||||
{
|
{
|
||||||
// save width and height
|
// save width and height
|
||||||
if (myParent.WindowState == FormWindowState.Normal && myResizable)
|
if (myParent.WindowState == FormWindowState.Normal && myResizable)
|
||||||
{
|
{
|
||||||
myNormalWidth = myParent.Width;
|
myNormalWidth = myParent.Width;
|
||||||
myNormalHeight = myParent.Height;
|
myNormalHeight = myParent.Height;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myOnMove(object sender, System.EventArgs e)
|
private void myOnMove(object sender, System.EventArgs e)
|
||||||
{
|
{
|
||||||
// save position
|
// save position
|
||||||
if(myParent.WindowState == FormWindowState.Normal)
|
if(myParent.WindowState == FormWindowState.Normal)
|
||||||
{
|
{
|
||||||
myNormalLeft = myParent.Left;
|
myNormalLeft = myParent.Left;
|
||||||
myNormalTop = myParent.Top;
|
myNormalTop = myParent.Top;
|
||||||
}
|
}
|
||||||
// save state
|
// save state
|
||||||
myWindowState = myParent.WindowState;
|
myWindowState = myParent.WindowState;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
|
private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
|
||||||
{
|
{
|
||||||
// save position, size and state
|
// save position, size and state
|
||||||
if (mySavePosition)
|
if (mySavePosition)
|
||||||
{
|
{
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
|
myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
|
myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (myResizable)
|
if (myResizable)
|
||||||
{
|
{
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
|
myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
|
myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
|
||||||
}
|
}
|
||||||
|
|
||||||
// check if we are allowed to save the state as minimized (not normally)
|
// check if we are allowed to save the state as minimized (not normally)
|
||||||
if (!myAllowSaveMinimized)
|
if (!myAllowSaveMinimized)
|
||||||
{
|
{
|
||||||
if(myWindowState == FormWindowState.Minimized)
|
if(myWindowState == FormWindowState.Minimized)
|
||||||
myWindowState = FormWindowState.Normal;
|
myWindowState = FormWindowState.Normal;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mySaveState)
|
if (mySaveState)
|
||||||
myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
|
myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
|
||||||
|
|
||||||
// fire SaveState event
|
// fire SaveState event
|
||||||
if (SaveState != null)
|
if (SaveState != null)
|
||||||
SaveState (this, myRegistry);
|
SaveState (this, myRegistry);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myOnLoad (object sender, System.EventArgs e)
|
private void myOnLoad (object sender, System.EventArgs e)
|
||||||
{
|
{
|
||||||
if (mySavePosition)
|
if (mySavePosition)
|
||||||
{
|
{
|
||||||
int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
|
int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
|
||||||
int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
|
int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
|
||||||
myParent.Location = new Point (left, top);
|
myParent.Location = new Point (left, top);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (myResizable)
|
if (myResizable)
|
||||||
{
|
{
|
||||||
int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
|
int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
|
||||||
int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
|
int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
|
||||||
myParent.Size = new Size (width, height);
|
myParent.Size = new Size (width, height);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mySaveState)
|
if (mySaveState)
|
||||||
{
|
{
|
||||||
FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState",
|
FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState",
|
||||||
(int) myParent.WindowState);
|
(int) myParent.WindowState);
|
||||||
myParent.WindowState = windowState;
|
myParent.WindowState = windowState;
|
||||||
}
|
}
|
||||||
|
|
||||||
// fire LoadState event
|
// fire LoadState event
|
||||||
if(LoadState != null)
|
if(LoadState != null)
|
||||||
LoadState (this, myRegistry);
|
LoadState (this, myRegistry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
|
||||||
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
|
|
||||||
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup Label="ColumnExpression">
|
|
||||||
<Scanner Include="ColumnExpressionTokens.lex" />
|
|
||||||
<Parser Include="ColumnExpressionGrammar.y" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
Whitespace-only changes.
@@ -1,62 +0,0 @@
|
|||||||
namespace Bts.ColumnExpressions;
|
|
||||||
|
|
||||||
public class ColumnExpression {
|
|
||||||
public ColumnExpression(string name, Type? type = null, object? value = null) {
|
|
||||||
Name = name;
|
|
||||||
Type = type
|
|
||||||
?? value?.GetType()
|
|
||||||
?? throw new ArgumentException(
|
|
||||||
"Type must be specified explicitly when value is null", nameof(type));
|
|
||||||
Value = value
|
|
||||||
?? DBNull.Value;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string Name { get; }
|
|
||||||
|
|
||||||
public Type Type { get; }
|
|
||||||
|
|
||||||
public object Value { get; }
|
|
||||||
|
|
||||||
public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
|
|
||||||
// Try to convert value to 'type' to verify they are compatible
|
|
||||||
// Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
|
|
||||||
// account for implicit conversions (e.g., int -> long), just inheritance.
|
|
||||||
if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
|
|
||||||
throw new ArgumentException(
|
|
||||||
$"Type '{type}' is not assignable from value type '{value?.GetType()}'");
|
|
||||||
}
|
|
||||||
return new(name, type ?? typeof(T), value);
|
|
||||||
}
|
|
||||||
|
|
||||||
internal static Type ParseType(string typeId) {
|
|
||||||
return Candidates().ExceptNull().FirstOrDefault() ??
|
|
||||||
throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
|
|
||||||
|
|
||||||
IEnumerable<Type?> Candidates() {
|
|
||||||
if (typeId.Length == 0) {
|
|
||||||
yield break;
|
|
||||||
}
|
|
||||||
yield return Type.GetType(typeId);
|
|
||||||
yield return Type.GetType($"System.{typeId}");
|
|
||||||
if (typeId.Length > 1) {
|
|
||||||
yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
|
|
||||||
}
|
|
||||||
yield return typeId.ToLower() switch {
|
|
||||||
"float64" or "double" or "real64" or "number" => typeof(double),
|
|
||||||
"float32" or "single" or "real32" or "real" or "float" => typeof(float),
|
|
||||||
"int16" or "short" or "integer16" or "signed16" => typeof(short),
|
|
||||||
"int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
|
|
||||||
"int64" or "long" or "integer64" or "signed64" => typeof(long),
|
|
||||||
"uint16" or "ushort" or "unsigned16" => typeof(ushort),
|
|
||||||
"uint32" or "uint" or "unsigned" => typeof(uint),
|
|
||||||
"uint64" or "ulong" or "unsigned64" => typeof(ulong),
|
|
||||||
"char" => typeof(char),
|
|
||||||
"byte" or "uint8" => typeof(byte),
|
|
||||||
"bool" or "boolean" => typeof(bool),
|
|
||||||
"string" or "text" => typeof(string),
|
|
||||||
"date" or "datetime" => typeof(DateTime),
|
|
||||||
_ => null
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
%namespace Bts.ColumnExpressions
|
|
||||||
%partial
|
|
||||||
%visibility internal
|
|
||||||
%tokentype ColumnExpressionToken
|
|
||||||
%parsertype ColumnExpressionParser
|
|
||||||
|
|
||||||
%union {
|
|
||||||
public int Integer;
|
|
||||||
public double Real;
|
|
||||||
public bool Boolean;
|
|
||||||
public string String;
|
|
||||||
public string Identifier;
|
|
||||||
public string TypeId;
|
|
||||||
public DateTime DateTime;
|
|
||||||
public Tuple<string, Type> ColumnDefinition;
|
|
||||||
public ColumnExpression ColumnExpression;
|
|
||||||
public ImmutableList<ColumnExpression> ColumnExpressions;
|
|
||||||
}
|
|
||||||
|
|
||||||
%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
|
|
||||||
%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
|
|
||||||
|
|
||||||
%nonassoc EQUALS
|
|
||||||
|
|
||||||
%type <ColumnDefinition> ColumnDefinition
|
|
||||||
%type <ColumnExpression> ColumnExpression
|
|
||||||
%type <ColumnExpressions> ColumnExpressions
|
|
||||||
|
|
||||||
%%
|
|
||||||
|
|
||||||
ColumnExpressions :
|
|
||||||
ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
|
|
||||||
| ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
|
|
||||||
;
|
|
||||||
|
|
||||||
ColumnExpression :
|
|
||||||
ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
|
|
||||||
| ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
|
|
||||||
| ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
|
|
||||||
| ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
|
|
||||||
| ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
|
|
||||||
| ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
|
|
||||||
;
|
|
||||||
|
|
||||||
ColumnDefinition :
|
|
||||||
IDENTIFIER { $$ = new($1.Identifier, null); }
|
|
||||||
| IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
|
|
||||||
| IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
|
|
||||||
;
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace Bts.ColumnExpressions;
|
|
||||||
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
public class ColumnExpressionInterpreter {
|
|
||||||
public static IList<ColumnExpression> Evaluate(Stream stream) {
|
|
||||||
var parser = new ColumnExpressionParser();
|
|
||||||
var res = parser.Parse(stream);
|
|
||||||
return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
|
|
||||||
res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IList<ColumnExpression> Evaluate(string str) {
|
|
||||||
using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
|
|
||||||
return Evaluate(ms);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace Bts.ColumnExpressions;
|
|
||||||
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
partial class ColumnExpressionParser {
|
|
||||||
public ColumnExpressionParser() : base(null) { }
|
|
||||||
|
|
||||||
public ValueType? Parse(string s) {
|
|
||||||
using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
|
|
||||||
return Parse(ms);
|
|
||||||
}
|
|
||||||
|
|
||||||
public ValueType? Parse(Stream stream) {
|
|
||||||
Scanner = new ColumnExpressionScanner(stream);
|
|
||||||
Parse();
|
|
||||||
return CurrentSemanticValue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
namespace Bts.ColumnExpressions;
|
|
||||||
|
|
||||||
partial class ColumnExpressionScanner {
|
|
||||||
public override void yyerror(string format, params object[] args) {
|
|
||||||
base.yyerror(format, args);
|
|
||||||
Console.WriteLine(format, args);
|
|
||||||
Console.WriteLine();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
%namespace Bts.ColumnExpressions
|
|
||||||
%scannertype ColumnExpressionScanner
|
|
||||||
%visibility internal
|
|
||||||
%tokentype ColumnExpressionToken
|
|
||||||
|
|
||||||
%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
|
|
||||||
|
|
||||||
/* Literal Constants */
|
|
||||||
Letter [a-zA-Z]
|
|
||||||
BooleanConstant true|false
|
|
||||||
StringConstant '([^\'\\]*|\\.)*'
|
|
||||||
DateTimeConstant ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
|
|
||||||
IntegerConstant -?[0-9]+
|
|
||||||
NullConstant 'null'
|
|
||||||
RealConstant -?[0-9]+\.[0-9]+
|
|
||||||
|
|
||||||
/* Relational Operators */
|
|
||||||
Equals =
|
|
||||||
OfType /
|
|
||||||
|
|
||||||
Identifier {Letter}({Letter}*[0-9]*_*)*
|
|
||||||
NamespacePrefix {Identifier}\.
|
|
||||||
TypeId {NamespacePrefix}*{Identifier}
|
|
||||||
|
|
||||||
/* Utility Tokens */
|
|
||||||
OpenParenthesis \(
|
|
||||||
CloseParenthesis \)
|
|
||||||
Comma ,
|
|
||||||
QuoteOpen \[
|
|
||||||
QuoteClose \]
|
|
||||||
|
|
||||||
/* Comments */
|
|
||||||
LineComment \/\/[^\n\r]*(\n|\r|\r\n)
|
|
||||||
|
|
||||||
%%
|
|
||||||
|
|
||||||
{Equals} {
|
|
||||||
return (int)ColumnExpressionToken.EQUALS;
|
|
||||||
}
|
|
||||||
|
|
||||||
{OfType} {
|
|
||||||
return (int)ColumnExpressionToken.OF_TYPE;
|
|
||||||
}
|
|
||||||
|
|
||||||
{StringConstant} {
|
|
||||||
yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
|
|
||||||
return (int)ColumnExpressionToken.STRING_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{BooleanConstant} {
|
|
||||||
yylval.Boolean = yytext.ToLower() switch {
|
|
||||||
"true" => true,
|
|
||||||
"false" => false,
|
|
||||||
_ => throw new Exception(
|
|
||||||
string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
|
|
||||||
};
|
|
||||||
return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{DateTimeConstant} {
|
|
||||||
yylval.DateTime = DateTime.Parse(yytext);
|
|
||||||
return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{IntegerConstant} {
|
|
||||||
yylval.Integer = int.Parse(yytext);
|
|
||||||
return (int)ColumnExpressionToken.INTEGER_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{RealConstant} {
|
|
||||||
yylval.Real = double.Parse(yytext);
|
|
||||||
return (int)ColumnExpressionToken.REAL_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{Identifier} {
|
|
||||||
yylval.Identifier = yytext;
|
|
||||||
return (int)ColumnExpressionToken.IDENTIFIER;
|
|
||||||
}
|
|
||||||
|
|
||||||
{TypeId} {
|
|
||||||
yylval.TypeId = yytext;
|
|
||||||
return (int)ColumnExpressionToken.TYPE_ID;
|
|
||||||
}
|
|
||||||
|
|
||||||
%%
|
|
||||||
@@ -1,720 +0,0 @@
|
|||||||
// ==============================================================
|
|
||||||
// <auto-generated>
|
|
||||||
// This code automatically produced from an embedded resource.
|
|
||||||
// Do not edit this file, or it will become incompatible with
|
|
||||||
// the specification from which it was generated.
|
|
||||||
// </auto-generated>
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Runtime.Serialization;
|
|
||||||
using System.Globalization;
|
|
||||||
|
|
||||||
namespace QUT.GplexBuffers
|
|
||||||
{
|
|
||||||
// Code copied from GPLEX embedded resource
|
|
||||||
[Serializable]
|
|
||||||
internal class BufferException : Exception
|
|
||||||
{
|
|
||||||
public BufferException() { }
|
|
||||||
public BufferException(string message) : base(message) { }
|
|
||||||
public BufferException(string message, Exception innerException)
|
|
||||||
: base(message, innerException) { }
|
|
||||||
protected BufferException(SerializationInfo info, StreamingContext context)
|
|
||||||
: base(info, context) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
internal abstract class ScanBuff
|
|
||||||
{
|
|
||||||
private string fileNm;
|
|
||||||
|
|
||||||
public const int EndOfFile = -1;
|
|
||||||
public const int UnicodeReplacementChar = 0xFFFD;
|
|
||||||
|
|
||||||
public bool IsFile { get { return (fileNm != null); } }
|
|
||||||
public string FileName { get { return fileNm; } set { fileNm = value; } }
|
|
||||||
|
|
||||||
public abstract int Pos { get; set; }
|
|
||||||
public abstract int Read();
|
|
||||||
public virtual void Mark() { }
|
|
||||||
|
|
||||||
public abstract string GetString(int begin, int limit);
|
|
||||||
|
|
||||||
public static ScanBuff GetBuffer(string source)
|
|
||||||
{
|
|
||||||
return new StringBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ScanBuff GetBuffer(IList<string> source)
|
|
||||||
{
|
|
||||||
return new LineBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!NOFILES)
|
|
||||||
public static ScanBuff GetBuffer(Stream source)
|
|
||||||
{
|
|
||||||
return new BuildBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
return new BuildBuffer(source, fallbackCodePage);
|
|
||||||
}
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endif // !NOFILES
|
|
||||||
}
|
|
||||||
|
|
||||||
#region Buffer classes
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// ===== Definitions for various ScanBuff derived classes ====
|
|
||||||
// ==============================================================
|
|
||||||
// =============== String input ================
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class reads characters from a single string as
|
|
||||||
/// required, for example, by Visual Studio language services
|
|
||||||
/// </summary>
|
|
||||||
sealed class StringBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
string str; // input buffer
|
|
||||||
int bPos; // current position in buffer
|
|
||||||
int sLen;
|
|
||||||
|
|
||||||
public StringBuffer(string source)
|
|
||||||
{
|
|
||||||
this.str = source;
|
|
||||||
this.sLen = source.Length;
|
|
||||||
this.FileName = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
if (bPos < sLen) return str[bPos++];
|
|
||||||
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
|
|
||||||
else { bPos++; return EndOfFile; } // two strikes and you're out!
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
// "limit" can be greater than sLen with the BABEL
|
|
||||||
// option set. Read returns a "virtual" EOL if
|
|
||||||
// an attempt is made to read past the end of the
|
|
||||||
// string buffer. Without the guard any attempt
|
|
||||||
// to fetch yytext for a token that includes the
|
|
||||||
// EOL will throw an index exception.
|
|
||||||
if (limit > sLen) limit = sLen;
|
|
||||||
if (limit <= begin) return "";
|
|
||||||
else return str.Substring(begin, limit - begin);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return bPos; }
|
|
||||||
set { bPos = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() { return "StringBuffer"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// The LineBuff class contributed by Nigel Horspool,
|
|
||||||
// nigelh@cs.uvic.cs
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
sealed class LineBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
IList<string> line; // list of source lines from a file
|
|
||||||
int numLines; // number of strings in line list
|
|
||||||
string curLine; // current line in that list
|
|
||||||
int cLine; // index of current line in the list
|
|
||||||
int curLen; // length of current line
|
|
||||||
int curLineStart; // position of line start in whole file
|
|
||||||
int curLineEnd; // position of line end in whole file
|
|
||||||
int maxPos; // max position ever visited in whole file
|
|
||||||
int cPos; // ordinal number of code in source
|
|
||||||
|
|
||||||
// Constructed from a list of strings, one per source line.
|
|
||||||
// The lines have had trailing '\n' characters removed.
|
|
||||||
public LineBuffer(IList<string> lineList)
|
|
||||||
{
|
|
||||||
line = lineList;
|
|
||||||
numLines = line.Count;
|
|
||||||
cPos = curLineStart = 0;
|
|
||||||
curLine = (numLines > 0 ? line[0] : "");
|
|
||||||
maxPos = curLineEnd = curLen = curLine.Length;
|
|
||||||
cLine = 1;
|
|
||||||
FileName = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
if (cPos < curLineEnd)
|
|
||||||
return curLine[cPos++ - curLineStart];
|
|
||||||
if (cPos++ == curLineEnd)
|
|
||||||
return '\n';
|
|
||||||
if (cLine >= numLines)
|
|
||||||
return EndOfFile;
|
|
||||||
curLine = line[cLine];
|
|
||||||
curLen = curLine.Length;
|
|
||||||
curLineStart = curLineEnd + 1;
|
|
||||||
curLineEnd = curLineStart + curLen;
|
|
||||||
if (curLineEnd > maxPos)
|
|
||||||
maxPos = curLineEnd;
|
|
||||||
cLine++;
|
|
||||||
return curLen > 0 ? curLine[0] : '\n';
|
|
||||||
}
|
|
||||||
|
|
||||||
// To speed up searches for the line containing a position
|
|
||||||
private int cachedPosition;
|
|
||||||
private int cachedIxdex;
|
|
||||||
private int cachedLineStart;
|
|
||||||
|
|
||||||
// Given a position pos within the entire source, the results are
|
|
||||||
// ix -- the index of the containing line
|
|
||||||
// lstart -- the position of the first character on that line
|
|
||||||
private void findIndex(int pos, out int ix, out int lstart)
|
|
||||||
{
|
|
||||||
if (pos >= cachedPosition)
|
|
||||||
{
|
|
||||||
ix = cachedIxdex; lstart = cachedLineStart;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ix = lstart = 0;
|
|
||||||
}
|
|
||||||
while (ix < numLines)
|
|
||||||
{
|
|
||||||
int len = line[ix].Length + 1;
|
|
||||||
if (pos < lstart + len) break;
|
|
||||||
lstart += len;
|
|
||||||
ix++;
|
|
||||||
}
|
|
||||||
cachedPosition = pos;
|
|
||||||
cachedIxdex = ix;
|
|
||||||
cachedLineStart = lstart;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
if (begin >= maxPos || limit <= begin) return "";
|
|
||||||
int endIx, begIx, endLineStart, begLineStart;
|
|
||||||
findIndex(begin, out begIx, out begLineStart);
|
|
||||||
int begCol = begin - begLineStart;
|
|
||||||
findIndex(limit, out endIx, out endLineStart);
|
|
||||||
int endCol = limit - endLineStart;
|
|
||||||
string s = line[begIx];
|
|
||||||
if (begIx == endIx)
|
|
||||||
{
|
|
||||||
// the usual case, substring all on one line
|
|
||||||
return (endCol <= s.Length) ?
|
|
||||||
s.Substring(begCol, endCol - begCol)
|
|
||||||
: s.Substring(begCol) + "\n";
|
|
||||||
}
|
|
||||||
// the string spans multiple lines, yuk!
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
if (begCol < s.Length)
|
|
||||||
sb.Append(s.Substring(begCol));
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
sb.Append("\n");
|
|
||||||
s = line[++begIx];
|
|
||||||
if (begIx >= endIx) break;
|
|
||||||
sb.Append(s);
|
|
||||||
}
|
|
||||||
if (endCol <= s.Length)
|
|
||||||
{
|
|
||||||
sb.Append(s.Substring(0, endCol));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
sb.Append(s);
|
|
||||||
sb.Append("\n");
|
|
||||||
}
|
|
||||||
return sb.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return cPos; }
|
|
||||||
set
|
|
||||||
{
|
|
||||||
cPos = value;
|
|
||||||
findIndex(cPos, out cLine, out curLineStart);
|
|
||||||
// cLine should be the *next* line after curLine.
|
|
||||||
curLine = (cLine < numLines ? line[cLine++] : "");
|
|
||||||
curLineEnd = curLineStart + curLine.Length;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() { return "LineBuffer"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!NOFILES)
|
|
||||||
// ==============================================================
|
|
||||||
// ===== class BuildBuff : for unicode text files ========
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
class BuildBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
// Double buffer for char stream.
|
|
||||||
class BufferElement
|
|
||||||
{
|
|
||||||
StringBuilder bldr = new StringBuilder();
|
|
||||||
StringBuilder next = new StringBuilder();
|
|
||||||
int minIx;
|
|
||||||
int maxIx;
|
|
||||||
int brkIx;
|
|
||||||
bool appendToNext;
|
|
||||||
|
|
||||||
internal BufferElement() { }
|
|
||||||
|
|
||||||
internal int MaxIndex { get { return maxIx; } }
|
|
||||||
// internal int MinIndex { get { return minIx; } }
|
|
||||||
|
|
||||||
internal char this[int index]
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
if (index < minIx || index >= maxIx)
|
|
||||||
throw new BufferException("Index was outside data buffer");
|
|
||||||
else if (index < brkIx)
|
|
||||||
return bldr[index - minIx];
|
|
||||||
else
|
|
||||||
return next[index - brkIx];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Append(char[] block, int count)
|
|
||||||
{
|
|
||||||
maxIx += count;
|
|
||||||
if (appendToNext)
|
|
||||||
this.next.Append(block, 0, count);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
this.bldr.Append(block, 0, count);
|
|
||||||
brkIx = maxIx;
|
|
||||||
appendToNext = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal string GetString(int start, int limit)
|
|
||||||
{
|
|
||||||
if (limit <= start)
|
|
||||||
return "";
|
|
||||||
if (start >= minIx && limit <= maxIx)
|
|
||||||
if (limit < brkIx) // String entirely in bldr builder
|
|
||||||
return bldr.ToString(start - minIx, limit - start);
|
|
||||||
else if (start >= brkIx) // String entirely in next builder
|
|
||||||
return next.ToString(start - brkIx, limit - start);
|
|
||||||
else // Must do a string-concatenation
|
|
||||||
return
|
|
||||||
bldr.ToString(start - minIx, brkIx - start) +
|
|
||||||
next.ToString(0, limit - brkIx);
|
|
||||||
else
|
|
||||||
throw new BufferException("String was outside data buffer");
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Mark(int limit)
|
|
||||||
{
|
|
||||||
if (limit > brkIx + 16) // Rotate blocks
|
|
||||||
{
|
|
||||||
StringBuilder temp = bldr;
|
|
||||||
bldr = next;
|
|
||||||
next = temp;
|
|
||||||
next.Length = 0;
|
|
||||||
minIx = brkIx;
|
|
||||||
brkIx = maxIx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
BufferElement data = new BufferElement();
|
|
||||||
|
|
||||||
int bPos; // Postion index in the StringBuilder
|
|
||||||
BlockReader NextBlk; // Delegate that serves char-arrays;
|
|
||||||
|
|
||||||
private string EncodingName
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
StreamReader rdr = NextBlk.Target as StreamReader;
|
|
||||||
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public BuildBuffer(Stream stream)
|
|
||||||
{
|
|
||||||
FileStream fStrm = (stream as FileStream);
|
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
|
||||||
NextBlk = BlockReaderFactory.Raw(stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public BuildBuffer(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
FileStream fStrm = (stream as FileStream);
|
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
|
||||||
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Marks a conservative lower bound for the buffer,
|
|
||||||
/// allowing space to be reclaimed. If an application
|
|
||||||
/// needs to call GetString at arbitrary past locations
|
|
||||||
/// in the input stream, Mark() is not called.
|
|
||||||
/// </summary>
|
|
||||||
public override void Mark() { data.Mark(bPos - 2); }
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return bPos; }
|
|
||||||
set { bPos = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Read returns the ordinal number of the next char, or
|
|
||||||
/// EOF (-1) for an end of stream. Note that the next
|
|
||||||
/// code point may require *two* calls of Read().
|
|
||||||
/// </summary>
|
|
||||||
/// <returns></returns>
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
//
|
|
||||||
// Characters at positions
|
|
||||||
// [data.offset, data.offset + data.bldr.Length)
|
|
||||||
// are available in data.bldr.
|
|
||||||
//
|
|
||||||
if (bPos < data.MaxIndex)
|
|
||||||
{
|
|
||||||
// ch0 cannot be EOF
|
|
||||||
return (int)data[bPos++];
|
|
||||||
}
|
|
||||||
else // Read from underlying stream
|
|
||||||
{
|
|
||||||
// Experimental code, blocks of page size
|
|
||||||
char[] chrs = new char[4096];
|
|
||||||
int count = NextBlk(chrs, 0, 4096);
|
|
||||||
if (count == 0)
|
|
||||||
return EndOfFile;
|
|
||||||
else
|
|
||||||
{
|
|
||||||
data.Append(chrs, count);
|
|
||||||
return (int)data[bPos++];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
return data.GetString(begin, limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString()
|
|
||||||
{
|
|
||||||
return "StringBuilder buffer, encoding: " + this.EncodingName;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// =============== End ScanBuff-derived classes ==================
|
|
||||||
|
|
||||||
internal delegate int BlockReader(char[] block, int index, int number);
|
|
||||||
|
|
||||||
// A delegate factory, serving up a delegate that
|
|
||||||
// reads a block of characters from the underlying
|
|
||||||
// encoded stream, via a StreamReader object.
|
|
||||||
//
|
|
||||||
internal static class BlockReaderFactory
|
|
||||||
{
|
|
||||||
public static BlockReader Raw(Stream stream)
|
|
||||||
{
|
|
||||||
return delegate(char[] block, int index, int number)
|
|
||||||
{
|
|
||||||
byte[] b = new byte[number];
|
|
||||||
int count = stream.Read(b, 0, number);
|
|
||||||
int i = 0;
|
|
||||||
int j = index;
|
|
||||||
for (; i < count; i++, j++)
|
|
||||||
block[j] = (char)b[i];
|
|
||||||
return count;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static BlockReader Get(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
Encoding encoding;
|
|
||||||
int preamble = Preamble(stream);
|
|
||||||
|
|
||||||
if (preamble != 0) // There is a valid BOM here!
|
|
||||||
encoding = Encoding.GetEncoding(preamble);
|
|
||||||
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
|
|
||||||
return Raw(stream);
|
|
||||||
else if (fallbackCodePage != -2) // Anything but "guess"
|
|
||||||
encoding = Encoding.GetEncoding(fallbackCodePage);
|
|
||||||
else // This is the "guess" option
|
|
||||||
{
|
|
||||||
int guess = new Guesser(stream).GuessCodePage();
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (guess == -1) // ==> this is a 7-bit file
|
|
||||||
encoding = Encoding.ASCII;
|
|
||||||
else if (guess == 65001)
|
|
||||||
encoding = Encoding.UTF8;
|
|
||||||
else // ==> use the machine default
|
|
||||||
encoding = Encoding.Default;
|
|
||||||
}
|
|
||||||
StreamReader reader = new StreamReader(stream, encoding);
|
|
||||||
return reader.Read;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int Preamble(Stream stream)
|
|
||||||
{
|
|
||||||
int b0 = stream.ReadByte();
|
|
||||||
int b1 = stream.ReadByte();
|
|
||||||
|
|
||||||
if (b0 == 0xfe && b1 == 0xff)
|
|
||||||
return 1201; // UTF16BE
|
|
||||||
if (b0 == 0xff && b1 == 0xfe)
|
|
||||||
return 1200; // UTF16LE
|
|
||||||
|
|
||||||
int b2 = stream.ReadByte();
|
|
||||||
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
|
||||||
return 65001; // UTF8
|
|
||||||
//
|
|
||||||
// There is no unicode preamble, so we
|
|
||||||
// return denoter for the machine default.
|
|
||||||
//
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
}
|
|
||||||
#endif // !NOFILES
|
|
||||||
#endregion Buffer classes
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// ============ class CodePageHandling =============
|
|
||||||
// ==============================================================
|
|
||||||
#if (!NOFILES)
|
|
||||||
public static class CodePageHandling
|
|
||||||
{
|
|
||||||
public static int GetCodePage(string option)
|
|
||||||
{
|
|
||||||
string command = option.ToUpperInvariant();
|
|
||||||
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
|
|
||||||
command = command.Substring(9);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (command.Equals("RAW"))
|
|
||||||
return -1;
|
|
||||||
else if (command.Equals("GUESS"))
|
|
||||||
return -2;
|
|
||||||
else if (command.Equals("DEFAULT"))
|
|
||||||
return 0;
|
|
||||||
else if (char.IsDigit(command[0]))
|
|
||||||
return int.Parse(command, CultureInfo.InvariantCulture);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Encoding enc = Encoding.GetEncoding(command);
|
|
||||||
return enc.CodePage;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (FormatException)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine(
|
|
||||||
"Invalid format \"{0}\", using machine default", option);
|
|
||||||
}
|
|
||||||
catch (ArgumentException)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine(
|
|
||||||
"Unknown code page \"{0}\", using machine default", option);
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#region guesser
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
// ==============================================================
|
|
||||||
// ============ Encoding Guesser =============
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class provides a simple finite state automaton that
|
|
||||||
/// scans the file looking for (1) valid UTF-8 byte patterns,
|
|
||||||
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
|
|
||||||
/// The method then guesses whether it is UTF-8 or maybe some
|
|
||||||
/// local machine default encoding. This works well for the
|
|
||||||
/// various Latin encodings.
|
|
||||||
/// </summary>
|
|
||||||
internal class Guesser
|
|
||||||
{
|
|
||||||
ScanBuff buffer;
|
|
||||||
|
|
||||||
public int GuessCodePage() { return Scan(); }
|
|
||||||
|
|
||||||
const int maxAccept = 10;
|
|
||||||
const int initial = 0;
|
|
||||||
const int eofNum = 0;
|
|
||||||
const int goStart = -1;
|
|
||||||
const int INITIAL = 0;
|
|
||||||
const int EndToken = 0;
|
|
||||||
|
|
||||||
#region user code
|
|
||||||
/*
|
|
||||||
* Reads the bytes of a file to determine if it is
|
|
||||||
* UTF-8 or a single-byte code page file.
|
|
||||||
*/
|
|
||||||
public long utfX;
|
|
||||||
public long uppr;
|
|
||||||
#endregion user code
|
|
||||||
|
|
||||||
int state;
|
|
||||||
int currentStart = startState[0];
|
|
||||||
int code;
|
|
||||||
|
|
||||||
#region ScannerTables
|
|
||||||
static int[] startState = new int[] { 11, 0 };
|
|
||||||
|
|
||||||
#region CharacterMap
|
|
||||||
static sbyte[] map = new sbyte[256] {
|
|
||||||
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
||||||
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
static sbyte[][] nextState = new sbyte[][] {
|
|
||||||
new sbyte[] {0, 0, 0, 0, 0, 0},
|
|
||||||
new sbyte[] {-1, -1, 10, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, -1, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 8, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 5, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 6, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 7, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, -1, 9, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, 1, 2, 3, 4, 2}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
|
|
||||||
// Reason for suppression: cannot have self-reference in array initializer.
|
|
||||||
static Guesser()
|
|
||||||
{
|
|
||||||
nextState[7] = nextState[2];
|
|
||||||
nextState[9] = nextState[2];
|
|
||||||
nextState[10] = nextState[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
int NextState()
|
|
||||||
{
|
|
||||||
if (code == ScanBuff.EndOfFile)
|
|
||||||
return eofNum;
|
|
||||||
else
|
|
||||||
return nextState[state][map[code]];
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
public Guesser(System.IO.Stream file) { SetSource(file); }
|
|
||||||
|
|
||||||
public void SetSource(System.IO.Stream source)
|
|
||||||
{
|
|
||||||
this.buffer = new BuildBuffer(source);
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
|
|
||||||
int Scan()
|
|
||||||
{
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
int next;
|
|
||||||
state = currentStart;
|
|
||||||
while ((next = NextState()) == goStart)
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
while ((next = NextState()) > eofNum)
|
|
||||||
{
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
if (state <= maxAccept)
|
|
||||||
{
|
|
||||||
#region ActionSwitch
|
|
||||||
#pragma warning disable 162
|
|
||||||
switch (state)
|
|
||||||
{
|
|
||||||
case eofNum:
|
|
||||||
switch (currentStart)
|
|
||||||
{
|
|
||||||
case 11:
|
|
||||||
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
|
|
||||||
else if (uppr * 10 > utfX) return 0; /* default code page */
|
|
||||||
else return 65001; /* UTF-8 encoding */
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return EndToken;
|
|
||||||
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
|
|
||||||
case 2: // Recognized '{Upper128}', Shortest string "\x80"
|
|
||||||
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
|
|
||||||
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
|
|
||||||
uppr++;
|
|
||||||
break;
|
|
||||||
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
|
|
||||||
uppr += 3;
|
|
||||||
break;
|
|
||||||
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
|
|
||||||
utfX += 3;
|
|
||||||
break;
|
|
||||||
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
|
|
||||||
utfX += 2;
|
|
||||||
break;
|
|
||||||
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
|
|
||||||
utfX++;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
#pragma warning restore 162
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // end class Guesser
|
|
||||||
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endregion
|
|
||||||
#endif // !NOFILES
|
|
||||||
|
|
||||||
// End of code copied from embedded resource
|
|
||||||
}
|
|
||||||
@@ -1,884 +0,0 @@
|
|||||||
// ==============================================================
|
|
||||||
// <auto-generated>
|
|
||||||
// This code automatically produced from an embedded resource.
|
|
||||||
// Do not edit this file, or it will become incompatible with
|
|
||||||
// the specification from which it was generated.
|
|
||||||
// </auto-generated>
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
// Gardens Point Parser Generator
|
|
||||||
// Copyright (c) Wayne Kelly, QUT 2005-2010
|
|
||||||
// (see accompanying GPPGcopyright.rtf)
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Text;
|
|
||||||
using System.Globalization;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Runtime.Serialization;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
|
||||||
namespace QUT.Gppg {
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract class for GPPG shift-reduce parsers.
|
|
||||||
/// Parsers generated by GPPG derive from this base
|
|
||||||
/// class, overriding the abstract Initialize() and
|
|
||||||
/// DoAction() methods.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TValue">Semantic value type</typeparam>
|
|
||||||
/// <typeparam name="TSpan">Location type</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public abstract class ShiftReduceParser<TValue, TSpan>
|
|
||||||
#else
|
|
||||||
internal abstract class ShiftReduceParser<TValue, TSpan>
|
|
||||||
#endif
|
|
||||||
where TSpan : IMerge<TSpan>, new() {
|
|
||||||
private AbstractScanner<TValue, TSpan> scanner;
|
|
||||||
/// <summary>
|
|
||||||
/// The abstract scanner for this parser.
|
|
||||||
/// </summary>
|
|
||||||
protected AbstractScanner<TValue, TSpan> Scanner {
|
|
||||||
get { return scanner; }
|
|
||||||
set { scanner = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Constructor for base class
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="scanner">Scanner instance for this parser</param>
|
|
||||||
protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
|
|
||||||
this.scanner = scanner;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// TECHNICAL EXPLANATION.
|
|
||||||
// Why the next two fields are not exposed via properties.
|
|
||||||
// ==============================================================
|
|
||||||
// These fields are of the generic parameter types, and are
|
|
||||||
// frequently instantiated as struct types in derived classes.
|
|
||||||
// Semantic actions are defined in the derived classes and refer
|
|
||||||
// to instance fields of these structs. In such cases the code
|
|
||||||
// "get_CurrentSemanticValue().myField = blah;" will fail since
|
|
||||||
// the getter pushes the value of the field, not the reference.
|
|
||||||
// So, in the presence of properties, gppg would need to encode
|
|
||||||
// such field accesses as ...
|
|
||||||
// "tmp = get_CurrentSemanticValue(); // Fetch value
|
|
||||||
// tmp.myField = blah; // update
|
|
||||||
// set_CurrentSemanticValue(tmp); " // Write update back.
|
|
||||||
// There is no issue if TValue is restricted to be a ref type.
|
|
||||||
// The same explanation applies to scanner.yylval.
|
|
||||||
// ==============================================================
|
|
||||||
/// <summary>
|
|
||||||
/// The current value of the "$$" symbolic variable in the parser
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
protected TValue CurrentSemanticValue;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The current value of the "@$" symbolic variable in the parser
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
protected TSpan CurrentLocationSpan;
|
|
||||||
protected int NextToken;
|
|
||||||
|
|
||||||
private TSpan LastSpan;
|
|
||||||
private State FsaState;
|
|
||||||
private bool recovering;
|
|
||||||
private int tokensSinceLastError;
|
|
||||||
|
|
||||||
private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
|
|
||||||
private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
|
|
||||||
private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The stack of semantic value (YYSTYPE) values.
|
|
||||||
/// </summary>
|
|
||||||
protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The stack of location value (YYLTYPE) varlues.
|
|
||||||
/// </summary>
|
|
||||||
protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
|
|
||||||
|
|
||||||
private int errorToken;
|
|
||||||
private int endOfFileToken;
|
|
||||||
private string[] nonTerminals;
|
|
||||||
private State[] states;
|
|
||||||
private Rule[] rules;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes
|
|
||||||
/// to insert the rule list into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="rules">The array of Rule objects</param>
|
|
||||||
protected void InitRules( Rule[] rules ) { this.rules = rules; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes to
|
|
||||||
/// insert the states table into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="states">The pre-initialized states table</param>
|
|
||||||
protected void InitStates( State[] states ) { this.states = states; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// OBSOLETE FOR VERSION 1.4.0
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="size"></param>
|
|
||||||
protected void InitStateTable( int size ) { states = new State[size]; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes
|
|
||||||
/// to insert the special value for the error and EOF tokens.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="err">The error state ordinal</param>
|
|
||||||
/// <param name="end">The EOF stat ordinal</param>
|
|
||||||
protected void InitSpecialTokens( int err, int end ) {
|
|
||||||
errorToken = err;
|
|
||||||
endOfFileToken = end;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes to
|
|
||||||
/// insert the non-terminal symbol names into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="names">Non-terminal symbol names</param>
|
|
||||||
protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
|
|
||||||
|
|
||||||
#region YYAbort, YYAccept etcetera.
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class AcceptException : Exception {
|
|
||||||
internal AcceptException() { }
|
|
||||||
protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class AbortException : Exception {
|
|
||||||
internal AbortException() { }
|
|
||||||
protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class ErrorException : Exception {
|
|
||||||
internal ErrorException() { }
|
|
||||||
protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
// The following methods are only called from within
|
|
||||||
// a semantic action. The thrown exceptions can never
|
|
||||||
// propagate outside the ShiftReduceParser class in
|
|
||||||
// which they are nested.
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning "true"
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYAccept() { throw new AcceptException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning "false"
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYAbort() { throw new AbortException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning
|
|
||||||
/// "false" if error recovery fails.
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYError() { throw new ErrorException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Check if parser in error recovery state.
|
|
||||||
/// </summary>
|
|
||||||
protected bool YYRecovering { get { return recovering; } }
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract base method. ShiftReduceParser calls this
|
|
||||||
/// to initialize the base class data structures. Concrete
|
|
||||||
/// parser classes must override this method.
|
|
||||||
/// </summary>
|
|
||||||
protected abstract void Initialize();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Main entry point of the Shift-Reduce Parser.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if parse succeeds, else false for
|
|
||||||
/// unrecoverable errors</returns>
|
|
||||||
public bool Parse() {
|
|
||||||
Initialize(); // allow derived classes to instantiate rules, states and nonTerminals
|
|
||||||
|
|
||||||
NextToken = 0;
|
|
||||||
FsaState = states[0];
|
|
||||||
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
valueStack.Push( CurrentSemanticValue );
|
|
||||||
LocationStack.Push( CurrentLocationSpan );
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
|
|
||||||
DisplayStack();
|
|
||||||
#endif
|
|
||||||
int action = FsaState.defaultAction;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable != null) {
|
|
||||||
if (NextToken == 0) {
|
|
||||||
// We save the last token span, so that the location span
|
|
||||||
// of production right hand sides that begin or end with a
|
|
||||||
// nullable production will be correct.
|
|
||||||
LastSpan = scanner.yylloc;
|
|
||||||
NextToken = scanner.yylex();
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
else
|
|
||||||
Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
|
|
||||||
#endif
|
|
||||||
if (FsaState.ParserTable.ContainsKey( NextToken ))
|
|
||||||
action = FsaState.ParserTable[NextToken];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (action > 0) // shift
|
|
||||||
{
|
|
||||||
Shift( action );
|
|
||||||
}
|
|
||||||
else if (action < 0) // reduce
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
Reduce( -action );
|
|
||||||
if (action == -1) // accept
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (AbortException) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
catch (AcceptException) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (ErrorException) {
|
|
||||||
if (!ErrorRecovery())
|
|
||||||
return false;
|
|
||||||
else
|
|
||||||
throw; // Rethrow x, preserving information.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (action == 0 && !ErrorRecovery()) // error
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Shift( int stateIndex ) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
FsaState = states[stateIndex];
|
|
||||||
|
|
||||||
valueStack.Push( scanner.yylval );
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
LocationStack.Push( scanner.yylloc );
|
|
||||||
|
|
||||||
if (recovering) {
|
|
||||||
if (NextToken != errorToken)
|
|
||||||
tokensSinceLastError++;
|
|
||||||
|
|
||||||
if (tokensSinceLastError > 5)
|
|
||||||
recovering = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (NextToken != endOfFileToken)
|
|
||||||
NextToken = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Reduce( int ruleNumber ) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
DisplayRule(ruleNumber);
|
|
||||||
#endif
|
|
||||||
Rule rule = rules[ruleNumber];
|
|
||||||
int rhLen = rule.RightHandSide.Length;
|
|
||||||
//
|
|
||||||
// Default actions for unit productions.
|
|
||||||
//
|
|
||||||
if (rhLen == 1) {
|
|
||||||
CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1;
|
|
||||||
CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
|
|
||||||
}
|
|
||||||
else if (rhLen == 0) {
|
|
||||||
// Create a new blank value.
|
|
||||||
// Explicit semantic action may mutate this value
|
|
||||||
CurrentSemanticValue = default( TValue );
|
|
||||||
// The location span for an empty production will start with the
|
|
||||||
// beginning of the next lexeme, and end with the finish of the
|
|
||||||
// previous lexeme. This gives the correct behaviour when this
|
|
||||||
// nonsense value is used in later Merge operations.
|
|
||||||
CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
|
|
||||||
scanner.yylloc.Merge( LastSpan ) :
|
|
||||||
default( TSpan ));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
// Default action: $$ = $1;
|
|
||||||
CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
|
|
||||||
// Default action "@$ = @1.Merge(@N)" for location info.
|
|
||||||
TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
|
|
||||||
TSpan atN = LocationStack[LocationStack.Depth - 1];
|
|
||||||
CurrentLocationSpan =
|
|
||||||
((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
|
|
||||||
}
|
|
||||||
|
|
||||||
DoAction( ruleNumber );
|
|
||||||
|
|
||||||
for (int i = 0; i < rule.RightHandSide.Length; i++) {
|
|
||||||
StateStack.Pop();
|
|
||||||
valueStack.Pop();
|
|
||||||
LocationStack.Pop();
|
|
||||||
}
|
|
||||||
FsaState = StateStack.TopElement();
|
|
||||||
|
|
||||||
if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
|
|
||||||
FsaState = states[FsaState.Goto[rule.LeftHandSide]];
|
|
||||||
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
valueStack.Push( CurrentSemanticValue );
|
|
||||||
LocationStack.Push( CurrentLocationSpan );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Execute the selected action from array.
|
|
||||||
/// Must be overriden in derived classes.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actionNumber">Index of the action to perform</param>
|
|
||||||
protected abstract void DoAction( int actionNumber );
|
|
||||||
|
|
||||||
private bool ErrorRecovery() {
|
|
||||||
bool discard;
|
|
||||||
|
|
||||||
if (!recovering) // if not recovering from previous error
|
|
||||||
ReportError();
|
|
||||||
|
|
||||||
if (!FindErrorRecoveryState())
|
|
||||||
return false;
|
|
||||||
//
|
|
||||||
// The interim fix for the "looping in error recovery"
|
|
||||||
// artifact involved moving the setting of the recovering
|
|
||||||
// bool until after invalid tokens have been discarded.
|
|
||||||
//
|
|
||||||
ShiftErrorToken();
|
|
||||||
discard = DiscardInvalidTokens();
|
|
||||||
recovering = true;
|
|
||||||
tokensSinceLastError = 0;
|
|
||||||
return discard;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ReportError() {
|
|
||||||
StringBuilder errorMsg = new StringBuilder();
|
|
||||||
errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
|
|
||||||
|
|
||||||
if (FsaState.ParserTable.Count < 7) {
|
|
||||||
bool first = true;
|
|
||||||
foreach (int terminal in FsaState.ParserTable.Keys) {
|
|
||||||
if (first)
|
|
||||||
errorMsg.Append( ", expecting " );
|
|
||||||
else
|
|
||||||
errorMsg.Append( ", or " );
|
|
||||||
|
|
||||||
errorMsg.Append( TerminalToString( terminal ) );
|
|
||||||
first = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
scanner.yyerror( errorMsg.ToString() );
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ShiftErrorToken() {
|
|
||||||
int old_next = NextToken;
|
|
||||||
NextToken = errorToken;
|
|
||||||
|
|
||||||
Shift( FsaState.ParserTable[NextToken] );
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
|
|
||||||
#endif
|
|
||||||
NextToken = old_next;
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool FindErrorRecoveryState() {
|
|
||||||
while (true) // pop states until one found that accepts error token
|
|
||||||
{
|
|
||||||
if (FsaState.ParserTable != null &&
|
|
||||||
FsaState.ParserTable.ContainsKey( errorToken ) &&
|
|
||||||
FsaState.ParserTable[errorToken] > 0) // shift
|
|
||||||
return true;
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
|
|
||||||
#endif
|
|
||||||
StateStack.Pop();
|
|
||||||
valueStack.Pop();
|
|
||||||
LocationStack.Pop();
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
DisplayStack();
|
|
||||||
#endif
|
|
||||||
if (StateStack.IsEmpty()) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
|
|
||||||
#endif
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
FsaState = StateStack.TopElement();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool DiscardInvalidTokens() {
|
|
||||||
|
|
||||||
int action = FsaState.defaultAction;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable != null) {
|
|
||||||
// Discard tokens until find one that works ...
|
|
||||||
while (true) {
|
|
||||||
if (NextToken == 0) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.Write("Reading a token: ");
|
|
||||||
#endif
|
|
||||||
NextToken = scanner.yylex();
|
|
||||||
}
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
if (NextToken == endOfFileToken)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable.ContainsKey( NextToken ))
|
|
||||||
action = FsaState.ParserTable[NextToken];
|
|
||||||
|
|
||||||
if (action != 0)
|
|
||||||
return true;
|
|
||||||
else {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
NextToken = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (recovering && tokensSinceLastError == 0) {
|
|
||||||
//
|
|
||||||
// Boolean recovering is not set until after the first
|
|
||||||
// error token has been shifted. Thus if we get back
|
|
||||||
// here with recovering set and no tokens read we are
|
|
||||||
// looping on the same error recovery action. This
|
|
||||||
// happens if current_state.ParserTable is null because
|
|
||||||
// the state has an LR(0) reduction, but not all
|
|
||||||
// lookahead tokens are valid. This only occurs for
|
|
||||||
// error productions that *end* on "error".
|
|
||||||
//
|
|
||||||
// This action discards tokens one at a time until
|
|
||||||
// the looping stops. Another attack would be to always
|
|
||||||
// use the LALR(1) table if a production ends on "error"
|
|
||||||
//
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
if (NextToken == endOfFileToken)
|
|
||||||
return false;
|
|
||||||
NextToken = 0;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Traditional YACC method. Discards the next input token.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
protected void yyclearin() { NextToken = 0; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Tradional YACC method. Clear the "recovering" flag.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
protected void yyerrok() { recovering = false; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// OBSOLETE FOR VERSION 1.4.0
|
|
||||||
/// Method used by derived types to insert new
|
|
||||||
/// state instances in the "states" array.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="stateNumber">index of the state</param>
|
|
||||||
/// <param name="state">data for the state</param>
|
|
||||||
protected void AddState( int stateNumber, State state ) {
|
|
||||||
states[stateNumber] = state;
|
|
||||||
state.number = stateNumber;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayStack() {
|
|
||||||
Console.Error.Write( "State stack is now:" );
|
|
||||||
for (int i = 0; i < StateStack.Depth; i++)
|
|
||||||
Console.Error.Write( " {0}", StateStack[i].number );
|
|
||||||
Console.Error.WriteLine();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayRule( int ruleNumber ) {
|
|
||||||
Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
|
|
||||||
DisplayProduction( rules[ruleNumber] );
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayProduction( Rule rule ) {
|
|
||||||
if (rule.RightHandSide.Length == 0)
|
|
||||||
Console.Error.Write( "/* empty */ " );
|
|
||||||
else
|
|
||||||
foreach (int symbol in rule.RightHandSide)
|
|
||||||
Console.Error.Write( "{0} ", SymbolToString( symbol ) );
|
|
||||||
|
|
||||||
Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract state class naming terminal symbols.
|
|
||||||
/// This is overridden by derived classes with the
|
|
||||||
/// name (or alias) to be used in error messages.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="terminal">The terminal ordinal</param>
|
|
||||||
/// <returns></returns>
|
|
||||||
protected abstract string TerminalToString( int terminal );
|
|
||||||
|
|
||||||
private string SymbolToString( int symbol ) {
|
|
||||||
if (symbol < 0)
|
|
||||||
return nonTerminals[-symbol - 1];
|
|
||||||
else
|
|
||||||
return TerminalToString( symbol );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Return text representation of argument character
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The character to convert</param>
|
|
||||||
/// <returns>String representation of the character</returns>
|
|
||||||
protected static string CharToString( char input ) {
|
|
||||||
switch (input) {
|
|
||||||
case '\a': return @"'\a'";
|
|
||||||
case '\b': return @"'\b'";
|
|
||||||
case '\f': return @"'\f'";
|
|
||||||
case '\n': return @"'\n'";
|
|
||||||
case '\r': return @"'\r'";
|
|
||||||
case '\t': return @"'\t'";
|
|
||||||
case '\v': return @"'\v'";
|
|
||||||
case '\0': return @"'\0'";
|
|
||||||
default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Classes implementing this interface must supply a
|
|
||||||
/// method that merges two location objects to return
|
|
||||||
/// a new object of the same type.
|
|
||||||
/// GPPG-generated parsers have the default location
|
|
||||||
/// action equivalent to "@$ = @1.Merge(@N);" where N
|
|
||||||
/// is the right-hand-side length of the production.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TSpan">The Location type</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public interface IMerge<TSpan>
|
|
||||||
#else
|
|
||||||
internal interface IMerge<TSpan>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Interface method that creates a location object from
|
|
||||||
/// the current and last object. Typically used to create
|
|
||||||
/// a location object extending from the start of the @1
|
|
||||||
/// object to the end of the @N object.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="last">The lexically last object to merge</param>
|
|
||||||
/// <returns>The merged location object</returns>
|
|
||||||
TSpan Merge( TSpan last );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This is the default class that carries location
|
|
||||||
/// information from the scanner to the parser.
|
|
||||||
/// If you don't declare "%YYLTYPE Foo" the parser
|
|
||||||
/// will expect to deal with this type.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class LexLocation : IMerge<LexLocation>
|
|
||||||
#else
|
|
||||||
[SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
|
|
||||||
internal class LexLocation : IMerge<LexLocation>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
private int startLine; // start line
|
|
||||||
private int startColumn; // start column
|
|
||||||
private int endLine; // end line
|
|
||||||
private int endColumn; // end column
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The line at which the text span starts.
|
|
||||||
/// </summary>
|
|
||||||
public int StartLine { get { return startLine; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The column at which the text span starts.
|
|
||||||
/// </summary>
|
|
||||||
public int StartColumn { get { return startColumn; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The line on which the text span ends.
|
|
||||||
/// </summary>
|
|
||||||
public int EndLine { get { return endLine; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The column of the first character
|
|
||||||
/// beyond the end of the text span.
|
|
||||||
/// </summary>
|
|
||||||
public int EndColumn { get { return endColumn; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Default no-arg constructor.
|
|
||||||
/// </summary>
|
|
||||||
public LexLocation() { }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Constructor for text-span with given start and end.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="sl">start line</param>
|
|
||||||
/// <param name="sc">start column</param>
|
|
||||||
/// <param name="el">end line </param>
|
|
||||||
/// <param name="ec">end column</param>
|
|
||||||
public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Create a text location which spans from the
|
|
||||||
/// start of "this" to the end of the argument "last"
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="last">The last location in the result span</param>
|
|
||||||
/// <returns>The merged span</returns>
|
|
||||||
public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract scanner class that GPPG expects its scanners to
|
|
||||||
/// extend.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
|
|
||||||
/// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public abstract class AbstractScanner<TValue, TSpan>
|
|
||||||
#else
|
|
||||||
internal abstract class AbstractScanner<TValue, TSpan>
|
|
||||||
#endif
|
|
||||||
where TSpan : IMerge<TSpan> {
|
|
||||||
/// <summary>
|
|
||||||
/// Lexical value optionally set by the scanner. The value
|
|
||||||
/// is of the %YYSTYPE type declared in the parser spec.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
// A field must be declared for this value of parametric type,
|
|
||||||
// since it may be instantiated by a value struct. If it were
|
|
||||||
// implemented as a property, machine generated code in derived
|
|
||||||
// types would not be able to select on the returned value.
|
|
||||||
#pragma warning disable 649
|
|
||||||
public TValue yylval; // Lexical value: set by scanner
|
|
||||||
#pragma warning restore 649
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Current scanner location property. The value is of the
|
|
||||||
/// type declared by %YYLTYPE in the parser specification.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public virtual TSpan yylloc {
|
|
||||||
get { return default( TSpan ); } // Empty implementation allowing
|
|
||||||
set { /* skip */ } // yylloc to be ignored entirely.
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Main call point for LEX-like scanners. Returns an int
|
|
||||||
/// corresponding to the token recognized by the scanner.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>An int corresponding to the token</returns>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public abstract int yylex();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Traditional error reporting provided by LEX-like scanners
|
|
||||||
/// to their YACC-like clients.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="format">Message format string</param>
|
|
||||||
/// <param name="args">Optional array of args</param>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public virtual void yyerror( string format, params object[] args ) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Encapsulated state for the parser.
|
|
||||||
/// Opaque to users, visible to the tool-generated code.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class State
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The number of states in the automaton.
|
|
||||||
/// </summary>
|
|
||||||
public int number;
|
|
||||||
#else
|
|
||||||
internal class State {
|
|
||||||
/// <summary>
|
|
||||||
/// The index of this state in the states array.
|
|
||||||
/// </summary>
|
|
||||||
internal int number;
|
|
||||||
#endif
|
|
||||||
internal Dictionary<int, int> ParserTable; // Terminal -> ParseAction
|
|
||||||
internal Dictionary<int, int> Goto; // NonTerminal -> State;
|
|
||||||
internal int defaultAction; // = 0; // ParseAction
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// State transition data for this state. Pairs of elements of the
|
|
||||||
/// goto array associate symbol ordinals with next state indices.
|
|
||||||
/// The actions array is passed to another constructor.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actions">The action list</param>c
|
|
||||||
/// <param name="goToList">Next state data</param>
|
|
||||||
public State( int[] actions, int[] goToList )
|
|
||||||
: this( actions ) {
|
|
||||||
Goto = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < goToList.Length; i += 2)
|
|
||||||
Goto.Add( goToList[i], goToList[i + 1] );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Action data for this state. Pairs of elements of the
|
|
||||||
/// action array associate action ordinals with each of
|
|
||||||
/// those symbols that have actions in the current state.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actions">The action array</param>
|
|
||||||
public State( int[] actions ) {
|
|
||||||
ParserTable = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < actions.Length; i += 2)
|
|
||||||
ParserTable.Add( actions[i], actions[i + 1] );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set the default action for this state.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="defaultAction">Ordinal of the default action</param>
|
|
||||||
public State( int defaultAction ) {
|
|
||||||
this.defaultAction = defaultAction;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set the default action and the state transition table.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="defaultAction">The default action</param>
|
|
||||||
/// <param name="goToList">Transitions from this state</param>
|
|
||||||
public State( int defaultAction, int[] goToList )
|
|
||||||
: this( defaultAction ) {
|
|
||||||
Goto = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < goToList.Length; i += 2)
|
|
||||||
Goto.Add( goToList[i], goToList[i + 1] );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rule representation at runtime.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class Rule
|
|
||||||
#else
|
|
||||||
internal class Rule
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
internal int LeftHandSide; // symbol
|
|
||||||
internal int[] RightHandSide; // symbols
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rule constructor. This holds the ordinal of
|
|
||||||
/// the left hand side symbol, and the list of
|
|
||||||
/// right hand side symbols, in lexical order.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="left">The LHS non-terminal</param>
|
|
||||||
/// <param name="right">The RHS symbols, in lexical order</param>
|
|
||||||
public Rule( int left, int[] right ) {
|
|
||||||
this.LeftHandSide = left;
|
|
||||||
this.RightHandSide = right;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Stack utility for the shift-reduce parser.
|
|
||||||
/// GPPG parsers have three instances:
|
|
||||||
/// (1) The parser state stack, T = QUT.Gppg.State,
|
|
||||||
/// (2) The semantic value stack, T = TValue,
|
|
||||||
/// (3) The location stack, T = TSpan.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="T"></typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class PushdownPrefixState<T>
|
|
||||||
#else
|
|
||||||
internal class PushdownPrefixState<T>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
// Note that we cannot use the BCL Stack<T> class
|
|
||||||
// here as derived types need to index into stacks.
|
|
||||||
//
|
|
||||||
private T[] array = new T[8];
|
|
||||||
private int tos = 0;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Indexer for values of the stack below the top.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="index">index of the element, starting from the bottom</param>
|
|
||||||
/// <returns>the selected element</returns>
|
|
||||||
public T this[int index] { get { return array[index]; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The current depth of the stack.
|
|
||||||
/// </summary>
|
|
||||||
public int Depth { get { return tos; } }
|
|
||||||
|
|
||||||
internal void Push( T value ) {
|
|
||||||
if (tos >= array.Length) {
|
|
||||||
T[] newarray = new T[array.Length * 2];
|
|
||||||
System.Array.Copy( array, newarray, tos );
|
|
||||||
array = newarray;
|
|
||||||
}
|
|
||||||
array[tos++] = value;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal T Pop() {
|
|
||||||
T rslt = array[--tos];
|
|
||||||
array[tos] = default( T );
|
|
||||||
return rslt;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal T TopElement() { return array[tos - 1]; }
|
|
||||||
|
|
||||||
internal bool IsEmpty() { return tos == 0; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
global using System;
|
|
||||||
global using System.Linq;
|
|
||||||
global using System.Collections.Generic;
|
|
||||||
global using System.Collections.Immutable;
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
|
||||||
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
|
|
||||||
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup Label="ColumnSortExpression">
|
|
||||||
<Scanner Include="ColumnSortExpressionTokens.lex" />
|
|
||||||
<Parser Include="ColumnSortExpressionGrammar.y" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
namespace Bts.ColumnSortExpressions;
|
|
||||||
|
|
||||||
public class ColumnSortExpression {
|
|
||||||
public ColumnSortExpression(string name, bool isAscending = true) {
|
|
||||||
Name = name;
|
|
||||||
IsAscending = isAscending;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string Name { get; }
|
|
||||||
public bool IsAscending { get; }
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
%namespace Bts.ColumnSortExpressions
|
|
||||||
%partial
|
|
||||||
%visibility internal
|
|
||||||
%tokentype ColumnSortExpressionToken
|
|
||||||
%parsertype ColumnSortExpressionParser
|
|
||||||
|
|
||||||
%union {
|
|
||||||
public string Identifier;
|
|
||||||
public ColumnSortExpression Column;
|
|
||||||
public ImmutableList<ColumnSortExpression> Columns;
|
|
||||||
}
|
|
||||||
|
|
||||||
%token ASCENDING, DESCENDING, IDENTIFIER
|
|
||||||
|
|
||||||
%type <Column> Column
|
|
||||||
%type <Columns> Columns
|
|
||||||
|
|
||||||
%%
|
|
||||||
|
|
||||||
Columns :
|
|
||||||
Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
|
|
||||||
| Columns Column { $$ = $1.Add($2); }
|
|
||||||
;
|
|
||||||
|
|
||||||
Column :
|
|
||||||
IDENTIFIER { $$ = new($1.Identifier, true); }
|
|
||||||
| ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
|
|
||||||
| DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
|
|
||||||
;
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace Bts.ColumnSortExpressions;
|
|
||||||
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
public class ColumnSortExpressionInterpreter {
|
|
||||||
public static IList<ColumnSortExpression> Evaluate(Stream stream) {
|
|
||||||
var parser = new ColumnSortExpressionParser();
|
|
||||||
var res = parser.Parse(stream);
|
|
||||||
return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
|
|
||||||
res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IList<ColumnSortExpression> Evaluate(string str) {
|
|
||||||
using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
|
|
||||||
return Evaluate(ms);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace Bts.ColumnSortExpressions;
|
|
||||||
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
partial class ColumnSortExpressionParser {
|
|
||||||
public ColumnSortExpressionParser() : base(null) { }
|
|
||||||
|
|
||||||
public ValueType? Parse(string s) {
|
|
||||||
using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
|
|
||||||
return Parse(ms);
|
|
||||||
}
|
|
||||||
|
|
||||||
public ValueType? Parse(Stream stream) {
|
|
||||||
Scanner = new ColumnSortExpressionScanner(stream);
|
|
||||||
Parse();
|
|
||||||
return CurrentSemanticValue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
namespace Bts.ColumnSortExpressions;
|
|
||||||
|
|
||||||
partial class ColumnSortExpressionScanner {
|
|
||||||
public override void yyerror(string format, params object[] args) {
|
|
||||||
base.yyerror(format, args);
|
|
||||||
Console.WriteLine(format, args);
|
|
||||||
Console.WriteLine();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
%namespace Bts.ColumnSortExpressions
|
|
||||||
%scannertype ColumnSortExpressionScanner
|
|
||||||
%visibility internal
|
|
||||||
%tokentype ColumnSortExpressionToken
|
|
||||||
|
|
||||||
%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
|
|
||||||
|
|
||||||
/* Literal Constants */
|
|
||||||
Letter [a-zA-Z]
|
|
||||||
|
|
||||||
Identifier {Letter}({Letter}*[0-9]*_*)*
|
|
||||||
Ascending +
|
|
||||||
Descending -
|
|
||||||
|
|
||||||
%%
|
|
||||||
|
|
||||||
{Ascending} {
|
|
||||||
return (int)ColumnSortExpressionToken.ASCENDING;
|
|
||||||
}
|
|
||||||
|
|
||||||
{Descending} {
|
|
||||||
return (int)ColumnSortExpressionToken.DESCENDING;
|
|
||||||
}
|
|
||||||
|
|
||||||
{Identifier} {
|
|
||||||
yylval.Identifier = yytext;
|
|
||||||
return (int)ColumnSortExpressionToken.IDENTIFIER;
|
|
||||||
}
|
|
||||||
|
|
||||||
%%
|
|
||||||
@@ -1,720 +0,0 @@
|
|||||||
// ==============================================================
|
|
||||||
// <auto-generated>
|
|
||||||
// This code automatically produced from an embedded resource.
|
|
||||||
// Do not edit this file, or it will become incompatible with
|
|
||||||
// the specification from which it was generated.
|
|
||||||
// </auto-generated>
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Runtime.Serialization;
|
|
||||||
using System.Globalization;
|
|
||||||
|
|
||||||
namespace QUT.GplexBuffers
|
|
||||||
{
|
|
||||||
// Code copied from GPLEX embedded resource
|
|
||||||
[Serializable]
|
|
||||||
internal class BufferException : Exception
|
|
||||||
{
|
|
||||||
public BufferException() { }
|
|
||||||
public BufferException(string message) : base(message) { }
|
|
||||||
public BufferException(string message, Exception innerException)
|
|
||||||
: base(message, innerException) { }
|
|
||||||
protected BufferException(SerializationInfo info, StreamingContext context)
|
|
||||||
: base(info, context) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
internal abstract class ScanBuff
|
|
||||||
{
|
|
||||||
private string fileNm;
|
|
||||||
|
|
||||||
public const int EndOfFile = -1;
|
|
||||||
public const int UnicodeReplacementChar = 0xFFFD;
|
|
||||||
|
|
||||||
public bool IsFile { get { return (fileNm != null); } }
|
|
||||||
public string FileName { get { return fileNm; } set { fileNm = value; } }
|
|
||||||
|
|
||||||
public abstract int Pos { get; set; }
|
|
||||||
public abstract int Read();
|
|
||||||
public virtual void Mark() { }
|
|
||||||
|
|
||||||
public abstract string GetString(int begin, int limit);
|
|
||||||
|
|
||||||
public static ScanBuff GetBuffer(string source)
|
|
||||||
{
|
|
||||||
return new StringBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ScanBuff GetBuffer(IList<string> source)
|
|
||||||
{
|
|
||||||
return new LineBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!NOFILES)
|
|
||||||
public static ScanBuff GetBuffer(Stream source)
|
|
||||||
{
|
|
||||||
return new BuildBuffer(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
return new BuildBuffer(source, fallbackCodePage);
|
|
||||||
}
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endif // !NOFILES
|
|
||||||
}
|
|
||||||
|
|
||||||
#region Buffer classes
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// ===== Definitions for various ScanBuff derived classes ====
|
|
||||||
// ==============================================================
|
|
||||||
// =============== String input ================
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class reads characters from a single string as
|
|
||||||
/// required, for example, by Visual Studio language services
|
|
||||||
/// </summary>
|
|
||||||
sealed class StringBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
string str; // input buffer
|
|
||||||
int bPos; // current position in buffer
|
|
||||||
int sLen;
|
|
||||||
|
|
||||||
public StringBuffer(string source)
|
|
||||||
{
|
|
||||||
this.str = source;
|
|
||||||
this.sLen = source.Length;
|
|
||||||
this.FileName = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
if (bPos < sLen) return str[bPos++];
|
|
||||||
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
|
|
||||||
else { bPos++; return EndOfFile; } // two strikes and you're out!
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
// "limit" can be greater than sLen with the BABEL
|
|
||||||
// option set. Read returns a "virtual" EOL if
|
|
||||||
// an attempt is made to read past the end of the
|
|
||||||
// string buffer. Without the guard any attempt
|
|
||||||
// to fetch yytext for a token that includes the
|
|
||||||
// EOL will throw an index exception.
|
|
||||||
if (limit > sLen) limit = sLen;
|
|
||||||
if (limit <= begin) return "";
|
|
||||||
else return str.Substring(begin, limit - begin);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return bPos; }
|
|
||||||
set { bPos = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() { return "StringBuffer"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// The LineBuff class contributed by Nigel Horspool,
|
|
||||||
// nigelh@cs.uvic.cs
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
sealed class LineBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
IList<string> line; // list of source lines from a file
|
|
||||||
int numLines; // number of strings in line list
|
|
||||||
string curLine; // current line in that list
|
|
||||||
int cLine; // index of current line in the list
|
|
||||||
int curLen; // length of current line
|
|
||||||
int curLineStart; // position of line start in whole file
|
|
||||||
int curLineEnd; // position of line end in whole file
|
|
||||||
int maxPos; // max position ever visited in whole file
|
|
||||||
int cPos; // ordinal number of code in source
|
|
||||||
|
|
||||||
// Constructed from a list of strings, one per source line.
|
|
||||||
// The lines have had trailing '\n' characters removed.
|
|
||||||
public LineBuffer(IList<string> lineList)
|
|
||||||
{
|
|
||||||
line = lineList;
|
|
||||||
numLines = line.Count;
|
|
||||||
cPos = curLineStart = 0;
|
|
||||||
curLine = (numLines > 0 ? line[0] : "");
|
|
||||||
maxPos = curLineEnd = curLen = curLine.Length;
|
|
||||||
cLine = 1;
|
|
||||||
FileName = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
if (cPos < curLineEnd)
|
|
||||||
return curLine[cPos++ - curLineStart];
|
|
||||||
if (cPos++ == curLineEnd)
|
|
||||||
return '\n';
|
|
||||||
if (cLine >= numLines)
|
|
||||||
return EndOfFile;
|
|
||||||
curLine = line[cLine];
|
|
||||||
curLen = curLine.Length;
|
|
||||||
curLineStart = curLineEnd + 1;
|
|
||||||
curLineEnd = curLineStart + curLen;
|
|
||||||
if (curLineEnd > maxPos)
|
|
||||||
maxPos = curLineEnd;
|
|
||||||
cLine++;
|
|
||||||
return curLen > 0 ? curLine[0] : '\n';
|
|
||||||
}
|
|
||||||
|
|
||||||
// To speed up searches for the line containing a position
|
|
||||||
private int cachedPosition;
|
|
||||||
private int cachedIxdex;
|
|
||||||
private int cachedLineStart;
|
|
||||||
|
|
||||||
// Given a position pos within the entire source, the results are
|
|
||||||
// ix -- the index of the containing line
|
|
||||||
// lstart -- the position of the first character on that line
|
|
||||||
private void findIndex(int pos, out int ix, out int lstart)
|
|
||||||
{
|
|
||||||
if (pos >= cachedPosition)
|
|
||||||
{
|
|
||||||
ix = cachedIxdex; lstart = cachedLineStart;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ix = lstart = 0;
|
|
||||||
}
|
|
||||||
while (ix < numLines)
|
|
||||||
{
|
|
||||||
int len = line[ix].Length + 1;
|
|
||||||
if (pos < lstart + len) break;
|
|
||||||
lstart += len;
|
|
||||||
ix++;
|
|
||||||
}
|
|
||||||
cachedPosition = pos;
|
|
||||||
cachedIxdex = ix;
|
|
||||||
cachedLineStart = lstart;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
if (begin >= maxPos || limit <= begin) return "";
|
|
||||||
int endIx, begIx, endLineStart, begLineStart;
|
|
||||||
findIndex(begin, out begIx, out begLineStart);
|
|
||||||
int begCol = begin - begLineStart;
|
|
||||||
findIndex(limit, out endIx, out endLineStart);
|
|
||||||
int endCol = limit - endLineStart;
|
|
||||||
string s = line[begIx];
|
|
||||||
if (begIx == endIx)
|
|
||||||
{
|
|
||||||
// the usual case, substring all on one line
|
|
||||||
return (endCol <= s.Length) ?
|
|
||||||
s.Substring(begCol, endCol - begCol)
|
|
||||||
: s.Substring(begCol) + "\n";
|
|
||||||
}
|
|
||||||
// the string spans multiple lines, yuk!
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
if (begCol < s.Length)
|
|
||||||
sb.Append(s.Substring(begCol));
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
sb.Append("\n");
|
|
||||||
s = line[++begIx];
|
|
||||||
if (begIx >= endIx) break;
|
|
||||||
sb.Append(s);
|
|
||||||
}
|
|
||||||
if (endCol <= s.Length)
|
|
||||||
{
|
|
||||||
sb.Append(s.Substring(0, endCol));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
sb.Append(s);
|
|
||||||
sb.Append("\n");
|
|
||||||
}
|
|
||||||
return sb.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return cPos; }
|
|
||||||
set
|
|
||||||
{
|
|
||||||
cPos = value;
|
|
||||||
findIndex(cPos, out cLine, out curLineStart);
|
|
||||||
// cLine should be the *next* line after curLine.
|
|
||||||
curLine = (cLine < numLines ? line[cLine++] : "");
|
|
||||||
curLineEnd = curLineStart + curLine.Length;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() { return "LineBuffer"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!NOFILES)
|
|
||||||
// ==============================================================
|
|
||||||
// ===== class BuildBuff : for unicode text files ========
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
class BuildBuffer : ScanBuff
|
|
||||||
{
|
|
||||||
// Double buffer for char stream.
|
|
||||||
class BufferElement
|
|
||||||
{
|
|
||||||
StringBuilder bldr = new StringBuilder();
|
|
||||||
StringBuilder next = new StringBuilder();
|
|
||||||
int minIx;
|
|
||||||
int maxIx;
|
|
||||||
int brkIx;
|
|
||||||
bool appendToNext;
|
|
||||||
|
|
||||||
internal BufferElement() { }
|
|
||||||
|
|
||||||
internal int MaxIndex { get { return maxIx; } }
|
|
||||||
// internal int MinIndex { get { return minIx; } }
|
|
||||||
|
|
||||||
internal char this[int index]
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
if (index < minIx || index >= maxIx)
|
|
||||||
throw new BufferException("Index was outside data buffer");
|
|
||||||
else if (index < brkIx)
|
|
||||||
return bldr[index - minIx];
|
|
||||||
else
|
|
||||||
return next[index - brkIx];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Append(char[] block, int count)
|
|
||||||
{
|
|
||||||
maxIx += count;
|
|
||||||
if (appendToNext)
|
|
||||||
this.next.Append(block, 0, count);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
this.bldr.Append(block, 0, count);
|
|
||||||
brkIx = maxIx;
|
|
||||||
appendToNext = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal string GetString(int start, int limit)
|
|
||||||
{
|
|
||||||
if (limit <= start)
|
|
||||||
return "";
|
|
||||||
if (start >= minIx && limit <= maxIx)
|
|
||||||
if (limit < brkIx) // String entirely in bldr builder
|
|
||||||
return bldr.ToString(start - minIx, limit - start);
|
|
||||||
else if (start >= brkIx) // String entirely in next builder
|
|
||||||
return next.ToString(start - brkIx, limit - start);
|
|
||||||
else // Must do a string-concatenation
|
|
||||||
return
|
|
||||||
bldr.ToString(start - minIx, brkIx - start) +
|
|
||||||
next.ToString(0, limit - brkIx);
|
|
||||||
else
|
|
||||||
throw new BufferException("String was outside data buffer");
|
|
||||||
}
|
|
||||||
|
|
||||||
internal void Mark(int limit)
|
|
||||||
{
|
|
||||||
if (limit > brkIx + 16) // Rotate blocks
|
|
||||||
{
|
|
||||||
StringBuilder temp = bldr;
|
|
||||||
bldr = next;
|
|
||||||
next = temp;
|
|
||||||
next.Length = 0;
|
|
||||||
minIx = brkIx;
|
|
||||||
brkIx = maxIx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
BufferElement data = new BufferElement();
|
|
||||||
|
|
||||||
int bPos; // Postion index in the StringBuilder
|
|
||||||
BlockReader NextBlk; // Delegate that serves char-arrays;
|
|
||||||
|
|
||||||
private string EncodingName
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
StreamReader rdr = NextBlk.Target as StreamReader;
|
|
||||||
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public BuildBuffer(Stream stream)
|
|
||||||
{
|
|
||||||
FileStream fStrm = (stream as FileStream);
|
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
|
||||||
NextBlk = BlockReaderFactory.Raw(stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public BuildBuffer(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
FileStream fStrm = (stream as FileStream);
|
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
|
||||||
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Marks a conservative lower bound for the buffer,
|
|
||||||
/// allowing space to be reclaimed. If an application
|
|
||||||
/// needs to call GetString at arbitrary past locations
|
|
||||||
/// in the input stream, Mark() is not called.
|
|
||||||
/// </summary>
|
|
||||||
public override void Mark() { data.Mark(bPos - 2); }
|
|
||||||
|
|
||||||
public override int Pos
|
|
||||||
{
|
|
||||||
get { return bPos; }
|
|
||||||
set { bPos = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Read returns the ordinal number of the next char, or
|
|
||||||
/// EOF (-1) for an end of stream. Note that the next
|
|
||||||
/// code point may require *two* calls of Read().
|
|
||||||
/// </summary>
|
|
||||||
/// <returns></returns>
|
|
||||||
public override int Read()
|
|
||||||
{
|
|
||||||
//
|
|
||||||
// Characters at positions
|
|
||||||
// [data.offset, data.offset + data.bldr.Length)
|
|
||||||
// are available in data.bldr.
|
|
||||||
//
|
|
||||||
if (bPos < data.MaxIndex)
|
|
||||||
{
|
|
||||||
// ch0 cannot be EOF
|
|
||||||
return (int)data[bPos++];
|
|
||||||
}
|
|
||||||
else // Read from underlying stream
|
|
||||||
{
|
|
||||||
// Experimental code, blocks of page size
|
|
||||||
char[] chrs = new char[4096];
|
|
||||||
int count = NextBlk(chrs, 0, 4096);
|
|
||||||
if (count == 0)
|
|
||||||
return EndOfFile;
|
|
||||||
else
|
|
||||||
{
|
|
||||||
data.Append(chrs, count);
|
|
||||||
return (int)data[bPos++];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string GetString(int begin, int limit)
|
|
||||||
{
|
|
||||||
return data.GetString(begin, limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString()
|
|
||||||
{
|
|
||||||
return "StringBuilder buffer, encoding: " + this.EncodingName;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// =============== End ScanBuff-derived classes ==================
|
|
||||||
|
|
||||||
internal delegate int BlockReader(char[] block, int index, int number);
|
|
||||||
|
|
||||||
// A delegate factory, serving up a delegate that
|
|
||||||
// reads a block of characters from the underlying
|
|
||||||
// encoded stream, via a StreamReader object.
|
|
||||||
//
|
|
||||||
internal static class BlockReaderFactory
|
|
||||||
{
|
|
||||||
public static BlockReader Raw(Stream stream)
|
|
||||||
{
|
|
||||||
return delegate(char[] block, int index, int number)
|
|
||||||
{
|
|
||||||
byte[] b = new byte[number];
|
|
||||||
int count = stream.Read(b, 0, number);
|
|
||||||
int i = 0;
|
|
||||||
int j = index;
|
|
||||||
for (; i < count; i++, j++)
|
|
||||||
block[j] = (char)b[i];
|
|
||||||
return count;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static BlockReader Get(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
|
||||||
Encoding encoding;
|
|
||||||
int preamble = Preamble(stream);
|
|
||||||
|
|
||||||
if (preamble != 0) // There is a valid BOM here!
|
|
||||||
encoding = Encoding.GetEncoding(preamble);
|
|
||||||
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
|
|
||||||
return Raw(stream);
|
|
||||||
else if (fallbackCodePage != -2) // Anything but "guess"
|
|
||||||
encoding = Encoding.GetEncoding(fallbackCodePage);
|
|
||||||
else // This is the "guess" option
|
|
||||||
{
|
|
||||||
int guess = new Guesser(stream).GuessCodePage();
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (guess == -1) // ==> this is a 7-bit file
|
|
||||||
encoding = Encoding.ASCII;
|
|
||||||
else if (guess == 65001)
|
|
||||||
encoding = Encoding.UTF8;
|
|
||||||
else // ==> use the machine default
|
|
||||||
encoding = Encoding.Default;
|
|
||||||
}
|
|
||||||
StreamReader reader = new StreamReader(stream, encoding);
|
|
||||||
return reader.Read;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int Preamble(Stream stream)
|
|
||||||
{
|
|
||||||
int b0 = stream.ReadByte();
|
|
||||||
int b1 = stream.ReadByte();
|
|
||||||
|
|
||||||
if (b0 == 0xfe && b1 == 0xff)
|
|
||||||
return 1201; // UTF16BE
|
|
||||||
if (b0 == 0xff && b1 == 0xfe)
|
|
||||||
return 1200; // UTF16LE
|
|
||||||
|
|
||||||
int b2 = stream.ReadByte();
|
|
||||||
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
|
||||||
return 65001; // UTF8
|
|
||||||
//
|
|
||||||
// There is no unicode preamble, so we
|
|
||||||
// return denoter for the machine default.
|
|
||||||
//
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
}
|
|
||||||
#endif // !NOFILES
|
|
||||||
#endregion Buffer classes
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// ============ class CodePageHandling =============
|
|
||||||
// ==============================================================
|
|
||||||
#if (!NOFILES)
|
|
||||||
public static class CodePageHandling
|
|
||||||
{
|
|
||||||
public static int GetCodePage(string option)
|
|
||||||
{
|
|
||||||
string command = option.ToUpperInvariant();
|
|
||||||
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
|
|
||||||
command = command.Substring(9);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (command.Equals("RAW"))
|
|
||||||
return -1;
|
|
||||||
else if (command.Equals("GUESS"))
|
|
||||||
return -2;
|
|
||||||
else if (command.Equals("DEFAULT"))
|
|
||||||
return 0;
|
|
||||||
else if (char.IsDigit(command[0]))
|
|
||||||
return int.Parse(command, CultureInfo.InvariantCulture);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Encoding enc = Encoding.GetEncoding(command);
|
|
||||||
return enc.CodePage;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (FormatException)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine(
|
|
||||||
"Invalid format \"{0}\", using machine default", option);
|
|
||||||
}
|
|
||||||
catch (ArgumentException)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine(
|
|
||||||
"Unknown code page \"{0}\", using machine default", option);
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#region guesser
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
// ==============================================================
|
|
||||||
// ============ Encoding Guesser =============
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class provides a simple finite state automaton that
|
|
||||||
/// scans the file looking for (1) valid UTF-8 byte patterns,
|
|
||||||
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
|
|
||||||
/// The method then guesses whether it is UTF-8 or maybe some
|
|
||||||
/// local machine default encoding. This works well for the
|
|
||||||
/// various Latin encodings.
|
|
||||||
/// </summary>
|
|
||||||
internal class Guesser
|
|
||||||
{
|
|
||||||
ScanBuff buffer;
|
|
||||||
|
|
||||||
public int GuessCodePage() { return Scan(); }
|
|
||||||
|
|
||||||
const int maxAccept = 10;
|
|
||||||
const int initial = 0;
|
|
||||||
const int eofNum = 0;
|
|
||||||
const int goStart = -1;
|
|
||||||
const int INITIAL = 0;
|
|
||||||
const int EndToken = 0;
|
|
||||||
|
|
||||||
#region user code
|
|
||||||
/*
|
|
||||||
* Reads the bytes of a file to determine if it is
|
|
||||||
* UTF-8 or a single-byte code page file.
|
|
||||||
*/
|
|
||||||
public long utfX;
|
|
||||||
public long uppr;
|
|
||||||
#endregion user code
|
|
||||||
|
|
||||||
int state;
|
|
||||||
int currentStart = startState[0];
|
|
||||||
int code;
|
|
||||||
|
|
||||||
#region ScannerTables
|
|
||||||
static int[] startState = new int[] { 11, 0 };
|
|
||||||
|
|
||||||
#region CharacterMap
|
|
||||||
static sbyte[] map = new sbyte[256] {
|
|
||||||
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
||||||
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
static sbyte[][] nextState = new sbyte[][] {
|
|
||||||
new sbyte[] {0, 0, 0, 0, 0, 0},
|
|
||||||
new sbyte[] {-1, -1, 10, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, -1, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 8, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 5, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 6, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 7, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, -1, 9, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, 1, 2, 3, 4, 2}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
|
|
||||||
// Reason for suppression: cannot have self-reference in array initializer.
|
|
||||||
static Guesser()
|
|
||||||
{
|
|
||||||
nextState[7] = nextState[2];
|
|
||||||
nextState[9] = nextState[2];
|
|
||||||
nextState[10] = nextState[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
int NextState()
|
|
||||||
{
|
|
||||||
if (code == ScanBuff.EndOfFile)
|
|
||||||
return eofNum;
|
|
||||||
else
|
|
||||||
return nextState[state][map[code]];
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
public Guesser(System.IO.Stream file) { SetSource(file); }
|
|
||||||
|
|
||||||
public void SetSource(System.IO.Stream source)
|
|
||||||
{
|
|
||||||
this.buffer = new BuildBuffer(source);
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
|
|
||||||
int Scan()
|
|
||||||
{
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
int next;
|
|
||||||
state = currentStart;
|
|
||||||
while ((next = NextState()) == goStart)
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
while ((next = NextState()) > eofNum)
|
|
||||||
{
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
if (state <= maxAccept)
|
|
||||||
{
|
|
||||||
#region ActionSwitch
|
|
||||||
#pragma warning disable 162
|
|
||||||
switch (state)
|
|
||||||
{
|
|
||||||
case eofNum:
|
|
||||||
switch (currentStart)
|
|
||||||
{
|
|
||||||
case 11:
|
|
||||||
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
|
|
||||||
else if (uppr * 10 > utfX) return 0; /* default code page */
|
|
||||||
else return 65001; /* UTF-8 encoding */
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return EndToken;
|
|
||||||
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
|
|
||||||
case 2: // Recognized '{Upper128}', Shortest string "\x80"
|
|
||||||
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
|
|
||||||
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
|
|
||||||
uppr++;
|
|
||||||
break;
|
|
||||||
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
|
|
||||||
uppr += 3;
|
|
||||||
break;
|
|
||||||
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
|
|
||||||
utfX += 3;
|
|
||||||
break;
|
|
||||||
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
|
|
||||||
utfX += 2;
|
|
||||||
break;
|
|
||||||
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
|
|
||||||
utfX++;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
#pragma warning restore 162
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // end class Guesser
|
|
||||||
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endregion
|
|
||||||
#endif // !NOFILES
|
|
||||||
|
|
||||||
// End of code copied from embedded resource
|
|
||||||
}
|
|
||||||
@@ -1,884 +0,0 @@
|
|||||||
// ==============================================================
|
|
||||||
// <auto-generated>
|
|
||||||
// This code automatically produced from an embedded resource.
|
|
||||||
// Do not edit this file, or it will become incompatible with
|
|
||||||
// the specification from which it was generated.
|
|
||||||
// </auto-generated>
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
// Gardens Point Parser Generator
|
|
||||||
// Copyright (c) Wayne Kelly, QUT 2005-2010
|
|
||||||
// (see accompanying GPPGcopyright.rtf)
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Text;
|
|
||||||
using System.Globalization;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Runtime.Serialization;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
|
||||||
namespace QUT.Gppg {
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract class for GPPG shift-reduce parsers.
|
|
||||||
/// Parsers generated by GPPG derive from this base
|
|
||||||
/// class, overriding the abstract Initialize() and
|
|
||||||
/// DoAction() methods.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TValue">Semantic value type</typeparam>
|
|
||||||
/// <typeparam name="TSpan">Location type</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public abstract class ShiftReduceParser<TValue, TSpan>
|
|
||||||
#else
|
|
||||||
internal abstract class ShiftReduceParser<TValue, TSpan>
|
|
||||||
#endif
|
|
||||||
where TSpan : IMerge<TSpan>, new() {
|
|
||||||
private AbstractScanner<TValue, TSpan> scanner;
|
|
||||||
/// <summary>
|
|
||||||
/// The abstract scanner for this parser.
|
|
||||||
/// </summary>
|
|
||||||
protected AbstractScanner<TValue, TSpan> Scanner {
|
|
||||||
get { return scanner; }
|
|
||||||
set { scanner = value; }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Constructor for base class
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="scanner">Scanner instance for this parser</param>
|
|
||||||
protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
|
|
||||||
this.scanner = scanner;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==============================================================
|
|
||||||
// TECHNICAL EXPLANATION.
|
|
||||||
// Why the next two fields are not exposed via properties.
|
|
||||||
// ==============================================================
|
|
||||||
// These fields are of the generic parameter types, and are
|
|
||||||
// frequently instantiated as struct types in derived classes.
|
|
||||||
// Semantic actions are defined in the derived classes and refer
|
|
||||||
// to instance fields of these structs. In such cases the code
|
|
||||||
// "get_CurrentSemanticValue().myField = blah;" will fail since
|
|
||||||
// the getter pushes the value of the field, not the reference.
|
|
||||||
// So, in the presence of properties, gppg would need to encode
|
|
||||||
// such field accesses as ...
|
|
||||||
// "tmp = get_CurrentSemanticValue(); // Fetch value
|
|
||||||
// tmp.myField = blah; // update
|
|
||||||
// set_CurrentSemanticValue(tmp); " // Write update back.
|
|
||||||
// There is no issue if TValue is restricted to be a ref type.
|
|
||||||
// The same explanation applies to scanner.yylval.
|
|
||||||
// ==============================================================
|
|
||||||
/// <summary>
|
|
||||||
/// The current value of the "$$" symbolic variable in the parser
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
protected TValue CurrentSemanticValue;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The current value of the "@$" symbolic variable in the parser
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
protected TSpan CurrentLocationSpan;
|
|
||||||
protected int NextToken;
|
|
||||||
|
|
||||||
private TSpan LastSpan;
|
|
||||||
private State FsaState;
|
|
||||||
private bool recovering;
|
|
||||||
private int tokensSinceLastError;
|
|
||||||
|
|
||||||
private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
|
|
||||||
private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
|
|
||||||
private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The stack of semantic value (YYSTYPE) values.
|
|
||||||
/// </summary>
|
|
||||||
protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The stack of location value (YYLTYPE) varlues.
|
|
||||||
/// </summary>
|
|
||||||
protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
|
|
||||||
|
|
||||||
private int errorToken;
|
|
||||||
private int endOfFileToken;
|
|
||||||
private string[] nonTerminals;
|
|
||||||
private State[] states;
|
|
||||||
private Rule[] rules;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes
|
|
||||||
/// to insert the rule list into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="rules">The array of Rule objects</param>
|
|
||||||
protected void InitRules( Rule[] rules ) { this.rules = rules; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes to
|
|
||||||
/// insert the states table into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="states">The pre-initialized states table</param>
|
|
||||||
protected void InitStates( State[] states ) { this.states = states; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// OBSOLETE FOR VERSION 1.4.0
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="size"></param>
|
|
||||||
protected void InitStateTable( int size ) { states = new State[size]; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes
|
|
||||||
/// to insert the special value for the error and EOF tokens.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="err">The error state ordinal</param>
|
|
||||||
/// <param name="end">The EOF stat ordinal</param>
|
|
||||||
protected void InitSpecialTokens( int err, int end ) {
|
|
||||||
errorToken = err;
|
|
||||||
endOfFileToken = end;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initialization method to allow derived classes to
|
|
||||||
/// insert the non-terminal symbol names into this base class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="names">Non-terminal symbol names</param>
|
|
||||||
protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
|
|
||||||
|
|
||||||
#region YYAbort, YYAccept etcetera.
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class AcceptException : Exception {
|
|
||||||
internal AcceptException() { }
|
|
||||||
protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class AbortException : Exception {
|
|
||||||
internal AbortException() { }
|
|
||||||
protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
[Serializable]
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This exception cannot escape from the local context
|
|
||||||
private class ErrorException : Exception {
|
|
||||||
internal ErrorException() { }
|
|
||||||
protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
// The following methods are only called from within
|
|
||||||
// a semantic action. The thrown exceptions can never
|
|
||||||
// propagate outside the ShiftReduceParser class in
|
|
||||||
// which they are nested.
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning "true"
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYAccept() { throw new AcceptException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning "false"
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYAbort() { throw new AbortException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Force parser to terminate, returning
|
|
||||||
/// "false" if error recovery fails.
|
|
||||||
/// </summary>
|
|
||||||
protected static void YYError() { throw new ErrorException(); }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Check if parser in error recovery state.
|
|
||||||
/// </summary>
|
|
||||||
protected bool YYRecovering { get { return recovering; } }
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract base method. ShiftReduceParser calls this
|
|
||||||
/// to initialize the base class data structures. Concrete
|
|
||||||
/// parser classes must override this method.
|
|
||||||
/// </summary>
|
|
||||||
protected abstract void Initialize();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Main entry point of the Shift-Reduce Parser.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if parse succeeds, else false for
|
|
||||||
/// unrecoverable errors</returns>
|
|
||||||
public bool Parse() {
|
|
||||||
Initialize(); // allow derived classes to instantiate rules, states and nonTerminals
|
|
||||||
|
|
||||||
NextToken = 0;
|
|
||||||
FsaState = states[0];
|
|
||||||
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
valueStack.Push( CurrentSemanticValue );
|
|
||||||
LocationStack.Push( CurrentLocationSpan );
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
|
|
||||||
DisplayStack();
|
|
||||||
#endif
|
|
||||||
int action = FsaState.defaultAction;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable != null) {
|
|
||||||
if (NextToken == 0) {
|
|
||||||
// We save the last token span, so that the location span
|
|
||||||
// of production right hand sides that begin or end with a
|
|
||||||
// nullable production will be correct.
|
|
||||||
LastSpan = scanner.yylloc;
|
|
||||||
NextToken = scanner.yylex();
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
else
|
|
||||||
Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
|
|
||||||
#endif
|
|
||||||
if (FsaState.ParserTable.ContainsKey( NextToken ))
|
|
||||||
action = FsaState.ParserTable[NextToken];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (action > 0) // shift
|
|
||||||
{
|
|
||||||
Shift( action );
|
|
||||||
}
|
|
||||||
else if (action < 0) // reduce
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
Reduce( -action );
|
|
||||||
if (action == -1) // accept
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (AbortException) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
catch (AcceptException) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (ErrorException) {
|
|
||||||
if (!ErrorRecovery())
|
|
||||||
return false;
|
|
||||||
else
|
|
||||||
throw; // Rethrow x, preserving information.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (action == 0 && !ErrorRecovery()) // error
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Shift( int stateIndex ) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
FsaState = states[stateIndex];
|
|
||||||
|
|
||||||
valueStack.Push( scanner.yylval );
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
LocationStack.Push( scanner.yylloc );
|
|
||||||
|
|
||||||
if (recovering) {
|
|
||||||
if (NextToken != errorToken)
|
|
||||||
tokensSinceLastError++;
|
|
||||||
|
|
||||||
if (tokensSinceLastError > 5)
|
|
||||||
recovering = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (NextToken != endOfFileToken)
|
|
||||||
NextToken = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Reduce( int ruleNumber ) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
DisplayRule(ruleNumber);
|
|
||||||
#endif
|
|
||||||
Rule rule = rules[ruleNumber];
|
|
||||||
int rhLen = rule.RightHandSide.Length;
|
|
||||||
//
|
|
||||||
// Default actions for unit productions.
|
|
||||||
//
|
|
||||||
if (rhLen == 1) {
|
|
||||||
CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1;
|
|
||||||
CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
|
|
||||||
}
|
|
||||||
else if (rhLen == 0) {
|
|
||||||
// Create a new blank value.
|
|
||||||
// Explicit semantic action may mutate this value
|
|
||||||
CurrentSemanticValue = default( TValue );
|
|
||||||
// The location span for an empty production will start with the
|
|
||||||
// beginning of the next lexeme, and end with the finish of the
|
|
||||||
// previous lexeme. This gives the correct behaviour when this
|
|
||||||
// nonsense value is used in later Merge operations.
|
|
||||||
CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
|
|
||||||
scanner.yylloc.Merge( LastSpan ) :
|
|
||||||
default( TSpan ));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
// Default action: $$ = $1;
|
|
||||||
CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
|
|
||||||
// Default action "@$ = @1.Merge(@N)" for location info.
|
|
||||||
TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
|
|
||||||
TSpan atN = LocationStack[LocationStack.Depth - 1];
|
|
||||||
CurrentLocationSpan =
|
|
||||||
((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
|
|
||||||
}
|
|
||||||
|
|
||||||
DoAction( ruleNumber );
|
|
||||||
|
|
||||||
for (int i = 0; i < rule.RightHandSide.Length; i++) {
|
|
||||||
StateStack.Pop();
|
|
||||||
valueStack.Pop();
|
|
||||||
LocationStack.Pop();
|
|
||||||
}
|
|
||||||
FsaState = StateStack.TopElement();
|
|
||||||
|
|
||||||
if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
|
|
||||||
FsaState = states[FsaState.Goto[rule.LeftHandSide]];
|
|
||||||
|
|
||||||
StateStack.Push( FsaState );
|
|
||||||
valueStack.Push( CurrentSemanticValue );
|
|
||||||
LocationStack.Push( CurrentLocationSpan );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Execute the selected action from array.
|
|
||||||
/// Must be overriden in derived classes.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actionNumber">Index of the action to perform</param>
|
|
||||||
protected abstract void DoAction( int actionNumber );
|
|
||||||
|
|
||||||
private bool ErrorRecovery() {
|
|
||||||
bool discard;
|
|
||||||
|
|
||||||
if (!recovering) // if not recovering from previous error
|
|
||||||
ReportError();
|
|
||||||
|
|
||||||
if (!FindErrorRecoveryState())
|
|
||||||
return false;
|
|
||||||
//
|
|
||||||
// The interim fix for the "looping in error recovery"
|
|
||||||
// artifact involved moving the setting of the recovering
|
|
||||||
// bool until after invalid tokens have been discarded.
|
|
||||||
//
|
|
||||||
ShiftErrorToken();
|
|
||||||
discard = DiscardInvalidTokens();
|
|
||||||
recovering = true;
|
|
||||||
tokensSinceLastError = 0;
|
|
||||||
return discard;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ReportError() {
|
|
||||||
StringBuilder errorMsg = new StringBuilder();
|
|
||||||
errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
|
|
||||||
|
|
||||||
if (FsaState.ParserTable.Count < 7) {
|
|
||||||
bool first = true;
|
|
||||||
foreach (int terminal in FsaState.ParserTable.Keys) {
|
|
||||||
if (first)
|
|
||||||
errorMsg.Append( ", expecting " );
|
|
||||||
else
|
|
||||||
errorMsg.Append( ", or " );
|
|
||||||
|
|
||||||
errorMsg.Append( TerminalToString( terminal ) );
|
|
||||||
first = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
scanner.yyerror( errorMsg.ToString() );
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ShiftErrorToken() {
|
|
||||||
int old_next = NextToken;
|
|
||||||
NextToken = errorToken;
|
|
||||||
|
|
||||||
Shift( FsaState.ParserTable[NextToken] );
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
|
|
||||||
#endif
|
|
||||||
NextToken = old_next;
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool FindErrorRecoveryState() {
|
|
||||||
while (true) // pop states until one found that accepts error token
|
|
||||||
{
|
|
||||||
if (FsaState.ParserTable != null &&
|
|
||||||
FsaState.ParserTable.ContainsKey( errorToken ) &&
|
|
||||||
FsaState.ParserTable[errorToken] > 0) // shift
|
|
||||||
return true;
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
|
|
||||||
#endif
|
|
||||||
StateStack.Pop();
|
|
||||||
valueStack.Pop();
|
|
||||||
LocationStack.Pop();
|
|
||||||
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
DisplayStack();
|
|
||||||
#endif
|
|
||||||
if (StateStack.IsEmpty()) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
|
|
||||||
#endif
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
FsaState = StateStack.TopElement();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool DiscardInvalidTokens() {
|
|
||||||
|
|
||||||
int action = FsaState.defaultAction;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable != null) {
|
|
||||||
// Discard tokens until find one that works ...
|
|
||||||
while (true) {
|
|
||||||
if (NextToken == 0) {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.Write("Reading a token: ");
|
|
||||||
#endif
|
|
||||||
NextToken = scanner.yylex();
|
|
||||||
}
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
if (NextToken == endOfFileToken)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (FsaState.ParserTable.ContainsKey( NextToken ))
|
|
||||||
action = FsaState.ParserTable[NextToken];
|
|
||||||
|
|
||||||
if (action != 0)
|
|
||||||
return true;
|
|
||||||
else {
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
NextToken = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (recovering && tokensSinceLastError == 0) {
|
|
||||||
//
|
|
||||||
// Boolean recovering is not set until after the first
|
|
||||||
// error token has been shifted. Thus if we get back
|
|
||||||
// here with recovering set and no tokens read we are
|
|
||||||
// looping on the same error recovery action. This
|
|
||||||
// happens if current_state.ParserTable is null because
|
|
||||||
// the state has an LR(0) reduction, but not all
|
|
||||||
// lookahead tokens are valid. This only occurs for
|
|
||||||
// error productions that *end* on "error".
|
|
||||||
//
|
|
||||||
// This action discards tokens one at a time until
|
|
||||||
// the looping stops. Another attack would be to always
|
|
||||||
// use the LALR(1) table if a production ends on "error"
|
|
||||||
//
|
|
||||||
#if TRACE_ACTIONS
|
|
||||||
Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
|
|
||||||
#endif
|
|
||||||
if (NextToken == endOfFileToken)
|
|
||||||
return false;
|
|
||||||
NextToken = 0;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Traditional YACC method. Discards the next input token.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
protected void yyclearin() { NextToken = 0; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Tradional YACC method. Clear the "recovering" flag.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
protected void yyerrok() { recovering = false; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// OBSOLETE FOR VERSION 1.4.0
|
|
||||||
/// Method used by derived types to insert new
|
|
||||||
/// state instances in the "states" array.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="stateNumber">index of the state</param>
|
|
||||||
/// <param name="state">data for the state</param>
|
|
||||||
protected void AddState( int stateNumber, State state ) {
|
|
||||||
states[stateNumber] = state;
|
|
||||||
state.number = stateNumber;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayStack() {
|
|
||||||
Console.Error.Write( "State stack is now:" );
|
|
||||||
for (int i = 0; i < StateStack.Depth; i++)
|
|
||||||
Console.Error.Write( " {0}", StateStack[i].number );
|
|
||||||
Console.Error.WriteLine();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayRule( int ruleNumber ) {
|
|
||||||
Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
|
|
||||||
DisplayProduction( rules[ruleNumber] );
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisplayProduction( Rule rule ) {
|
|
||||||
if (rule.RightHandSide.Length == 0)
|
|
||||||
Console.Error.Write( "/* empty */ " );
|
|
||||||
else
|
|
||||||
foreach (int symbol in rule.RightHandSide)
|
|
||||||
Console.Error.Write( "{0} ", SymbolToString( symbol ) );
|
|
||||||
|
|
||||||
Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract state class naming terminal symbols.
|
|
||||||
/// This is overridden by derived classes with the
|
|
||||||
/// name (or alias) to be used in error messages.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="terminal">The terminal ordinal</param>
|
|
||||||
/// <returns></returns>
|
|
||||||
protected abstract string TerminalToString( int terminal );
|
|
||||||
|
|
||||||
private string SymbolToString( int symbol ) {
|
|
||||||
if (symbol < 0)
|
|
||||||
return nonTerminals[-symbol - 1];
|
|
||||||
else
|
|
||||||
return TerminalToString( symbol );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Return text representation of argument character
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The character to convert</param>
|
|
||||||
/// <returns>String representation of the character</returns>
|
|
||||||
protected static string CharToString( char input ) {
|
|
||||||
switch (input) {
|
|
||||||
case '\a': return @"'\a'";
|
|
||||||
case '\b': return @"'\b'";
|
|
||||||
case '\f': return @"'\f'";
|
|
||||||
case '\n': return @"'\n'";
|
|
||||||
case '\r': return @"'\r'";
|
|
||||||
case '\t': return @"'\t'";
|
|
||||||
case '\v': return @"'\v'";
|
|
||||||
case '\0': return @"'\0'";
|
|
||||||
default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Classes implementing this interface must supply a
|
|
||||||
/// method that merges two location objects to return
|
|
||||||
/// a new object of the same type.
|
|
||||||
/// GPPG-generated parsers have the default location
|
|
||||||
/// action equivalent to "@$ = @1.Merge(@N);" where N
|
|
||||||
/// is the right-hand-side length of the production.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TSpan">The Location type</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public interface IMerge<TSpan>
|
|
||||||
#else
|
|
||||||
internal interface IMerge<TSpan>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Interface method that creates a location object from
|
|
||||||
/// the current and last object. Typically used to create
|
|
||||||
/// a location object extending from the start of the @1
|
|
||||||
/// object to the end of the @N object.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="last">The lexically last object to merge</param>
|
|
||||||
/// <returns>The merged location object</returns>
|
|
||||||
TSpan Merge( TSpan last );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This is the default class that carries location
|
|
||||||
/// information from the scanner to the parser.
|
|
||||||
/// If you don't declare "%YYLTYPE Foo" the parser
|
|
||||||
/// will expect to deal with this type.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class LexLocation : IMerge<LexLocation>
|
|
||||||
#else
|
|
||||||
[SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
|
|
||||||
internal class LexLocation : IMerge<LexLocation>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
private int startLine; // start line
|
|
||||||
private int startColumn; // start column
|
|
||||||
private int endLine; // end line
|
|
||||||
private int endColumn; // end column
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The line at which the text span starts.
|
|
||||||
/// </summary>
|
|
||||||
public int StartLine { get { return startLine; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The column at which the text span starts.
|
|
||||||
/// </summary>
|
|
||||||
public int StartColumn { get { return startColumn; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The line on which the text span ends.
|
|
||||||
/// </summary>
|
|
||||||
public int EndLine { get { return endLine; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The column of the first character
|
|
||||||
/// beyond the end of the text span.
|
|
||||||
/// </summary>
|
|
||||||
public int EndColumn { get { return endColumn; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Default no-arg constructor.
|
|
||||||
/// </summary>
|
|
||||||
public LexLocation() { }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Constructor for text-span with given start and end.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="sl">start line</param>
|
|
||||||
/// <param name="sc">start column</param>
|
|
||||||
/// <param name="el">end line </param>
|
|
||||||
/// <param name="ec">end column</param>
|
|
||||||
public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Create a text location which spans from the
|
|
||||||
/// start of "this" to the end of the argument "last"
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="last">The last location in the result span</param>
|
|
||||||
/// <returns>The merged span</returns>
|
|
||||||
public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Abstract scanner class that GPPG expects its scanners to
|
|
||||||
/// extend.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
|
|
||||||
/// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public abstract class AbstractScanner<TValue, TSpan>
|
|
||||||
#else
|
|
||||||
internal abstract class AbstractScanner<TValue, TSpan>
|
|
||||||
#endif
|
|
||||||
where TSpan : IMerge<TSpan> {
|
|
||||||
/// <summary>
|
|
||||||
/// Lexical value optionally set by the scanner. The value
|
|
||||||
/// is of the %YYSTYPE type declared in the parser spec.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
// A field must be declared for this value of parametric type,
|
|
||||||
// since it may be instantiated by a value struct. If it were
|
|
||||||
// implemented as a property, machine generated code in derived
|
|
||||||
// types would not be able to select on the returned value.
|
|
||||||
#pragma warning disable 649
|
|
||||||
public TValue yylval; // Lexical value: set by scanner
|
|
||||||
#pragma warning restore 649
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Current scanner location property. The value is of the
|
|
||||||
/// type declared by %YYLTYPE in the parser specification.
|
|
||||||
/// </summary>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public virtual TSpan yylloc {
|
|
||||||
get { return default( TSpan ); } // Empty implementation allowing
|
|
||||||
set { /* skip */ } // yylloc to be ignored entirely.
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Main call point for LEX-like scanners. Returns an int
|
|
||||||
/// corresponding to the token recognized by the scanner.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>An int corresponding to the token</returns>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public abstract int yylex();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Traditional error reporting provided by LEX-like scanners
|
|
||||||
/// to their YACC-like clients.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="format">Message format string</param>
|
|
||||||
/// <param name="args">Optional array of args</param>
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
|
|
||||||
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
|
|
||||||
// Reason for FxCop message suppression -
|
|
||||||
// This is a traditional name for YACC-like functionality
|
|
||||||
public virtual void yyerror( string format, params object[] args ) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Encapsulated state for the parser.
|
|
||||||
/// Opaque to users, visible to the tool-generated code.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class State
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The number of states in the automaton.
|
|
||||||
/// </summary>
|
|
||||||
public int number;
|
|
||||||
#else
|
|
||||||
internal class State {
|
|
||||||
/// <summary>
|
|
||||||
/// The index of this state in the states array.
|
|
||||||
/// </summary>
|
|
||||||
internal int number;
|
|
||||||
#endif
|
|
||||||
internal Dictionary<int, int> ParserTable; // Terminal -> ParseAction
|
|
||||||
internal Dictionary<int, int> Goto; // NonTerminal -> State;
|
|
||||||
internal int defaultAction; // = 0; // ParseAction
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// State transition data for this state. Pairs of elements of the
|
|
||||||
/// goto array associate symbol ordinals with next state indices.
|
|
||||||
/// The actions array is passed to another constructor.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actions">The action list</param>c
|
|
||||||
/// <param name="goToList">Next state data</param>
|
|
||||||
public State( int[] actions, int[] goToList )
|
|
||||||
: this( actions ) {
|
|
||||||
Goto = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < goToList.Length; i += 2)
|
|
||||||
Goto.Add( goToList[i], goToList[i + 1] );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Action data for this state. Pairs of elements of the
|
|
||||||
/// action array associate action ordinals with each of
|
|
||||||
/// those symbols that have actions in the current state.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="actions">The action array</param>
|
|
||||||
public State( int[] actions ) {
|
|
||||||
ParserTable = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < actions.Length; i += 2)
|
|
||||||
ParserTable.Add( actions[i], actions[i + 1] );
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set the default action for this state.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="defaultAction">Ordinal of the default action</param>
|
|
||||||
public State( int defaultAction ) {
|
|
||||||
this.defaultAction = defaultAction;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set the default action and the state transition table.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="defaultAction">The default action</param>
|
|
||||||
/// <param name="goToList">Transitions from this state</param>
|
|
||||||
public State( int defaultAction, int[] goToList )
|
|
||||||
: this( defaultAction ) {
|
|
||||||
Goto = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < goToList.Length; i += 2)
|
|
||||||
Goto.Add( goToList[i], goToList[i + 1] );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rule representation at runtime.
|
|
||||||
/// </summary>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class Rule
|
|
||||||
#else
|
|
||||||
internal class Rule
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
internal int LeftHandSide; // symbol
|
|
||||||
internal int[] RightHandSide; // symbols
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rule constructor. This holds the ordinal of
|
|
||||||
/// the left hand side symbol, and the list of
|
|
||||||
/// right hand side symbols, in lexical order.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="left">The LHS non-terminal</param>
|
|
||||||
/// <param name="right">The RHS symbols, in lexical order</param>
|
|
||||||
public Rule( int left, int[] right ) {
|
|
||||||
this.LeftHandSide = left;
|
|
||||||
this.RightHandSide = right;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Stack utility for the shift-reduce parser.
|
|
||||||
/// GPPG parsers have three instances:
|
|
||||||
/// (1) The parser state stack, T = QUT.Gppg.State,
|
|
||||||
/// (2) The semantic value stack, T = TValue,
|
|
||||||
/// (3) The location stack, T = TSpan.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="T"></typeparam>
|
|
||||||
#if EXPORT_GPPG
|
|
||||||
public class PushdownPrefixState<T>
|
|
||||||
#else
|
|
||||||
internal class PushdownPrefixState<T>
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
// Note that we cannot use the BCL Stack<T> class
|
|
||||||
// here as derived types need to index into stacks.
|
|
||||||
//
|
|
||||||
private T[] array = new T[8];
|
|
||||||
private int tos = 0;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Indexer for values of the stack below the top.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="index">index of the element, starting from the bottom</param>
|
|
||||||
/// <returns>the selected element</returns>
|
|
||||||
public T this[int index] { get { return array[index]; } }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The current depth of the stack.
|
|
||||||
/// </summary>
|
|
||||||
public int Depth { get { return tos; } }
|
|
||||||
|
|
||||||
internal void Push( T value ) {
|
|
||||||
if (tos >= array.Length) {
|
|
||||||
T[] newarray = new T[array.Length * 2];
|
|
||||||
System.Array.Copy( array, newarray, tos );
|
|
||||||
array = newarray;
|
|
||||||
}
|
|
||||||
array[tos++] = value;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal T Pop() {
|
|
||||||
T rslt = array[--tos];
|
|
||||||
array[tos] = default( T );
|
|
||||||
return rslt;
|
|
||||||
}
|
|
||||||
|
|
||||||
internal T TopElement() { return array[tos - 1]; }
|
|
||||||
|
|
||||||
internal bool IsEmpty() { return tos == 0; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
global using System;
|
|
||||||
global using System.Linq;
|
|
||||||
global using System.Collections.Generic;
|
|
||||||
global using System.Collections.Immutable;
|
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
<PackageReference Include="YaccLexTools" Version="1.2.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -9,8 +9,6 @@
|
|||||||
<ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
|
<ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
|
||||||
<ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
|
<ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
|
||||||
<ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
|
<ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
|
||||||
<ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
|
|
||||||
<ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
|
|
||||||
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
||||||
<ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
|
<ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
|
||||||
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
||||||
|
|||||||
@@ -1,26 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Reactive.Linq;
|
|
||||||
|
|
||||||
public static class DataStreamExtensions {
|
|
||||||
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
|
||||||
source as ViewStream ?? new ObservableViewStream(source);
|
|
||||||
|
|
||||||
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
|
||||||
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
|
||||||
|
|
||||||
IDisposable Subscribe(IObserver<DataPacket> observer) {
|
|
||||||
source.onChange += OnChange;
|
|
||||||
return Disposable.Create(() => source.onChange -= OnChange);
|
|
||||||
|
|
||||||
void OnChange(object? newValue) =>
|
|
||||||
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
|
|
||||||
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
|
|
||||||
public class PandasScriptStream : ViewStream {
|
|
||||||
public PandasScriptStream(ViewStream vs, string file)
|
|
||||||
: this(vs, file, null) { }
|
|
||||||
|
|
||||||
public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
|
|
||||||
: this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
|
|
||||||
mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
|
|
||||||
|
|
||||||
string readFile() {
|
|
||||||
try {
|
|
||||||
return File.ReadAllText(file);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Log.Gen.error($"PandasScriptStream error reading file: {ex}");
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
using BCG.Extensions;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Runtime.CompilerServices;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
public class PandasStream : ViewStream, IWorker {
|
|
||||||
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
|
||||||
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
|
||||||
readonly string pythonExe;
|
|
||||||
readonly ViewStreamDatabase vsd;
|
|
||||||
readonly CSVSaverStream inputCsvStream;
|
|
||||||
readonly Lazy<string> pythonCode;
|
|
||||||
IDictionary<int, IBlueRow>? lastRowMap = null;
|
|
||||||
int inputVersion = 0;
|
|
||||||
int outputVersion = 0;
|
|
||||||
DateTime lastFireTime = DateTime.MinValue;
|
|
||||||
TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
|
|
||||||
IDisposable? dispose;
|
|
||||||
|
|
||||||
public PandasStream(ViewStream vs, string dataFrameExpression)
|
|
||||||
: this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
|
|
||||||
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
|
|
||||||
this.pythonExe = pythonExe;
|
|
||||||
this.vsd = vsd;
|
|
||||||
// This should fire an update when it writes to the output file
|
|
||||||
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
|
|
||||||
pythonCode = new(getPythonCode);
|
|
||||||
|
|
||||||
string getPythonCode() {
|
|
||||||
var imports =
|
|
||||||
@"import numpy as np
|
|
||||||
import pandas as pd
|
|
||||||
import sys
|
|
||||||
";
|
|
||||||
var bd = new StringBuilder(imports);
|
|
||||||
bd.AppendLine($"df=pd.read_csv('{inputFile}')");
|
|
||||||
bd.AppendLine(dataFrameExpression);
|
|
||||||
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
|
|
||||||
return bd.ToString();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
vsd.addWorker(this);
|
|
||||||
inputCsvStream.onChange += onInputChanged;
|
|
||||||
this.dispose = Disposable.Create(dispose);
|
|
||||||
|
|
||||||
void onInputChanged(object newValue) =>
|
|
||||||
inputVersion++;
|
|
||||||
|
|
||||||
void dispose() {
|
|
||||||
inputCsvStream.onChange -= onInputChanged;
|
|
||||||
vsd.removeWorker(this);
|
|
||||||
deleteFile(inputFile);
|
|
||||||
deleteFile(outputFile);
|
|
||||||
}
|
|
||||||
|
|
||||||
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
|
|
||||||
if (!File.Exists(file)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
File.Delete(file);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose(ref dispose);
|
|
||||||
|
|
||||||
static void executePythonCode(string pythonCode, string? pythonExe = null) {
|
|
||||||
pythonExe ??= "python";
|
|
||||||
var start = new ProcessStartInfo {
|
|
||||||
FileName = pythonExe,
|
|
||||||
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
|
|
||||||
UseShellExecute = false,
|
|
||||||
RedirectStandardOutput = true
|
|
||||||
};
|
|
||||||
using var process = Process.Start(start)
|
|
||||||
?? throw new InvalidOperationException("Failed to launch python");
|
|
||||||
process.WaitForExit();
|
|
||||||
}
|
|
||||||
|
|
||||||
void IWorker.doWork() {
|
|
||||||
if (outputVersion == inputVersion) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
var now = DateTime.Now;
|
|
||||||
var elapsedSinceLastFire = now - lastFireTime;
|
|
||||||
if (elapsedSinceLastFire < fireThrottle) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
outputVersion = inputVersion;
|
|
||||||
executePythonCode(pythonCode.Value, pythonExe);
|
|
||||||
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
|
|
||||||
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
|
|
||||||
var comp = BlueRowEqualityComparer.Instance;
|
|
||||||
var deletedRows = new BlueTable(schema);
|
|
||||||
var addedRows = new BlueTable(schema);
|
|
||||||
if (lastRowMap is not null) {
|
|
||||||
deletedRows.SetItems(lastRowMap.Where(oldRow =>
|
|
||||||
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
|
|
||||||
addedRows.SetItems(rowMap.Where(newRow =>
|
|
||||||
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
|
|
||||||
}
|
|
||||||
lastRowMap = rowMap;
|
|
||||||
lastFireTime = DateTime.Now;
|
|
||||||
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
|
|
||||||
class State {
|
|
||||||
public int n_sends = 0;
|
|
||||||
public int n_recvs = 0;
|
|
||||||
}
|
|
||||||
private readonly string _ip;
|
|
||||||
private readonly int _port;
|
|
||||||
private readonly ViewStreamDatabase _vsd;
|
|
||||||
State _state = new State();
|
|
||||||
private readonly int _msg_size;
|
|
||||||
|
|
||||||
private IBlueTableSchema _schema;
|
|
||||||
private TimeSpan _send_period;
|
|
||||||
private ITCPConnection _connection = null;
|
|
||||||
private byte[] _buff;
|
|
||||||
|
|
||||||
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
|
|
||||||
this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
|
|
||||||
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
|
|
||||||
_ip = ip;
|
|
||||||
_port = port;
|
|
||||||
_msg_size = msg_size;
|
|
||||||
_vsd = vsd;
|
|
||||||
_schema = BlueDataUtility.form_schema("TCPClient",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("n_sends", TypeCode.UInt32),
|
|
||||||
Tuple.Create("n_recvs", TypeCode.UInt32));
|
|
||||||
_send_period = TimeSpan.FromSeconds(send_period_seconds);
|
|
||||||
_buff = new byte[msg_size];
|
|
||||||
for (int i = 0; i < _buff.Length; i++) {
|
|
||||||
_buff[i] = (byte)i;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
|
||||||
var server_rci = new RemoteConnectionInfo(_ip, _port);
|
|
||||||
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
MasterWorker.getInstance(false).removeWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork() {
|
|
||||||
if (_connection != null) {
|
|
||||||
_connection.send(_buff);
|
|
||||||
_state.n_sends++;
|
|
||||||
}
|
|
||||||
IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
|
|
||||||
BlueTable rows = new BlueTable(r.schema);
|
|
||||||
rows.addRow(r);
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void onConnection(ITCPConnection connection) {
|
|
||||||
_connection = connection;
|
|
||||||
_connection.register(this);
|
|
||||||
}
|
|
||||||
public bool isDisposed { get; }
|
|
||||||
|
|
||||||
public void onData(RemoteMessage msg) {
|
|
||||||
bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
|
|
||||||
_state.n_recvs++;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
file_path = r"C:\Temp\test2.txt" # r prefix for raw string to avoid escape characters
|
|
||||||
with open(file_path, "w") as f:
|
|
||||||
f.write("Hello, world!\n")
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
|
|
||||||
readonly IObservable<DataPacket> source;
|
|
||||||
IDisposable? subscription;
|
|
||||||
|
|
||||||
public ObservableViewStream(IObservable<DataPacket> source) =>
|
|
||||||
this.source = source;
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
if (subscription is not null) {
|
|
||||||
subscription.Dispose();
|
|
||||||
subscription = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
Log.isnull(subscription);
|
|
||||||
subscription = source.Subscribe(myFireEvent);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
Log.notnull(subscription);
|
|
||||||
subscription.Dispose();
|
|
||||||
subscription = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
|
|
||||||
source.Subscribe(observer);
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,13 @@
|
|||||||
|
using BCG;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using mbts.config;
|
||||||
|
using bts.utils;
|
||||||
|
using System.Threading.Tasks;
|
||||||
using Bts.DataStreamExpressions;
|
using Bts.DataStreamExpressions;
|
||||||
|
using System.Collections.Generic;
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
using bts;
|
using bts;
|
||||||
|
|
||||||
@@ -24,10 +30,8 @@ namespace BCG {
|
|||||||
view_publisher = view_publisher_;
|
view_publisher = view_publisher_;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
static ViewStream? build_stream(string stream_expression)
|
static ViewStream build_stream(string stream_expression) {
|
||||||
{
|
try {
|
||||||
try
|
|
||||||
{
|
|
||||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||||
return rv;
|
return rv;
|
||||||
@@ -51,13 +55,10 @@ namespace BCG {
|
|||||||
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
private void _on_str(object obj)
|
// add a RemoteQuery sub, additionally and subscribe to it
|
||||||
{
|
private void _on_str(object obj) {
|
||||||
if (obj is not DataPacket dp) {
|
DataPacket dp = obj as DataPacket;
|
||||||
return;
|
foreach (IBlueRow r in dp.newRows) {
|
||||||
}
|
|
||||||
foreach (IBlueRow r in dp.newRows)
|
|
||||||
{
|
|
||||||
string name = r.get<string>("name");
|
string name = r.get<string>("name");
|
||||||
bool persist = r.get<bool>("persist");
|
bool persist = r.get<bool>("persist");
|
||||||
string stream_exp = r.get<string>("stream_exp");
|
string stream_exp = r.get<string>("stream_exp");
|
||||||
@@ -87,4 +88,4 @@ namespace BCG {
|
|||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,10 @@
|
|||||||
using System;
|
using System;
|
||||||
|
using System.Data;
|
||||||
|
using System.Collections;
|
||||||
|
using BCG.Utilities;
|
||||||
|
using BCG.Utilities.General;
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
|
|
||||||
public class AddColumnsStream : ViewStream {
|
public class AddColumnsStream : ViewStream {
|
||||||
public static Tuple<string, Type, object> parse_exp(string exp) {
|
public static Tuple<string, Type, object> parse_exp(string exp) {
|
||||||
string[] toks = exp.Split(':');
|
string[] toks = exp.Split(':');
|
||||||
@@ -31,4 +35,4 @@ namespace BCG {
|
|||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
namespace bts {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
|
|
||||||
public class AddConstantColumnRow {
|
|
||||||
AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +1,63 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections;
|
||||||
using System.Diagnostics.CodeAnalysis;
|
// using BCG.Utilities;
|
||||||
using System.Linq;
|
// using BCG.Evaluator.AddColumnsExpression;
|
||||||
|
|
||||||
using Bts.ColumnExpressions;
|
// namespace BCG {
|
||||||
|
// /// <summary>
|
||||||
namespace BCG;
|
// /// Provides methods to add columns to the schema
|
||||||
|
// /// </summary>
|
||||||
/// <summary>
|
// public class ColumnAdder : IStreamMapper {
|
||||||
/// Provides methods to add columns to the schema
|
// protected IDictionary myNewColumns;
|
||||||
/// </summary>
|
// protected IBlueTableSchema mySchema;
|
||||||
/// <param name="columnsExpression">
|
|
||||||
/// String represting the columns to add to the output table
|
// static syntax myParser =
|
||||||
/// </param>
|
// new syntax();
|
||||||
public class ColumnAdder : IStreamMapper {
|
|
||||||
// Store tuples of (column, index) to preserve order
|
// /// <summary>
|
||||||
protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
|
// /// string represting the columns to be added to the output table.
|
||||||
protected IBlueTableSchema? mySchema;
|
// /// </summary>
|
||||||
|
// /// <param name="colAddExpression"></param>
|
||||||
public ColumnAdder(string columnsExpression) =>
|
// public ColumnAdder(string columnDef) {
|
||||||
myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
|
// AddColumnsExpression ace;
|
||||||
throw new Exception("Invalid columnn expression: " + columnsExpression))
|
|
||||||
.Select((c, i) => (c, i))
|
// if (DataStreamLoadFileConfig.instance.useStaticParser) {
|
||||||
.ToDictionary(c => c.c.Name, c => (c.c, c.i));
|
// lock (myParser) {
|
||||||
|
// ace =
|
||||||
[MemberNotNull(nameof(mySchema))]
|
// (AddColumnsExpression)myParser.Parse(columnDef);
|
||||||
public virtual IBlueRow map(IBlueRow row) {
|
// }
|
||||||
// Only rebuild schema if necessary (i.e., if it's null)
|
// } else {
|
||||||
mySchema ??= buildSchema(row.schema, myNewColumns);
|
// syntax parser = new syntax();
|
||||||
return new AddColumnsRow(mySchema, row);
|
// ace =
|
||||||
}
|
// (AddColumnsExpression)parser.Parse(columnDef);
|
||||||
|
// }
|
||||||
[MemberNotNull(nameof(mySchema))]
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
|
// if (ace == null)
|
||||||
// Always rebuild schema (even if it's already non-null)
|
// throw new Exception("Invalid columnn expression: " + columnDef);
|
||||||
mySchema = buildSchema(inputSchema, myNewColumns);
|
// myNewColumns = ace.columns;
|
||||||
|
// }
|
||||||
[MemberNotNull(nameof(mySchema))]
|
|
||||||
static BlueTableSchema buildSchema(
|
// public virtual IBlueRow map(IBlueRow row) {
|
||||||
IBlueTableSchema schema,
|
// if (mySchema == null)
|
||||||
IDictionary<string, (ColumnExpression column, int index)> newColumns
|
// myConstructSchema(row.schema);
|
||||||
) {
|
// return new AddColumnsRow(mySchema, row);
|
||||||
ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
|
// }
|
||||||
schema.columnInfo.CopyTo(cis, 0);
|
|
||||||
foreach (var (col, index) in newColumns.Values) {
|
// public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||||
var schemaIdx = schema.columnInfo.Length + index;
|
// myConstructSchema(inputSchema);
|
||||||
cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
|
// return mySchema;
|
||||||
}
|
// }
|
||||||
return new BlueTableSchema(cis, schema.tableName);
|
|
||||||
}
|
// private void myConstructSchema(IBlueTableSchema schema) {
|
||||||
}
|
// ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
|
||||||
|
// int ciIndex = 0;
|
||||||
|
// foreach (ColumnInfo ci in schema.columnInfo)
|
||||||
|
// cis[ciIndex++] = ci;
|
||||||
|
// foreach (DictionaryEntry de in myNewColumns)
|
||||||
|
// cis[ciIndex] = new ColumnInfo((string)de.Key, ciIndex++,
|
||||||
|
// ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
|
||||||
|
// mySchema = new BlueTableSchema(cis, schema.tableName);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,24 +1,23 @@
|
|||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Provides a stream that with distinct rows with respect to the columns
|
/// Provides a stream that with distinct rows with respect to the columns
|
||||||
/// the user specifies in the argument
|
/// the user specifies in the argument
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[DataStreamUserRespresentation("distinct")]
|
[DataStreamUserRespresentation ("distinct")]
|
||||||
public class DistinctRowStream : ViewStream {
|
public class DistinctRowStream : ViewStream
|
||||||
public DistinctRowStream(ViewStream stream, params string[] cols) {
|
{
|
||||||
var g_by_eval = new DistinctRowEvaluator(cols);
|
public DistinctRowStream (ViewStream stream, params string[] cols)
|
||||||
ViewStream r = new GroupByStream(stream, g_by_eval);
|
{
|
||||||
var proj_exp = String.Empty;
|
ArrayList groupByCols = new ArrayList ();
|
||||||
foreach (var c in cols)
|
foreach (string col in cols)
|
||||||
proj_exp += $"{c} ";
|
foreach (string c in col.Split (' '))
|
||||||
proj_exp = proj_exp.Trim();
|
groupByCols.Add (c);
|
||||||
r = new NewProjectStream(r, proj_exp);
|
mySetRealSource (new GroupByStream (stream,
|
||||||
mySetRealSource(r);
|
new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
// using System;
|
||||||
|
// using System.Collections.Generic;
|
||||||
|
// using System.Text;
|
||||||
|
// using System.IO;
|
||||||
|
|
||||||
|
// using System.Runtime.Serialization.Formatters.Binary;
|
||||||
|
|
||||||
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// public class DumpLoaderStream : ViewStream, ISideEffectStream
|
||||||
|
// {
|
||||||
|
// //wtf
|
||||||
|
// //public class Exception : System.Exception
|
||||||
|
// //{public Exception(string msg) : base(msg) { }}
|
||||||
|
|
||||||
|
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
|
||||||
|
// {
|
||||||
|
// mySetRealSource(
|
||||||
|
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
||||||
|
// ViewStreamDatabase.instance));
|
||||||
|
// }
|
||||||
|
|
||||||
|
// public DumpLoaderStream(string fileName)
|
||||||
|
// {
|
||||||
|
// if (fileName == null)
|
||||||
|
// throw new LoaderStream.Exception("File name cannot be null");
|
||||||
|
|
||||||
|
// for (int i = 0; i < 5; i++)
|
||||||
|
// try
|
||||||
|
// {
|
||||||
|
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
||||||
|
// {
|
||||||
|
// IClientDeserializer deser = new ClientUniversalDeserializer();
|
||||||
|
// byte[] the_ops;
|
||||||
|
// Array the_rows;
|
||||||
|
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
|
||||||
|
|
||||||
|
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
|
||||||
|
// }
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// catch (IOException)
|
||||||
|
// {
|
||||||
|
// System.Threading.Thread.Sleep(50);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
// This is the other side of the SaverStream, its "loads" the stream and fires once
|
|
||||||
[DataStreamUserRespresentation("loader")]
|
[DataStreamUserRespresentation("loader")]
|
||||||
public class LoaderStream : ViewStream {
|
public class LoaderStream : ViewStream {
|
||||||
|
|
||||||
|
|||||||
@@ -5,11 +5,10 @@ using System.Text;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using BCG;
|
using bts;
|
||||||
|
|
||||||
namespace bts {
|
namespace BCG {
|
||||||
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
|
// Writes all the udpates to a stream in a file.
|
||||||
// which just writes one update, as a create)
|
|
||||||
// File naming:
|
// File naming:
|
||||||
// token_epoch_yyyyMMdd_HHmmss
|
// token_epoch_yyyyMMdd_HHmmss
|
||||||
// File location: directory(passed)\token\
|
// File location: directory(passed)\token\
|
||||||
@@ -17,11 +16,6 @@ namespace bts {
|
|||||||
// data:
|
// data:
|
||||||
// epochtime, size, serialized data
|
// epochtime, size, serialized data
|
||||||
[DataStreamUserRespresentation("pers")]
|
[DataStreamUserRespresentation("pers")]
|
||||||
// Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
|
|
||||||
// which maps the name of the directory where the streams are saved. The directory (3rd arg) is the
|
|
||||||
// parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
|
|
||||||
// file is created. The file names are token_epoch_datetime
|
|
||||||
// the first entry in a file is the complete state
|
|
||||||
public class PersistorStream : ViewStream {
|
public class PersistorStream : ViewStream {
|
||||||
private ViewStream _source_stream;
|
private ViewStream _source_stream;
|
||||||
private string _token;
|
private string _token;
|
||||||
@@ -95,10 +89,10 @@ namespace bts {
|
|||||||
void _on_change(object new_value) {
|
void _on_change(object new_value) {
|
||||||
DataPacket dp = new_value as DataPacket;
|
DataPacket dp = new_value as DataPacket;
|
||||||
byte[] serialized_update = null;
|
byte[] serialized_update = null;
|
||||||
bool first_write = _serializer == null || _n_updates_in_current_file == 0;
|
bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
|
||||||
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
||||||
if (first_write) {
|
if (_write_create) {
|
||||||
// first file always starts with create
|
// new files always starts with create
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
||||||
} else
|
} else
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
||||||
@@ -106,8 +100,8 @@ namespace bts {
|
|||||||
long utc_epoch = epoch_now();
|
long utc_epoch = epoch_now();
|
||||||
|
|
||||||
//time, length of data, data
|
//time, length of data, data
|
||||||
//bw.Write(utc_epoch);
|
bw.Write(utc_epoch);
|
||||||
//bw.Write(serialized_update.Length);
|
bw.Write(serialized_update.Length);
|
||||||
bw.Write(serialized_update);
|
bw.Write(serialized_update);
|
||||||
_out_fs.Flush();
|
_out_fs.Flush();
|
||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
|
|||||||
@@ -0,0 +1,190 @@
|
|||||||
|
// using System;
|
||||||
|
// using System.Collections.Generic;
|
||||||
|
// using System.Linq;
|
||||||
|
// using System.Text;
|
||||||
|
// using System.IO;
|
||||||
|
// using System.Runtime.Serialization.Formatters.Binary;
|
||||||
|
// using BCG.Utilities.General;
|
||||||
|
// using bts.collections;
|
||||||
|
// using System.Reflection.Metadata;
|
||||||
|
|
||||||
|
|
||||||
|
// namespace BCG {
|
||||||
|
|
||||||
|
// public class PlaybackStream : ViewStream, IWorker {
|
||||||
|
// private readonly PlaybackType _type;
|
||||||
|
// private BlueTable _current_table;
|
||||||
|
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||||
|
// // these are the packets already downloaded from the current file
|
||||||
|
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
|
||||||
|
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
|
||||||
|
// private IEnumerable<string> _file_names;
|
||||||
|
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
|
||||||
|
// private DateTime _playback_t0, _playback_t1;
|
||||||
|
// private bool _all_files_read = false;
|
||||||
|
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
|
||||||
|
|
||||||
|
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
|
||||||
|
// _type = PlaybackType.Streaming;
|
||||||
|
// _playback_t0 = PersistorStream.parse_datetime(datetime_start);
|
||||||
|
// _playback_t1 = PersistorStream.parse_datetime(datetime_end);
|
||||||
|
// string token_directory_name = $"{directory_name}\\{token}";
|
||||||
|
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
|
||||||
|
// List<string> files_in_range = new List<string>();
|
||||||
|
// if (Directory.Exists(token_directory_name)) {
|
||||||
|
// _file_names =
|
||||||
|
// Directory.GetFiles(token_directory_name).Where(f => {
|
||||||
|
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
|
||||||
|
// return file_time > _playback_t0 && file_time < _playback_t1;
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnFirstSubscriber() {
|
||||||
|
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
|
||||||
|
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
|
||||||
|
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
|
||||||
|
// if (snapshot_packet != null) {
|
||||||
|
// foreach (PlaybackPacket p in _snapshot_data.packets) {
|
||||||
|
// byte[] ops;
|
||||||
|
// Array row_array;
|
||||||
|
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
||||||
|
// for (int i = 0; i < ops.Length; i++) {
|
||||||
|
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
|
// byte op = ops[i];
|
||||||
|
// if (op == StreamOpCode.create) {
|
||||||
|
// IBlueRowCollection full_table = rows_args[0];
|
||||||
|
// _current_table = new BlueTable(full_table);
|
||||||
|
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
|
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
|
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
// } else {
|
||||||
|
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
|
// }
|
||||||
|
// if (p.idx == snapshot_packet.idx) {
|
||||||
|
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
// private void _load_packets_from_current_file() {
|
||||||
|
// int curr_idx = _current_fs.Item1;
|
||||||
|
// FileStream curr_fs = _current_fs.Item2;
|
||||||
|
// _packets = new List<PlaybackPacket>();
|
||||||
|
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private bool current_file_all_replayed {
|
||||||
|
// get {
|
||||||
|
// bool rv = true;
|
||||||
|
// _packets.ForEach(p => rv = rv && p.replayed);
|
||||||
|
// return rv;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnLastSubscriber() {
|
||||||
|
// var fs = _current_fs.Item2;
|
||||||
|
// if (fs != null) {
|
||||||
|
// fs.Close();
|
||||||
|
// }
|
||||||
|
// ViewStreamDatabase.instance.removeWorker(this);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // public event PlaybackStreamProgress onProgress;
|
||||||
|
// // public event Action onFileExhausted;
|
||||||
|
// private PlaybackPacket _locate_packet_to_replay() {
|
||||||
|
// foreach (var p in _packets) {
|
||||||
|
// // returns the first non replayed packet
|
||||||
|
// if (!p.replayed)
|
||||||
|
// return p;
|
||||||
|
// }
|
||||||
|
// //else
|
||||||
|
// {
|
||||||
|
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
|
||||||
|
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
|
||||||
|
// if (onProgress != null)
|
||||||
|
// onProgress(myFileReader.Position, myFileReader.Length);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// #region IWorker Members
|
||||||
|
|
||||||
|
// public void doWork() {
|
||||||
|
// if (_all_files_read)
|
||||||
|
// return;
|
||||||
|
// if (current_file_all_replayed) {
|
||||||
|
// _load_next_file();
|
||||||
|
// }
|
||||||
|
// PlaybackPacket curr_packet = _locate_packet_to_replay();
|
||||||
|
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
|
||||||
|
// byte[] ops;
|
||||||
|
// Array row_array;
|
||||||
|
// // what a stupid interface!
|
||||||
|
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
||||||
|
// _fire_packet(ops, row_array);
|
||||||
|
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
||||||
|
// } else {
|
||||||
|
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
|
||||||
|
// // this is the time we need to fire (replay) this packet
|
||||||
|
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
|
||||||
|
// long now_epoch = PersistorStream.epoch_now();
|
||||||
|
// // fire when we are past the
|
||||||
|
// if (now_epoch >= time_to_fire) {
|
||||||
|
// byte[] ops;
|
||||||
|
// Array row_array;
|
||||||
|
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
||||||
|
// _fire_packet(ops, row_array);
|
||||||
|
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void _fire_packet(byte[] ops, Array row_array) {
|
||||||
|
// for (int i = 0; i < ops.Length; i++) {
|
||||||
|
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
|
// byte op = ops[i];
|
||||||
|
// if (op == StreamOpCode.create) {
|
||||||
|
// IBlueRowCollection full_table = rows_args[0];
|
||||||
|
// _current_table = new BlueTable(full_table);
|
||||||
|
// //_current_table.addRows(full_table);
|
||||||
|
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
||||||
|
// (myLastValue as DataPacket).newRows));
|
||||||
|
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
|
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
|
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
||||||
|
// } else {
|
||||||
|
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void _load_next_file() {
|
||||||
|
// var new_idx = _current_fs.Item1 + 1;
|
||||||
|
// if (new_idx >= _file_names.Count()) {
|
||||||
|
// _all_files_read = true;
|
||||||
|
// bts.Log.Gen.info("All files read");
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
// var prev_fs = _current_fs.Item2;
|
||||||
|
// if (prev_fs != null)
|
||||||
|
// prev_fs.Close();
|
||||||
|
// var new_filename = _file_names.ElementAt(new_idx);
|
||||||
|
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
|
||||||
|
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
|
||||||
|
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
|
||||||
|
// _load_packets_from_current_file();
|
||||||
|
// }
|
||||||
|
|
||||||
|
// #endregion
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
// using System;
|
||||||
|
// using System.Collections.Generic;
|
||||||
|
// using System.Linq;
|
||||||
|
// using System.Text;
|
||||||
|
// using System.IO;
|
||||||
|
|
||||||
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// public class RecorderStream : ViewStream
|
||||||
|
// {
|
||||||
|
// private ViewStream mySource;
|
||||||
|
// private DateTime myStartTime;
|
||||||
|
// private FileStream myOutFile;
|
||||||
|
// private string myFileName;
|
||||||
|
// private IBlueDataSerializer mySerializer;
|
||||||
|
|
||||||
|
// public RecorderStream(ViewStream src, string recordFile)
|
||||||
|
// {
|
||||||
|
// mySource = src;
|
||||||
|
// myFileName = recordFile;
|
||||||
|
|
||||||
|
// myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnFirstSubscriber()
|
||||||
|
// {
|
||||||
|
// myStartTime = DateTime.Now;
|
||||||
|
// mySource.onChange += new ChangeDelegate(mySource_onChange);
|
||||||
|
// myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnLastSubscriber()
|
||||||
|
// {
|
||||||
|
// mySource.onChange -= mySource_onChange;
|
||||||
|
// myOutFile.Close();
|
||||||
|
// }
|
||||||
|
|
||||||
|
// void mySource_onChange(object newValue)
|
||||||
|
// {
|
||||||
|
// DataPacket dp = (DataPacket)newValue;
|
||||||
|
|
||||||
|
// BlueTable toWrite = new BlueTable(dp.schema);
|
||||||
|
|
||||||
|
// if (mySerializer == null)
|
||||||
|
// {
|
||||||
|
// mySerializer = new UniversalSerializer(dp.schema);
|
||||||
|
// }
|
||||||
|
// foreach (IBlueRow r in dp.newRows)
|
||||||
|
// toWrite.addRow(r);
|
||||||
|
|
||||||
|
// byte[] result;
|
||||||
|
|
||||||
|
// if (myLastValue == null)
|
||||||
|
// mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
|
||||||
|
// else
|
||||||
|
// mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
|
||||||
|
|
||||||
|
// BinaryWriter bw = new BinaryWriter(myOutFile);
|
||||||
|
// //write down what time this happened
|
||||||
|
// bw.Write((DateTime.Now - myStartTime).TotalSeconds);
|
||||||
|
// bw.Write(result.Length);
|
||||||
|
// bw.Write(result);
|
||||||
|
// myOutFile.Flush();
|
||||||
|
|
||||||
|
// myFireEvent(dp);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -5,8 +5,7 @@ using System.IO;
|
|||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
// this saves the updates as a Create on the same file, each update calls -save, which
|
// this saves the total state in the filr provided in the cons
|
||||||
// reqrites the whole data to the packet. The other side of this stream is the Loader
|
|
||||||
[DataStreamUserRespresentation("saver")]
|
[DataStreamUserRespresentation("saver")]
|
||||||
public class SaverStream : ViewStream {
|
public class SaverStream : ViewStream {
|
||||||
private ViewStream _stream;
|
private ViewStream _stream;
|
||||||
@@ -32,6 +31,7 @@ namespace bts {
|
|||||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||||
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||||
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -9,29 +9,15 @@
|
|||||||
// using System.Reflection.Metadata;
|
// using System.Reflection.Metadata;
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG {
|
||||||
public class SnapshotStream : ViewStream {
|
// public class SnapshotStream : ViewStream {
|
||||||
public enum SpecificSnapshot {
|
|
||||||
FirstOfDay,
|
|
||||||
LastOfDay
|
|
||||||
};
|
|
||||||
// private BlueTable _current_table;
|
// private BlueTable _current_table;
|
||||||
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||||
// public DateTime _snapshot_time;
|
// public DateTime _snapshot_time;
|
||||||
// public string _snapshot_file;
|
// public string _snapshot_file;
|
||||||
// public IEnumerable<PlaybackPacket> _packets;
|
// public IEnumerable<PlaybackPacket> _packets;
|
||||||
// public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
|
|
||||||
// DateTime dt;
|
|
||||||
// var parsed = DateTime.TryParse(snapshot_date, out dt);
|
|
||||||
// dt = dt.Date;
|
|
||||||
// bts.Log.assert(parsed);
|
|
||||||
// SpecificSnapshot specific_snapshot_enum;
|
|
||||||
// parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
|
|
||||||
// bts.Log.assert(parsed);
|
|
||||||
// string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
|
|
||||||
// ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
|
|
||||||
// mySetRealSource(rs);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public SnapshotStream(string snapshot_dt, string directory_name, string token) {
|
// public SnapshotStream(string snapshot_dt, string directory_name, string token) {
|
||||||
// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
|
// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
|
||||||
@@ -125,5 +111,5 @@ namespace BCG {
|
|||||||
|
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -3,7 +3,6 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using bts;
|
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
@@ -38,20 +37,6 @@ namespace BCG {
|
|||||||
public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
|
public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
|
||||||
}
|
}
|
||||||
public static class StreamPersistorUtils {
|
public static class StreamPersistorUtils {
|
||||||
public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
|
|
||||||
DateTime max_dt = DateTime.MinValue;
|
|
||||||
string rv = String.Empty;
|
|
||||||
string[] file_names = Directory.GetFiles(directory_name);
|
|
||||||
for (int i = 0; i < file_names.Count() - 1; i++) {
|
|
||||||
string fn = file_names[i];
|
|
||||||
var dt = parse_datetime_from_filename(fn);
|
|
||||||
if (dt > max_dt) {
|
|
||||||
max_dt = dt;
|
|
||||||
rv = fn;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
|
public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
|
||||||
string[] file_names = Directory.GetFiles(directory_name);
|
string[] file_names = Directory.GetFiles(directory_name);
|
||||||
for (int i = 0; i < file_names.Count() - 1; i++) {
|
for (int i = 0; i < file_names.Count() - 1; i++) {
|
||||||
|
|||||||
@@ -1,10 +1,16 @@
|
|||||||
|
using BCG;
|
||||||
|
using System;
|
||||||
|
using System.Data;
|
||||||
|
using System.Collections;
|
||||||
|
using BCG.Utilities;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
|
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
|
|
||||||
namespace BCG.StreamWorld.Addition
|
namespace BCG.StreamWorld.Addition
|
||||||
{
|
{
|
||||||
[DataStreamUserRespresentation (false)]
|
[DataStreamUserRespresentation (false)]
|
||||||
@@ -145,6 +151,7 @@ namespace BCG
|
|||||||
private string myFilterString;
|
private string myFilterString;
|
||||||
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
||||||
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
|
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
|
||||||
|
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||||
|
|
||||||
#region constructors
|
#region constructors
|
||||||
public delegate bool OnFilter(IBlueRow r);
|
public delegate bool OnFilter(IBlueRow r);
|
||||||
@@ -219,6 +226,7 @@ namespace BCG
|
|||||||
|
|
||||||
private void myHandleCallbacks(object newValue)
|
private void myHandleCallbacks(object newValue)
|
||||||
{
|
{
|
||||||
|
_timer.start();
|
||||||
DataPacket retVal;
|
DataPacket retVal;
|
||||||
DataPacket arrivedPacket = newValue as DataPacket;
|
DataPacket arrivedPacket = newValue as DataPacket;
|
||||||
myLastReceivedRows = arrivedPacket.newRows;
|
myLastReceivedRows = arrivedPacket.newRows;
|
||||||
@@ -247,6 +255,7 @@ namespace BCG
|
|||||||
if (myFilter.filter(r))
|
if (myFilter.filter(r))
|
||||||
myLastFiredRows.addRow(r);
|
myLastFiredRows.addRow(r);
|
||||||
}
|
}
|
||||||
|
_timer.stop();
|
||||||
myFireEvent(retVal);
|
myFireEvent(retVal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,42 +1,47 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections;
|
// using System.Collections;
|
||||||
|
// using BCG.Utilities;
|
||||||
|
|
||||||
using BCG.Utilities;
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// /// <summary>
|
||||||
|
// /// Computes the max for a given column
|
||||||
|
// /// </summary>
|
||||||
|
// public class MaxGroupByEvaluator : GroupByEvaluator
|
||||||
|
// {
|
||||||
|
|
||||||
namespace BCG {
|
// public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
|
||||||
/// <summary>
|
// {}
|
||||||
/// Computes the max for a given column
|
|
||||||
/// </summary>
|
|
||||||
public class MaxGroupByEvaluator : GroupByEvaluator {
|
|
||||||
|
|
||||||
public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
|
// public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
|
||||||
|
// : base (colMapEpression, outputCol, groupByCols) {}
|
||||||
|
|
||||||
public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
// public override IBlueRow evaluate (DataPacket dp)
|
||||||
: base(colMapEpression, outputCol, groupByCols) { }
|
// {
|
||||||
|
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
|
||||||
public override IBlueRow evaluate(DataPacket dp) {
|
// myOutputTable;
|
||||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
|
// string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
|
||||||
myOutputTable;
|
// IBlueRow currMaxRow = myOutputTable.getRow (hashString);
|
||||||
string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
// IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
|
||||||
IBlueRow currMaxRow = myOutputTable.getRow(hashString);
|
// if (maxRow != null)
|
||||||
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
|
// {
|
||||||
if (maxRow != null) {
|
// BlueEventLog.logAssert (currMaxRow != null);
|
||||||
bts.Log.assert(currMaxRow != null);
|
// BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
|
||||||
bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
|
// if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
|
||||||
if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
|
// {
|
||||||
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
|
// maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
|
||||||
if (maxRow == null)
|
// if (maxRow == null)
|
||||||
myOutputTable.removeRow(hashString);
|
// myOutputTable.removeRow (hashString);
|
||||||
else
|
// else
|
||||||
myOutputTable.addRow(maxRow, hashString);
|
// myOutputTable.addRow (maxRow, hashString);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
|
// maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
|
||||||
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
|
// // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
|
||||||
// otherwise if maxRow > currMaxRow then its replaced
|
// // otherwise if maxRow > currMaxRow then its replaced
|
||||||
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
|
// if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
|
||||||
myOutputTable.addRow(maxRow, hashString);
|
// myOutputTable.addRow (maxRow, hashString);
|
||||||
return myProject(myOutputTable.getRow(hashString));
|
// return myProject (myOutputTable.getRow (hashString));
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -1,15 +1,20 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
public class MaxStream : ViewStream {
|
// {
|
||||||
public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
|
// public class MaxStream : ViewStream
|
||||||
}
|
// {
|
||||||
public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
|
// public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// {
|
||||||
new MaxGroupByEvaluator(outputCol, inputCols)));
|
// }
|
||||||
}
|
// public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
|
||||||
|
// {
|
||||||
|
// mySetRealSource (new GroupByStream (stream,
|
||||||
|
// new MaxGroupByEvaluator (outputCol, inputCols)));
|
||||||
|
// }
|
||||||
|
|
||||||
public MaxStream(ViewStream stream, string outputCol)
|
// public MaxStream (ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
|
||||||
}
|
// {}
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
@@ -1,36 +1,40 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
// using BCG.Utilities;
|
||||||
|
|
||||||
using BCG.Utilities;
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// public class MinGroupByEvaluator : MaxGroupByEvaluator
|
||||||
|
// {
|
||||||
|
// public MinGroupByEvaluator (string outputCol, string[] groupByCols)
|
||||||
|
// : base (outputCol, groupByCols) {}
|
||||||
|
|
||||||
namespace BCG {
|
// public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
|
||||||
public class MinGroupByEvaluator : MaxGroupByEvaluator {
|
// base (colMapEpression, outputCol, groupByCols) {}
|
||||||
public MinGroupByEvaluator(string outputCol, string[] groupByCols)
|
|
||||||
: base(outputCol, groupByCols) { }
|
|
||||||
|
|
||||||
public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
|
// public override IBlueRow evaluate (DataPacket dp)
|
||||||
base(colMapEpression, outputCol, groupByCols) { }
|
// {
|
||||||
|
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
|
||||||
|
// string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
|
||||||
|
// IBlueRow currMinRow = myOutputTable.getRow (stringRep);
|
||||||
|
// IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
|
||||||
|
// if (minRow != null)
|
||||||
|
// {
|
||||||
|
// // BlueEventLog.logAssert (currMinRow != null);
|
||||||
|
// // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
|
||||||
|
// if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
|
||||||
|
// {
|
||||||
|
// minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
|
||||||
|
// if (minRow == null)
|
||||||
|
// myOutputTable.removeRow (stringRep);
|
||||||
|
// else
|
||||||
|
// myOutputTable.addRow (minRow, stringRep);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
|
||||||
|
// if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
|
||||||
|
// myOutputTable.addRow (minRow, stringRep);
|
||||||
|
// return myOutputTable.getRow (stringRep);
|
||||||
|
// }
|
||||||
|
|
||||||
public override IBlueRow evaluate(DataPacket dp) {
|
// }
|
||||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
// }
|
||||||
string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
|
||||||
IBlueRow currMinRow = myOutputTable.getRow(stringRep);
|
|
||||||
IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
|
|
||||||
if (minRow != null) {
|
|
||||||
// BlueEventLog.logAssert (currMinRow != null);
|
|
||||||
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
|
|
||||||
if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
|
|
||||||
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
|
|
||||||
if (minRow == null)
|
|
||||||
myOutputTable.removeRow(stringRep);
|
|
||||||
else
|
|
||||||
myOutputTable.addRow(minRow, stringRep);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
|
|
||||||
if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
|
|
||||||
myOutputTable.addRow(minRow, stringRep);
|
|
||||||
return myOutputTable.getRow(stringRep);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +1,17 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
public class MinStream : ViewStream {
|
// {
|
||||||
public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
|
// public class MinStream : ViewStream
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// {
|
||||||
new MinGroupByEvaluator(outputCol, inputCol)));
|
// public MinStream (ViewStream stream, string outputCol, string[] inputCol)
|
||||||
}
|
// {
|
||||||
|
// mySetRealSource (new GroupByStream (stream,
|
||||||
|
// new MinGroupByEvaluator (outputCol, inputCol)));
|
||||||
|
// }
|
||||||
|
|
||||||
public MinStream(ViewStream stream, string outputCol)
|
// public MinStream (ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
|
||||||
}
|
// {}
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
+14
-18
@@ -1,8 +1,10 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
using bts;
|
using System.Threading.Tasks;
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -12,7 +14,7 @@ namespace BCG
|
|||||||
private INonIncGroupByEvaluator _groupby_func;
|
private INonIncGroupByEvaluator _groupby_func;
|
||||||
private BlueRowHasher _row_hasher;
|
private BlueRowHasher _row_hasher;
|
||||||
private string[] _groupby_cols;
|
private string[] _groupby_cols;
|
||||||
private Stopwatch _timer = new();
|
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
|
|
||||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
||||||
@@ -31,27 +33,21 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
DataPacket dp = nv as DataPacket;
|
DataPacket dp = nv as DataPacket;
|
||||||
///grouping together
|
///grouping together
|
||||||
_timer.Restart();
|
_timer.start();
|
||||||
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
||||||
_timer.Stop();
|
_timer.stop();
|
||||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||||
if (groupElapsedMs >= 50) {
|
|
||||||
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
|
||||||
}
|
|
||||||
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
|
||||||
/// reducing
|
/// reducing
|
||||||
_timer.Restart();
|
_timer.start();
|
||||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||||
_timer.Stop();
|
_timer.stop();
|
||||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (applyElapsedMs >= 50)
|
if (_timer.lastIntervalTime > 50)
|
||||||
{
|
{
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
Log.DataFlow.info(
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
|
||||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
|
||||||
}
|
}
|
||||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,82 +1,94 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
/// <summary>
|
// {
|
||||||
/// Computes the sum for a given column
|
// /// <summary>
|
||||||
/// </summary>
|
// /// Computes the sum for a given column
|
||||||
public class SumGroupByEvaluator : GroupByEvaluator {
|
// /// </summary>
|
||||||
public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
|
// public class SumGroupByEvaluator : GroupByEvaluator
|
||||||
|
// {
|
||||||
|
// public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
|
||||||
|
// {}
|
||||||
|
|
||||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
// public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
|
||||||
: base(colMapEpression, outputCol, groupByCols) { }
|
// : base (colMapEpression, outputCol, groupByCols) {}
|
||||||
|
|
||||||
public override IBlueRow evaluate(DataPacket dp) {
|
// public override IBlueRow evaluate (DataPacket dp)
|
||||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
// {
|
||||||
|
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
|
||||||
IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
|
|
||||||
return sumRow;
|
// IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
|
||||||
}
|
// return sumRow;
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
|
// public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
|
||||||
private string[] myOutputCols, myGroupByCols;
|
// {
|
||||||
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
|
// private string[] myOutputCols, myGroupByCols;
|
||||||
myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
|
||||||
myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
// {
|
||||||
}
|
// myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
||||||
|
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
||||||
|
// }
|
||||||
|
|
||||||
public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
|
// public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
|
||||||
myOutputCols = output_columns;
|
// {
|
||||||
myGroupByCols = groupby_columns;
|
// myOutputCols = output_columns;
|
||||||
}
|
// myGroupByCols = groupby_columns;
|
||||||
|
// }
|
||||||
|
|
||||||
#region IGroupByEvaluator Members
|
// #region IGroupByEvaluator Members
|
||||||
|
|
||||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
|
// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
|
||||||
|
|
||||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
// public string[] groupByColumns { get { return myGroupByCols; } }
|
||||||
|
|
||||||
public void clear() {/*stateless*/}
|
// public void clear() {/*stateless*/}
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
|
// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
|
||||||
|
|
||||||
#endregion
|
// #endregion
|
||||||
}
|
// }
|
||||||
|
|
||||||
public class SumGroupByEvaluator<T> : GroupByEvaluator {
|
// public class SumGroupByEvaluator<T> : GroupByEvaluator
|
||||||
public SumGroupByEvaluator(string outputCol, string[] groupByCols)
|
// {
|
||||||
: base(outputCol, groupByCols) { }
|
// public SumGroupByEvaluator(string outputCol, string[] groupByCols)
|
||||||
|
// : base(outputCol, groupByCols)
|
||||||
|
// { }
|
||||||
|
|
||||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
// public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||||
: base(colMapEpression, outputCol, groupByCols) { }
|
// : base(colMapEpression, outputCol, groupByCols) { }
|
||||||
|
|
||||||
public override IBlueRow evaluate(DataPacket dp) {
|
// public override IBlueRow evaluate(DataPacket dp)
|
||||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
// {
|
||||||
|
// myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||||
|
|
||||||
IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
|
// IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
|
||||||
return sumRow;
|
// return sumRow;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
|
// public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
|
||||||
private string[] myOutputCols, myGroupByCols;
|
// {
|
||||||
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
|
// private string[] myOutputCols, myGroupByCols;
|
||||||
myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
|
||||||
myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
// {
|
||||||
}
|
// myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
||||||
|
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
||||||
|
// }
|
||||||
|
|
||||||
#region IGroupByEvaluator Members
|
// #region IGroupByEvaluator Members
|
||||||
|
|
||||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
|
// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
|
||||||
|
|
||||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
// public string[] groupByColumns { get { return myGroupByCols; } }
|
||||||
|
|
||||||
public void clear() {/*stateless*/}
|
// public void clear() {/*stateless*/}
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
|
// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
|
||||||
|
|
||||||
#endregion
|
// #endregion
|
||||||
}
|
// }
|
||||||
|
|
||||||
}
|
// }
|
||||||
@@ -1,89 +1,107 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections.Generic;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
public class NullMapper : IStreamMapper {
|
// {
|
||||||
private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
|
// public class NullMapper : IStreamMapper
|
||||||
|
// {
|
||||||
|
// private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
|
||||||
|
|
||||||
public NullMapper(string[] valid_cols) {
|
// public NullMapper(string[] valid_cols)
|
||||||
foreach (string c in valid_cols)
|
// {
|
||||||
_valid_cols[c] = true;
|
// foreach (string c in valid_cols)
|
||||||
_valid_cols["id"] = true;
|
// _valid_cols[c] = true;
|
||||||
}
|
// _valid_cols["id"] = true;
|
||||||
|
// }
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
|
// public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row) {
|
// public IBlueRow map(IBlueRow row)
|
||||||
MutableBlueRow out_row = new MutableBlueRow(row);
|
// {
|
||||||
foreach (string c in row.schema.columnNames)
|
// MutableBlueRow out_row = new MutableBlueRow(row);
|
||||||
if (!_valid_cols.ContainsKey(c))
|
// foreach (string c in row.schema.columnNames)
|
||||||
out_row.setValue(c, DBNull.Value);
|
// if (!_valid_cols.ContainsKey(c))
|
||||||
return out_row;
|
// out_row.setValue(c, DBNull.Value);
|
||||||
}
|
// return out_row;
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
public class SumStream : ViewStream {
|
// public class SumStream : ViewStream
|
||||||
public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
|
// {
|
||||||
string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
// public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
|
||||||
string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
// {
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
||||||
outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
|
// string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
||||||
new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
// mySetRealSource(new GroupByStream(stream,
|
||||||
}
|
// outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
|
||||||
|
// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
||||||
|
// }
|
||||||
|
|
||||||
public SumStream(ViewStream stream, string[] output_columns) {
|
// public SumStream(ViewStream stream, string[] output_columns)
|
||||||
ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
|
// {
|
||||||
rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
|
// ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
|
||||||
rs = new ProjectStream(rs, "!fake_col");
|
// rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
|
||||||
mySetRealSource(rs);
|
// rs = new ProjectStream(rs, "!fake_col");
|
||||||
}
|
// mySetRealSource(rs);
|
||||||
|
// }
|
||||||
|
|
||||||
public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
|
// public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
|
||||||
string[] output_columns = space_seperated_output_columns.Split(' ');
|
// {
|
||||||
string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
|
// string[] output_columns = space_seperated_output_columns.Split(' ');
|
||||||
mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
|
// string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
|
||||||
}
|
// mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
|
||||||
|
// }
|
||||||
|
|
||||||
public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
|
// public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
|
||||||
ViewStream rs = null;
|
// {
|
||||||
groupby_columns = groupby_columns ?? new string[] { };
|
// ViewStream rs = null;
|
||||||
if (groupby_columns.Length == 0) {
|
// groupby_columns = groupby_columns ?? new string[] {};
|
||||||
// rs = new AddColumnsStream(stream, "fake_col=0");
|
// if (groupby_columns.Length == 0)
|
||||||
// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
|
// {
|
||||||
// rs = new ProjectStream(rs, "!fake_col");
|
// rs = new AddColumnsStream(stream, "fake_col=0");
|
||||||
} else {
|
// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
|
||||||
rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
|
// rs = new ProjectStream(rs, "!fake_col");
|
||||||
}
|
// }
|
||||||
if (force_null_into_other_columns) {
|
// else
|
||||||
List<string> valid_cols = new List<string>(output_columns);
|
// {
|
||||||
valid_cols.AddRange(groupby_columns);
|
// rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
|
||||||
rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
|
// }
|
||||||
}
|
// if (force_null_into_other_columns)
|
||||||
mySetRealSource(rs);
|
// {
|
||||||
}
|
// List<string> valid_cols = new List<string>(output_columns);
|
||||||
|
// valid_cols.AddRange(groupby_columns);
|
||||||
|
// rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
|
||||||
|
// }
|
||||||
|
// mySetRealSource(rs);
|
||||||
|
// }
|
||||||
|
|
||||||
public SumStream(ViewStream stream, string outputCol)
|
// public SumStream (ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
|
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
|
||||||
}
|
// {
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
public class IntSumStream : ViewStream {
|
// public class IntSumStream : ViewStream
|
||||||
public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
|
// {
|
||||||
string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
// public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
|
||||||
string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
// {
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
||||||
outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
|
// string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
||||||
new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
// mySetRealSource(new GroupByStream(stream,
|
||||||
}
|
// outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
|
||||||
|
// new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
||||||
|
// }
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
public class SumStream<T> : ViewStream {
|
// public class SumStream<T> : ViewStream
|
||||||
public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
|
// {
|
||||||
string[] inputCols = spaceSeparatedInputCols.Split(' ');
|
// public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
|
||||||
mySetRealSource(new GroupByStream(stream,
|
// {
|
||||||
new SumGroupByEvaluator<T>(outputCol, inputCols)));
|
// string[] inputCols = spaceSeparatedInputCols.Split(' ');
|
||||||
}
|
// mySetRealSource(new GroupByStream(stream,
|
||||||
}
|
// new SumGroupByEvaluator<T>(outputCol, inputCols)));
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
using ZstdSharp.Unsafe;
|
|
||||||
|
|
||||||
namespace bts {
|
|
||||||
public class RowDiffStream : ViewStream {
|
|
||||||
private ViewStream _vs;
|
|
||||||
private readonly string _order_col, _col_1, _col_2;
|
|
||||||
private readonly int _diff;
|
|
||||||
private IBlueTableSchema _out_schema;
|
|
||||||
private readonly string _diff_col_name;
|
|
||||||
public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
|
|
||||||
_vs = vs;
|
|
||||||
_order_col = order_col;
|
|
||||||
_col_1 = col_1;
|
|
||||||
_col_2 = col_2;
|
|
||||||
_diff = diff;
|
|
||||||
_diff_col_name = diff_col_name;
|
|
||||||
}
|
|
||||||
|
|
||||||
private string _diff_col_name_perc {
|
|
||||||
get {
|
|
||||||
return $"{_diff_col_name}_percentage";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_vs.onChange += _handle_vs;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_vs.onChange -= _handle_vs;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _handle_vs(object dp_obj) {
|
|
||||||
var dp = dp_obj as DataPacket;
|
|
||||||
IBlueRowCollection nrows = dp.newRows;
|
|
||||||
_out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
|
|
||||||
_out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
|
|
||||||
List<IBlueRow> row_list = new List<IBlueRow>();
|
|
||||||
foreach (IBlueRow r in nrows)
|
|
||||||
row_list.Add(r);
|
|
||||||
//OrderBy sorts in ascedning order
|
|
||||||
var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
|
|
||||||
var sorted_row_array = sorted_rows.ToArray();
|
|
||||||
// reverse it to make it descending
|
|
||||||
sorted_rows.Reverse();
|
|
||||||
//sort rows so we can do the diff
|
|
||||||
OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
|
|
||||||
int i = 0;
|
|
||||||
foreach (IBlueRow r in sorted_rows) {
|
|
||||||
MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
|
|
||||||
//So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
|
|
||||||
// we assume x - y, and name variables like that
|
|
||||||
double x = r.get<double>(_col_1);
|
|
||||||
int y_row_idx = i + _diff;
|
|
||||||
double diff = System.Double.NaN;
|
|
||||||
double perc = System.Double.NaN;
|
|
||||||
if (y_row_idx < sorted_rows.Count()) {
|
|
||||||
var y = sorted_rows[y_row_idx].get<double>(_col_2);
|
|
||||||
diff = x - y;
|
|
||||||
perc = 100.0 * diff / y;
|
|
||||||
}
|
|
||||||
nr.setValue(_diff_col_name, diff);
|
|
||||||
nr.setValue(_diff_col_name_perc, perc);
|
|
||||||
rv.addRow(nr);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -27,8 +27,7 @@ namespace BCG
|
|||||||
string prependString = isLeft ? "left" : "right";
|
string prependString = isLeft ? "left" : "right";
|
||||||
return String.Concat (prependString, colName).GetHashCode ();
|
return String.Concat (prependString, colName).GetHashCode ();
|
||||||
}
|
}
|
||||||
public override bool Equals (object? obj) =>
|
public override bool Equals (object obj) {return GetHashCode () == obj.GetHashCode ();}
|
||||||
GetHashCode() == obj?.GetHashCode();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
public class JoinStream : ViewStream
|
public class JoinStream : ViewStream
|
||||||
{
|
{
|
||||||
protected InternalJoinStream? myIjs;
|
protected InternalJoinStream myIjs;
|
||||||
protected ProjectStream? myPs;
|
protected ProjectStream myPs;
|
||||||
protected ColumnMap? myLeftCm, myRightCm;
|
protected ColumnMap myLeftCm, myRightCm;
|
||||||
protected bool myIsProject = true;
|
protected bool myIsProject = true;
|
||||||
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
||||||
|
|
||||||
|
|||||||
+13
-11
@@ -1,10 +1,10 @@
|
|||||||
using bts;
|
using System;
|
||||||
|
|
||||||
using mbts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -40,7 +40,8 @@ namespace BCG
|
|||||||
|
|
||||||
public void _on_stream_cb(object dp_obj)
|
public void _on_stream_cb(object dp_obj)
|
||||||
{
|
{
|
||||||
var timer = Stopwatch.StartNew();
|
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
|
timer.start();
|
||||||
DataPacket dp = (DataPacket)dp_obj;
|
DataPacket dp = (DataPacket)dp_obj;
|
||||||
if (_schema == null)
|
if (_schema == null)
|
||||||
{
|
{
|
||||||
@@ -66,12 +67,13 @@ namespace BCG
|
|||||||
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
timer.Stop();
|
timer.stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (timer.ElapsedMilliseconds >= 50) {
|
if (timer.lastIntervalTime > 50)
|
||||||
|
{
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
|
string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
||||||
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||||
}
|
}
|
||||||
if (on_new_data != null)
|
if (on_new_data != null)
|
||||||
on_new_data();
|
on_new_data();
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
using bts;
|
using System;
|
||||||
|
|
||||||
using mbts;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -114,7 +114,8 @@ namespace BCG
|
|||||||
if (_is_ready())
|
if (_is_ready())
|
||||||
{
|
{
|
||||||
_construct_schema();
|
_construct_schema();
|
||||||
var timer = Stopwatch.StartNew();
|
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
|
timer.start();
|
||||||
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
||||||
List<object[]> output_rows = new List<object[]>();
|
List<object[]> output_rows = new List<object[]>();
|
||||||
|
|
||||||
@@ -137,12 +138,13 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
||||||
timer.Stop();
|
timer.stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (timer.ElapsedMilliseconds >= 250) {
|
if (timer.lastIntervalTime >= 250)
|
||||||
|
{
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
||||||
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||||
}
|
}
|
||||||
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
@@ -152,12 +154,15 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
{
|
{
|
||||||
|
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
IBlueRowCollection output_rows;
|
IBlueRowCollection output_rows;
|
||||||
var timer = Stopwatch.StartNew();
|
timer.start();
|
||||||
output_rows = _output_mapper.map_rows(unmapped_rows);
|
output_rows = _output_mapper.map_rows(unmapped_rows);
|
||||||
timer.Stop();
|
timer.stop();
|
||||||
if (timer.ElapsedMilliseconds >= 50) {
|
if (timer.lastIntervalTime >= 50)
|
||||||
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
{
|
||||||
|
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
|
||||||
|
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||||
}
|
}
|
||||||
return output_rows;
|
return output_rows;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Text;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using BCG.Utilities;
|
||||||
using mbts;
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
using Spark.Core.Collections;
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
@@ -222,8 +226,7 @@ namespace BCG
|
|||||||
|
|
||||||
#region IDictionary Members
|
#region IDictionary Members
|
||||||
|
|
||||||
public void Add(object key, object? value) =>
|
public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
|
||||||
throw new NotSupportedException("IBlueRow is readonly");
|
|
||||||
|
|
||||||
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
|
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
|
||||||
|
|
||||||
@@ -253,9 +256,10 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public object? this[object key] {
|
public object this[object key]
|
||||||
get => key is string str ? this[str] : this[(int)key];
|
{
|
||||||
set => throw new NotSupportedException("IBlueRow is readonly");
|
get { return key is string ? this[key.ToString()] : this[(int)key]; }
|
||||||
|
set { throw new NotSupportedException("IBlueRow is readonly"); }
|
||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
@@ -290,6 +294,7 @@ namespace BCG
|
|||||||
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
||||||
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
||||||
private bool myManageUnmappedLeftRows = false;
|
private bool myManageUnmappedLeftRows = false;
|
||||||
|
protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
|
||||||
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
||||||
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
||||||
protected JoinProjectHelper myJoinProjectHelper;
|
protected JoinProjectHelper myJoinProjectHelper;
|
||||||
@@ -356,6 +361,7 @@ namespace BCG
|
|||||||
|
|
||||||
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
||||||
{
|
{
|
||||||
|
myTimer.start();
|
||||||
myInitSchema(dp.schema, isLeft);
|
myInitSchema(dp.schema, isLeft);
|
||||||
BlueTable dels = null, adds = null;
|
BlueTable dels = null, adds = null;
|
||||||
if (myState != null)
|
if (myState != null)
|
||||||
@@ -365,6 +371,7 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
||||||
myAddRows(dp.addedRows, isLeft, myState, adds);
|
myAddRows(dp.addedRows, isLeft, myState, adds);
|
||||||
|
myTimer.stop();
|
||||||
if (myState != null)
|
if (myState != null)
|
||||||
{
|
{
|
||||||
myFireEvent(new DataPacket(myState, adds, dels));
|
myFireEvent(new DataPacket(myState, adds, dels));
|
||||||
|
|||||||
@@ -91,8 +91,7 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Add (object key, object? value) =>
|
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
throw new NotSupportedException ("The IBlueRow is readonly");
|
|
||||||
|
|
||||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,13 @@
|
|||||||
|
using System;
|
||||||
|
using System.Data;
|
||||||
|
using System.Collections;
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
|
|
||||||
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
|
|
||||||
|
|
||||||
@@ -13,9 +17,9 @@ namespace BCG {
|
|||||||
/// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id
|
/// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id
|
||||||
/// is preserved across the rows, ie row["id"] = f(row)["id"]
|
/// is preserved across the rows, ie row["id"] = f(row)["id"]
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[DataStreamUserRespresentation (false)]
|
[DataStreamUserRespresentation(false)]
|
||||||
public class MapStream : MethodDrivenStream
|
public class MapStream : MethodDrivenStream {
|
||||||
{
|
private HighPerformanceTimer t = new HighPerformanceTimer();
|
||||||
protected IStreamMapper myMapper;
|
protected IStreamMapper myMapper;
|
||||||
protected IChangeableStreamMapper myChangeableMapper = null;
|
protected IChangeableStreamMapper myChangeableMapper = null;
|
||||||
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
||||||
|
|||||||
@@ -8,33 +8,26 @@ using System;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace bts
|
namespace bts {
|
||||||
{
|
public class PubSubTestingStream : ViewStream, IWorker {
|
||||||
public class PubSubTestingStream : ViewStream, IWorker
|
public class PayloadMessage {
|
||||||
{
|
|
||||||
public class PayloadMessage
|
|
||||||
{
|
|
||||||
public int msg_id = 0;
|
public int msg_id = 0;
|
||||||
public string guid;
|
public string guid;
|
||||||
public Int64 travel_epoch_ms = 0;
|
public Int64 travel_epoch_ms = 0;
|
||||||
public byte[] payload = null;
|
public byte[] payload = null;
|
||||||
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
|
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
|
||||||
{
|
|
||||||
msg_id = msg_id_args;
|
msg_id = msg_id_args;
|
||||||
travel_epoch_ms = send_epoch_args;
|
travel_epoch_ms = send_epoch_args;
|
||||||
payload = new byte[payload_size];
|
payload = new byte[payload_size];
|
||||||
for (int i = 0; i < payload_size; i++)
|
for (int i = 0; i < payload_size; i++) {
|
||||||
{
|
|
||||||
payload[i] = payload_byte;
|
payload[i] = payload_byte;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
private PayloadMessage() { }
|
private PayloadMessage() { }
|
||||||
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
|
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
|
||||||
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
|
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public byte[] serialize()
|
public byte[] serialize() {
|
||||||
{
|
|
||||||
var str = new MemoryStream();
|
var str = new MemoryStream();
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
bw.Write(msg_id);
|
bw.Write(msg_id);
|
||||||
@@ -47,8 +40,7 @@ namespace bts
|
|||||||
return bytes;
|
return bytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
public static PayloadMessage deserialize(byte[] ser_bytes) {
|
||||||
{
|
|
||||||
var str = new MemoryStream(ser_bytes);
|
var str = new MemoryStream(ser_bytes);
|
||||||
BinaryReader br = new BinaryReader(str);
|
BinaryReader br = new BinaryReader(str);
|
||||||
int msg_id = br.ReadInt32();
|
int msg_id = br.ReadInt32();
|
||||||
@@ -56,8 +48,7 @@ namespace bts
|
|||||||
Int64 epoch = br.ReadInt64();
|
Int64 epoch = br.ReadInt64();
|
||||||
int len = br.ReadInt32();
|
int len = br.ReadInt32();
|
||||||
var payload = br.ReadBytes(len);
|
var payload = br.ReadBytes(len);
|
||||||
return new PayloadMessage()
|
return new PayloadMessage() {
|
||||||
{
|
|
||||||
msg_id = msg_id,
|
msg_id = msg_id,
|
||||||
guid = guid,
|
guid = guid,
|
||||||
travel_epoch_ms = epoch,
|
travel_epoch_ms = epoch,
|
||||||
@@ -82,8 +73,7 @@ namespace bts
|
|||||||
private readonly int _payload_sz;
|
private readonly int _payload_sz;
|
||||||
private const byte _payload_byte = (byte)'A';
|
private const byte _payload_byte = (byte)'A';
|
||||||
private readonly string _tag_name;
|
private readonly string _tag_name;
|
||||||
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
|
||||||
{
|
|
||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
_payload_sz = payload_size;
|
_payload_sz = payload_size;
|
||||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||||
@@ -106,53 +96,43 @@ namespace bts
|
|||||||
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
||||||
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
|
||||||
_vsd.addWorker(this);
|
_vsd.addWorker(this);
|
||||||
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
|
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
|
||||||
th.IsBackground = true;
|
th.IsBackground = true;
|
||||||
th.Start();
|
th.Start();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
|
||||||
_vsd.removeWorker(this);
|
_vsd.removeWorker(this);
|
||||||
_continue_sub_thread = false;
|
_continue_sub_thread = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void _log_updates(IEnumerable<PubSubServerMessage> updates)
|
void _log_updates(IEnumerable<PubSubServerMessage> updates) {
|
||||||
{
|
|
||||||
var idx = 0;
|
var idx = 0;
|
||||||
foreach (var msg in updates)
|
foreach (var msg in updates) {
|
||||||
{
|
|
||||||
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
|
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
|
||||||
idx++;
|
idx++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _subscribe_thread_func()
|
private void _subscribe_thread_func() {
|
||||||
{
|
|
||||||
// subscription is done in another thread
|
// subscription is done in another thread
|
||||||
SubscriberCore sub_core = new SubscriberCore();
|
SubscriberCore sub_core = new SubscriberCore();
|
||||||
sub_core.subscribe(_tag_name);
|
sub_core.subscribe(_tag_name);
|
||||||
while (_continue_sub_thread)
|
while (_continue_sub_thread) {
|
||||||
{
|
|
||||||
// TODO recv_id
|
// TODO recv_id
|
||||||
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
||||||
if (updates == null || updates.Count() == 0)
|
if (updates == null || updates.Count() == 0)
|
||||||
continue;
|
continue;
|
||||||
_log_updates(updates);
|
_log_updates(updates);
|
||||||
_n_updates += updates.Count();
|
_n_updates += updates.Count();
|
||||||
if (updates.Count() > 0)
|
if (updates.Count() > 0) {
|
||||||
{
|
|
||||||
var server_msg = updates.Last();
|
var server_msg = updates.Last();
|
||||||
if (server_msg.data != null && server_msg.data.Length > 0)
|
if (server_msg.data != null && server_msg.data.Length > 0) {
|
||||||
{
|
|
||||||
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
||||||
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
// why are we getting empty or null data ? (server is sending data regardless?)
|
// why are we getting empty or null data ? (server is sending data regardless?)
|
||||||
}
|
}
|
||||||
_fire_dirty = true;
|
_fire_dirty = true;
|
||||||
@@ -160,36 +140,28 @@ namespace bts
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _publish_next_packet()
|
private void _publish_next_packet() {
|
||||||
{
|
if (_last_sent_msg == null) {
|
||||||
if (_last_sent_msg == null)
|
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
|
||||||
{
|
|
||||||
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
|
|
||||||
{
|
|
||||||
guid = Guid.NewGuid().ToString()
|
guid = Guid.NewGuid().ToString()
|
||||||
};
|
};
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
_last_sent_msg.msg_id++;
|
_last_sent_msg.msg_id++;
|
||||||
_last_sent_msg.guid = Guid.NewGuid().ToString();
|
_last_sent_msg.guid = Guid.NewGuid().ToString();
|
||||||
_last_sent_msg.travel_epoch_ms = epoch_now();
|
_last_sent_msg.travel_epoch_ms = epoch_now();
|
||||||
}
|
}
|
||||||
var ser_bytes = _last_sent_msg.serialize();
|
var ser_bytes = _last_sent_msg.serialize();
|
||||||
_n_pubs++;
|
_n_pubs++;
|
||||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
_pub_core.publish(String.Empty, _tag_name, ser_bytes);
|
||||||
_fire_dirty = true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _fire_dp()
|
private void _fire_dp() {
|
||||||
{
|
|
||||||
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
|
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
|
||||||
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
|
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
|
||||||
object echo_time = DBNull.Value;
|
object echo_time = DBNull.Value;
|
||||||
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
|
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
|
||||||
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
|
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
|
||||||
if (_last_sent_msg != null && _last_recvd_msg != null)
|
if (_last_sent_msg != null && _last_recvd_msg != null) {
|
||||||
{
|
|
||||||
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
|
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
|
||||||
}
|
}
|
||||||
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
|
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
|
||||||
@@ -201,25 +173,21 @@ namespace bts
|
|||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void doWork()
|
public void doWork() {
|
||||||
{
|
|
||||||
if (DateTime.Now - _last_publish_time > _publish_period)
|
if (DateTime.Now - _last_publish_time > _publish_period)
|
||||||
if (_n_pubs <= _max_pubs)
|
if (_n_pubs <= _max_pubs) {
|
||||||
{
|
|
||||||
_publish_next_packet();
|
_publish_next_packet();
|
||||||
_last_publish_time = DateTime.Now;
|
_last_publish_time = DateTime.Now;
|
||||||
}
|
}
|
||||||
// publish
|
// publish
|
||||||
// TODO: send id
|
// TODO: send id
|
||||||
if (_fire_dirty)
|
if (_fire_dirty) {
|
||||||
{
|
|
||||||
_fire_dirty = false;
|
_fire_dirty = false;
|
||||||
_fire_dp();
|
_fire_dp();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static long epoch_now()
|
public static long epoch_now() {
|
||||||
{
|
|
||||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,223 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Runtime.CompilerServices;
|
|
||||||
using bts.api;
|
|
||||||
|
|
||||||
namespace bts
|
|
||||||
{
|
|
||||||
[DataStreamUserRespresentation("pss_echo")]
|
|
||||||
public class PubsubEchoTestStream : ViewStream, IWorker
|
|
||||||
{
|
|
||||||
public class PayloadMessage
|
|
||||||
{
|
|
||||||
public readonly string msg_id;
|
|
||||||
public readonly Int64 timestamp = 0;
|
|
||||||
public readonly byte[] payload = null;
|
|
||||||
|
|
||||||
public static byte[] create_data(int payload_size, byte payload_byte)
|
|
||||||
{
|
|
||||||
var rv = new byte[payload_size];
|
|
||||||
for (int i = 0; i < payload_size; i++)
|
|
||||||
{
|
|
||||||
rv[i] = payload_byte;
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
|
|
||||||
// {
|
|
||||||
// guid = guid_;
|
|
||||||
// timestamp = timestamp_;
|
|
||||||
// payload = new byte[payload_size];
|
|
||||||
// for (int i = 0; i < payload_size; i++)
|
|
||||||
// {
|
|
||||||
// payload[i] = payload_byte;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
|
|
||||||
{
|
|
||||||
msg_id = guid_;
|
|
||||||
timestamp = timestamp_;
|
|
||||||
payload = payload_;
|
|
||||||
}
|
|
||||||
|
|
||||||
private PayloadMessage() { }
|
|
||||||
public byte[] serialize()
|
|
||||||
{
|
|
||||||
var str = new MemoryStream();
|
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
|
||||||
bw.Write(msg_id);
|
|
||||||
bw.Write(timestamp);
|
|
||||||
bw.Write(payload.Length);
|
|
||||||
bw.Write(payload);
|
|
||||||
str.Position = 0;
|
|
||||||
var bytes = str.ToArray();
|
|
||||||
return bytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
|
||||||
{
|
|
||||||
var str = new MemoryStream(ser_bytes);
|
|
||||||
BinaryReader br = new BinaryReader(str);
|
|
||||||
string guid = br.ReadString();
|
|
||||||
Int64 timestamp_ = br.ReadInt64();
|
|
||||||
int len = br.ReadInt32();
|
|
||||||
var payload = br.ReadBytes(len);
|
|
||||||
return new PayloadMessage(guid, timestamp_, payload);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
//public void
|
|
||||||
private readonly int _msg_size;
|
|
||||||
private readonly TimeSpan _publish_period;
|
|
||||||
private DateTime _last_publish_time = DateTime.MinValue;
|
|
||||||
private PublisherCore _pub_core;
|
|
||||||
private ISubscriberCore _sub_core;
|
|
||||||
private BlueTable _table;
|
|
||||||
private int _n_pubs = 0;
|
|
||||||
private readonly int _max_pubs;
|
|
||||||
private int _n_updates = 0;
|
|
||||||
private ViewStreamDatabase _vsd;
|
|
||||||
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
|
||||||
private const byte _payload_byte = (byte)'A';
|
|
||||||
private readonly string _tag_name;
|
|
||||||
private readonly int _payload_sz;
|
|
||||||
private readonly IPubSubRouter _router;
|
|
||||||
private enum IOState
|
|
||||||
{
|
|
||||||
SEND = 0,
|
|
||||||
RECV = 1
|
|
||||||
};
|
|
||||||
IOState _state = IOState.SEND;
|
|
||||||
|
|
||||||
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
|
||||||
{
|
|
||||||
var route = new PubSubRoute(server_url, ".*");
|
|
||||||
_router = new PubSubRouter(route);
|
|
||||||
_vsd = vsd;
|
|
||||||
_payload_sz = payload_size;
|
|
||||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("sent_id", TypeCode.String),
|
|
||||||
Tuple.Create("sent_time", TypeCode.Int64),
|
|
||||||
Tuple.Create("recv_id", TypeCode.String),
|
|
||||||
Tuple.Create("recv_time", TypeCode.Int64),
|
|
||||||
Tuple.Create("match", TypeCode.Boolean),
|
|
||||||
Tuple.Create("echo_time", TypeCode.Int32));
|
|
||||||
_table = new BlueTable(schema);
|
|
||||||
_publish_period = publish_period;
|
|
||||||
_tag_name = tag_name;
|
|
||||||
}
|
|
||||||
|
|
||||||
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
|
||||||
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
|
||||||
{
|
|
||||||
_pub_core = new PublisherCore(_router);
|
|
||||||
_sub_core = new SubscriberCore(_router);
|
|
||||||
_sub_core.subscribe(_tag_name);
|
|
||||||
_vsd.addWorker(this);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
|
||||||
{
|
|
||||||
_vsd.removeWorker(this);
|
|
||||||
_sub_core.unsubscribe(_tag_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _fire_dp_after_send()
|
|
||||||
{
|
|
||||||
// as soon as the data is published, its reflected in the stream state
|
|
||||||
Log.assert(_last_sent_msg != null);
|
|
||||||
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
|
||||||
DBNull.Value, DBNull.Value);
|
|
||||||
_table.addRow(r);
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _fire_dp_after_recv()
|
|
||||||
{
|
|
||||||
Log.assert(_last_recvd_msg != null);
|
|
||||||
// update the last publish row
|
|
||||||
int id = _table.Count - 1;
|
|
||||||
IBlueRow last_row = _table[id];
|
|
||||||
_table.deleteRow(last_row);
|
|
||||||
var recv_ts = epoch_now();
|
|
||||||
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
|
|
||||||
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
|
|
||||||
_table.addRow(nr);
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
|
||||||
|
|
||||||
private string _get_next_id()
|
|
||||||
{
|
|
||||||
return _table.Count.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _publish_next_packet()
|
|
||||||
{
|
|
||||||
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
|
||||||
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
|
||||||
var ser_bytes = _last_sent_msg.serialize();
|
|
||||||
_n_pubs++;
|
|
||||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
|
||||||
_last_publish_time = DateTime.Now;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork()
|
|
||||||
{
|
|
||||||
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
|
|
||||||
{
|
|
||||||
foreach (var u in updates)
|
|
||||||
{
|
|
||||||
var recv_msg = PayloadMessage.deserialize(u.data);
|
|
||||||
if (recv_msg.msg_id == _last_sent_msg.msg_id)
|
|
||||||
{
|
|
||||||
return recv_msg;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
switch (_state)
|
|
||||||
{
|
|
||||||
case IOState.SEND:
|
|
||||||
if (DateTime.Now - _last_publish_time >= _publish_period)
|
|
||||||
{
|
|
||||||
_publish_next_packet();
|
|
||||||
_last_publish_time = DateTime.Now;
|
|
||||||
_fire_dp_after_send();
|
|
||||||
_state = IOState.RECV;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case IOState.RECV:
|
|
||||||
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
|
||||||
updates = updates.Where(u => u.tag == _tag_name);
|
|
||||||
var recv_msg = find_packet(updates);
|
|
||||||
if (recv_msg != null)
|
|
||||||
{
|
|
||||||
_last_recvd_msg = recv_msg;
|
|
||||||
_fire_dp_after_recv();
|
|
||||||
_state = IOState.SEND;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static long epoch_now()
|
|
||||||
{
|
|
||||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,14 +14,11 @@ namespace bts {
|
|||||||
private BlueTable _table;
|
private BlueTable _table;
|
||||||
private SyncQueue<string> _action_queue = new SyncQueue<string>();
|
private SyncQueue<string> _action_queue = new SyncQueue<string>();
|
||||||
|
|
||||||
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
|
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) {
|
||||||
: this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
|
|
||||||
|
|
||||||
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
|
|
||||||
_ip = ip;
|
_ip = ip;
|
||||||
_port = port;
|
_port = port;
|
||||||
_table = new BlueTable(_schema);
|
_table = new BlueTable(_schema);
|
||||||
_tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
|
_tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
|
|||||||
@@ -1,8 +1,17 @@
|
|||||||
|
|
||||||
|
|
||||||
|
using Amazon.Runtime.Internal;
|
||||||
|
|
||||||
using bts;
|
using bts;
|
||||||
|
|
||||||
|
using DnsClient.Protocol;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
using System;
|
||||||
|
using Microsoft.VisualBasic;
|
||||||
|
using Spark.Core.Collections;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
@@ -28,8 +37,7 @@ namespace BCG {
|
|||||||
|
|
||||||
private IBlueTableSchema _schema;
|
private IBlueTableSchema _schema;
|
||||||
class TestResult {
|
class TestResult {
|
||||||
public string test_name = MethodBase.GetCurrentMethod()?.Name
|
public string test_name = MethodBase.GetCurrentMethod().Name;
|
||||||
?? throw new InvalidOperationException("Failed to get test name from method");
|
|
||||||
public bool test_result;
|
public bool test_result;
|
||||||
}
|
}
|
||||||
private List<Func<TestResult>> _tests = new
|
private List<Func<TestResult>> _tests = new
|
||||||
@@ -43,11 +51,11 @@ namespace BCG {
|
|||||||
Tuple.Create("exception", TypeCode.String)
|
Tuple.Create("exception", TypeCode.String)
|
||||||
);
|
);
|
||||||
// register tests here
|
// register tests here
|
||||||
_tests.Add(new Func<TestResult>(_run_sub_msg_test));
|
// _tests.Add(new Func<TestResult>(_run_sub_msg_test));
|
||||||
_tests.Add(new Func<TestResult>(_run_pub_msg_test));
|
// _tests.Add(new Func<TestResult>(_run_pub_msg_test));
|
||||||
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
|
// _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
|
||||||
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
|
// _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
|
||||||
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
|
// _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
|
||||||
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
|
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
|
||||||
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
|
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
|
||||||
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
|
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
|
||||||
@@ -806,4 +814,4 @@ namespace BCG {
|
|||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
namespace BCG {
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class TickingTestStream : ViewStream {
|
|
||||||
public TickingTestStream(int ticking_period_msecs)
|
|
||||||
:
|
|
||||||
this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
|
|
||||||
|
|
||||||
public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
|
|
||||||
ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
|
|
||||||
vs = new MathStream(vs, "x=id*123 y=x*x");
|
|
||||||
mySetRealSource(vs);
|
|
||||||
//vs.onChange += _on_str_update;
|
|
||||||
}
|
|
||||||
|
|
||||||
// private void _on_str_update(object dp_obj) {
|
|
||||||
// DataPacket dp = dp_obj as DataPacket;
|
|
||||||
// IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
|
|
||||||
// BlueTable state = new BlueTable(out_schema);
|
|
||||||
// foreach (IBlueRow r in dp.newRows) {
|
|
||||||
// MutableBlueRow nr = new MutableBlueRow(out_schema, r);
|
|
||||||
// DateTime time = r.get_datetime("time");
|
|
||||||
// var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
|
||||||
// nr.setValue("time_txt", time_txt);
|
|
||||||
// state.addRow(nr);
|
|
||||||
// }
|
|
||||||
// var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
|
|
||||||
// myFireEvent(dp_out);
|
|
||||||
// }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+8
-29
@@ -7,45 +7,26 @@ namespace bts {
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.Json.Nodes;
|
using System.Text.Json.Nodes;
|
||||||
[DataStreamUserRespresentation("benzingaNews")]
|
[DataStreamUserRespresentation("benzingaNews")]
|
||||||
public class BenzingaNewsStream : ViewStream {
|
public class BenzingaNewStream : ViewStream {
|
||||||
private IBlueTableSchema _schema = null;
|
private IBlueTableSchema _schema = null;
|
||||||
private int _id = 0;
|
private int _id = 0;
|
||||||
|
// constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
|
||||||
|
public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
|
||||||
private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
|
|
||||||
ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
|
|
||||||
r = new DateStandardizationStream(r, "created_at", "ISO8601");
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
|
|
||||||
public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
|
|
||||||
: this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
|
|
||||||
}
|
|
||||||
|
|
||||||
// public BenzingaNewsStream()
|
|
||||||
// : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
|
|
||||||
// }
|
|
||||||
|
|
||||||
// assumes it gets a raw mongodb_raw_kv_stream with key value pairs
|
|
||||||
public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
|
|
||||||
ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
|
|
||||||
// Hiding body (with project, columns can be hidden, or renamed)
|
// Hiding body (with project, columns can be hidden, or renamed)
|
||||||
output_stream = new ProjectStream(output_stream, "!body");
|
ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
|
||||||
// add column news_source = 'Benzinga', so we can append other news sources
|
// add column news_source = 'Benzinga', so we can append other news sources
|
||||||
output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
|
output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
|
||||||
// now SuperMapStream is used to map one row into multiple, one per symbol,
|
// now SuperMapStream is used to map one row into multiple, one per symbol,
|
||||||
// (SuperMapStream takes 2 lambdas, one to get the enw schema and another to
|
// (SuperMapStream takes 2 lambdas, one to get the enw schema and another to
|
||||||
// return an array of rows, per one row)
|
// return an array of rows, per one row)
|
||||||
output_stream = new SuperMapStream(output_stream,
|
output_stream = new SuperMapStream(mongodb_raw_stream,
|
||||||
|
|
||||||
(IBlueTableSchema s) => {
|
(IBlueTableSchema s) => {
|
||||||
// this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
|
// this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
|
||||||
if (_schema == null) {
|
if (_schema == null) {
|
||||||
var out_schema = s;
|
var out_schema = s;
|
||||||
out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
|
out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
|
||||||
out_schema = BlueDataUtility.remove_columns(out_schema, "body");
|
out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
|
||||||
out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
|
|
||||||
out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
|
|
||||||
out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
|
out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
|
||||||
out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
|
out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
|
||||||
out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
|
out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
|
||||||
@@ -63,13 +44,11 @@ namespace bts {
|
|||||||
}
|
}
|
||||||
foreach (JsonNode jnode in json_objs) {
|
foreach (JsonNode jnode in json_objs) {
|
||||||
if (jnode is JsonObject json_object) {
|
if (jnode is JsonObject json_object) {
|
||||||
string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
|
|
||||||
string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
|
|
||||||
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
||||||
nr.setValue("id", _id++);
|
nr.setValue("id", _id++);
|
||||||
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
|
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
|
||||||
nr.setValue("symbol", symbol);
|
nr.setValue("symbol", jnode["symbol"]);
|
||||||
nr.setValue("exchange", exchange);
|
nr.setValue("exchange", jnode["exchange"]);
|
||||||
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
|
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
|
||||||
expanded_rows.Add(nr);
|
expanded_rows.Add(nr);
|
||||||
}
|
}
|
||||||
@@ -1,194 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using mbts;
|
|
||||||
|
|
||||||
using MongoDB.Bson;
|
|
||||||
using MongoDB.Driver;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Reactive.Subjects;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
|
|
||||||
readonly string collectionName;
|
|
||||||
readonly string databaseName;
|
|
||||||
readonly string url;
|
|
||||||
readonly object syncRoot = new();
|
|
||||||
readonly CancellationTokenSource cts = new();
|
|
||||||
readonly Subject<DataPacket> subject = new();
|
|
||||||
|
|
||||||
bool hasStarted = false;
|
|
||||||
IBlueRowFactory<BsonDocument>? rowFactory;
|
|
||||||
MongoClient? clientBacking;
|
|
||||||
IMongoDatabase? databaseBacking;
|
|
||||||
IMongoCollection<BsonDocument>? collectionBacking = null;
|
|
||||||
BlueTableImmutable? table = null;
|
|
||||||
|
|
||||||
public MongoCollectionStream(string collectionName)
|
|
||||||
: this(
|
|
||||||
collectionName,
|
|
||||||
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
|
|
||||||
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public MongoCollectionStream(string collectionName, string databaseName)
|
|
||||||
: this(
|
|
||||||
collectionName,
|
|
||||||
databaseName,
|
|
||||||
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public MongoCollectionStream(string collectionName, string databaseName, string url)
|
|
||||||
: this(collectionName, databaseName, url, null) { }
|
|
||||||
|
|
||||||
internal MongoCollectionStream(
|
|
||||||
string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
|
|
||||||
this.collectionName = collectionName;
|
|
||||||
this.databaseName = databaseName;
|
|
||||||
this.url = url;
|
|
||||||
this.rowFactory = rowFactory;
|
|
||||||
mySetRealSource(new ObservableViewStream(this));
|
|
||||||
}
|
|
||||||
|
|
||||||
MongoClient client => clientBacking ??= new(url);
|
|
||||||
IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
|
|
||||||
IMongoCollection<BsonDocument>? collection {
|
|
||||||
get => Volatile.Read(ref collectionBacking);
|
|
||||||
set => Volatile.Write(ref collectionBacking, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IDisposable Subscribe(IObserver<DataPacket> observer) {
|
|
||||||
lock (syncRoot) {
|
|
||||||
if (hasStarted) {
|
|
||||||
if (table is not null) {
|
|
||||||
// The first update has already been fired. Seed observer w/ current state.
|
|
||||||
observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
|
|
||||||
}
|
|
||||||
return subject.Subscribe(observer);
|
|
||||||
}
|
|
||||||
hasStarted = true;
|
|
||||||
}
|
|
||||||
var subscription = subject.Subscribe(observer);
|
|
||||||
// Retrieve collection snapshot to bootstrap sequence
|
|
||||||
initTable().ignore();
|
|
||||||
// Watch changes
|
|
||||||
Task.Run(watchChanges);
|
|
||||||
return Disposable.Create(Dispose);
|
|
||||||
|
|
||||||
void Dispose() {
|
|
||||||
Log.Gen.info("Disposing MongoDbChangeObservable subscription");
|
|
||||||
subscription.Dispose();
|
|
||||||
cts.Cancel();
|
|
||||||
cts.Dispose();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
|
|
||||||
MongoSchemaAdaptor.InferSchema(docs);
|
|
||||||
|
|
||||||
async Task watchChanges() {
|
|
||||||
Log.Gen.info(
|
|
||||||
$"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}");
|
|
||||||
while (collection is null) {
|
|
||||||
await Task.Delay(100);
|
|
||||||
}
|
|
||||||
Log.notnull(collection);
|
|
||||||
using var cursor = collection.Watch(new ChangeStreamOptions {
|
|
||||||
FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
|
|
||||||
});
|
|
||||||
try {
|
|
||||||
while (await cursor.MoveNextAsync(cts.Token)) {
|
|
||||||
/// Ignore while waiting for init.
|
|
||||||
lock (syncRoot) {
|
|
||||||
/// Note: <see cref="rowFactory"/> and <see cref="table"/> are initialized together
|
|
||||||
/// in the <see cref="initTable"/> fn since both depend on the schema, which is
|
|
||||||
/// inferred from the first incoming BsonDocument(s).
|
|
||||||
if (table is null || rowFactory is null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var bd = table.ToDataPacketBuilder();
|
|
||||||
foreach (var ch in cursor.Current) {
|
|
||||||
var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
|
|
||||||
var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
|
|
||||||
Log.Gen.info(
|
|
||||||
$"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
|
|
||||||
$"changes:\n\t\t{updDescJson}");
|
|
||||||
switch (ch.OperationType) {
|
|
||||||
case ChangeStreamOperationType.Insert: {
|
|
||||||
bd.AddOrUpdate(Row());
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case ChangeStreamOperationType.Update: {
|
|
||||||
bd.Remove(OldRow().Id());
|
|
||||||
bd.AddOrUpdate(Row());
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case ChangeStreamOperationType.Replace: {
|
|
||||||
bd.Remove(OldRow().Id());
|
|
||||||
bd.AddOrUpdate(Row());
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case ChangeStreamOperationType.Delete: {
|
|
||||||
bd.Remove(OldRow().Id());
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
IBlueRow Row() =>
|
|
||||||
rowFactory.GetBlueRow(ch.FullDocument);
|
|
||||||
|
|
||||||
IBlueRow OldRow() =>
|
|
||||||
rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
|
|
||||||
}
|
|
||||||
var dp = bd.ToDataPacket();
|
|
||||||
if (!dp.HasChanges()) {
|
|
||||||
Log.Gen.warn(
|
|
||||||
$"MongoCollectionStream. Ignoring update because it didn't change any rows.");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
lock (syncRoot) {
|
|
||||||
table = (BlueTableImmutable)dp.newRows;
|
|
||||||
subject.OnNext(dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (TaskCanceledException ex) {
|
|
||||||
Log.Gen.warn(
|
|
||||||
$"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
[MemberNotNull(nameof(collection))]
|
|
||||||
async Task initTable() {
|
|
||||||
collection = database.GetCollection<BsonDocument>(collectionName);
|
|
||||||
var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
|
||||||
var docList = await cursor.ToListAsync();
|
|
||||||
rowFactory ??= getRowFactory(docList);
|
|
||||||
var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
|
|
||||||
foreach (var doc in docList) {
|
|
||||||
var row = rowFactory.GetBlueRow(doc);
|
|
||||||
firstTable.AddOrUpdate(row);
|
|
||||||
}
|
|
||||||
var dp = firstTable.ToDataPacket();
|
|
||||||
lock (syncRoot) {
|
|
||||||
table = (BlueTableImmutable)dp.newRows;
|
|
||||||
subject.OnNext(dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,133 +1,33 @@
|
|||||||
using BCG;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using MongoDB.Bson;
|
|
||||||
using MongoDB.Driver;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using MongoDB.Driver;
|
||||||
using System.Threading.Tasks;
|
using MongoDB.Bson;
|
||||||
using System.Threading;
|
using BCG;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using BCG;
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
// [DataStreamUserRespresentation("mongodb_sql")]
|
[DataStreamUserRespresentation("mongodb_sql")]
|
||||||
// public class MongoDBSQLStream : ViewStream, IWorker {
|
public class MongoDBSQLStream : ViewStream, IWorker {
|
||||||
// private ViewStreamDatabase _vsd;
|
|
||||||
// private readonly string _connection_string, _db_name, _collection_name;
|
|
||||||
// private MongoClient _client;
|
|
||||||
// // make this a sorted table
|
|
||||||
// private BlueTable _table;
|
|
||||||
|
|
||||||
|
|
||||||
// // Access the database
|
|
||||||
// private IMongoDatabase _database;
|
|
||||||
// private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
|
||||||
// // mongodb://192.168.1.80:27017
|
|
||||||
|
|
||||||
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
|
||||||
// : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
|
||||||
|
|
||||||
// }
|
|
||||||
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
|
||||||
// bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
|
||||||
// $"db_name: {db_name}");
|
|
||||||
// _connection_string = connection_string;
|
|
||||||
// _db_name = db_name;
|
|
||||||
// _collection_name = collection_name;
|
|
||||||
// _vsd = vsd;
|
|
||||||
// _client = new MongoClient(_connection_string);
|
|
||||||
// _database = _client.GetDatabase(_db_name);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
|
||||||
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
|
||||||
// MasterWorker.getInstance(false).removeWorker(this);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public void doWork() {
|
|
||||||
// _read_mogodb();
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _read_mogodb() {
|
|
||||||
// var timer = Stopwatch.StartNew();
|
|
||||||
// // Access the collection
|
|
||||||
// IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
|
||||||
|
|
||||||
// // Fetch all documents from the collection
|
|
||||||
// var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
|
||||||
// _table = _table == null ? null : new BlueTable(_table.schema);
|
|
||||||
// // Iterate through the documents and print values
|
|
||||||
// foreach (var document in documents.ToEnumerable()) {
|
|
||||||
// bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
|
||||||
// if (_table == null) {
|
|
||||||
// _init_table(document);
|
|
||||||
// }
|
|
||||||
// _add_row(document);
|
|
||||||
// }
|
|
||||||
// timer.Stop();
|
|
||||||
// Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
|
||||||
// timer.Restart();
|
|
||||||
// var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
|
||||||
// timer.Stop();
|
|
||||||
// Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
|
||||||
// if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
|
||||||
// return;
|
|
||||||
// myFireEvent(dp);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _init_table(BsonDocument doc_row) {
|
|
||||||
// bts.Log.assert(_table == null);
|
|
||||||
// List<ColumnInfo> cis = new List<ColumnInfo>();
|
|
||||||
// cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
|
||||||
// foreach (var n in doc_row.Names) {
|
|
||||||
// cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
|
||||||
// }
|
|
||||||
// IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
|
||||||
// _table = new BlueTable(schema);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _add_row(BsonDocument doc) {
|
|
||||||
// var key_val = doc["_id"];
|
|
||||||
// var key = key_val.ToString();
|
|
||||||
// int id;
|
|
||||||
// if (!_id_map.TryGetValue(key, out id)) {
|
|
||||||
// id = _id_map.Count;
|
|
||||||
// _id_map.Add(key, id);
|
|
||||||
// }
|
|
||||||
// MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
|
||||||
// nr.setValue("id", id);
|
|
||||||
// foreach (var n in doc.Names) {
|
|
||||||
// var val = doc[n].ToString();
|
|
||||||
// nr.setValue(n, val);
|
|
||||||
// }
|
|
||||||
// _table.addRow(nr);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("mongodb")]
|
|
||||||
public class MongoDBStream : ViewStream, IWorker {
|
|
||||||
private ViewStreamDatabase _vsd;
|
private ViewStreamDatabase _vsd;
|
||||||
private readonly string _connection_string, _db_name, _collection_name;
|
private readonly string _connection_string, _db_name, _collection_name;
|
||||||
private readonly TimeSpan _poll_period;
|
|
||||||
private MongoClient _client;
|
private MongoClient _client;
|
||||||
|
// make this a sorted table
|
||||||
private BlueTable _table;
|
private BlueTable _table;
|
||||||
private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
|
|
||||||
private BlockingQueue _work_queue = new BlockingQueue();
|
|
||||||
|
|
||||||
// Access the database
|
// Access the database
|
||||||
private IMongoDatabase _database;
|
private IMongoDatabase _database;
|
||||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||||
// mongodb://192.168.1.80:27017
|
// mongodb://192.168.1.80:27017
|
||||||
|
|
||||||
public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
|
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||||
: this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
|
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||||
|
|
||||||
}
|
}
|
||||||
public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
|
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||||
$"db_name: {db_name}");
|
$"db_name: {db_name}");
|
||||||
_connection_string = connection_string;
|
_connection_string = connection_string;
|
||||||
@@ -135,34 +35,121 @@ namespace bts {
|
|||||||
_collection_name = collection_name;
|
_collection_name = collection_name;
|
||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
_client = new MongoClient(_connection_string);
|
_client = new MongoClient(_connection_string);
|
||||||
_poll_period = poll_period;
|
|
||||||
_database = _client.GetDatabase(_db_name);
|
_database = _client.GetDatabase(_db_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
_vsd.addWorker(this);
|
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||||
//MasterWorker.getInstance(false).addWorker(this, _poll_period);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
_vsd.removeWorker(this);
|
MasterWorker.getInstance(false).removeWorker(this);
|
||||||
//MasterWorker.getInstance(false).removeWorker(this);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private DateTime _last_poll_time = DateTime.MinValue;
|
|
||||||
public void doWork() {
|
public void doWork() {
|
||||||
var now = DateTime.Now;
|
_read_mogodb();
|
||||||
if (now - _last_poll_time > _poll_period) {
|
|
||||||
ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
|
|
||||||
_last_poll_time = DateTime.Now;
|
|
||||||
}
|
|
||||||
var dp = _table_queue.dequeue(0);
|
|
||||||
if (dp != null)
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _read_mogodb() {
|
private void _read_mogodb() {
|
||||||
var timer = Stopwatch.StartNew();
|
|
||||||
|
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
|
timer.start();
|
||||||
|
// Access the collection
|
||||||
|
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||||
|
|
||||||
|
// Fetch all documents from the collection
|
||||||
|
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||||
|
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||||
|
// Iterate through the documents and print values
|
||||||
|
foreach (var document in documents.ToEnumerable()) {
|
||||||
|
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||||
|
if (_table == null) {
|
||||||
|
_init_table(document);
|
||||||
|
}
|
||||||
|
_add_row(document);
|
||||||
|
}
|
||||||
|
timer.stop();
|
||||||
|
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||||
|
timer.start();
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||||
|
timer.stop();
|
||||||
|
bts.Log.Gen.info($"MongoDBStream time to compare tables: {timer.lastIntervalTime} msecs");
|
||||||
|
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||||
|
return;
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _init_table(BsonDocument doc_row) {
|
||||||
|
bts.Log.assert(_table == null);
|
||||||
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
|
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||||
|
foreach (var n in doc_row.Names) {
|
||||||
|
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||||
|
}
|
||||||
|
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||||
|
_table = new BlueTable(schema);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _add_row(BsonDocument doc) {
|
||||||
|
var key_val = doc["_id"];
|
||||||
|
var key = key_val.ToString();
|
||||||
|
int id;
|
||||||
|
if (!_id_map.TryGetValue(key, out id)) {
|
||||||
|
id = _id_map.Count;
|
||||||
|
_id_map.Add(key, id);
|
||||||
|
}
|
||||||
|
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||||
|
nr.setValue("id", id);
|
||||||
|
foreach (var n in doc.Names) {
|
||||||
|
var val = doc[n].ToString();
|
||||||
|
nr.setValue(n, val);
|
||||||
|
}
|
||||||
|
_table.addRow(nr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("mongodb")]
|
||||||
|
public class MongoDBStream : ViewStream, IWorker {
|
||||||
|
private ViewStreamDatabase _vsd;
|
||||||
|
private readonly string _connection_string, _db_name, _collection_name;
|
||||||
|
private MongoClient _client;
|
||||||
|
private BlueTable _table;
|
||||||
|
|
||||||
|
// Access the database
|
||||||
|
private IMongoDatabase _database;
|
||||||
|
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||||
|
// mongodb://192.168.1.80:27017
|
||||||
|
|
||||||
|
public MongoDBStream(string connection_string, string db_name, string collection_name)
|
||||||
|
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||||
|
|
||||||
|
}
|
||||||
|
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||||
|
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||||
|
$"db_name: {db_name}");
|
||||||
|
_connection_string = connection_string;
|
||||||
|
_db_name = db_name;
|
||||||
|
_collection_name = collection_name;
|
||||||
|
_vsd = vsd;
|
||||||
|
_client = new MongoClient(_connection_string);
|
||||||
|
_database = _client.GetDatabase(_db_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
MasterWorker.getInstance(false).removeWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
_read_mogodb();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _read_mogodb() {
|
||||||
|
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
|
timer.start();
|
||||||
// Access the collection
|
// Access the collection
|
||||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||||
|
|
||||||
@@ -177,16 +164,15 @@ namespace bts {
|
|||||||
}
|
}
|
||||||
_add_doc(document);
|
_add_doc(document);
|
||||||
}
|
}
|
||||||
timer.Stop();
|
timer.stop();
|
||||||
Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
|
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||||
timer.Restart();
|
timer.start();
|
||||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||||
timer.Stop();
|
timer.stop();
|
||||||
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
|
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||||
return;
|
return;
|
||||||
_table_queue.enqueue(dp);
|
myFireEvent(dp);
|
||||||
//myFireEvent(dp);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _init_table(BsonDocument doc_row) {
|
private void _init_table(BsonDocument doc_row) {
|
||||||
@@ -225,46 +211,46 @@ namespace bts {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// [DataStreamUserRespresentation("mongodb_flat")]
|
[DataStreamUserRespresentation("mongodb_flat")]
|
||||||
// public class MongoDBFlatStream : ViewStream {
|
public class MongoDBFlatStream : ViewStream {
|
||||||
// public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
||||||
// :
|
:
|
||||||
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
||||||
|
|
||||||
// public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
||||||
// ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||||
// rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
||||||
// rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
||||||
// //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
//rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
||||||
// mySetRealSource(rs);
|
mySetRealSource(rs);
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// [DataStreamUserRespresentation("newsreplay")]
|
[DataStreamUserRespresentation("newsreplay")]
|
||||||
// public class NewsReplayStream : ViewStream {
|
public class NewsReplayStream : ViewStream {
|
||||||
// public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
||||||
// int buffer_sz, string fire_period) :
|
int buffer_sz, string fire_period) :
|
||||||
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
||||||
|
|
||||||
// public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
||||||
// int buffer_sz, TimeSpan fire_period) {
|
int buffer_sz, TimeSpan fire_period) {
|
||||||
// ViewStream r;
|
ViewStream r;
|
||||||
|
|
||||||
// r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||||
// r = new KeyValueToRelationalStream(r,
|
r = new KeyValueToRelationalStream(r,
|
||||||
// "doc_id",
|
"doc_id",
|
||||||
// "key",
|
"key",
|
||||||
// "value");
|
"value");
|
||||||
// r = new DateStandardizationStream(r,
|
r = new DateStandardizationStream(r,
|
||||||
// "created_at",
|
"created_at",
|
||||||
// "ISO8601");
|
"ISO8601");
|
||||||
// r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
||||||
|
|
||||||
// mySetRealSource(r);
|
mySetRealSource(r);
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using MongoDB.Bson;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public static class MongoDbExtensions {
|
|
||||||
public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
|
|
||||||
doc[elementName].AsBsonArray;
|
|
||||||
|
|
||||||
public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
|
|
||||||
doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
|
|
||||||
val.AsBsonArray : null;
|
|
||||||
|
|
||||||
public static string GetString(this BsonDocument doc, string elementName) =>
|
|
||||||
doc[elementName].AsString;
|
|
||||||
|
|
||||||
public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
|
|
||||||
doc.TryGetValue(elementName, out var val) && val.IsString ?
|
|
||||||
val.AsString : null;
|
|
||||||
|
|
||||||
public static int GetInt32(this BsonDocument doc, string elementName) =>
|
|
||||||
doc[elementName].AsInt32;
|
|
||||||
|
|
||||||
public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
|
|
||||||
doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
|
|
||||||
val.AsInt32 : null;
|
|
||||||
|
|
||||||
public static long GetInt64(this BsonDocument doc, string elementName) =>
|
|
||||||
doc[elementName].AsInt64;
|
|
||||||
|
|
||||||
public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
|
|
||||||
doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
|
|
||||||
val.AsInt64 : null;
|
|
||||||
|
|
||||||
public static double GetDouble(this BsonDocument doc, string elementName) =>
|
|
||||||
doc[elementName].AsDouble;
|
|
||||||
|
|
||||||
public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
|
|
||||||
doc.TryGetValue(elementName, out var val) && val.IsDouble ?
|
|
||||||
val.AsDouble : null;
|
|
||||||
|
|
||||||
public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
|
|
||||||
doc[elementName].ToUniversalTime();
|
|
||||||
|
|
||||||
public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
|
|
||||||
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
|
|
||||||
val.ToUniversalTime() : null;
|
|
||||||
|
|
||||||
public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
|
|
||||||
doc[elementName].ToUniversalTime();
|
|
||||||
|
|
||||||
public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
|
|
||||||
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
|
|
||||||
val.ToUniversalTime() : null;
|
|
||||||
|
|
||||||
public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
|
|
||||||
doc[elementName].AsObjectId;
|
|
||||||
|
|
||||||
public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
|
|
||||||
doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
|
|
||||||
val.AsObjectId : null;
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using MongoDB.Bson;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public static class MongoSchemaAdaptor {
|
|
||||||
public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
|
|
||||||
var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
|
|
||||||
foreach (var doc in docs) {
|
|
||||||
foreach (var el in doc) {
|
|
||||||
if (schemaAdaptor.Contains(el.Name)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
switch (el.Value.BsonType) {
|
|
||||||
case BsonType.Array:
|
|
||||||
schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
|
|
||||||
break;
|
|
||||||
case BsonType.String:
|
|
||||||
schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
|
|
||||||
break;
|
|
||||||
case BsonType.Int32:
|
|
||||||
schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
|
|
||||||
break;
|
|
||||||
case BsonType.Int64:
|
|
||||||
schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
|
|
||||||
break;
|
|
||||||
case BsonType.Double:
|
|
||||||
schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
|
|
||||||
break;
|
|
||||||
case BsonType.DateTime:
|
|
||||||
schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
|
|
||||||
break;
|
|
||||||
case BsonType.Timestamp:
|
|
||||||
schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
|
|
||||||
break;
|
|
||||||
case BsonType.ObjectId:
|
|
||||||
schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
|
|
||||||
break;
|
|
||||||
case BsonType.Null:
|
|
||||||
Log.Gen.warn(
|
|
||||||
$"Ignoring column '{el.Name}' because at least one element is null");
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new InvalidOperationException(
|
|
||||||
$"Unsupported BsonType '{el.Value.BsonType}'");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return schemaAdaptor;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -6,7 +6,6 @@ namespace bts {
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class ProjectData {
|
public class ProjectData {
|
||||||
// schema from the base stream
|
// schema from the base stream
|
||||||
@@ -148,23 +147,12 @@ namespace bts {
|
|||||||
mapped_to_orig_map.Add(mapped_col, orig_col);
|
mapped_to_orig_map.Add(mapped_col, orig_col);
|
||||||
orig_to_mapped.Add(orig_col, mapped_col);
|
orig_to_mapped.Add(orig_col, mapped_col);
|
||||||
} else {
|
} else {
|
||||||
if (input_schema.columnNames.Contains(p_tok)) {
|
Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
|
||||||
string orig_col = p_tok;
|
|
||||||
string mapped_col = orig_col;
|
|
||||||
orig_to_mapped.Add(orig_col, mapped_col);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// make the new schema
|
// make the new schema
|
||||||
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
||||||
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
||||||
if (ci.colName == "id") {
|
|
||||||
// if there's a hidden col, other columns will show by default, but if cols are being
|
|
||||||
// projected epxlicitely, than others wil be hidden
|
|
||||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
|
||||||
new_cols.Add(nc);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (removed_cols.Contains(ci.colName)) {
|
if (removed_cols.Contains(ci.colName)) {
|
||||||
continue;
|
continue;
|
||||||
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
||||||
@@ -172,12 +160,8 @@ namespace bts {
|
|||||||
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
||||||
new_cols.Add(nc);
|
new_cols.Add(nc);
|
||||||
} else {
|
} else {
|
||||||
if (ci.colName == "id" || removed_cols.Count > 0) {
|
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||||
// if there's a hidden col, other columns will show by default, but if cols are being
|
new_cols.Add(nc);
|
||||||
// projected epxlicitely, than others wil be hidden
|
|
||||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
|
||||||
new_cols.Add(nc);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
||||||
|
|||||||
@@ -1,23 +1,25 @@
|
|||||||
namespace BCG {
|
namespace BCG {
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
[DataStreamUserRespresentation("oldproject")]
|
[DataStreamUserRespresentation("oldproject")]
|
||||||
public class ProjectStream : ViewStream
|
public class ProjectStream : ViewStream {
|
||||||
{
|
|
||||||
private ProjectHelper myHelper;
|
private ProjectHelper myHelper;
|
||||||
private string myColExp, myTableName;
|
private string myColExp, myTableName;
|
||||||
private ViewStream myVs;
|
private ViewStream myVs;
|
||||||
private IDictionary<string, string> myDic;
|
private IDictionary<string, string> myDic;
|
||||||
|
private HighPerformanceTimer _timer = new();
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
||||||
/// <param name="colExp">expression defining the column mapping</param>
|
/// <param name="colExp">expression defining the column mapping</param>
|
||||||
public ProjectStream (ViewStream vs, string colExp)
|
public ProjectStream(ViewStream vs, string colExp) {
|
||||||
{
|
|
||||||
myVs = vs;
|
myVs = vs;
|
||||||
myColExp = colExp;
|
myColExp = colExp;
|
||||||
}
|
}
|
||||||
@@ -27,12 +29,10 @@ namespace BCG {
|
|||||||
/// <param name="vs">stream to subscribe to as the input stream</param>
|
/// <param name="vs">stream to subscribe to as the input stream</param>
|
||||||
/// <param name="colExp">expression defining the column mapping</param>
|
/// <param name="colExp">expression defining the column mapping</param>
|
||||||
/// <param name="tableName">name of the mapped table</param>
|
/// <param name="tableName">name of the mapped table</param>
|
||||||
public ProjectStream (ViewStream vs, string colExp, string tableName)
|
public ProjectStream(ViewStream vs, string colExp, string tableName) {
|
||||||
{
|
|
||||||
if (colExp == "!")
|
if (colExp == "!")
|
||||||
mySetRealSource (new ChangeTableNameStream (vs, tableName));
|
mySetRealSource(new ChangeTableNameStream(vs, tableName));
|
||||||
else
|
else {
|
||||||
{
|
|
||||||
myVs = vs;
|
myVs = vs;
|
||||||
myColExp = colExp;
|
myColExp = colExp;
|
||||||
myTableName = tableName;
|
myTableName = tableName;
|
||||||
@@ -43,57 +43,51 @@ namespace BCG {
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="vs">stream to subscribe to as the input stream</param>
|
/// <param name="vs">stream to subscribe to as the input stream</param>
|
||||||
/// <param name="dic">dictionary mapping the aliased to the original</param>
|
/// <param name="dic">dictionary mapping the aliased to the original</param>
|
||||||
public ProjectStream (ViewStream vs, IDictionary dic)
|
public ProjectStream(ViewStream vs, IDictionary dic)
|
||||||
: this(vs, GenericDictionaryWrapper.Create<string, string>(dic)) {}
|
: this(vs, GenericDictionaryWrapper.Create<string, string>(dic)) { }
|
||||||
|
|
||||||
public ProjectStream (ViewStream vs, IEnumerable<KeyValuePair<string, string>> dic)
|
public ProjectStream(ViewStream vs, IEnumerable<KeyValuePair<string, string>> dic) {
|
||||||
{
|
|
||||||
myVs = vs;
|
myVs = vs;
|
||||||
myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key);
|
myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
myVs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChange);
|
||||||
myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
myVs.onChange -= new BCG.DataStream.ChangeDelegate(myHandleChange); ;
|
||||||
myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
internal void myHandleChange (object newValue)
|
internal void myHandleChange(object newValue) {
|
||||||
{
|
|
||||||
DataPacket dp = newValue as DataPacket;
|
DataPacket dp = newValue as DataPacket;
|
||||||
if (myHelper != null)
|
if (myHelper != null) {
|
||||||
{
|
_timer.start();
|
||||||
IBlueRowCollection dels, adds;
|
IBlueRowCollection dels, adds;
|
||||||
BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
|
BlueDataUtility.getTrueDeletesAndAdds(dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
|
||||||
|
_timer.stop();
|
||||||
if (dels.Count + adds.Count > 0)
|
if (dels.Count + adds.Count > 0)
|
||||||
myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
|
myFireEvent(new DataPacket(new ProjectedTable(dp.newRows, myHelper),
|
||||||
new ProjectedTable (adds, myHelper),
|
new ProjectedTable(adds, myHelper),
|
||||||
new ProjectedTable (dels, myHelper)
|
new ProjectedTable(dels, myHelper)
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
} else { //first time a callback is received
|
||||||
else
|
|
||||||
{ //first time a callback is received
|
|
||||||
myHelper = myDic == null ?
|
myHelper = myDic == null ?
|
||||||
new ProjectHelper (dp.schema, myColExp, myTableName) :
|
new ProjectHelper(dp.schema, myColExp, myTableName) :
|
||||||
new ProjectHelper (dp.schema, myDic, myTableName);
|
new ProjectHelper(dp.schema, myDic, myTableName);
|
||||||
|
|
||||||
IBlueRowCollection temp;
|
IBlueRowCollection temp;
|
||||||
myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper),
|
myFireEvent(new DataPacket(temp = new ProjectedTable(dp.newRows, myHelper),
|
||||||
temp,
|
temp,
|
||||||
new BlueTable (temp.schema)
|
new BlueTable(temp.schema)
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
public void myHandleChangeTest(object newValue)
|
public void myHandleChangeTest(object newValue) {
|
||||||
{
|
myHandleChange(newValue);
|
||||||
myHandleChange(newValue);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,33 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
namespace bts {
|
|
||||||
public class SchemaGuesserStream : ViewStream {
|
|
||||||
|
|
||||||
private IBlueRowCollection _rows = null;
|
|
||||||
private ViewStream _vs;
|
|
||||||
public SchemaGuesserStream(ViewStream vs) {
|
|
||||||
_vs = vs;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_vs.onChange += _on_data;
|
|
||||||
}
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_vs.onChange -= _on_data;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _on_data(object obj) {
|
|
||||||
DataPacket dp = obj as DataPacket;
|
|
||||||
if (_rows == null) {
|
|
||||||
_rows = BlueDataUtility.get_typed_table(dp.newRows);
|
|
||||||
}
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,103 +1,115 @@
|
|||||||
using Bts.ColumnSortExpressions;
|
// using System;
|
||||||
|
// using System.Collections;
|
||||||
using System;
|
// using BCG.Utilities;
|
||||||
using System.Collections;
|
// using BCG.Evaluator.SortExpression;
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
// namespace BCG
|
||||||
namespace BCG;
|
// {
|
||||||
|
// public class SortRow : IComparable, IBlueRow
|
||||||
public class SortRow : IComparable, IBlueRow {
|
// {
|
||||||
readonly IBlueRow row;
|
// private IBlueRow myRow;
|
||||||
readonly IEnumerable<ColumnSortExpression>? columns;
|
// private IEnumerable myColumns;
|
||||||
readonly IComparer? comparer;
|
// private IComparer myComparer;
|
||||||
|
|
||||||
public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) {
|
// public SortRow (IBlueRow row, SortExpression sortExpression)
|
||||||
this.row = row;
|
// {
|
||||||
this.columns = columns;
|
// myRow = row;
|
||||||
}
|
// myColumns = sortExpression.columns;
|
||||||
|
// }
|
||||||
public SortRow(IBlueRow row, IComparer comparer) {
|
|
||||||
this.row = row;
|
// public SortRow (IBlueRow row, IEnumerable columns)
|
||||||
this.comparer = comparer;
|
// {
|
||||||
}
|
// myRow = row;
|
||||||
|
// myColumns = columns;
|
||||||
public IBlueRow internal_row { get { return row; } }
|
// }
|
||||||
|
|
||||||
#region IComparable method
|
// public SortRow (IBlueRow row, IComparer comparer)
|
||||||
public int CompareTo(object? obj) {
|
// {
|
||||||
if (obj is not IBlueRow otherRow) {
|
// myRow = row;
|
||||||
throw new ArgumentException(
|
// myComparer = comparer;
|
||||||
$"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})",
|
// }
|
||||||
nameof(obj));
|
|
||||||
}
|
// public IBlueRow internal_row { get { return myRow; } }
|
||||||
if (comparer is not null) {
|
|
||||||
return comparer.Compare(this, otherRow);
|
// #region IComparable method
|
||||||
}
|
// public int CompareTo (object obj)
|
||||||
return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0);
|
// {
|
||||||
int Compare(ColumnSortExpression col) {
|
// IBlueRow otherRow = (IBlueRow)obj;
|
||||||
var otherVal = otherRow[col.Name];
|
// if (myComparer == null)
|
||||||
int rv = myIsNull(this[col.Name]) ?
|
// {
|
||||||
(myIsNull(otherVal) ? 0 : -1) :
|
// foreach (SortExpression.ColumnCriteria cc in myColumns)
|
||||||
(myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
|
// {
|
||||||
rv *= col.IsAscending ? 1 : -1;
|
// int retVal = myIsNull (this[cc.columnName]) ?
|
||||||
return rv;
|
// myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
|
||||||
}
|
// myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
|
||||||
}
|
// retVal *= cc.ascending ? 1 : -1;
|
||||||
#endregion
|
// if (retVal != 0)
|
||||||
|
// return retVal;
|
||||||
#region IBlueRow members
|
// }
|
||||||
public IBlueTableSchema schema { get { return row.schema; } }
|
// return 0;
|
||||||
#endregion
|
// }
|
||||||
|
// else
|
||||||
#region ICollection members
|
// return myComparer.Compare (this, otherRow);
|
||||||
public int Count { get { return row.Count; } }
|
// }
|
||||||
|
// #endregion
|
||||||
public bool IsSynchronized { get { return false; } }
|
|
||||||
|
// #region IBlueRow members
|
||||||
public object SyncRoot { get { return this; } }
|
// public IBlueTableSchema schema{get{return myRow.schema;}}
|
||||||
|
// #endregion
|
||||||
public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
|
|
||||||
#endregion
|
// #region ICollection members
|
||||||
|
// public int Count {get{return myRow.Count;}}
|
||||||
#region IDictionary members
|
|
||||||
public bool IsFixedSize { get { return true; } }
|
// public bool IsSynchronized {get{return false;}}
|
||||||
|
|
||||||
public bool IsReadOnly { get { return true; } }
|
// public object SyncRoot {get{return this;}}
|
||||||
|
|
||||||
/// <summary>
|
// public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
|
||||||
/// The key can either be the string corrsponding to the column name or
|
// #endregion
|
||||||
/// the order of the column
|
|
||||||
/// </summary>
|
// #region IDictionary members
|
||||||
public object? this[object key] {
|
// public bool IsFixedSize {get{return true;}}
|
||||||
get => key is string str ? this[str] : this[(int)key];
|
|
||||||
set => throw new NotSupportedException("The IBlueRow is readonly");
|
// public bool IsReadOnly {get{return true;}}
|
||||||
}
|
|
||||||
|
// /// <summary>
|
||||||
public object? this[int index] => row[index];
|
// /// The key can either be the string corrsponding to the column name or
|
||||||
|
// /// the order of the column
|
||||||
public object? this[string colName] => row[colName];
|
// /// </summary>
|
||||||
|
// public object this [object key]
|
||||||
public ICollection Keys => row.Keys;
|
// {
|
||||||
|
// get
|
||||||
public ICollection Values => row.Values;
|
// {
|
||||||
|
// if (key is string)
|
||||||
public bool Contains(object key) => row.Contains(key);
|
// return this[(string)key];
|
||||||
|
// return this[(int)key];
|
||||||
void IDictionary.Add(object key, object? value) =>
|
// }
|
||||||
throw new NotSupportedException("The IBlueRow is readonly");
|
// set {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
// }
|
||||||
void IDictionary.Clear() =>
|
|
||||||
throw new NotSupportedException("The IBlueRow is readonly");
|
// public object this [int index] {get{return myRow[index];}}
|
||||||
|
|
||||||
IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); }
|
// public object this [string colName] {get{return myRow[colName];}}
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); }
|
// public ICollection Keys{get{return myRow.Keys;}}
|
||||||
|
|
||||||
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
// public ICollection Values{get{return myRow.Values;}}
|
||||||
#endregion
|
|
||||||
|
// public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
bool myIsNull(object? o) =>
|
|
||||||
o is null || o == null || o is DBNull;
|
// public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
|
||||||
}
|
// public bool Contains (object key){return myRow.Contains (key);}
|
||||||
|
|
||||||
|
// IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
|
||||||
|
|
||||||
|
// IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
|
||||||
|
|
||||||
|
// public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
// #endregion
|
||||||
|
|
||||||
|
// private bool myIsNull (object o) {return o == null || o is DBNull;}
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,99 +1,129 @@
|
|||||||
using Bts.ColumnSortExpressions;
|
// using System;
|
||||||
|
// using System.Collections;
|
||||||
|
// using BCG.Evaluator.SortExpression;
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
|
||||||
namespace BCG;
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// /// <summary>
|
||||||
|
// /// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
|
||||||
|
// /// to merge the additions to the accumulates table
|
||||||
|
// /// </summary>
|
||||||
|
// public class SortStream : ViewStream
|
||||||
|
// {
|
||||||
|
// private ViewStream myStream;
|
||||||
|
// private IEnumerable mySortCols;
|
||||||
|
// private IComparer myComparer;
|
||||||
|
// private DataPacket myLastFiredPacket;
|
||||||
|
// private DataPacket myLastRecievedPacket;
|
||||||
|
|
||||||
/// <summary>
|
// public SortStream (ViewStream stream, string sortExpression)
|
||||||
/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
|
// {
|
||||||
/// to merge the additions to the accumulates table
|
// myStream = stream;
|
||||||
/// </summary>
|
// mySetSortExpression (sortExpression);
|
||||||
public class SortStream : ViewStream {
|
// }
|
||||||
ViewStream stream;
|
|
||||||
IList<ColumnSortExpression> sortCols;
|
|
||||||
IComparer? comparer;
|
|
||||||
DataPacket? lastFiredPacket;
|
|
||||||
DataPacket? lastReceivedPacket;
|
|
||||||
|
|
||||||
public SortStream(ViewStream stream, string sortExpression) {
|
// public SortStream (ViewStream vs, string colName, bool isAsc)
|
||||||
this.stream = stream;
|
// {
|
||||||
SortExpression = sortExpression;
|
// myStream = vs;
|
||||||
}
|
// string sortExpression = isAsc ? "+" + colName : "-" + colName;
|
||||||
|
// mySetSortExpression (sortExpression);
|
||||||
|
// }
|
||||||
|
|
||||||
public SortStream(ViewStream vs, string colName, bool isAsc) {
|
// public SortStream (ViewStream vs, IComparer comparer)
|
||||||
stream = vs;
|
// {
|
||||||
SortExpression = isAsc ? "+" + colName : "-" + colName;
|
// myStream = vs;
|
||||||
}
|
// myComparer = comparer;
|
||||||
|
// }
|
||||||
|
|
||||||
public SortStream(ViewStream stream, IComparer comparer) {
|
// public string sortExpression
|
||||||
this.stream = stream;
|
// {
|
||||||
this.comparer = comparer;
|
// set
|
||||||
sortCols = Array.Empty<ColumnSortExpression>();
|
// {
|
||||||
}
|
// myComparer = null;
|
||||||
|
// myLastFiredPacket = null;
|
||||||
|
// mySetSortExpression (value);
|
||||||
|
// myChangeHandler (myLastRecievedPacket);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
[MemberNotNull(nameof(sortCols))]
|
// public IComparer comparer
|
||||||
public string SortExpression {
|
// {
|
||||||
set {
|
// set
|
||||||
comparer = null;
|
// {
|
||||||
lastFiredPacket = null;
|
// myComparer = value;
|
||||||
sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
|
// myLastFiredPacket = null;
|
||||||
if (lastReceivedPacket is not null) {
|
// myChangeHandler(myLastRecievedPacket);
|
||||||
myChangeHandler(lastReceivedPacket);
|
// }
|
||||||
}
|
// }
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IComparer Comparer {
|
// private void mySetSortExpression (string sortExpression)
|
||||||
set {
|
// {
|
||||||
comparer = value;
|
// syntax parser =
|
||||||
lastFiredPacket = null;
|
// new syntax();
|
||||||
if (lastReceivedPacket is not null) {
|
|
||||||
myChangeHandler(lastReceivedPacket);
|
// mySortCols = ((SortExpression)
|
||||||
}
|
// parser.Parse (sortExpression)).columns;
|
||||||
}
|
// }
|
||||||
}
|
|
||||||
|
|
||||||
public void myChangeHandlerTest(object o) =>
|
// protected override void myOnFirstSubscriber()
|
||||||
myChangeHandler(o);
|
// {
|
||||||
|
// myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
|
||||||
|
// }
|
||||||
|
|
||||||
[MemberNotNull(nameof(lastFiredPacket))]
|
// protected override void myOnLastSubscriber()
|
||||||
internal void myChangeHandler(object o) {
|
// {
|
||||||
lastReceivedPacket = (DataPacket)o;
|
// myInit ();
|
||||||
// First callback, so we need to sort the whole table, this is the
|
// myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
||||||
// only time we will do that for all the other sorting operations would
|
// }
|
||||||
// be sorting the increments and inserting them into the whole list
|
|
||||||
if (lastFiredPacket is null) {
|
|
||||||
lastFiredPacket = new() {
|
|
||||||
newRows = comparer is null ?
|
|
||||||
new SortedTable(lastReceivedPacket.newRows, sortCols) :
|
|
||||||
new SortedTable(lastReceivedPacket.newRows, comparer)
|
|
||||||
};
|
|
||||||
lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
|
|
||||||
lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
|
|
||||||
myFireEvent(lastFiredPacket.Clone());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Deletes
|
|
||||||
lastFiredPacket.deletedRows =
|
|
||||||
comparer == null ?
|
|
||||||
new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
|
|
||||||
new SortedTable(lastReceivedPacket.deletedRows, comparer);
|
|
||||||
((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
|
|
||||||
// Adds
|
|
||||||
lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
|
|
||||||
((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
|
|
||||||
myFireEvent(lastFiredPacket.Clone());
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() =>
|
// internal void myChangeHandler (object o)
|
||||||
stream.onChange += myChangeHandler;
|
// {
|
||||||
|
// myLastRecievedPacket = (DataPacket)o;
|
||||||
|
// //first callback, so we need to sort the whole table, this is the
|
||||||
|
// //only time we will do that for all the other sorting operations would
|
||||||
|
// //be sorting the increments and inserting them into the whole list
|
||||||
|
// if (myLastFiredPacket == null)
|
||||||
|
// {
|
||||||
|
// myLastFiredPacket = new DataPacket ();
|
||||||
|
// myLastFiredPacket.newRows =
|
||||||
|
// myComparer == null ?
|
||||||
|
// new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
|
||||||
|
// new SortedTable (myLastRecievedPacket.newRows, myComparer);
|
||||||
|
// myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
|
||||||
|
// myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
|
||||||
|
// myFireEvent (myLastFiredPacket.Clone());
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
// myHandleDeletes (myLastRecievedPacket);
|
||||||
|
// myHandleAdds (myLastRecievedPacket);
|
||||||
|
// myFireEvent (myLastFiredPacket.Clone());
|
||||||
|
// }
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
// public void myChangeHandlerTest(object o)
|
||||||
lastFiredPacket = null;
|
// {
|
||||||
lastReceivedPacket = null;
|
// myChangeHandler(o);
|
||||||
stream.onChange -= myChangeHandler;
|
// }
|
||||||
}
|
|
||||||
}
|
// private void myHandleDeletes (DataPacket dp)
|
||||||
|
// {
|
||||||
|
// myLastFiredPacket.deletedRows =
|
||||||
|
// myComparer == null ?
|
||||||
|
// new SortedTable (dp.deletedRows, mySortCols) :
|
||||||
|
// new SortedTable (dp.deletedRows, myComparer);
|
||||||
|
// ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void myHandleAdds (DataPacket dp)
|
||||||
|
// {
|
||||||
|
// myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
|
||||||
|
// ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void myInit ()
|
||||||
|
// {
|
||||||
|
// myLastFiredPacket = null;
|
||||||
|
// myLastRecievedPacket = null;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,108 +1,136 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections;
|
// using System.Collections;
|
||||||
using System.Collections.Generic;
|
// using BCG.Evaluator.SortExpression;
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
// namespace BCG
|
||||||
using Bts.ColumnSortExpressions;
|
// {
|
||||||
|
// /// <summary>
|
||||||
namespace BCG;
|
// /// Represents an IBlueRowCollection with the rows sorted according to the
|
||||||
|
// ///
|
||||||
/// <summary>
|
// /// </summary>
|
||||||
/// Represents an IBlueRowCollection with the rows sorted according to the
|
// public class SortedTable : IBlueRowCollection
|
||||||
/// </summary>
|
// {
|
||||||
public class SortedTable : IBlueRowCollection {
|
// private ArrayList myRows = new ArrayList ();
|
||||||
readonly List<SortRow> rows = new(0);
|
// private Hashtable myIndex = new Hashtable ();
|
||||||
readonly Dictionary<int, IBlueRow> rowIndex = new(0);
|
// private IBlueTableSchema mySchema;
|
||||||
readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>();
|
// private IEnumerable mySortCols;
|
||||||
readonly IComparer? comparer;
|
// private IComparer myComparer;
|
||||||
|
|
||||||
/// <summary>
|
// #region SortedTable method(s)
|
||||||
/// Sorts the rows using sortExpression as the sorting criteria
|
// public void addRows (IBlueRowCollection rows)
|
||||||
/// </summary>
|
// {
|
||||||
/// <param name="rows">Rows to be inserted into the table</param>
|
// //requires merger of the addedRows to the already existing ones
|
||||||
/// <param name="sortCols">Sorting criteria</param>
|
// if (rows.Count == 0)
|
||||||
public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) {
|
// //at times there are calls with 0 additions, in that case
|
||||||
this.sortCols = sortCols;
|
// //we ofcourse need no processing
|
||||||
init(rows);
|
// return;
|
||||||
}
|
// SortedTable addedRows =
|
||||||
|
// myComparer == null ? new SortedTable (rows, mySortCols) :
|
||||||
public SortedTable(IBlueRowCollection rows, IComparer comparer) {
|
// new SortedTable (rows, myComparer);
|
||||||
this.comparer = comparer;
|
// int addedRowsIndex = 0;
|
||||||
init(rows);
|
// SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
|
||||||
}
|
// for (int i = 0; i < myRows.Count; i ++)
|
||||||
|
// if (newRow.CompareTo(myRows[i]) <= 0)
|
||||||
public int Count => rows.Count;
|
// {
|
||||||
public bool IsSynchronized => false;
|
// myRows.Insert (i, newRow);
|
||||||
public IBlueTableSchema schema { get; private set; }
|
// myIndex[newRow["id"]] = newRow;
|
||||||
public object SyncRoot => this;
|
// if (++addedRowsIndex == addedRows.Count)
|
||||||
public IBlueRow this[int id] => rowIndex[id]
|
// return;
|
||||||
?? throw new BlueDataException(
|
// newRow = addedRows.getRowByIndex (addedRowsIndex);
|
||||||
$"ID '{id}' does not exist in table '{schema?.tableName}'");
|
// }
|
||||||
|
// if (addedRowsIndex < addedRows.Count)
|
||||||
public void addRows(IBlueRowCollection rows) {
|
// for (int i = addedRowsIndex; i < addedRows.Count; i++)
|
||||||
// Requires merger of the addedRows to the already existing ones
|
// {
|
||||||
if (rows.Count == 0) {
|
// newRow = addedRows.getRowByIndex (i);
|
||||||
// At times there are calls with 0 additions, in that case
|
// myRows.Add (newRow);
|
||||||
// we of course need no processing
|
// myIndex[newRow["id"]] = newRow;
|
||||||
return;
|
// }
|
||||||
}
|
// }
|
||||||
SortedTable addedRows =
|
|
||||||
comparer == null ? new SortedTable(rows, sortCols) :
|
// public void deleteRows (IBlueRowCollection deletedRows)
|
||||||
new SortedTable(rows, comparer);
|
// {
|
||||||
int addedRowsIndex = 0;
|
// foreach (IBlueRow deletedRow in deletedRows)
|
||||||
SortRow newRow = addedRows.getRowByIndex(addedRowsIndex);
|
// {
|
||||||
for (int i = 0; i < this.rows.Count; i++) {
|
// IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
|
||||||
if (newRow.CompareTo(this.rows[i]) <= 0) {
|
// myRows.Remove (row);
|
||||||
this.rows.Insert(i, newRow);
|
// myIndex.Remove (row["id"]);
|
||||||
rowIndex[(int)newRow["id"]] = newRow;
|
// }
|
||||||
if (++addedRowsIndex == addedRows.Count) {
|
// }
|
||||||
return;
|
|
||||||
}
|
// public SortRow getRowByIndex (int index) {return (SortRow)myRows[index];}
|
||||||
newRow = addedRows.getRowByIndex(addedRowsIndex);
|
// #endregion
|
||||||
}
|
|
||||||
}
|
// #region Constructor(s)
|
||||||
if (addedRowsIndex < addedRows.Count) {
|
// /// <summary>
|
||||||
for (int i = addedRowsIndex; i < addedRows.Count; i++) {
|
// /// Sorts the rows using sortExpression as the sorting criteria
|
||||||
newRow = addedRows.getRowByIndex(i);
|
// /// </summary>
|
||||||
this.rows.Add(newRow);
|
// /// <param name="rows">
|
||||||
rowIndex[(int)newRow["id"]] = newRow;
|
// /// Rows to be inserted into the table
|
||||||
}
|
// /// </param>
|
||||||
}
|
// /// <param name="sortExpression">
|
||||||
}
|
// /// Sorting criteria
|
||||||
|
// /// </param>
|
||||||
public void deleteRows(IBlueRowCollection deletedRows) {
|
// public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
|
||||||
foreach (IBlueRow deletedRow in deletedRows) {
|
// {
|
||||||
var row = rowIndex[(int)deletedRow["id"]!] as SortRow
|
// mySortCols = sortExpression.columns;
|
||||||
?? throw new InvalidOperationException("Index did not return IBlueRow");
|
// myConstructorHelper (rows);
|
||||||
rows.Remove(row);
|
// }
|
||||||
rowIndex.Remove((int)row["id"]!);
|
|
||||||
}
|
// public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
|
||||||
}
|
// {
|
||||||
|
// mySortCols = sortCols;
|
||||||
public SortRow getRowByIndex(int index) =>
|
// myConstructorHelper (rows);
|
||||||
rows[index]!;
|
// }
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
// public SortedTable (IBlueRowCollection rows, IComparer comparer)
|
||||||
rowIndex[id] is not null;
|
// {
|
||||||
|
// myComparer = comparer;
|
||||||
public IEnumerator GetEnumerator() =>
|
// myConstructorHelper (rows);
|
||||||
rows.GetEnumerator();
|
// }
|
||||||
|
|
||||||
public void CopyTo(Array array, int index) {
|
// private void myConstructorHelper (IBlueRowCollection rows)
|
||||||
for (int i = 0; i < Count; i++) {
|
// {
|
||||||
array.SetValue(rows[i], index + i);
|
// mySchema = rows.schema;
|
||||||
}
|
// foreach (IBlueRow row in rows)
|
||||||
}
|
// {
|
||||||
|
// SortRow newRow =
|
||||||
[MemberNotNull(nameof(schema))]
|
// myComparer == null ?
|
||||||
void init(IBlueRowCollection rows) {
|
// new SortRow (row, mySortCols) : new SortRow (row, myComparer);
|
||||||
schema = rows.schema;
|
// myRows.Add (newRow);
|
||||||
foreach (IBlueRow row in rows) {
|
// myIndex[row["id"]] = newRow;
|
||||||
SortRow newRow = comparer is null ?
|
// }
|
||||||
new(row, sortCols) : new(row, comparer);
|
// myRows.Sort ();
|
||||||
this.rows.Add(newRow);
|
// }
|
||||||
rowIndex[(int)row["id"]!] = newRow;
|
|
||||||
}
|
// #endregion
|
||||||
this.rows.Sort();
|
|
||||||
}
|
// #region IBlueRowCollections members
|
||||||
}
|
// public IBlueRow this[int id]
|
||||||
|
// {
|
||||||
|
// get
|
||||||
|
// {
|
||||||
|
// if (myIndex[id] == null)
|
||||||
|
// throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||||
|
// return (IBlueRow)myIndex[id];
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// public IBlueTableSchema schema {get{return mySchema;}}
|
||||||
|
|
||||||
|
// public bool containsID (int id) {return myIndex[id] != null;}
|
||||||
|
|
||||||
|
// #endregion
|
||||||
|
|
||||||
|
// #region ICollection members
|
||||||
|
// public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
|
||||||
|
|
||||||
|
// public bool IsSynchronized {get{return false;}}
|
||||||
|
|
||||||
|
// public object SyncRoot {get{return this;}}
|
||||||
|
|
||||||
|
// public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
|
||||||
|
|
||||||
|
// public int Count{get{return myRows.Count;}}
|
||||||
|
// #endregion
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
@@ -22,12 +22,9 @@ namespace BCG
|
|||||||
/// <param name="rows"></param>
|
/// <param name="rows"></param>
|
||||||
public void loadCollection (IBlueRowCollection rows)
|
public void loadCollection (IBlueRowCollection rows)
|
||||||
{
|
{
|
||||||
myFireEvent(new DataPacket(
|
myFireEvent (new DataPacket (rows, new BlueTable(rows),
|
||||||
rows,
|
myLastValue != null ?
|
||||||
new BlueTable(rows),
|
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
|
||||||
myLastValue is DataPacket dp
|
|
||||||
? new BlueTable(dp.newRows)
|
|
||||||
: new BlueTable(rows.schema)));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ namespace BCG
|
|||||||
public Guid subscription;
|
public Guid subscription;
|
||||||
}
|
}
|
||||||
private ThreadSafeQueue<Update> _queued_updates;
|
private ThreadSafeQueue<Update> _queued_updates;
|
||||||
TargetDoWorkRelay? _do_work_relay;
|
TargetDoWorkRelay _do_work_relay;
|
||||||
Guid _current_source_subscription;
|
Guid _current_source_subscription;
|
||||||
Guid _current_target_subscription;
|
Guid _current_target_subscription;
|
||||||
int _source_thread;
|
int _source_thread;
|
||||||
|
|||||||
@@ -1,71 +1,29 @@
|
|||||||
using System;
|
using System;
|
||||||
|
|
||||||
using BCG.Utilities.General;
|
namespace BCG
|
||||||
|
{
|
||||||
|
public class MSecondStream : TimeUnitStream
|
||||||
|
{
|
||||||
|
public MSecondStream (int intervalInms, int size)
|
||||||
|
: base (DateTime.Now, new TimeSpan (0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance)
|
||||||
|
{}
|
||||||
|
|
||||||
using bts;
|
protected override DateTime myRoundTime (DateTime dt) {return dt;}
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
[DataStreamUserRespresentation("milliseconds")]
|
|
||||||
public class MillisecondStream : ViewStream, IWorker {
|
|
||||||
private DateTime _last_fire_time = DateTime.MinValue;
|
|
||||||
private readonly TimeSpan _period;
|
|
||||||
private readonly int _size;
|
|
||||||
private readonly ViewStreamDatabase _vsd;
|
|
||||||
private IBlueTableSchema _schema;
|
|
||||||
private bts.CircularBufferTable _table;
|
|
||||||
|
|
||||||
public MillisecondStream(int interval_msecs, int size)
|
|
||||||
: this(ViewStreamDatabase.instance, interval_msecs, size) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
|
|
||||||
_vsd = vsd;
|
|
||||||
_period = TimeSpan.FromMilliseconds(interval_msecs);
|
|
||||||
_size = size;
|
|
||||||
_schema = BlueDataUtility.form_schema("Milliseconds",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("time", TypeCode.DateTime),
|
|
||||||
Tuple.Create("display_time", TypeCode.String));
|
|
||||||
_table = new CircularBufferTable(_schema, _size);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_vsd.addWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_vsd.removeWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
private int _id = 0;
|
|
||||||
public void doWork() {
|
|
||||||
if (DateTime.Now - _last_fire_time >= _period) {
|
|
||||||
_last_fire_time = DateTime.Now;
|
|
||||||
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
|
|
||||||
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
|
|
||||||
_table.add_row(nr);
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
public class MSecondStream : TimeUnitStream {
|
|
||||||
public MSecondStream(int intervalInms, int size)
|
|
||||||
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
protected override DateTime myRoundTime(DateTime dt) { return dt; }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
|
public class DisplayableMillisecondStream : ViewStream, IStreamMapper
|
||||||
private IBlueTableSchema mySchema = new BlueTableSchema("ms",
|
{
|
||||||
new ColumnInfo("id", 0, typeof(int)),
|
private IBlueTableSchema mySchema = new BlueTableSchema ("ms",
|
||||||
new ColumnInfo("time", 1, typeof(string)));
|
new ColumnInfo ("id", 0, typeof(int)),
|
||||||
public DisplayableMillisecondStream(int intervalInMs, int sz) {
|
new ColumnInfo ("time", 1, typeof(string)));
|
||||||
mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
|
public DisplayableMillisecondStream (int intervalInMs, int sz)
|
||||||
|
{
|
||||||
|
mySetRealSource (new MapStream (new MSecondStream (intervalInMs, sz), this));
|
||||||
}
|
}
|
||||||
#region IStreamMapper Members
|
#region IStreamMapper Members
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
|
||||||
|
{
|
||||||
return mySchema;
|
return mySchema;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,15 +31,15 @@ namespace BCG {
|
|||||||
|
|
||||||
#region IDynamicStreamMapper Members
|
#region IDynamicStreamMapper Members
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row) {
|
public IBlueRow map (IBlueRow row)
|
||||||
DateTime time = row.get_datetime("time");
|
{
|
||||||
var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
return new BlueRow (mySchema, row["id"], myToString ((DateTime)row["time"]));
|
||||||
return new BlueRow(mySchema, row["id"], time_txt);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
private string myToString(DateTime dt) {
|
private string myToString (DateTime dt)
|
||||||
|
{
|
||||||
string sep = ":";
|
string sep = ":";
|
||||||
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
|
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,43 +1,49 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
|
||||||
namespace BCG {
|
// namespace BCG
|
||||||
public class TopStream : MethodDrivenStream {
|
// {
|
||||||
private int myTopCount;
|
// public class TopStream : MethodDrivenStream
|
||||||
|
// {
|
||||||
|
// private int myTopCount;
|
||||||
|
|
||||||
public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
|
// public TopStream (ViewStream stream, int topCount) : base (stream) {myTopCount = topCount;}
|
||||||
|
|
||||||
public TopStream(ViewStream stream,
|
// public TopStream (ViewStream stream,
|
||||||
int topCount,
|
// int topCount,
|
||||||
string sortExpression)
|
// string sortExpression)
|
||||||
:
|
// :
|
||||||
this(sortExpression == null ?
|
// this (sortExpression == null?
|
||||||
stream :
|
// stream :
|
||||||
new SortStream(stream, sortExpression), topCount) { }
|
// new SortStream (stream, sortExpression), topCount)
|
||||||
|
// {}
|
||||||
|
|
||||||
protected override void myOnTableClear() {
|
// protected override void myOnTableClear ()
|
||||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
// {
|
||||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
|
// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||||
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
|
// myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
|
||||||
}
|
// myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
|
||||||
|
// }
|
||||||
|
|
||||||
protected override void myHandleFirstCall() {
|
// protected override void myHandleFirstCall ()
|
||||||
myLastFiredPacket = new DataPacket();
|
// {
|
||||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
|
// myLastFiredPacket = new DataPacket ();
|
||||||
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
|
// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
|
||||||
myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
|
// myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
|
||||||
}
|
// myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
|
||||||
|
// }
|
||||||
|
|
||||||
protected override void myHandleIncrementalAdd() { }
|
// protected override void myHandleIncrementalAdd () {}
|
||||||
|
|
||||||
protected override void myHandleIncrementalDelete() {
|
// protected override void myHandleIncrementalDelete()
|
||||||
IBlueRowCollection oldRows = myLastFiredPacket.newRows;
|
// {
|
||||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
|
// IBlueRowCollection oldRows = myLastFiredPacket.newRows;
|
||||||
BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
|
// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
|
||||||
oldRows,
|
// BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
|
||||||
myLastFiredPacket.schema.columnNames,
|
// oldRows,
|
||||||
out myLastFiredPacket.deletedRows,
|
// myLastFiredPacket.schema.columnNames,
|
||||||
out myLastFiredPacket.addedRows);
|
// out myLastFiredPacket.deletedRows,
|
||||||
myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
|
// out myLastFiredPacket.addedRows);
|
||||||
}
|
// myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
// }
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Text;
|
||||||
using mbts;
|
using mbts;
|
||||||
|
using mbts.spark;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -181,6 +183,7 @@ namespace BCG
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public class MutableAppendStream : ViewStream
|
public class MutableAppendStream : ViewStream
|
||||||
{
|
{
|
||||||
|
HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||||
private bool mySubscriptionActive = false;
|
private bool mySubscriptionActive = false;
|
||||||
private MutableAppendTable myTable;
|
private MutableAppendTable myTable;
|
||||||
private int myMovingIndex;
|
private int myMovingIndex;
|
||||||
@@ -269,6 +272,7 @@ namespace BCG
|
|||||||
private void myHandleStreamChanges(object newValue)
|
private void myHandleStreamChanges(object newValue)
|
||||||
{
|
{
|
||||||
DataPacket dp = newValue as DataPacket;
|
DataPacket dp = newValue as DataPacket;
|
||||||
|
_timer.start();
|
||||||
myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
|
myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
|
||||||
BlueTable adds = new BlueTable(myTable.schema),
|
BlueTable adds = new BlueTable(myTable.schema),
|
||||||
dels = new BlueTable(myTable.schema);
|
dels = new BlueTable(myTable.schema);
|
||||||
@@ -283,6 +287,7 @@ namespace BCG
|
|||||||
IBlueRow nr = myTable.addRow(r, index);
|
IBlueRow nr = myTable.addRow(r, index);
|
||||||
adds.addRow(nr);
|
adds.addRow(nr);
|
||||||
}
|
}
|
||||||
|
_timer.stop();
|
||||||
myFireEvent(new DataPacket(myTable, adds, dels));
|
myFireEvent(new DataPacket(myTable, adds, dels));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -348,64 +348,6 @@ namespace BCG {
|
|||||||
return new BlueTableSchema(table_name, cis.ToArray());
|
return new BlueTableSchema(table_name, cis.ToArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
|
|
||||||
// just use one row to guess
|
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
|
||||||
foreach (IBlueRow r in rows) {
|
|
||||||
foreach (var ci in r.schema.columnInfo) {
|
|
||||||
if (ci.colType != typeof(System.String)) {
|
|
||||||
cis.Add(ci);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
var (type, _) = guess_type(r[ci.colName] as string);
|
|
||||||
cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
|
|
||||||
}
|
|
||||||
return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// converts a row where all columns are string types to a schema where the best types are guesed
|
|
||||||
public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
|
|
||||||
IBlueTableSchema typed_schema = get_typed_schema(string_rows);
|
|
||||||
var rv = new BlueTable(typed_schema);
|
|
||||||
foreach (IBlueRow r in string_rows) {
|
|
||||||
var typed_row = get_typed_row(r, typed_schema);
|
|
||||||
rv.addRow(typed_row);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
|
|
||||||
static IBlueRow get_typed_row(IBlueRow string_row, IBlueTableSchema typed_schema) {
|
|
||||||
var rv = new MutableBlueRow(typed_schema);
|
|
||||||
foreach (var ci in typed_schema.columnInfo) {
|
|
||||||
var typed_val = convert(ci.colType, string_row[ci.colName].ToString());
|
|
||||||
rv.setValue(ci.colName, typed_val);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
static object convert(Type t, string value) {
|
|
||||||
// Attempt to convert the string to the specified type
|
|
||||||
try {
|
|
||||||
return Convert.ChangeType(value, t);
|
|
||||||
} catch (Exception) {
|
|
||||||
if (t == typeof(double)) {
|
|
||||||
// some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
|
|
||||||
string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
|
|
||||||
cleaned_val = cleaned_val.Replace('−', '-');
|
|
||||||
double rv_d;
|
|
||||||
if (Double.TryParse(cleaned_val, out rv_d))
|
|
||||||
return rv_d;
|
|
||||||
else
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
// Handle conversion failure gracefully
|
|
||||||
// throw new ArgumentException("Failed to convert the string to the specified type.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
|
/// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
|
||||||
/// be alphabetic or not
|
/// be alphabetic or not
|
||||||
@@ -1628,13 +1570,6 @@ namespace BCG {
|
|||||||
return sb.ToString();
|
return sb.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static string id_string(IBlueRowCollection table) {
|
|
||||||
string rv = String.Empty;
|
|
||||||
foreach (IBlueRow r in table)
|
|
||||||
rv += $"{r["id"]} ";
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
|
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
|
||||||
BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
|
BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
|
||||||
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
|
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
|
||||||
@@ -1783,18 +1718,6 @@ namespace BCG {
|
|||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
|
|
||||||
public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
|
|
||||||
if (rows == null || rows.Count() == 0)
|
|
||||||
return null;
|
|
||||||
IBlueRow max_row = rows.ElementAt(0);
|
|
||||||
foreach (IBlueRow r in rows) {
|
|
||||||
int c = compare_rows(r, max_row, cols_to_compare);
|
|
||||||
if (c > 0)
|
|
||||||
max_row = r;
|
|
||||||
}
|
|
||||||
return max_row;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
|
public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
|
||||||
foreach (KeyValuePair<string, bool> p in sort_map) {
|
foreach (KeyValuePair<string, bool> p in sort_map) {
|
||||||
int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
|
int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
|
||||||
@@ -2229,10 +2152,7 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) =>
|
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) {
|
||||||
createRowsFromCSVFileWithoutTypes(fileAddress, out var _);
|
|
||||||
|
|
||||||
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress, out IBlueTableSchema schema) {
|
|
||||||
StreamReader sr = new StreamReader(fileAddress);
|
StreamReader sr = new StreamReader(fileAddress);
|
||||||
string cols = sr.ReadLine();
|
string cols = sr.ReadLine();
|
||||||
string[] columns = cols.Split(',');
|
string[] columns = cols.Split(',');
|
||||||
@@ -2240,11 +2160,11 @@ namespace BCG {
|
|||||||
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
||||||
for (int i = 1; i < ci.Length; i++)
|
for (int i = 1; i < ci.Length; i++)
|
||||||
ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
|
ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
|
||||||
schema = new BlueTableSchema(ci, "csvTable");
|
IBlueTableSchema bts = new BlueTableSchema(ci, "csvTable");
|
||||||
BlueTable result = new BlueTable(schema);
|
BlueTable result = new BlueTable(bts);
|
||||||
int id = 0;
|
int id = 0;
|
||||||
while (!sr.EndOfStream)
|
while (!sr.EndOfStream)
|
||||||
result.addRow(createRowFromCSVFileRow(sr.ReadLine(), schema, id++));
|
result.addRow(createRowFromCSVFileRow(sr.ReadLine(), bts, id++));
|
||||||
sr.Close();
|
sr.Close();
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,35 +0,0 @@
|
|||||||
#nullable enable
|
|
||||||
namespace Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class BlueRowCollectionEqualityComparer : IEqualityComparer<IBlueRowCollection?> {
|
|
||||||
readonly IEqualityComparer<IBlueRow> rowComparer = BlueRowEqualityComparer.Instance;
|
|
||||||
|
|
||||||
public static BlueRowCollectionEqualityComparer Instance { get; } = new();
|
|
||||||
|
|
||||||
public bool Equals(IBlueRowCollection? x, IBlueRowCollection? y) {
|
|
||||||
if (x is null) {
|
|
||||||
return y is null;
|
|
||||||
}
|
|
||||||
if (x.Count != y?.Count) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return x.schema.columnInfo.SequenceEqual(y.schema.columnInfo)
|
|
||||||
&& x.Cast<IBlueRow>().OrderBy(r => r.Id()).SequenceEqual(
|
|
||||||
y.Cast<IBlueRow>().OrderBy(r => r.Id()),
|
|
||||||
rowComparer);
|
|
||||||
}
|
|
||||||
|
|
||||||
public int GetHashCode([DisallowNull] IBlueRowCollection rows) {
|
|
||||||
var hash = new HashCode();
|
|
||||||
hash.Add(rows.schema.columnInfo.Length);
|
|
||||||
hash.Add(rows.Count);
|
|
||||||
return hash.ToHashCode();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
namespace Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class BlueRowEqualityComparer : IEqualityComparer<IBlueRow> {
|
|
||||||
public static BlueRowEqualityComparer Instance { get; } = new();
|
|
||||||
|
|
||||||
public bool Equals(IBlueRow x, IBlueRow y) =>
|
|
||||||
x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) &&
|
|
||||||
x.Values.Cast<object>().SequenceEqual(y.Values.Cast<object>());
|
|
||||||
|
|
||||||
public int GetHashCode([DisallowNull] IBlueRow obj) {
|
|
||||||
var hash = new HashCode();
|
|
||||||
foreach (var v in obj.Values) {
|
|
||||||
hash.Add(v?.GetHashCode() ?? 0);
|
|
||||||
}
|
|
||||||
return hash.ToHashCode();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,15 +1,8 @@
|
|||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.DataStream;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Data;
|
using System.Data;
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
|
||||||
//using BCG.Utilities;
|
//using BCG.Utilities;
|
||||||
|
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
@@ -17,7 +10,7 @@ namespace BCG {
|
|||||||
/// Anologue of the dotNet DataTable.
|
/// Anologue of the dotNet DataTable.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class BlueTable : IBlueRowCollection, IDictionary<int, IBlueRow> {
|
public class BlueTable : IBlueRowCollection {
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Hashtable indexed on "id" to speed up removes
|
/// Hashtable indexed on "id" to speed up removes
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -93,12 +86,6 @@ namespace BCG {
|
|||||||
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||||
return (IBlueRow)row;
|
return (IBlueRow)row;
|
||||||
}
|
}
|
||||||
set {
|
|
||||||
if ((int)value["id"] != id) {
|
|
||||||
throw new ArgumentException("id arg not equal to row's actual id", nameof(id));
|
|
||||||
}
|
|
||||||
changeRow(value);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -215,8 +202,6 @@ namespace BCG {
|
|||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
public IDictionary<int, IBlueRow> AsDictionary() => this;
|
|
||||||
|
|
||||||
#region ICollection members
|
#region ICollection members
|
||||||
public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
|
public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
|
||||||
{ return myIdIndex.Values.GetEnumerator(); }
|
{ return myIdIndex.Values.GetEnumerator(); }
|
||||||
@@ -229,55 +214,5 @@ namespace BCG {
|
|||||||
|
|
||||||
public virtual int Count { get { return myIdIndex.Count; } }
|
public virtual int Count { get { return myIdIndex.Count; } }
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IDictionary<int, IBlueRow> members
|
|
||||||
public void Add(int key, IBlueRow value) {
|
|
||||||
throw new NotImplementedException();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool IDictionary<int, IBlueRow>.ContainsKey(int key) =>
|
|
||||||
containsID(key);
|
|
||||||
|
|
||||||
bool IDictionary<int, IBlueRow>.Remove(int key) =>
|
|
||||||
deleteRow(key);
|
|
||||||
|
|
||||||
bool IDictionary<int, IBlueRow>.TryGetValue(
|
|
||||||
int key, [MaybeNullWhen(false)] out IBlueRow value) =>
|
|
||||||
tryGetValue(key, out value);
|
|
||||||
|
|
||||||
void ICollection<KeyValuePair<int, IBlueRow>>.Add(KeyValuePair<int, IBlueRow> item) =>
|
|
||||||
addRow(item.Value);
|
|
||||||
|
|
||||||
void ICollection<KeyValuePair<int, IBlueRow>>.Clear() =>
|
|
||||||
clear();
|
|
||||||
|
|
||||||
bool ICollection<KeyValuePair<int, IBlueRow>>.Contains(KeyValuePair<int, IBlueRow> item) =>
|
|
||||||
tryGetValue(item.Key, out var r)
|
|
||||||
&& BlueRowEqualityComparer.Instance.Equals(r, item.Value);
|
|
||||||
|
|
||||||
void ICollection<KeyValuePair<int, IBlueRow>>.CopyTo(
|
|
||||||
KeyValuePair<int, IBlueRow>[] array, int arrayIndex) {
|
|
||||||
foreach (DictionaryEntry de in myIdIndex) {
|
|
||||||
array[arrayIndex++] = new((int)de.Key, (IBlueRow)de.Value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bool ICollection<KeyValuePair<int, IBlueRow>>.Remove(KeyValuePair<int, IBlueRow> item) =>
|
|
||||||
deleteRow(item.Key);
|
|
||||||
|
|
||||||
IEnumerator<KeyValuePair<int, IBlueRow>> IEnumerable<KeyValuePair<int, IBlueRow>>.GetEnumerator() {
|
|
||||||
foreach (DictionaryEntry de in myIdIndex) {
|
|
||||||
yield return new((int)de.Key, (IBlueRow)de.Value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ICollection<int> IDictionary<int, IBlueRow>.Keys =>
|
|
||||||
GenericCollectionWrapper.Create<int>(myIdIndex.Keys);
|
|
||||||
|
|
||||||
ICollection<IBlueRow> IDictionary<int, IBlueRow>.Values =>
|
|
||||||
GenericCollectionWrapper.Create<IBlueRow>(myIdIndex.Values);
|
|
||||||
|
|
||||||
bool ICollection<KeyValuePair<int, IBlueRow>>.IsReadOnly => false;
|
|
||||||
#endregion
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Loaded 100 of 697 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user