Compare commits
279
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
11e6d25001 | ||
|
|
83d66ae9d7 | ||
|
|
e5e74b7b42 | ||
|
|
6ce2fb2cc5 | ||
|
|
9d63a004ec | ||
|
|
ccd53d71ea | ||
|
|
74a0235227 | ||
|
|
42bb101b43 | ||
|
|
c53561c6ef | ||
|
|
3701a70bd9 | ||
|
|
df56de3dd2 | ||
|
|
c9a418ce4b | ||
|
|
aaad404507 | ||
|
|
45acaa20b7 | ||
|
|
96a08500fb | ||
|
|
510e7b0050 | ||
|
|
2ab76c753e | ||
|
|
a02c6b77d4 | ||
|
|
6a289a5561 | ||
|
|
fddd310a03 | ||
|
|
d2a2d5fb37 | ||
|
|
4b76442a8f | ||
|
|
09b863f2ab | ||
|
|
dd221aba10 | ||
|
|
5ef862b026 | ||
|
|
cc43e9d3a4 | ||
|
|
b6eb727bc9 | ||
|
|
8ace69ab74 | ||
|
|
dc88b4ee10 | ||
|
|
dee32dead4 | ||
|
|
b2e022505f | ||
|
|
ac9b3ec365 | ||
|
|
ed89da7f38 | ||
|
|
a334d9be88 | ||
|
|
aee56569d1 | ||
|
|
7fdd3f6b1b | ||
|
|
3e1949d94f | ||
|
|
4cb3dbc313 | ||
|
|
0268bd476f | ||
|
|
2511b0d966 | ||
|
|
92456649df | ||
|
|
ec32078dd5 | ||
|
|
6bcce832ac | ||
|
|
85529b2232 | ||
|
|
c3cb8995bf | ||
|
|
ab2f0f975e | ||
|
|
2e133ea182 | ||
|
|
30615f3429 | ||
|
|
61371804a2 | ||
|
|
02d82e39bc | ||
|
|
50188e6688 | ||
|
|
5137785f7f | ||
|
|
def021612c | ||
|
|
217c5be131 | ||
|
|
a75441cd07 | ||
|
|
78eb096d6f | ||
|
|
2b66f46db1 | ||
|
|
ae3029d9ed | ||
|
|
d4a0603933 | ||
|
|
db6e96cf00 | ||
|
|
371db4afd2 | ||
|
|
08484a0954 | ||
|
|
39db44ed63 | ||
|
|
b4dca8e23f | ||
|
|
b4bf06d17a | ||
|
|
8c17705f02 | ||
|
|
551f26e5f3 | ||
|
|
f4adf35e49 | ||
|
|
90128863d9 | ||
|
|
300831c97a | ||
|
|
f3f6a82d9a | ||
|
|
7c8a41b2b7 | ||
|
|
408114c1f6 | ||
|
|
5c9be0bfb9 | ||
|
|
c0f970ab89 | ||
|
|
25e1804286 | ||
|
|
3c84dc48f6 | ||
|
|
9ae20f0c4d | ||
|
|
e8a72e7bd9 | ||
|
|
e99e2d817a | ||
|
|
7f4468c812 | ||
|
|
6206cbc425 | ||
|
|
21e50745e2 | ||
|
|
990e21e387 | ||
|
|
357be39db3 | ||
|
|
68e04cb3f5 | ||
|
|
b4a5d5c832 | ||
|
|
242fd758f4 | ||
|
|
3c9755f0dd | ||
|
|
61ed847d7d | ||
|
|
fc9094fea8 | ||
|
|
e36c0efc4b | ||
|
|
4158a14cff | ||
|
|
201a45cba0 | ||
|
|
ab31b9b253 | ||
|
|
780ddc79a7 | ||
|
|
d5a2adaed6 | ||
|
|
008ab3f068 | ||
|
|
7d926f588d | ||
|
|
707cd07876 | ||
|
|
f7ecb9090c | ||
|
|
a19e8a7315 | ||
|
|
cf189ecf81 | ||
|
|
4405b7adcb | ||
|
|
e51ee7b7b9 | ||
|
|
dfaa74f426 | ||
|
|
1d2ce59009 | ||
|
|
3602cc05bb | ||
|
|
ac88a81ac6 | ||
|
|
01fa8e0c63 | ||
|
|
59547e0f37 | ||
|
|
3ce100a9cf | ||
|
|
6513fb0cd5 | ||
|
|
fd0fb1f760 | ||
|
|
5cf7eacd82 | ||
|
|
087d8ab8c9 | ||
|
|
b69b7b3350 | ||
|
|
5620f33524 | ||
|
|
f3e33fa390 | ||
|
|
3b631e0d5f | ||
|
|
f93d28ee76 | ||
|
|
23d7d3775d | ||
|
|
4940771166 | ||
|
|
c1967a8f6a | ||
|
|
8c797ee76b | ||
|
|
ce80957d07 | ||
|
|
c60018b59e | ||
|
|
3a27af87d1 | ||
|
|
068c1c6973 | ||
|
|
645a58c97d | ||
|
|
b7f455b29a | ||
|
|
4a2735839e | ||
|
|
d2e9fe8237 | ||
|
|
cfc60766d8 | ||
|
|
b03fec7f79 | ||
|
|
fb8e7c5b81 | ||
|
|
e44601d1b1 | ||
|
|
07bade53c0 | ||
|
|
fac0a9c25b | ||
|
|
1d1de3dd18 | ||
|
|
cfcbf004ba | ||
|
|
78c7dadbbe | ||
|
|
7b5a6031af | ||
|
|
c13a7782c8 | ||
|
|
45f02391d1 | ||
|
|
a3ca06addc | ||
|
|
32869e54ec | ||
|
|
ca73220a2e | ||
|
|
9fc3c2618e | ||
|
|
e72e0c0a30 | ||
|
|
a5bfc75278 | ||
|
|
261a184d25 | ||
|
|
54e96a06cb | ||
|
|
17e60df32b | ||
|
|
c99d5fe24c | ||
|
|
6049826e90 | ||
|
|
672149dd89 | ||
|
|
8c99147f04 | ||
|
|
4d4b0b7ae3 | ||
|
|
1a7955c1b1 | ||
|
|
f8fc2a1d96 | ||
|
|
9a7b3cb77c | ||
|
|
8a3e4edb4d | ||
|
|
e59183bb1e | ||
|
|
dd4d4e8dc6 | ||
|
|
f1744450ca | ||
|
|
3a7827a75b | ||
|
|
8d22e91c0d | ||
|
|
c425145fc9 | ||
|
|
4855935ae0 | ||
|
|
65f412ec9f | ||
|
|
40681c6093 | ||
|
|
ab2d36e523 | ||
|
|
b1ab3a66d2 | ||
|
|
5e7e7e8ef4 | ||
|
|
9291d30ae9 | ||
|
|
549708fcc2 | ||
|
|
1d5551f301 | ||
|
|
283db0f9ea | ||
|
|
9fb04b2db9 | ||
|
|
ceca96e10a | ||
|
|
fc48cc64af | ||
|
|
015d846941 | ||
|
|
e39108bbda | ||
|
|
6240d89a08 | ||
|
|
c76700e5e5 | ||
|
|
a8a0e8dea0 | ||
|
|
d65a2d6ed6 | ||
|
|
1824d5f9a4 | ||
|
|
d3eb29087b | ||
|
|
d6fae238f9 | ||
|
|
44b46033e3 | ||
|
|
5741ec9ab2 | ||
|
|
596975b309 | ||
|
|
9050318ceb | ||
|
|
7ee6933789 | ||
|
|
306c572b53 | ||
|
|
63306173b1 | ||
|
|
df1b45ca3e | ||
|
|
c31a1f1bf3 | ||
|
|
e95997ea17 | ||
|
|
12a070fd8d | ||
|
|
e0459c54fd | ||
|
|
15262a5a95 | ||
|
|
90ec64d25c | ||
|
|
061481419e | ||
|
|
8035255868 | ||
|
|
7feaf869dc | ||
|
|
7104e90fc9 | ||
|
|
52c1e500e6 | ||
|
|
07274f897a | ||
|
|
f626e53415 | ||
|
|
5a7b2965be | ||
|
|
0aab1628bc | ||
|
|
b9649e1d8e | ||
|
|
3bc9bf2207 | ||
|
|
888c06262b | ||
|
|
03e80dabcb | ||
|
|
54184d0875 | ||
|
|
90bcf5a455 | ||
|
|
d32d5ef40c | ||
|
|
2e0e0658c6 | ||
|
|
bbdb50a41e | ||
|
|
90705cac83 | ||
|
|
b91cdd6d91 | ||
|
|
64f2bd7a88 | ||
|
|
3e32f684e3 | ||
|
|
ff73756a55 | ||
|
|
1e64811d93 | ||
|
|
2488597cf2 | ||
|
|
2d8047ba80 | ||
|
|
3cb602b77e | ||
|
|
51e1005e79 | ||
|
|
ef7ea64973 | ||
|
|
0267c16e5a | ||
|
|
ca40e278ff | ||
|
|
1c5da7ad48 | ||
|
|
9bb3d5c569 | ||
|
|
92ba43bfad | ||
|
|
ddeb5e59f0 | ||
|
|
6b0b3e31cc | ||
|
|
3f6c904ccc | ||
|
|
d433919f1b | ||
|
|
75da9af39d | ||
|
|
22eff44926 | ||
|
|
cd6a052d0c | ||
|
|
3b6cf0b187 | ||
|
|
c170dd7a9e | ||
|
|
154775bc70 | ||
|
|
7f936f3ba9 | ||
|
|
9b30a947a8 | ||
|
|
2ee58d8f99 | ||
|
|
0537b3e9c7 | ||
|
|
8ae940bcbc | ||
|
|
966ccb5f2c | ||
|
|
fae9c3e59e | ||
|
|
cb1dfb257d | ||
|
|
76c3fe86c2 | ||
|
|
7e3b1de70f | ||
|
|
2a07e3cf3c | ||
|
|
6d0249825d | ||
|
|
63810e3966 | ||
|
|
4c013a8886 | ||
|
|
9509818978 | ||
|
|
4fdf3b8b60 | ||
|
|
ee9a58486d | ||
|
|
75d6939c8b | ||
|
|
fb1697b571 | ||
|
|
4080d606d8 | ||
|
|
7655fd47dd | ||
|
|
d029b0fce7 | ||
|
|
60df5b2a68 | ||
|
|
bada5dc961 | ||
|
|
4510e120ab | ||
|
|
da2af06ee6 | ||
|
|
8c1342230b | ||
|
|
8dd2775092 | ||
|
|
8966b73ac9 | ||
|
|
ddb2a91763 |
No files matched your search
@@ -0,0 +1,125 @@
|
||||
FROM ubuntu:22.04
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Note installations that take a while (more than 10s)
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y ca-certificates
|
||||
RUN apt-get install -y ccache
|
||||
RUN apt-get install -y curl
|
||||
RUN apt-get install -y fish
|
||||
RUN apt-get install -y gdb
|
||||
# git-all: ~250s
|
||||
RUN apt-get install -y git-all
|
||||
RUN apt-get install -y gnupg
|
||||
# liblttng-ust-dev: ~30s
|
||||
RUN apt-get install -y liblttng-ust-dev
|
||||
RUN apt-get install -y libssh2-1-dev
|
||||
RUN apt-get install -y locales
|
||||
RUN apt-get install -y ninja-build
|
||||
# openssh-server: ~40s
|
||||
RUN apt-get install -y openssh-server
|
||||
RUN apt-get install -y pkg-config
|
||||
RUN apt-get install -y software-properties-common
|
||||
RUN apt-get install -y sudo
|
||||
RUN apt-get install -y tzdata
|
||||
RUN apt-get install -y vim
|
||||
RUN apt-get install -y zip
|
||||
|
||||
# Python: ~30s
|
||||
RUN apt-get install -y \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
&& python3 -m pip install --upgrade pip
|
||||
|
||||
# GCC
|
||||
ARG gcc_major=13
|
||||
RUN \
|
||||
add-apt-repository ppa:ubuntu-toolchain-r/test \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y g++-$gcc_major
|
||||
|
||||
# Clang
|
||||
ARG clang_major=17
|
||||
RUN \
|
||||
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
|
||||
| gpg --dearmor - \
|
||||
| tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
|
||||
&& echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
|
||||
| tee -a "/etc/apt/sources.list.d/llvm.list" \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y \
|
||||
clang-$clang_major \
|
||||
clang-format-$clang_major \
|
||||
clangd-$clang_major
|
||||
|
||||
# CMake
|
||||
RUN \
|
||||
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
|
||||
| gpg --dearmor - \
|
||||
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
|
||||
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
|
||||
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y cmake
|
||||
|
||||
# Dotnet
|
||||
RUN apt-get update && apt-get install -y dotnet7
|
||||
|
||||
# Node
|
||||
ARG node_major=18
|
||||
RUN mkdir -p /etc/apt/keyrings \
|
||||
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
|
||||
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
|
||||
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
|
||||
| tee /etc/apt/sources.list.d/nodesource.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y nodejs
|
||||
|
||||
# Emumba (deps)
|
||||
RUN apt-get update && apt-get -y install rapidjson-dev
|
||||
# Emumba
|
||||
COPY emumba /emumba
|
||||
RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
|
||||
&& rm -r /emumba
|
||||
|
||||
# Pcap
|
||||
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
||||
|
||||
# Databento (deps)
|
||||
RUN apt-get update && apt-get -y install libzstd-dev
|
||||
|
||||
# Set up default executables (update-alternatives)
|
||||
RUN \
|
||||
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
||||
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
|
||||
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
|
||||
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
|
||||
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
|
||||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
|
||||
|
||||
# Add compiler symlinks for ccache
|
||||
RUN \
|
||||
ln -s /usr/bin/ccache /usr/local/bin/clang; \
|
||||
ln -s /usr/bin/ccache /usr/local/bin/clang++; \
|
||||
ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
|
||||
ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
|
||||
ln -s /usr/bin/ccache /usr/local/bin/gcc; \
|
||||
ln -s /usr/bin/ccache /usr/local/bin/g++
|
||||
|
||||
RUN locale-gen en_US.UTF-8
|
||||
|
||||
ARG docker_user="bts"
|
||||
ARG docker_uid="1000"
|
||||
ARG docker_pass="bts"
|
||||
|
||||
RUN \
|
||||
groupadd -g "$docker_uid" "$docker_user" \
|
||||
&& useradd \
|
||||
-m \
|
||||
-u "$docker_uid" \
|
||||
-g "$docker_uid" \
|
||||
-G sudo \
|
||||
--password "$(openssl passwd -1 "$docker_pass")" \
|
||||
"$docker_user"
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"dockerFile": "Dockerfile",
|
||||
// Creates local dir to mount to HOME in docker
|
||||
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
|
||||
"hostRequirements": {
|
||||
"memory": "4gb",
|
||||
"storage": "6gb"
|
||||
},
|
||||
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
|
||||
"containerUser": "bts",
|
||||
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
|
||||
"workspaceFolder": "/home/bts/spark",
|
||||
"updateRemoteUserUID": true,
|
||||
"shutdownAction": "none",
|
||||
// Note: If running multiple containers on the same host, each must use a
|
||||
// unique set of ports. Usually we just add 1000 to each port as needed. For
|
||||
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
||||
"runArgs": [
|
||||
"--shm-size",
|
||||
"2gb",
|
||||
"-p",
|
||||
"8000:8000",
|
||||
"-p",
|
||||
"8001:8001",
|
||||
"-p",
|
||||
"8002:8002",
|
||||
"-p",
|
||||
"8003:8003",
|
||||
"-p",
|
||||
"12345:12345"
|
||||
],
|
||||
"mounts": [
|
||||
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
|
||||
],
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"settings": {
|
||||
"clangd.arguments": [
|
||||
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
|
||||
]
|
||||
},
|
||||
"extensions": [
|
||||
"eamodio.gitlens",
|
||||
"llvm-vs-code-extensions.vscode-clangd",
|
||||
"ms-vscode.cmake-tools",
|
||||
"ms-vscode.cpptools"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
|
||||
sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
|
||||
@@ -0,0 +1,3 @@
|
||||
*.cmd text=auto eol=crlf
|
||||
*.cs text=auto eol=crlf
|
||||
*.sh text=auto eol=lf
|
||||
+4
-2
@@ -8,7 +8,9 @@ cmake-build-debug/
|
||||
.idea/
|
||||
.dir-locals.el
|
||||
*.pyc
|
||||
**.vscode
|
||||
**.vscode/*
|
||||
!.vscode/launch.json
|
||||
!.vscode/tasks.json
|
||||
.vs
|
||||
obj/
|
||||
*.csproj.user
|
||||
@@ -54,6 +56,6 @@ trace_data/
|
||||
/src/Spark.Admin.WebApp/src/bin/
|
||||
**/CMakeDeps.cmake
|
||||
**/CMakeDeps_win.cmake
|
||||
**/.devcontainer
|
||||
.devcontainer/
|
||||
*.Generated.cs
|
||||
*.lst
|
||||
Vendored
+158
@@ -0,0 +1,158 @@
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": ".NET Core Attach",
|
||||
"type": "coreclr",
|
||||
"request": "attach"
|
||||
},
|
||||
{
|
||||
"name": "DataStreamPublisher",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"preLaunchTask": "Build DataStream (Debug)",
|
||||
"program": "",
|
||||
"linux": {
|
||||
"program": "DataStreamPublisher",
|
||||
},
|
||||
"windows": {
|
||||
"program": "DataStreamPublisher.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
||||
],
|
||||
"cwd": "build/Debug/bin/net7.0",
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "(gdb) Launch",
|
||||
"type": "cppdbg",
|
||||
"request": "launch",
|
||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||
"args": ["run"],
|
||||
"stopAtEntry": false,
|
||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||
"environment": [],
|
||||
"externalConsole": false,
|
||||
"MIMode": "gdb",
|
||||
},
|
||||
{
|
||||
"name": "DataStreamQueryAnalyzer",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
||||
"program": "",
|
||||
"windows": {
|
||||
"program": "DataStreamQueryAnalyzer.exe",
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
||||
],
|
||||
"cwd": "build/Debug/bin/net7.0-windows",
|
||||
"env": {
|
||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
||||
},
|
||||
},
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal",
|
||||
},
|
||||
{
|
||||
"name": "PubSubServer",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"preLaunchTask": "Build DataStream (Debug)",
|
||||
"program": "",
|
||||
"linux": {
|
||||
"program": "PubSubServer",
|
||||
},
|
||||
"windows": {
|
||||
"program": "PubSubServer.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
||||
],
|
||||
"cwd": "build/Debug/bin/net7.0",
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "Spark",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"program": "",
|
||||
"windows": {
|
||||
"preLaunchTask": "Build Spark (Debug)",
|
||||
"program": "Spark.exe",
|
||||
"cwd": "build/Debug/bin/net7.0-windows",
|
||||
},
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "Spark (Parameterized)",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"preLaunchTask": "Build Spark (Debug)",
|
||||
"program": "",
|
||||
"windows": {
|
||||
"program": "Spark.exe",
|
||||
"args": [
|
||||
"--sparkd",
|
||||
"${input:sparkConfigServer}"
|
||||
],
|
||||
"cwd": "build/Debug/bin/net7.0-windows",
|
||||
},
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "Sparkd",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"preLaunchTask": "Build Sparkd (Debug | Clang)",
|
||||
"program": "",
|
||||
"linux": {
|
||||
"program": "sparkd",
|
||||
},
|
||||
"args": [
|
||||
"run"
|
||||
],
|
||||
"cwd": "build/sparkd-debug-clang/server",
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
}
|
||||
],
|
||||
"inputs": [
|
||||
{
|
||||
"id": "dataStreamPublisherConfig",
|
||||
"type": "pickString",
|
||||
"description": "Publisher Configuration",
|
||||
"options": [
|
||||
"MarketDataPublisher",
|
||||
"SimpleEchoClient",
|
||||
"SimpleTestDataPublisher",
|
||||
"TestDataPublisher",
|
||||
],
|
||||
"default": "MarketDataPublisher",
|
||||
},
|
||||
{
|
||||
"id": "pubSubServerConfig",
|
||||
"type": "pickString",
|
||||
"description": "PubSub Configuration",
|
||||
"options": [
|
||||
"PubSubServer",
|
||||
"SimpleEchoClient",
|
||||
],
|
||||
"default": "PubSubServer",
|
||||
},
|
||||
{
|
||||
"id": "sparkConfigServer",
|
||||
"type": "promptString",
|
||||
"description": "Config Server URL",
|
||||
"default": "http://localhost:8000/config",
|
||||
},
|
||||
]
|
||||
}
|
||||
Vendored
+828
@@ -0,0 +1,828 @@
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "Build Api Server",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_api_server.sh",
|
||||
},
|
||||
"windows": {
|
||||
"command": "tools/build_api_server.cmd",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"${input:buildConfiguration}",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Api Server (Debug)",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_api_server.sh",
|
||||
},
|
||||
"windows": {
|
||||
"command": "tools/build_api_server.cmd",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"Debug",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Api Server (Release)",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "tools/build_api_server.cmd",
|
||||
},
|
||||
"linux": {
|
||||
"command": "tools/build_api_server.sh",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"Release",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build DataStream",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_data_stream.sh",
|
||||
},
|
||||
"windows": {
|
||||
"command": "tools/build_data_stream.cmd",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"${input:buildConfiguration}",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build DataStream (Debug)",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_data_stream.sh",
|
||||
},
|
||||
"windows": {
|
||||
"command": "tools/build_data_stream.cmd",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"Debug",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build DataStream (Release)",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_data_stream.sh",
|
||||
},
|
||||
"windows": {
|
||||
"command": "tools/build_data_stream.cmd",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"Release",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Spark",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "tools/build_spark.cmd",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"${input:buildConfiguration}",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Spark (Debug)",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "tools/build_spark.cmd",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"Debug",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Spark (Release)",
|
||||
"group": "build",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "tools/build_spark.cmd",
|
||||
},
|
||||
"args": [
|
||||
"--configuration",
|
||||
"Release",
|
||||
],
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Sparkd",
|
||||
"group": "build",
|
||||
"type": "shell",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_sparkd.sh",
|
||||
"args": [
|
||||
"--preset",
|
||||
"${input:sparkdCmakePreset}",
|
||||
],
|
||||
},
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Sparkd (Debug)",
|
||||
"group": "build",
|
||||
"type": "shell",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_sparkd.sh",
|
||||
"args": [
|
||||
"--build-type",
|
||||
"Debug",
|
||||
"--compiler",
|
||||
"${input:sparkdCompiler}"
|
||||
],
|
||||
},
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Sparkd (Debug | Gcc)",
|
||||
"group": "build",
|
||||
"type": "shell",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_sparkd.sh",
|
||||
"args": [
|
||||
"--build-type",
|
||||
"Debug",
|
||||
"--compiler",
|
||||
"gcc"
|
||||
],
|
||||
},
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Sparkd (Debug | Clang)",
|
||||
"group": "build",
|
||||
"type": "shell",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_sparkd.sh",
|
||||
"args": [
|
||||
"--build-type",
|
||||
"Debug",
|
||||
"--compiler",
|
||||
"clang"
|
||||
],
|
||||
},
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Sparkd (Release)",
|
||||
"group": "build",
|
||||
"type": "shell",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_sparkd.sh",
|
||||
"args": [
|
||||
"--build-type",
|
||||
"Release",
|
||||
"--compiler",
|
||||
"${input:sparkdCompiler}"
|
||||
],
|
||||
},
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Sparkd (Release | Gcc)",
|
||||
"group": "build",
|
||||
"type": "shell",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_sparkd.sh",
|
||||
"args": [
|
||||
"--build-type",
|
||||
"Release",
|
||||
"--compiler",
|
||||
"gcc"
|
||||
],
|
||||
},
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build Sparkd (Release | Clang)",
|
||||
"group": "build",
|
||||
"type": "shell",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/build_sparkd.sh",
|
||||
"args": [
|
||||
"--build-type",
|
||||
"Release",
|
||||
"--compiler",
|
||||
"clang"
|
||||
],
|
||||
},
|
||||
"icon": {
|
||||
"id": "terminal",
|
||||
"color": "terminal.ansiCyan"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Edit DataStream Config",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "editor",
|
||||
"args": [
|
||||
"${env:HOME}/.local/share/DataStream/conf/spark.conf"
|
||||
],
|
||||
},
|
||||
"windows": {
|
||||
"command": "vim",
|
||||
"args": [
|
||||
"${env:APPDATA}/bts/DataStream/conf/spark.conf"
|
||||
],
|
||||
},
|
||||
"presentation": {
|
||||
"reveal": "always",
|
||||
"focus": true,
|
||||
"panel": "shared",
|
||||
"showReuseMessage": false,
|
||||
"close": true,
|
||||
},
|
||||
"icon": {
|
||||
"id": "wrench"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Edit Spark Config",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "vim",
|
||||
"args": [
|
||||
"${env:APPDATA}/bts/conf/spark.conf"
|
||||
],
|
||||
},
|
||||
"presentation": {
|
||||
"reveal": "always",
|
||||
"focus": true,
|
||||
"panel": "dedicated",
|
||||
"showReuseMessage": false,
|
||||
"close": true,
|
||||
},
|
||||
"icon": {
|
||||
"id": "wrench"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run DataStreamQueryAnalyzer",
|
||||
"dependsOn": "Build DataStream",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "DataStreamQueryAnalyzer.exe",
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||
},
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run DataStreamQueryAnalyzer (Debug)",
|
||||
"dependsOn": "Build DataStream (Debug)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "DataStreamQueryAnalyzer.exe",
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
||||
},
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run DataStreamQueryAnalyzer (Release)",
|
||||
"dependsOn": "Build DataStream (Release)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "DataStreamQueryAnalyzer.exe",
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
||||
},
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run DataStreamPublisher",
|
||||
"dependsOn": "Build DataStream",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "DataStreamPublisher",
|
||||
},
|
||||
"windows": {
|
||||
"command": "DataStreamPublisher.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run DataStreamPublisher (Debug)",
|
||||
"dependsOn": "Build DataStream (Debug)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "DataStreamPublisher",
|
||||
},
|
||||
"windows": {
|
||||
"command": "DataStreamPublisher.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run DataStreamPublisher (Release)",
|
||||
"dependsOn": "Build DataStream (Release)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "DataStreamPublisher",
|
||||
},
|
||||
"windows": {
|
||||
"command": "DataStreamPublisher.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run PubSubServer",
|
||||
"dependsOn": "Build DataStream",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "PubSubServer",
|
||||
},
|
||||
"windows": {
|
||||
"command": "PubSubServer.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run PubSubServer (Debug)",
|
||||
"dependsOn": "Build DataStream (Debug)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "PubSubServer",
|
||||
},
|
||||
"windows": {
|
||||
"command": "PubSubServer.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run PubSubServer (Debug | Skip Build)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "PubSubServer",
|
||||
},
|
||||
"windows": {
|
||||
"command": "PubSubServer.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run PubSubServer (Release)",
|
||||
"dependsOn": "Build DataStream (Release)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "PubSubServer",
|
||||
},
|
||||
"windows": {
|
||||
"command": "PubSubServer.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run PubSubServer (Release | Skip Build)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "PubSubServer",
|
||||
},
|
||||
"windows": {
|
||||
"command": "PubSubServer.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run Spark",
|
||||
"dependsOn": "Build Spark",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "Spark.exe",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||
},
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "Run Spark (Debug)",
|
||||
"dependsOn": "Build Spark (Debug)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "Spark.exe",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
||||
},
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "Run Spark (Release)",
|
||||
"dependsOn": "Build Spark (Release)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "Spark.exe",
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
||||
},
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "Run Spark Config Query",
|
||||
"dependsOn": "Build Spark",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "Spark.Console",
|
||||
},
|
||||
"windows": {
|
||||
"command": "Spark.Console.exe",
|
||||
},
|
||||
"args": [
|
||||
"config",
|
||||
"--query",
|
||||
"${input:sparkConfigQuery}",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run Sparkd",
|
||||
"dependsOn": "Build Sparkd",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "sparkd",
|
||||
},
|
||||
"args": [
|
||||
"run"
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Run Sparkd Config Query",
|
||||
"dependsOn": "Build Sparkd",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "sparkd",
|
||||
},
|
||||
"args": [
|
||||
"config",
|
||||
"--query",
|
||||
"${input:sparkdConfigQuery}",
|
||||
],
|
||||
"options": {
|
||||
"cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
|
||||
},
|
||||
"icon": {
|
||||
"id": "run",
|
||||
"color": "terminal.ansiGreen",
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"windows": {
|
||||
"command": "tools/logging/show_last_run.cmd",
|
||||
"args": [
|
||||
"DataStream",
|
||||
"DataStreamQueryAnalyzer.exe",
|
||||
"'Starting DataStreamQueryAnalyzer'"
|
||||
]
|
||||
},
|
||||
"presentation": {
|
||||
"reveal": "always",
|
||||
"focus": true,
|
||||
"panel": "shared",
|
||||
"showReuseMessage": false,
|
||||
"close": true,
|
||||
},
|
||||
"icon": {
|
||||
"id": "wrench"
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "View PubSubServer Logs (Last Run)",
|
||||
"type": "process",
|
||||
"command": "",
|
||||
"linux": {
|
||||
"command": "tools/logging/show_last_run.sh",
|
||||
"args": [
|
||||
"${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
|
||||
"Starting PubSubServer"
|
||||
]
|
||||
},
|
||||
"windows": {
|
||||
"command": "tools/logging/show_last_run.cmd",
|
||||
"args": [
|
||||
"DataStream",
|
||||
"PubSubServer.exe",
|
||||
"Starting PubSubServer"
|
||||
]
|
||||
},
|
||||
"presentation": {
|
||||
"reveal": "always",
|
||||
"focus": true,
|
||||
"panel": "shared",
|
||||
"showReuseMessage": false,
|
||||
"close": true,
|
||||
},
|
||||
"icon": {
|
||||
"id": "wrench"
|
||||
},
|
||||
},
|
||||
],
|
||||
"inputs": [
|
||||
{
|
||||
"id": "buildConfiguration",
|
||||
"type": "pickString",
|
||||
"description": "Configuration",
|
||||
"options": [
|
||||
"Debug",
|
||||
"Release",
|
||||
],
|
||||
"default": "Debug",
|
||||
},
|
||||
{
|
||||
"id": "sparkdCmakePreset",
|
||||
"type": "pickString",
|
||||
"description": "Configuration",
|
||||
"options": [
|
||||
"sparkd-debug-clang",
|
||||
"sparkd-debug-gcc",
|
||||
"sparkd-release-clang",
|
||||
"sparkd-release-gcc",
|
||||
],
|
||||
"default": "sparkd-debug-clang",
|
||||
},
|
||||
{
|
||||
"id": "sparkdCompiler",
|
||||
"type": "pickString",
|
||||
"description": "Configuration",
|
||||
"options": [
|
||||
"clang",
|
||||
"gcc",
|
||||
],
|
||||
"default": "sparkd-debug-clang",
|
||||
},
|
||||
{
|
||||
"id": "dataStreamPublisherConfig",
|
||||
"type": "pickString",
|
||||
"description": "Publisher Configuration",
|
||||
"options": [
|
||||
"MarketDataPublisher",
|
||||
"SimpleEchoClient",
|
||||
"SimpleTestDataPublisher",
|
||||
"TestDataPublisher",
|
||||
],
|
||||
"default": "MarketDataPublisher",
|
||||
},
|
||||
{
|
||||
"id": "pubSubServerConfig",
|
||||
"type": "pickString",
|
||||
"description": "PubSub Configuration",
|
||||
"options": [
|
||||
"PubSubServer",
|
||||
"SimpleEchoClient",
|
||||
],
|
||||
"default": "PubSubServer",
|
||||
},
|
||||
{
|
||||
"id": "sparkConfigQuery",
|
||||
"type": "promptString",
|
||||
"description": "Config Type(s)"
|
||||
},
|
||||
{
|
||||
"id": "sparkdConfigQuery",
|
||||
"type": "promptString",
|
||||
"description": "Config Type(s)"
|
||||
}
|
||||
]
|
||||
}
|
||||
+14
-13
@@ -14,12 +14,17 @@ set(
|
||||
CACHE
|
||||
STRING
|
||||
"External dependencies root folder (should contain folders: lib, bin, include")
|
||||
|
||||
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
|
||||
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
|
||||
# api server, etc.).
|
||||
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
|
||||
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
|
||||
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
|
||||
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
||||
set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
|
||||
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
||||
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
||||
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
|
||||
if (WIN32)
|
||||
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
|
||||
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
|
||||
@@ -31,13 +36,13 @@ endif()
|
||||
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
||||
link_directories(${SPARK_LIB})
|
||||
|
||||
if(BUILD_API_SERVER)
|
||||
add_compile_definitions(SPARK_ENABLE_API_SERVER)
|
||||
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
||||
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
||||
if(DEFINED APP_DATA_DIR_OVERRIDE)
|
||||
add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
|
||||
endif()
|
||||
|
||||
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
||||
if(WIN32)
|
||||
add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
|
||||
add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
|
||||
# subdirs
|
||||
add_subdirectory(src/lib)
|
||||
else()
|
||||
@@ -47,13 +52,7 @@ else()
|
||||
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
||||
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
||||
|
||||
if(BUILD_API_SERVER)
|
||||
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
|
||||
add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
|
||||
elseif(BUILD_SPARKD)
|
||||
add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
|
||||
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
|
||||
|
||||
if(BUILD_SPARKD)
|
||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||
# read by their code). We override some below.
|
||||
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
||||
@@ -133,6 +132,8 @@ else()
|
||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
||||
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
|
||||
|
||||
COMMAND
|
||||
ln ARGS -sf
|
||||
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
||||
|
||||
+113
-62
@@ -15,11 +15,13 @@
|
||||
"rhs": "Windows"
|
||||
},
|
||||
"generator": "Visual Studio 17 2022",
|
||||
"binaryDir": "${sourceDir}/build",
|
||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||
"environment": {
|
||||
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
||||
},
|
||||
"cacheVariables": {
|
||||
"APP_DATA_DIR": "${dollar}{APPDATA}/bts",
|
||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
|
||||
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||
@@ -30,7 +32,12 @@
|
||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
|
||||
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
||||
"EXPORT_MD_CONSUMER": {
|
||||
"type": "BOOL",
|
||||
"value": "ON"
|
||||
},
|
||||
"SHARED_MEMORY_PREFIX": "spark"
|
||||
}
|
||||
},
|
||||
|
||||
@@ -46,6 +53,8 @@
|
||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||
"cacheVariables": {
|
||||
"platform": "",
|
||||
"APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
|
||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
|
||||
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
||||
"type": "BOOL",
|
||||
"value": "ON"
|
||||
@@ -62,7 +71,8 @@
|
||||
"BUILD_ADMIN_WEB_APP": {
|
||||
"type": "BOOL",
|
||||
"value": "ON"
|
||||
}
|
||||
},
|
||||
"SHARED_MEMORY_PREFIX": "sparkd"
|
||||
}
|
||||
},
|
||||
|
||||
@@ -95,98 +105,72 @@
|
||||
{
|
||||
"name": "sparkd-debug-clang",
|
||||
"inherits": "sparkd-debug",
|
||||
"displayName": "Debug (clang)",
|
||||
"displayName": "Sparkd (Debug | Clang)",
|
||||
"description": "Debug build compiled w/ clang",
|
||||
"binaryDir": "${sourceDir}/build/clang/debug",
|
||||
"cacheVariables": {
|
||||
"CMAKE_C_COMPILER": "clang",
|
||||
"CMAKE_CXX_COMPILER": "clang++",
|
||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
|
||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"name": "sparkd-debug-gcc",
|
||||
"inherits": "sparkd-debug",
|
||||
"displayName": "Debug (gcc)",
|
||||
"displayName": "Sparkd (Debug | Gcc)",
|
||||
"description": "Debug build compiled w/ gcc",
|
||||
"binaryDir": "${sourceDir}/build/gcc/debug",
|
||||
"cacheVariables": {
|
||||
"CMAKE_C_COMPILER": "gcc",
|
||||
"CMAKE_CXX_COMPILER": "g++",
|
||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
|
||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
|
||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"name": "sparkd-release-clang",
|
||||
"inherits": "sparkd-release",
|
||||
"displayName": "Release (clang)",
|
||||
"displayName": "Sparkd (Release | Clang)",
|
||||
"description": "Release build compiled w/ clang",
|
||||
"binaryDir": "${sourceDir}/build/clang/release",
|
||||
"cacheVariables": {
|
||||
"CMAKE_C_COMPILER": "clang",
|
||||
"CMAKE_CXX_COMPILER": "clang++",
|
||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
|
||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"name": "sparkd-release-gcc",
|
||||
"inherits": "sparkd-release",
|
||||
"displayName": "Release (gcc)",
|
||||
"displayName": "Sparkd (Release | Gcc)",
|
||||
"description": "Release build compiled w/ gcc",
|
||||
"binaryDir": "${sourceDir}/build/gcc/release",
|
||||
"cacheVariables": {
|
||||
"CMAKE_C_COMPILER": "gcc",
|
||||
"CMAKE_CXX_COMPILER": "g++",
|
||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"name": "spark-api-server-base",
|
||||
"hidden": true,
|
||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||
"cacheVariables": {
|
||||
"BUILD_API_SERVER": {
|
||||
"type": "bool",
|
||||
"value": "ON"
|
||||
},
|
||||
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
|
||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"name": "spark-api-server-windows",
|
||||
"inherits": "spark",
|
||||
"displayName": "Spark Api Server",
|
||||
"displayName": "Spark Api Server (Windows)",
|
||||
"cacheVariables": {
|
||||
"BUILD_API_SERVER": {
|
||||
"type": "bool",
|
||||
"value": "ON"
|
||||
}
|
||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
||||
"SHARED_MEMORY_PREFIX": "spark_api_server"
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"name": "spark-api-server-linux",
|
||||
"inherits": "sparkd",
|
||||
"displayName": "Spark Api Server",
|
||||
"displayName": "Spark Api Server (Linux)",
|
||||
"generator": "Ninja Multi-Config",
|
||||
"cacheVariables": {
|
||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
||||
"BUILD_ADMIN_WEB_APP": {
|
||||
"type": "BOOL",
|
||||
"value": "OFF"
|
||||
},
|
||||
"BUILD_API_SERVER": {
|
||||
"type": "bool",
|
||||
"value": "ON"
|
||||
},
|
||||
"BUILD_SPARKD": {
|
||||
"type": "BOOL",
|
||||
"value": "OFF"
|
||||
@@ -197,7 +181,11 @@
|
||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
|
||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||
"EXPORT_MD_CONSUMER": {
|
||||
"type": "BOOL",
|
||||
"value": "ON"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -220,13 +208,62 @@
|
||||
},
|
||||
|
||||
{
|
||||
"name": "data-stream-suite",
|
||||
"name": "data-stream-suite-windows",
|
||||
"condition": {
|
||||
"type": "equals",
|
||||
"lhs": "${hostSystemName}",
|
||||
"rhs": "Windows"
|
||||
},
|
||||
"inherits": "spark",
|
||||
"displayName": "Data Stream Suite",
|
||||
"displayName": "Data Stream Suite (Windows)",
|
||||
"description": "Data Stream utils ported over from Edge",
|
||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||
"cacheVariables": {
|
||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
|
||||
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
||||
"SHARED_MEMORY_PREFIX": "data_stream"
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"name": "data-stream-suite-linux",
|
||||
"displayName": "Data Stream Suite (Linux)",
|
||||
"description": "Data Stream utils ported over from Edge",
|
||||
"condition": {
|
||||
"type": "equals",
|
||||
"lhs": "${hostSystemName}",
|
||||
"rhs": "Linux"
|
||||
},
|
||||
"inherits": "sparkd",
|
||||
"generator": "Ninja Multi-Config",
|
||||
"cacheVariables": {
|
||||
"APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
|
||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
||||
"BUILD_ADMIN_WEB_APP": {
|
||||
"type": "BOOL",
|
||||
"value": "OFF"
|
||||
},
|
||||
"BUILD_SDK": {
|
||||
"type": "BOOL",
|
||||
"value": "OFF"
|
||||
},
|
||||
"BUILD_SPARKD": {
|
||||
"type": "BOOL",
|
||||
"value": "OFF"
|
||||
},
|
||||
"CMAKE_C_COMPILER": "clang",
|
||||
"CMAKE_CXX_COMPILER": "clang++",
|
||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||
"EXPORT_MD_CONSUMER": {
|
||||
"type": "BOOL",
|
||||
"value": "ON"
|
||||
},
|
||||
"SHARED_MEMORY_PREFIX": "data_stream"
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -244,7 +281,7 @@
|
||||
|
||||
{
|
||||
"name": "spark-debug",
|
||||
"displayName": "Debug",
|
||||
"displayName": "Spark (Debug)",
|
||||
"inherits": "spark",
|
||||
"configurePreset": "spark",
|
||||
"configuration": "Debug"
|
||||
@@ -252,7 +289,7 @@
|
||||
|
||||
{
|
||||
"name": "spark-release",
|
||||
"displayName": "Release",
|
||||
"displayName": "Spark (Release)",
|
||||
"inherits": "spark",
|
||||
"configurePreset": "spark",
|
||||
"configuration": "Release"
|
||||
@@ -280,28 +317,28 @@
|
||||
{
|
||||
"name": "sparkd-debug-clang",
|
||||
"configurePreset": "sparkd-debug-clang",
|
||||
"displayName": "Debug (clang)",
|
||||
"displayName": "Sparkd (Debug | Clang)",
|
||||
"inherits": "sparkd"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "sparkd-debug-gcc",
|
||||
"configurePreset": "sparkd-debug-gcc",
|
||||
"displayName": "Debug (gcc)",
|
||||
"displayName": "Sparkd (Debug | Gcc)",
|
||||
"inherits": "sparkd"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "sparkd-release-clang",
|
||||
"configurePreset": "sparkd-release-clang",
|
||||
"displayName": "Release (clang)",
|
||||
"displayName": "Sparkd (Release | Clang)",
|
||||
"inherits": "sparkd"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "sparkd-release-gcc",
|
||||
"configurePreset": "sparkd-release-gcc",
|
||||
"displayName": "Release (gcc)",
|
||||
"displayName": "Sparkd (Release | Gcc)",
|
||||
"inherits": "sparkd"
|
||||
},
|
||||
|
||||
@@ -318,14 +355,14 @@
|
||||
|
||||
{
|
||||
"name": "spark-api-server-windows-debug",
|
||||
"displayName": "Spark Api Server (Debug)",
|
||||
"displayName": "Spark Api Server (Windows | Debug)",
|
||||
"inherits": "spark-api-server-windows",
|
||||
"configuration": "Debug"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "spark-api-server-windows-release",
|
||||
"displayName": "Spark Api Server (Debug)",
|
||||
"displayName": "Spark Api Server (Windows | Release)",
|
||||
"inherits": "spark-api-server-windows",
|
||||
"configuration": "Release"
|
||||
},
|
||||
@@ -344,14 +381,14 @@
|
||||
{
|
||||
"name": "spark-api-server-linux-debug",
|
||||
"inherits": "spark-api-server-linux",
|
||||
"displayName": "Spark Api Server (Debug)",
|
||||
"displayName": "Spark Api Server (Linux | Debug)",
|
||||
"configuration": "Debug"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "spark-api-server-linux-release",
|
||||
"inherits": "spark-api-server-linux",
|
||||
"displayName": "Spark Api Server (Release)",
|
||||
"displayName": "Spark Api Server (Linux | Release)",
|
||||
"configuration": "Release"
|
||||
},
|
||||
|
||||
@@ -371,16 +408,30 @@
|
||||
},
|
||||
|
||||
{
|
||||
"name": "data-stream-suite-debug",
|
||||
"displayName": "DataStreamSuite (Debug)",
|
||||
"configurePreset": "data-stream-suite",
|
||||
"name": "data-stream-suite-windows-debug",
|
||||
"displayName": "Data Stream Suite (Windows | Debug)",
|
||||
"configurePreset": "data-stream-suite-windows",
|
||||
"configuration": "Debug"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "data-stream-suite-release",
|
||||
"displayName": "DataStreamSuite (Release)",
|
||||
"configurePreset": "data-stream-suite",
|
||||
"name": "data-stream-suite-windows-release",
|
||||
"displayName": "Data Stream Suite (Windows | Release)",
|
||||
"configurePreset": "data-stream-suite-windows",
|
||||
"configuration": "Release"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "data-stream-suite-linux-debug",
|
||||
"displayName": "Data Stream Suite (Linux | Debug)",
|
||||
"configurePreset": "data-stream-suite-linux",
|
||||
"configuration": "Debug"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "data-stream-suite-linux-release",
|
||||
"displayName": "Data Stream Suite (Linux | Release)",
|
||||
"configurePreset": "data-stream-suite-linux",
|
||||
"configuration": "Release"
|
||||
}
|
||||
]
|
||||
|
||||
Submodule
+1
Submodule docker/linux added at 56c61231f3.
Submodule
+1
Submodule docker/windows added at de7d40c09d.
+1
-1
Submodule externals updated: cae5fd20da...8069cc893f.
@@ -1,45 +0,0 @@
|
||||
/* global use, db */
|
||||
// MongoDB Playground
|
||||
// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
|
||||
// Make sure you are connected to enable completions and to be able to run a playground.
|
||||
// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
|
||||
// The result of the last command run in a playground is shown on the results panel.
|
||||
// By default the first 20 documents will be returned with a cursor.
|
||||
// Use 'console.log()' to print to the debug output.
|
||||
// For more documentation on playgrounds please refer to
|
||||
// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
|
||||
|
||||
// Select the database to use.
|
||||
use('mongodbVSCodePlaygroundDB');
|
||||
|
||||
// Insert a few documents into the sales collection.
|
||||
db.getCollection('sales').insertMany([
|
||||
{ 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
|
||||
{ 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
|
||||
{ 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
|
||||
{ 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
|
||||
{ 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
|
||||
{ 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
|
||||
{ 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
|
||||
{ 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
|
||||
]);
|
||||
|
||||
// Run a find command to view items sold on April 4th, 2014.
|
||||
const salesOnApril4th = db.getCollection('sales').find({
|
||||
date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
|
||||
}).count();
|
||||
|
||||
// Print a message to the output window.
|
||||
console.log(`${salesOnApril4th} sales occurred in 2014.`);
|
||||
|
||||
// Here we run an aggregation and open a cursor to the results.
|
||||
// Use '.toArray()' to exhaust the cursor to return the whole result set.
|
||||
// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
|
||||
db.getCollection('sales').aggregate([
|
||||
// Find all of the sales that occurred in 2014.
|
||||
{ $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
|
||||
// Group the total sales for each product.
|
||||
{ $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
|
||||
]);
|
||||
|
||||
|
||||
@@ -1,41 +1,22 @@
|
||||
using mbts.spark;
|
||||
using System.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Runtime;
|
||||
namespace BCG.Utilities.General;
|
||||
|
||||
namespace BCG.Utilities.General
|
||||
{
|
||||
/// <summary>
|
||||
/// Summary description for Crono.
|
||||
/// </summary>
|
||||
public class Crono
|
||||
{
|
||||
public Crono()
|
||||
{
|
||||
myTimer = new HighPerformanceTimer();
|
||||
}
|
||||
/// <summary>
|
||||
/// Summary description for Crono.
|
||||
/// </summary>
|
||||
public class Crono {
|
||||
readonly Stopwatch myTimer = new();
|
||||
|
||||
public void start()
|
||||
{myTimer.start();}
|
||||
public void start() =>
|
||||
myTimer.Start();
|
||||
|
||||
public void stop()
|
||||
{myTimer.stop();}
|
||||
public void stop() =>
|
||||
myTimer.Stop();
|
||||
|
||||
public TimeSpan time
|
||||
{
|
||||
get
|
||||
{
|
||||
return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
|
||||
}
|
||||
}
|
||||
|
||||
public double miliseconds
|
||||
{
|
||||
get
|
||||
{return myTimer.totalMilliseconds;}
|
||||
}
|
||||
|
||||
private HighPerformanceTimer myTimer;
|
||||
}
|
||||
public TimeSpan time =>
|
||||
myTimer.Elapsed;
|
||||
|
||||
public double miliseconds =>
|
||||
myTimer.Elapsed.TotalMilliseconds;
|
||||
}
|
||||
@@ -76,7 +76,7 @@ namespace BCG
|
||||
}
|
||||
|
||||
#region Generic Methods
|
||||
internal virtual void setValue(RegistryRoot root, string name, object value)
|
||||
public virtual void setValue(RegistryRoot root, string name, object value)
|
||||
{
|
||||
using (RegistryKey rk = myGetRegistryKey(root))
|
||||
{
|
||||
@@ -100,7 +100,7 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
|
||||
public virtual object getValue(RegistryRoot root, string name, object defaultValue)
|
||||
{
|
||||
using (RegistryKey rk = myGetRegistryKey(root))
|
||||
{
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0-windows</TargetFramework>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<UseWindowsForms>true</UseWindowsForms>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,26 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Reactive.Linq;
|
||||
|
||||
public static class DataStreamExtensions {
|
||||
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
||||
source as ViewStream ?? new ObservableViewStream(source);
|
||||
|
||||
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
||||
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
||||
|
||||
IDisposable Subscribe(IObserver<DataPacket> observer) {
|
||||
source.onChange += OnChange;
|
||||
return Disposable.Create(() => source.onChange -= OnChange);
|
||||
|
||||
void OnChange(object? newValue) =>
|
||||
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
|
||||
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
public class PandasScriptStream : ViewStream {
|
||||
public PandasScriptStream(ViewStream vs, string file)
|
||||
: this(vs, file, null) { }
|
||||
|
||||
public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
|
||||
: this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
|
||||
|
||||
public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
|
||||
mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
|
||||
|
||||
string readFile() {
|
||||
try {
|
||||
return File.ReadAllText(file);
|
||||
} catch (Exception ex) {
|
||||
Log.Gen.error($"PandasScriptStream error reading file: {ex}");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
using BCG.Extensions;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
|
||||
public class PandasStream : ViewStream, IWorker {
|
||||
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
||||
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
||||
readonly string pythonExe;
|
||||
readonly ViewStreamDatabase vsd;
|
||||
readonly CSVSaverStream inputCsvStream;
|
||||
readonly Lazy<string> pythonCode;
|
||||
IDictionary<int, IBlueRow>? lastRowMap = null;
|
||||
int inputVersion = 0;
|
||||
int outputVersion = 0;
|
||||
DateTime lastFireTime = DateTime.MinValue;
|
||||
TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
|
||||
IDisposable? dispose;
|
||||
|
||||
public PandasStream(ViewStream vs, string dataFrameExpression)
|
||||
: this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
|
||||
|
||||
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
|
||||
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
|
||||
this.pythonExe = pythonExe;
|
||||
this.vsd = vsd;
|
||||
// This should fire an update when it writes to the output file
|
||||
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
|
||||
pythonCode = new(getPythonCode);
|
||||
|
||||
string getPythonCode() {
|
||||
var imports =
|
||||
@"import numpy as np
|
||||
import pandas as pd
|
||||
import sys
|
||||
";
|
||||
var bd = new StringBuilder(imports);
|
||||
bd.AppendLine($"df=pd.read_csv('{inputFile}')");
|
||||
bd.AppendLine(dataFrameExpression);
|
||||
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
|
||||
return bd.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
vsd.addWorker(this);
|
||||
inputCsvStream.onChange += onInputChanged;
|
||||
this.dispose = Disposable.Create(dispose);
|
||||
|
||||
void onInputChanged(object newValue) =>
|
||||
inputVersion++;
|
||||
|
||||
void dispose() {
|
||||
inputCsvStream.onChange -= onInputChanged;
|
||||
vsd.removeWorker(this);
|
||||
deleteFile(inputFile);
|
||||
deleteFile(outputFile);
|
||||
}
|
||||
|
||||
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
|
||||
if (!File.Exists(file)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
File.Delete(file);
|
||||
} catch (Exception ex) {
|
||||
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() =>
|
||||
Disposable.dispose(ref dispose);
|
||||
|
||||
static void executePythonCode(string pythonCode, string? pythonExe = null) {
|
||||
pythonExe ??= "python";
|
||||
var start = new ProcessStartInfo {
|
||||
FileName = pythonExe,
|
||||
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true
|
||||
};
|
||||
using var process = Process.Start(start)
|
||||
?? throw new InvalidOperationException("Failed to launch python");
|
||||
process.WaitForExit();
|
||||
}
|
||||
|
||||
void IWorker.doWork() {
|
||||
if (outputVersion == inputVersion) {
|
||||
return;
|
||||
}
|
||||
var now = DateTime.Now;
|
||||
var elapsedSinceLastFire = now - lastFireTime;
|
||||
if (elapsedSinceLastFire < fireThrottle) {
|
||||
return;
|
||||
}
|
||||
outputVersion = inputVersion;
|
||||
executePythonCode(pythonCode.Value, pythonExe);
|
||||
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
|
||||
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
|
||||
var comp = BlueRowEqualityComparer.Instance;
|
||||
var deletedRows = new BlueTable(schema);
|
||||
var addedRows = new BlueTable(schema);
|
||||
if (lastRowMap is not null) {
|
||||
deletedRows.SetItems(lastRowMap.Where(oldRow =>
|
||||
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
|
||||
addedRows.SetItems(rowMap.Where(newRow =>
|
||||
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
|
||||
}
|
||||
lastRowMap = rowMap;
|
||||
lastFireTime = DateTime.Now;
|
||||
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
|
||||
}
|
||||
}
|
||||
@@ -39,8 +39,8 @@ namespace BCG {
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
||||
var server_address = new BlutoSubAddress(_ip, _port);
|
||||
bts.AsyncTCPConnector_APM.connect_to(server_address, this);
|
||||
var server_rci = new RemoteConnectionInfo(_ip, _port);
|
||||
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
file_path = r"C:\Temp\test2.txt" # r prefix for raw string to avoid escape characters
|
||||
with open(file_path, "w") as f:
|
||||
f.write("Hello, world!\n")
|
||||
@@ -0,0 +1,37 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
|
||||
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
|
||||
readonly IObservable<DataPacket> source;
|
||||
IDisposable? subscription;
|
||||
|
||||
public ObservableViewStream(IObservable<DataPacket> source) =>
|
||||
this.source = source;
|
||||
|
||||
public void Dispose() {
|
||||
GC.SuppressFinalize(this);
|
||||
if (subscription is not null) {
|
||||
subscription.Dispose();
|
||||
subscription = null;
|
||||
}
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
Log.isnull(subscription);
|
||||
subscription = source.Subscribe(myFireEvent);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
Log.notnull(subscription);
|
||||
subscription.Dispose();
|
||||
subscription = null;
|
||||
}
|
||||
|
||||
IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
|
||||
source.Subscribe(observer);
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
using BCG;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
||||
[DataStreamUserRespresentation("add_to_cloud")]
|
||||
public class AddToCloudStream : ViewStream
|
||||
{
|
||||
private readonly string _name, _stream_exp;
|
||||
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
|
||||
public AddToCloudStream(string name, params string[] lines)
|
||||
: this(name, 0, lines)
|
||||
{
|
||||
}
|
||||
|
||||
public AddToCloudStream(string name, int persist, params string[] lines)
|
||||
{
|
||||
_name = name;
|
||||
_stream_exp = getExpression(lines);
|
||||
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
|
||||
mySetRealSource(new TableStream("cloud_streams"));
|
||||
}
|
||||
|
||||
static string getExpression(params string[] lines) {
|
||||
for (int i = 0; i < lines.Length; i++) {
|
||||
// unescape quotes
|
||||
lines[i] = lines[i].Replace("\\\"", "\"");
|
||||
}
|
||||
return string.Join("\n", lines);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,31 +1,22 @@
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using mbts.config;
|
||||
using bts.utils;
|
||||
using System.Threading.Tasks;
|
||||
using Bts.DataStreamExpressions;
|
||||
using System.Collections.Generic;
|
||||
using BCG.Utilities.General;
|
||||
using bts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
// listens to the requests posted on "cloud_streams" request table, and
|
||||
// if unfulfilled starts a ViewPubsliher
|
||||
public class CloudStreamManager : IWorker
|
||||
{
|
||||
public class StreamPubUnit
|
||||
{
|
||||
public class RemoteQueryManager : IWorker {
|
||||
public class StreamPubUnit {
|
||||
public string name;
|
||||
public string stream_expression;
|
||||
public ViewStream stream;
|
||||
public bool persist;
|
||||
public ViewPublisher view_publisher;
|
||||
|
||||
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
|
||||
{
|
||||
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
|
||||
name = name_;
|
||||
stream_expression = stream_expression_;
|
||||
stream = stream_;
|
||||
@@ -33,7 +24,7 @@ namespace BCG
|
||||
view_publisher = view_publisher_;
|
||||
}
|
||||
}
|
||||
static ViewStream build_stream(string stream_expression)
|
||||
static ViewStream? build_stream(string stream_expression)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -41,9 +32,7 @@ namespace BCG
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
} catch (Exception ex) {
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
@@ -52,34 +41,37 @@ namespace BCG
|
||||
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
||||
private ViewStream _stream = new TableStream("cloud_streams");
|
||||
private PubTableBroker _broker = new PubTableBroker();
|
||||
private ManagedSubscriber _sub = ManagedSubscriber.instance;
|
||||
|
||||
public CloudStreamManager()
|
||||
{
|
||||
|
||||
public RemoteQueryManager() {
|
||||
_stream.onChange += _on_str;
|
||||
|
||||
_sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
|
||||
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
|
||||
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
||||
});
|
||||
}
|
||||
private void _on_str(object obj)
|
||||
{
|
||||
DataPacket dp = obj as DataPacket;
|
||||
if (obj is not DataPacket dp) {
|
||||
return;
|
||||
}
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string name = r.get<string>("name");
|
||||
bool persist = r.get<bool>("persist");
|
||||
string stream_exp = r.get<string>("stream_exp");
|
||||
if (!_stream_pub_map.ContainsKey(name))
|
||||
{
|
||||
_assign(name, stream_exp, persist);
|
||||
if (!_stream_pub_map.ContainsKey(name)) {
|
||||
_spawn_remote_stream(name, stream_exp, persist);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _assign(string name, string stream_expression, bool persist)
|
||||
{
|
||||
private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
|
||||
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
||||
ViewStream stream = build_stream(stream_expression);
|
||||
_expose_to_ws(name, stream);
|
||||
if (persist)
|
||||
{
|
||||
if (persist) {
|
||||
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
||||
}
|
||||
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
||||
@@ -87,13 +79,11 @@ namespace BCG
|
||||
_stream_pub_map.Add(name, u);
|
||||
}
|
||||
|
||||
private void _expose_to_ws(string name, ViewStream stream)
|
||||
{
|
||||
private void _expose_to_ws(string name, ViewStream stream) {
|
||||
// TODO: john
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
public void doWork() {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
|
||||
namespace BCG {
|
||||
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
||||
[DataStreamUserRespresentation("remoteQuery")]
|
||||
public class RemoteQueryStream : ViewStream {
|
||||
private readonly string _name, _stream_exp;
|
||||
private PublisherCore _publisher_core = new PublisherCore();
|
||||
private ViewStream _source_str;
|
||||
|
||||
public RemoteQueryStream(params string[] lines) {
|
||||
_stream_exp = get_expression(lines);
|
||||
_name = Guid.NewGuid().ToString();
|
||||
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
|
||||
//_name.Replace("-", "_");
|
||||
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
|
||||
byte[] rem_req_bytes = rem_req.serialize();
|
||||
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
|
||||
_source_str = new TableStream(_name);
|
||||
_source_str.onChange += new ChangeDelegate(_handle_data);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
//maybe there should be a way to turn remote requests off ?
|
||||
_source_str.onChange -= new ChangeDelegate(_handle_data);
|
||||
}
|
||||
|
||||
static string get_expression(params string[] lines) {
|
||||
for (int i = 0; i < lines.Length; i++) {
|
||||
// unescape quotes
|
||||
lines[i] = lines[i].Replace("\\\"", "\"");
|
||||
}
|
||||
return string.Join("\n", lines);
|
||||
}
|
||||
|
||||
private void _handle_data(object o) {
|
||||
myFireEvent(o);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace bts {
|
||||
using System.IO;
|
||||
using BCG;
|
||||
|
||||
using System.IO;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
|
||||
public class RemoteRequest {
|
||||
public readonly string stream_expression;
|
||||
public readonly string published_table_name;
|
||||
public RemoteRequest(string stream_expression_, string published_table_name_) {
|
||||
stream_expression = stream_expression_;
|
||||
published_table_name = published_table_name_;
|
||||
}
|
||||
public byte[] serialize() {
|
||||
var str = new MemoryStream();
|
||||
var bw = new BinaryWriter(str);
|
||||
bw.write_string_cpp(stream_expression);
|
||||
bw.write_string_cpp(published_table_name);
|
||||
return str.ToArray();
|
||||
}
|
||||
|
||||
public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
|
||||
BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
|
||||
string stream_expression_ = br.read_string_cpp();
|
||||
string published_table_name_ = br.read_string_cpp();
|
||||
return new RemoteRequest(stream_expression_, published_table_name_);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,59 +1,54 @@
|
||||
using BCG;
|
||||
// using BCG;
|
||||
|
||||
using System.Linq;
|
||||
// using bts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class StreamOffloadRequestor
|
||||
{
|
||||
public class OffloadOp
|
||||
{
|
||||
string name;
|
||||
string stream_exp;
|
||||
}
|
||||
private IBlueTableSchema _schema;
|
||||
private IBlueRowCollection _rows;
|
||||
private PubTableBroker _broker = new PubTableBroker();
|
||||
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
||||
public StreamOffloadRequestor()
|
||||
{
|
||||
_schema = new BlueTableSchema("cloud_streams",
|
||||
new ColumnInfo("id", 0, typeof(int)),
|
||||
new ColumnInfo("name", 1, typeof(string)),
|
||||
new ColumnInfo("persist", 2, typeof(bool)),
|
||||
new ColumnInfo("stream_exp", 3, typeof(string)));
|
||||
_existing_offloads_vs.onChange += _on_str;
|
||||
}
|
||||
private void _on_str(object obj)
|
||||
{
|
||||
DataPacket dp = obj as DataPacket;
|
||||
_rows = dp.newRows;
|
||||
// this is needed so the broker knows the state of the table
|
||||
_broker.createTable(_rows);
|
||||
}
|
||||
public void add_to_cloud(string name, string stream_exp, bool persist)
|
||||
{
|
||||
// check if it exists
|
||||
if (_rows == null)
|
||||
{
|
||||
BlueTable table = new BlueTable(_schema);
|
||||
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
||||
table.addRow(nr);
|
||||
_broker.createTable(table);
|
||||
// create table
|
||||
}
|
||||
else
|
||||
{
|
||||
if (exists(name))
|
||||
return;
|
||||
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
||||
_broker.addRow(nr);
|
||||
}
|
||||
}
|
||||
// using System.Linq;
|
||||
|
||||
bool exists(string name)
|
||||
{
|
||||
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
// namespace BCG {
|
||||
// // used on the client
|
||||
// public class StreamOffloadRequestor {
|
||||
// public class OffloadOp {
|
||||
// string name;
|
||||
// string stream_exp;
|
||||
// }
|
||||
// private IBlueTableSchema _schema;
|
||||
// private IBlueRowCollection _rows;
|
||||
// private PubTableBroker _broker = new PubTableBroker();
|
||||
// ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
||||
// public StreamOffloadRequestor() {
|
||||
// _schema = new BlueTableSchema("cloud_streams",
|
||||
// new ColumnInfo("id", 0, typeof(int)),
|
||||
// new ColumnInfo("name", 1, typeof(string)),
|
||||
// new ColumnInfo("persist", 2, typeof(bool)),
|
||||
// new ColumnInfo("stream_exp", 3, typeof(string)));
|
||||
// _existing_offloads_vs.onChange += _on_str;
|
||||
// }
|
||||
// private void _on_str(object obj) {
|
||||
// DataPacket dp = obj as DataPacket;
|
||||
// _rows = dp.newRows;
|
||||
// // this is needed so the broker knows the state of the table
|
||||
// _broker.create_table(_rows);
|
||||
// }
|
||||
// public void add_to_cloud(string name, string stream_exp, bool persist) {
|
||||
// // check if it exists
|
||||
// if (_rows == null) {
|
||||
// BlueTable table = new BlueTable(_schema);
|
||||
// IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
||||
// table.addRow(nr);
|
||||
// _broker.create_table(table);
|
||||
// // create table
|
||||
// } else {
|
||||
// if (exists(name))
|
||||
// return;
|
||||
// IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
||||
// // TODO: use make_batch_changes instead
|
||||
// Log.assert(false, "Fix addRow");
|
||||
// //_broker.addRow(nr);
|
||||
// }
|
||||
// }
|
||||
|
||||
// bool exists(string name) {
|
||||
// return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
@@ -1,17 +1,14 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
internal class AddColumnsRow : IBlueRow
|
||||
{
|
||||
namespace BCG {
|
||||
internal class AddColumnsRow : IBlueRow {
|
||||
private IBlueTableSchema mySchema;
|
||||
private IBlueRow myRow;
|
||||
//private object[] myCachedValues;
|
||||
|
||||
#region Constructors
|
||||
public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
|
||||
{
|
||||
public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
|
||||
mySchema = schema;
|
||||
myRow = row;
|
||||
//myCachedValues = new object[schema.columnNames.Length];
|
||||
@@ -19,85 +16,78 @@ namespace BCG
|
||||
#endregion
|
||||
|
||||
#region IBlueRow members
|
||||
public IBlueTableSchema schema{get{return mySchema;}}
|
||||
public IBlueTableSchema schema { get { return mySchema; } }
|
||||
#endregion
|
||||
|
||||
#region ICollection members
|
||||
public int Count {get{return mySchema.columnNames.Length;}}
|
||||
public int Count { get { return mySchema.columnNames.Length; } }
|
||||
|
||||
public bool IsSynchronized {get{return false;}}
|
||||
public bool IsSynchronized { get { return false; } }
|
||||
|
||||
public object SyncRoot {get{return this;}}
|
||||
public object SyncRoot { get { return this; } }
|
||||
|
||||
public void CopyTo (Array array, int index)
|
||||
{
|
||||
public void CopyTo(Array array, int index) {
|
||||
int i = index;
|
||||
foreach (object o in this as IEnumerable)
|
||||
array.SetValue (o, i++);
|
||||
array.SetValue(o, i++);
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region IDictionary members
|
||||
public bool IsFixedSize {get{return true;}}
|
||||
public bool IsFixedSize { get { return true; } }
|
||||
|
||||
public bool IsReadOnly {get{return true;}}
|
||||
public bool IsReadOnly { get { return true; } }
|
||||
|
||||
/// <summary>
|
||||
/// The key can either be the string corrsponding to the column name or
|
||||
/// the order of the column
|
||||
/// </summary>
|
||||
public object this [object key]
|
||||
{
|
||||
get
|
||||
{
|
||||
public object this[object key] {
|
||||
get {
|
||||
if (key is string)
|
||||
return this[(string)key];
|
||||
return this[(int)key];
|
||||
}
|
||||
set {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
}
|
||||
|
||||
public object this[int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
public object this[int index] {
|
||||
get {
|
||||
//if (myCachedValues[index] == null)
|
||||
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
|
||||
//return myCachedValues[index];
|
||||
return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
|
||||
return mySchema[index].isConstant ? DBNull.Value : myRow[index];
|
||||
}
|
||||
}
|
||||
|
||||
public object this[string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
|
||||
|
||||
public ICollection Keys{get{return mySchema.columnNames;}}
|
||||
public ICollection Keys { get { return mySchema.columnNames; } }
|
||||
|
||||
public ICollection Values
|
||||
{
|
||||
get
|
||||
{
|
||||
public ICollection Values {
|
||||
get {
|
||||
object[] retVal = new object[mySchema.columnNames.Length];
|
||||
CopyTo (retVal, 0);
|
||||
CopyTo(retVal, 0);
|
||||
return retVal;
|
||||
}
|
||||
}
|
||||
|
||||
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
|
||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
|
||||
public bool Contains (object key){return mySchema[(string)key] != null;}
|
||||
public bool Contains(object key) { return mySchema[(string)key] != null; }
|
||||
|
||||
/// <summary>
|
||||
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
||||
/// iterate through the values, typecast this into an IEnumerable
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
|
||||
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
|
||||
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
||||
|
||||
public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
#endregion
|
||||
|
||||
}
|
||||
|
||||
@@ -1,6 +1,34 @@
|
||||
namespace BCG;
|
||||
using System;
|
||||
|
||||
public class AddColumnsStream : MapStream {
|
||||
public AddColumnsStream(ViewStream vs, string columnDef)
|
||||
: base(vs, new ColumnAdder(columnDef)) { }
|
||||
namespace BCG {
|
||||
public class AddColumnsStream : ViewStream {
|
||||
public static Tuple<string, Type, object> parse_exp(string exp) {
|
||||
string[] toks = exp.Split(':');
|
||||
string col_name = toks[0];
|
||||
string col_value = toks[1];
|
||||
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
|
||||
return Tuple.Create(col_name, tup.Item1, tup.Item2);
|
||||
}
|
||||
|
||||
private Tuple<string, Type, object> _new_col;
|
||||
private IBlueTableSchema _out_schema;
|
||||
|
||||
public AddColumnsStream(ViewStream vs, string col_def) {
|
||||
_new_col = parse_exp(col_def);
|
||||
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
|
||||
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
|
||||
if (_out_schema == null)
|
||||
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
|
||||
return _out_schema;
|
||||
}
|
||||
|
||||
private IBlueRow _map(IBlueRow r) {
|
||||
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
|
||||
rv.setValue(_new_col.Item1, _new_col.Item3);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
public class AddConstantColumnRow {
|
||||
AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,23 +1,24 @@
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Provides a stream that with distinct rows with respect to the columns
|
||||
/// the user specifies in the argument
|
||||
/// </summary>
|
||||
[DataStreamUserRespresentation ("distinct")]
|
||||
public class DistinctRowStream : ViewStream
|
||||
{
|
||||
public DistinctRowStream (ViewStream stream, params string[] cols)
|
||||
{
|
||||
ArrayList groupByCols = new ArrayList ();
|
||||
foreach (string col in cols)
|
||||
foreach (string c in col.Split (' '))
|
||||
groupByCols.Add (c);
|
||||
mySetRealSource (new GroupByStream (stream,
|
||||
new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
|
||||
[DataStreamUserRespresentation("distinct")]
|
||||
public class DistinctRowStream : ViewStream {
|
||||
public DistinctRowStream(ViewStream stream, params string[] cols) {
|
||||
var g_by_eval = new DistinctRowEvaluator(cols);
|
||||
ViewStream r = new GroupByStream(stream, g_by_eval);
|
||||
var proj_exp = String.Empty;
|
||||
foreach (var c in cols)
|
||||
proj_exp += $"{c} ";
|
||||
proj_exp = proj_exp.Trim();
|
||||
r = new NewProjectStream(r, proj_exp);
|
||||
mySetRealSource(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
// using System.Text;
|
||||
// using System.IO;
|
||||
|
||||
// using System.Runtime.Serialization.Formatters.Binary;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class DumpLoaderStream : ViewStream, ISideEffectStream
|
||||
// {
|
||||
// //wtf
|
||||
// //public class Exception : System.Exception
|
||||
// //{public Exception(string msg) : base(msg) { }}
|
||||
|
||||
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
|
||||
// {
|
||||
// mySetRealSource(
|
||||
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
||||
// ViewStreamDatabase.instance));
|
||||
// }
|
||||
|
||||
// public DumpLoaderStream(string fileName)
|
||||
// {
|
||||
// if (fileName == null)
|
||||
// throw new LoaderStream.Exception("File name cannot be null");
|
||||
|
||||
// for (int i = 0; i < 5; i++)
|
||||
// try
|
||||
// {
|
||||
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
||||
// {
|
||||
// IClientDeserializer deser = new ClientUniversalDeserializer();
|
||||
// byte[] the_ops;
|
||||
// Array the_rows;
|
||||
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
|
||||
|
||||
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
|
||||
// }
|
||||
// break;
|
||||
// }
|
||||
// catch (IOException)
|
||||
// {
|
||||
// System.Threading.Thread.Sleep(50);
|
||||
// }
|
||||
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
@@ -1,48 +1,25 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
using BCG;
|
||||
|
||||
using System.Runtime.Serialization.Formatters.Binary;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace bts {
|
||||
// This is the other side of the SaverStream, its "loads" the stream and fires once
|
||||
[DataStreamUserRespresentation("loader")]
|
||||
public class LoaderStream : ViewStream, ISideEffectStream
|
||||
{
|
||||
public class Exception : System.Exception
|
||||
{public Exception(string msg) : base(msg) { }}
|
||||
public class LoaderStream : ViewStream {
|
||||
|
||||
public LoaderStream(string fileName, int secondsBetweenPolls)
|
||||
{
|
||||
mySetRealSource(
|
||||
new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
||||
ViewStreamDatabase.instance));
|
||||
private string _filename;
|
||||
private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
|
||||
|
||||
|
||||
public LoaderStream(string filename) {
|
||||
_filename = filename;
|
||||
}
|
||||
|
||||
public LoaderStream(string fileName)
|
||||
{
|
||||
if (fileName == null)
|
||||
throw new LoaderStream.Exception("File name cannot be null");
|
||||
|
||||
for (int i = 0; i < 5; i++)
|
||||
try
|
||||
{
|
||||
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
||||
{
|
||||
BinaryFormatter bf = new BinaryFormatter();
|
||||
#pragma warning disable SYSLIB0011
|
||||
BlueTable bt = (BlueTable)bf.Deserialize(fs);
|
||||
#pragma warning restore SYSLIB0011
|
||||
mySetRealSource(new BlueRowCollectionStaticStream(bt));
|
||||
}
|
||||
break;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
System.Threading.Thread.Sleep(50);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
|
||||
StreamIOMessage msg = _deser.deserialize(ser_msg);
|
||||
Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
|
||||
IBlueRowCollection rows = msg.create_message.rows;
|
||||
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,11 @@ using System.Text;
|
||||
using System.IO;
|
||||
using Google.Protobuf;
|
||||
using System.Globalization;
|
||||
using BCG;
|
||||
|
||||
namespace BCG {
|
||||
// Writes all the udpates to a stream in a file.
|
||||
namespace bts {
|
||||
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
|
||||
// which just writes one update, as a create)
|
||||
// File naming:
|
||||
// token_epoch_yyyyMMdd_HHmmss
|
||||
// File location: directory(passed)\token\
|
||||
@@ -15,11 +17,16 @@ namespace BCG {
|
||||
// data:
|
||||
// epochtime, size, serialized data
|
||||
[DataStreamUserRespresentation("pers")]
|
||||
// Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
|
||||
// which maps the name of the directory where the streams are saved. The directory (3rd arg) is the
|
||||
// parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
|
||||
// file is created. The file names are token_epoch_datetime
|
||||
// the first entry in a file is the complete state
|
||||
public class PersistorStream : ViewStream {
|
||||
private ViewStream _source_stream;
|
||||
private string _token;
|
||||
private FileStream _out_fs;
|
||||
private IBlueDataSerializer _serializer;
|
||||
private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
|
||||
private string _file_name = String.Empty;
|
||||
private long _rollover_size;
|
||||
private readonly string _directory_path;
|
||||
@@ -88,20 +95,19 @@ namespace BCG {
|
||||
void _on_change(object new_value) {
|
||||
DataPacket dp = new_value as DataPacket;
|
||||
byte[] serialized_update = null;
|
||||
bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
|
||||
_serializer = _serializer ?? new UniversalSerializer(dp.schema);
|
||||
if (_write_create) {
|
||||
// new files always starts with create
|
||||
_serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
|
||||
bool first_write = _serializer == null || _n_updates_in_current_file == 0;
|
||||
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
||||
if (first_write) {
|
||||
// first file always starts with create
|
||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
||||
} else
|
||||
_serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
|
||||
|
||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
||||
BinaryWriter bw = new BinaryWriter(_out_fs);
|
||||
long utc_epoch = epoch_now();
|
||||
|
||||
//time, length of data, data
|
||||
bw.Write(utc_epoch);
|
||||
bw.Write(serialized_update.Length);
|
||||
//bw.Write(utc_epoch);
|
||||
//bw.Write(serialized_update.Length);
|
||||
bw.Write(serialized_update);
|
||||
_out_fs.Flush();
|
||||
myFireEvent(dp);
|
||||
|
||||
@@ -1,190 +0,0 @@
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
// using System.Linq;
|
||||
// using System.Text;
|
||||
// using System.IO;
|
||||
// using System.Runtime.Serialization.Formatters.Binary;
|
||||
// using BCG.Utilities.General;
|
||||
// using bts.collections;
|
||||
// using System.Reflection.Metadata;
|
||||
|
||||
|
||||
// namespace BCG {
|
||||
|
||||
// public class PlaybackStream : ViewStream, IWorker {
|
||||
// private readonly PlaybackType _type;
|
||||
// private BlueTable _current_table;
|
||||
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||
// // these are the packets already downloaded from the current file
|
||||
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
|
||||
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
|
||||
// private IEnumerable<string> _file_names;
|
||||
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
|
||||
// private DateTime _playback_t0, _playback_t1;
|
||||
// private bool _all_files_read = false;
|
||||
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
|
||||
|
||||
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
|
||||
// _type = PlaybackType.Streaming;
|
||||
// _playback_t0 = PersistorStream.parse_datetime(datetime_start);
|
||||
// _playback_t1 = PersistorStream.parse_datetime(datetime_end);
|
||||
// string token_directory_name = $"{directory_name}\\{token}";
|
||||
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
|
||||
// List<string> files_in_range = new List<string>();
|
||||
// if (Directory.Exists(token_directory_name)) {
|
||||
// _file_names =
|
||||
// Directory.GetFiles(token_directory_name).Where(f => {
|
||||
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
|
||||
// return file_time > _playback_t0 && file_time < _playback_t1;
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
|
||||
// protected override void myOnFirstSubscriber() {
|
||||
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
|
||||
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
|
||||
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
|
||||
// if (snapshot_packet != null) {
|
||||
// foreach (PlaybackPacket p in _snapshot_data.packets) {
|
||||
// byte[] ops;
|
||||
// Array row_array;
|
||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
||||
// for (int i = 0; i < ops.Length; i++) {
|
||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||
// byte op = ops[i];
|
||||
// if (op == StreamOpCode.create) {
|
||||
// IBlueRowCollection full_table = rows_args[0];
|
||||
// _current_table = new BlueTable(full_table);
|
||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||
// } else {
|
||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||
// }
|
||||
// if (p.idx == snapshot_packet.idx) {
|
||||
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
// private void _load_packets_from_current_file() {
|
||||
// int curr_idx = _current_fs.Item1;
|
||||
// FileStream curr_fs = _current_fs.Item2;
|
||||
// _packets = new List<PlaybackPacket>();
|
||||
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
|
||||
// }
|
||||
|
||||
// private bool current_file_all_replayed {
|
||||
// get {
|
||||
// bool rv = true;
|
||||
// _packets.ForEach(p => rv = rv && p.replayed);
|
||||
// return rv;
|
||||
// }
|
||||
// }
|
||||
|
||||
// protected override void myOnLastSubscriber() {
|
||||
// var fs = _current_fs.Item2;
|
||||
// if (fs != null) {
|
||||
// fs.Close();
|
||||
// }
|
||||
// ViewStreamDatabase.instance.removeWorker(this);
|
||||
// }
|
||||
|
||||
// // public event PlaybackStreamProgress onProgress;
|
||||
// // public event Action onFileExhausted;
|
||||
// private PlaybackPacket _locate_packet_to_replay() {
|
||||
// foreach (var p in _packets) {
|
||||
// // returns the first non replayed packet
|
||||
// if (!p.replayed)
|
||||
// return p;
|
||||
// }
|
||||
// //else
|
||||
// {
|
||||
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
|
||||
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
|
||||
// if (onProgress != null)
|
||||
// onProgress(myFileReader.Position, myFileReader.Length);
|
||||
// }
|
||||
// }
|
||||
|
||||
// #region IWorker Members
|
||||
|
||||
// public void doWork() {
|
||||
// if (_all_files_read)
|
||||
// return;
|
||||
// if (current_file_all_replayed) {
|
||||
// _load_next_file();
|
||||
// }
|
||||
// PlaybackPacket curr_packet = _locate_packet_to_replay();
|
||||
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
|
||||
// byte[] ops;
|
||||
// Array row_array;
|
||||
// // what a stupid interface!
|
||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
||||
// _fire_packet(ops, row_array);
|
||||
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
||||
// } else {
|
||||
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
|
||||
// // this is the time we need to fire (replay) this packet
|
||||
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
|
||||
// long now_epoch = PersistorStream.epoch_now();
|
||||
// // fire when we are past the
|
||||
// if (now_epoch >= time_to_fire) {
|
||||
// byte[] ops;
|
||||
// Array row_array;
|
||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
||||
// _fire_packet(ops, row_array);
|
||||
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// private void _fire_packet(byte[] ops, Array row_array) {
|
||||
// for (int i = 0; i < ops.Length; i++) {
|
||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||
// byte op = ops[i];
|
||||
// if (op == StreamOpCode.create) {
|
||||
// IBlueRowCollection full_table = rows_args[0];
|
||||
// _current_table = new BlueTable(full_table);
|
||||
// //_current_table.addRows(full_table);
|
||||
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
||||
// (myLastValue as DataPacket).newRows));
|
||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
||||
// } else {
|
||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||
// }
|
||||
|
||||
// }
|
||||
// }
|
||||
|
||||
// private void _load_next_file() {
|
||||
// var new_idx = _current_fs.Item1 + 1;
|
||||
// if (new_idx >= _file_names.Count()) {
|
||||
// _all_files_read = true;
|
||||
// bts.Log.Gen.info("All files read");
|
||||
// return;
|
||||
// }
|
||||
// var prev_fs = _current_fs.Item2;
|
||||
// if (prev_fs != null)
|
||||
// prev_fs.Close();
|
||||
// var new_filename = _file_names.ElementAt(new_idx);
|
||||
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
|
||||
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
|
||||
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
|
||||
// _load_packets_from_current_file();
|
||||
// }
|
||||
|
||||
// #endregion
|
||||
// }
|
||||
// }
|
||||
@@ -1,68 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class RecorderStream : ViewStream
|
||||
{
|
||||
private ViewStream mySource;
|
||||
private DateTime myStartTime;
|
||||
private FileStream myOutFile;
|
||||
private string myFileName;
|
||||
private IBlueDataSerializer mySerializer;
|
||||
|
||||
public RecorderStream(ViewStream src, string recordFile)
|
||||
{
|
||||
mySource = src;
|
||||
myFileName = recordFile;
|
||||
|
||||
myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myStartTime = DateTime.Now;
|
||||
mySource.onChange += new ChangeDelegate(mySource_onChange);
|
||||
myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
mySource.onChange -= mySource_onChange;
|
||||
myOutFile.Close();
|
||||
}
|
||||
|
||||
void mySource_onChange(object newValue)
|
||||
{
|
||||
DataPacket dp = (DataPacket)newValue;
|
||||
|
||||
BlueTable toWrite = new BlueTable(dp.schema);
|
||||
|
||||
if (mySerializer == null)
|
||||
{
|
||||
mySerializer = new UniversalSerializer(dp.schema);
|
||||
}
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
toWrite.addRow(r);
|
||||
|
||||
byte[] result;
|
||||
|
||||
if (myLastValue == null)
|
||||
mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
|
||||
else
|
||||
mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
|
||||
|
||||
BinaryWriter bw = new BinaryWriter(myOutFile);
|
||||
//write down what time this happened
|
||||
bw.Write((DateTime.Now - myStartTime).TotalSeconds);
|
||||
bw.Write(result.Length);
|
||||
bw.Write(result);
|
||||
myOutFile.Flush();
|
||||
|
||||
myFireEvent(dp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,64 +1,37 @@
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
// using System.Text;
|
||||
// using System.IO;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
using BCG;
|
||||
|
||||
// using System.Runtime.Serialization.Formatters.Binary;
|
||||
namespace bts {
|
||||
// this saves the updates as a Create on the same file, each update calls -save, which
|
||||
// reqrites the whole data to the packet. The other side of this stream is the Loader
|
||||
[DataStreamUserRespresentation("saver")]
|
||||
public class SaverStream : ViewStream {
|
||||
private ViewStream _stream;
|
||||
private string _filename;
|
||||
private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// [DataStreamUserRespresentation("saver")]
|
||||
// public class SaverStream : ViewStream, ISideEffectStream
|
||||
// {
|
||||
// private ViewStream myStream;
|
||||
// private string myFileName;
|
||||
public SaverStream(ViewStream vs, string filename) {
|
||||
_stream = vs;
|
||||
_filename = filename;
|
||||
}
|
||||
|
||||
// public class Exception : System.Exception
|
||||
// {
|
||||
// public Exception(string msg) : base(msg) { }
|
||||
// }
|
||||
protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
|
||||
|
||||
// public SaverStream(ViewStream vs, string fileName)
|
||||
// {
|
||||
// if (fileName == null)
|
||||
// throw new SaverStream.Exception("File name cannot be null");
|
||||
// myStream = vs;
|
||||
// myFileName = fileName;
|
||||
void _handle_changes(object dp_obj) {
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
_save(dp);
|
||||
myFireEvent(dp_obj);
|
||||
}
|
||||
|
||||
// }
|
||||
protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
|
||||
|
||||
// protected override void myOnFirstSubscriber()
|
||||
// { myStream.onChange += new ChangeDelegate(myHandleChanges); }
|
||||
|
||||
// void myHandleChanges(object newValue)
|
||||
// {
|
||||
// DataPacket dp = newValue as DataPacket;
|
||||
// mySave(dp);
|
||||
// myFireEvent(newValue);
|
||||
// }
|
||||
|
||||
// protected override void myOnLastSubscriber()
|
||||
// { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
|
||||
|
||||
// protected void mySave(DataPacket dp)
|
||||
// {
|
||||
// for (int i = 0; i < 5 ; i++)
|
||||
// try
|
||||
// {
|
||||
// using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
|
||||
// {
|
||||
// BinaryFormatter bf = new BinaryFormatter();
|
||||
// BlueTable t = new BlueTable(dp.schema);
|
||||
// foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
|
||||
// t.addRow(new BlueRow(r));
|
||||
// bf.Serialize(fs, t);
|
||||
// }
|
||||
// break;
|
||||
// }
|
||||
// catch (IOException)
|
||||
// {
|
||||
// System.Threading.Thread.Sleep(5);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
protected void _save(DataPacket dp) {
|
||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,115 +1,129 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
using System.Runtime.Serialization.Formatters.Binary;
|
||||
using BCG.Utilities.General;
|
||||
using bts.collections;
|
||||
using System.Reflection.Metadata;
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
// using System.Linq;
|
||||
// using System.Text;
|
||||
// using System.IO;
|
||||
// using System.Runtime.Serialization.Formatters.Binary;
|
||||
// using BCG.Utilities.General;
|
||||
// using bts.collections;
|
||||
// using System.Reflection.Metadata;
|
||||
|
||||
|
||||
namespace BCG {
|
||||
public class SnapshotStream : ViewStream {
|
||||
public enum SpecificSnapshot {
|
||||
FirstOfDay,
|
||||
LastOfDay
|
||||
};
|
||||
// private BlueTable _current_table;
|
||||
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||
// public DateTime _snapshot_time;
|
||||
// public string _snapshot_file;
|
||||
// public IEnumerable<PlaybackPacket> _packets;
|
||||
// public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
|
||||
// DateTime dt;
|
||||
// var parsed = DateTime.TryParse(snapshot_date, out dt);
|
||||
// dt = dt.Date;
|
||||
// bts.Log.assert(parsed);
|
||||
// SpecificSnapshot specific_snapshot_enum;
|
||||
// parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
|
||||
// bts.Log.assert(parsed);
|
||||
// string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
|
||||
// ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
|
||||
// mySetRealSource(rs);
|
||||
// }
|
||||
|
||||
private BlueTable _current_table;
|
||||
private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||
public DateTime _snapshot_time;
|
||||
public string _snapshot_file;
|
||||
public IEnumerable<PlaybackPacket> _packets;
|
||||
// public SnapshotStream(string snapshot_dt, string directory_name, string token) {
|
||||
// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
|
||||
// string token_directory_name = $"{directory_name}\\{token}";
|
||||
// _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
|
||||
// }
|
||||
|
||||
// protected override void myOnFirstSubscriber() {
|
||||
// FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
|
||||
// _packets = StreamPersistorUtils.load_packets_from_file(fs);
|
||||
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
|
||||
// if (snapshot_packet != null) {
|
||||
// foreach (PlaybackPacket p in _packets) {
|
||||
// byte[] ops;
|
||||
// Array row_array;
|
||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
||||
// for (int i = 0; i < ops.Length; i++) {
|
||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||
// byte op = ops[i];
|
||||
// if (op == StreamOpCode.create) {
|
||||
// IBlueRowCollection full_table = rows_args[0];
|
||||
// _current_table = new BlueTable(full_table);
|
||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||
// } else {
|
||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||
// }
|
||||
// if (p.idx == snapshot_packet.idx) {
|
||||
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
public SnapshotStream(string snapshot_dt, string directory_name, string token) {
|
||||
_snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
|
||||
string token_directory_name = $"{directory_name}\\{token}";
|
||||
_snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
|
||||
}
|
||||
// private void _absorb_packet(byte[] ops, Array row_array) {
|
||||
// for (int i = 0; i < ops.Length; i++) {
|
||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||
// byte op = ops[i];
|
||||
// if (op == StreamOpCode.create) {
|
||||
// IBlueRowCollection full_table = rows_args[0];
|
||||
// _current_table = new BlueTable(full_table);
|
||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||
// } else {
|
||||
// bts.Log.assert(false, $"Operator: {op} is not expected");
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
|
||||
_packets = StreamPersistorUtils.load_packets_from_file(fs);
|
||||
PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
|
||||
if (snapshot_packet != null) {
|
||||
foreach (PlaybackPacket p in _packets) {
|
||||
byte[] ops;
|
||||
Array row_array;
|
||||
_deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
||||
for (int i = 0; i < ops.Length; i++) {
|
||||
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||
byte op = ops[i];
|
||||
if (op == StreamOpCode.create) {
|
||||
IBlueRowCollection full_table = rows_args[0];
|
||||
_current_table = new BlueTable(full_table);
|
||||
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||
} else {
|
||||
bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||
}
|
||||
if (p.idx == snapshot_packet.idx) {
|
||||
myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// protected override void myOnLastSubscriber() {
|
||||
// }
|
||||
|
||||
private void _absorb_packet(byte[] ops, Array row_array) {
|
||||
for (int i = 0; i < ops.Length; i++) {
|
||||
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||
byte op = ops[i];
|
||||
if (op == StreamOpCode.create) {
|
||||
IBlueRowCollection full_table = rows_args[0];
|
||||
_current_table = new BlueTable(full_table);
|
||||
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||
} else {
|
||||
bts.Log.assert(false, $"Operator: {op} is not expected");
|
||||
}
|
||||
}
|
||||
}
|
||||
// // public event SnapshotStream.Progress onProgress;
|
||||
// // public event Action onFileExhausted;
|
||||
// private PlaybackPacket _locate_packet_to_replay() {
|
||||
// foreach (var p in _packets) {
|
||||
// // returns the first non replayed packet
|
||||
// if (!p.replayed)
|
||||
// return p;
|
||||
// }
|
||||
// return null;
|
||||
// }
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
}
|
||||
// private void _fire_packet(byte[] ops, Array row_array) {
|
||||
// for (int i = 0; i < ops.Length; i++) {
|
||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||
// byte op = ops[i];
|
||||
// if (op == StreamOpCode.create) {
|
||||
// IBlueRowCollection full_table = rows_args[0];
|
||||
// _current_table = new BlueTable(full_table);
|
||||
// //_current_table.addRows(full_table);
|
||||
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
||||
// (myLastValue as DataPacket).newRows));
|
||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
||||
// } else {
|
||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||
// }
|
||||
|
||||
// public event SnapshotStream.Progress onProgress;
|
||||
// public event Action onFileExhausted;
|
||||
private PlaybackPacket _locate_packet_to_replay() {
|
||||
foreach (var p in _packets) {
|
||||
// returns the first non replayed packet
|
||||
if (!p.replayed)
|
||||
return p;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void _fire_packet(byte[] ops, Array row_array) {
|
||||
for (int i = 0; i < ops.Length; i++) {
|
||||
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||
byte op = ops[i];
|
||||
if (op == StreamOpCode.create) {
|
||||
IBlueRowCollection full_table = rows_args[0];
|
||||
_current_table = new BlueTable(full_table);
|
||||
//_current_table.addRows(full_table);
|
||||
myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
||||
(myLastValue as DataPacket).newRows));
|
||||
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||
myFireEvent(new DataPacket(_current_table, adds, dels));
|
||||
} else {
|
||||
bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using bts;
|
||||
|
||||
|
||||
namespace BCG {
|
||||
@@ -37,6 +38,20 @@ namespace BCG {
|
||||
public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
|
||||
}
|
||||
public static class StreamPersistorUtils {
|
||||
public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
|
||||
DateTime max_dt = DateTime.MinValue;
|
||||
string rv = String.Empty;
|
||||
string[] file_names = Directory.GetFiles(directory_name);
|
||||
for (int i = 0; i < file_names.Count() - 1; i++) {
|
||||
string fn = file_names[i];
|
||||
var dt = parse_datetime_from_filename(fn);
|
||||
if (dt > max_dt) {
|
||||
max_dt = dt;
|
||||
rv = fn;
|
||||
}
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
|
||||
string[] file_names = Directory.GetFiles(directory_name);
|
||||
for (int i = 0; i < file_names.Count() - 1; i++) {
|
||||
|
||||
@@ -1,16 +1,10 @@
|
||||
using BCG;
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
using Bts.DataStream;
|
||||
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
using mbts.config;
|
||||
|
||||
using System;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BCG.StreamWorld.Addition
|
||||
{
|
||||
[DataStreamUserRespresentation (false)]
|
||||
@@ -151,7 +145,6 @@ namespace BCG
|
||||
private string myFilterString;
|
||||
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
||||
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
|
||||
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||
|
||||
#region constructors
|
||||
public delegate bool OnFilter(IBlueRow r);
|
||||
@@ -226,7 +219,6 @@ namespace BCG
|
||||
|
||||
private void myHandleCallbacks(object newValue)
|
||||
{
|
||||
_timer.start();
|
||||
DataPacket retVal;
|
||||
DataPacket arrivedPacket = newValue as DataPacket;
|
||||
myLastReceivedRows = arrivedPacket.newRows;
|
||||
@@ -255,7 +247,6 @@ namespace BCG
|
||||
if (myFilter.filter(r))
|
||||
myLastFiredRows.addRow(r);
|
||||
}
|
||||
_timer.stop();
|
||||
myFireEvent(retVal);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,37 +1,66 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
|
||||
{
|
||||
namespace BCG {
|
||||
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
|
||||
|
||||
public static TimeSpan parse_duration(string input) {
|
||||
int days = 0, hours = 0, minutes = 0, seconds = 0;
|
||||
|
||||
// Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
|
||||
Regex regex = new Regex(@"(\d+)([dhms])");
|
||||
MatchCollection matches = regex.Matches(input);
|
||||
|
||||
foreach (Match match in matches) {
|
||||
int value = int.Parse(match.Groups[1].Value);
|
||||
string unit = match.Groups[2].Value;
|
||||
|
||||
switch (unit) {
|
||||
case "d":
|
||||
days = value;
|
||||
break;
|
||||
case "h":
|
||||
hours = value;
|
||||
break;
|
||||
case "m":
|
||||
minutes = value;
|
||||
break;
|
||||
case "s":
|
||||
seconds = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return new TimeSpan(days, hours, minutes, seconds);
|
||||
}
|
||||
|
||||
#region constructors
|
||||
|
||||
public TimeBasedFilter(string col_name, string row_time_to_live)
|
||||
: this(col_name, parse_duration(row_time_to_live)) {
|
||||
|
||||
}
|
||||
|
||||
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
|
||||
: this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
|
||||
{
|
||||
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
|
||||
}
|
||||
|
||||
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
|
||||
: this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
|
||||
{
|
||||
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
|
||||
}
|
||||
|
||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
||||
: this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
|
||||
{
|
||||
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
|
||||
}
|
||||
|
||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
|
||||
{
|
||||
myVsd = vsd;
|
||||
myRowTimeColumnName = rowTimeColumnName;
|
||||
myRowTimeToLive = rowTimeToLive;
|
||||
myFilterFrequency = filterFrequency;
|
||||
myNextFilter = DateTime.MinValue;
|
||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
|
||||
myVsd = vsd;
|
||||
_time_col = rowTimeColumnName;
|
||||
_ttl = rowTimeToLive;
|
||||
myFilterFrequency = filterFrequency;
|
||||
myNextFilter = DateTime.MinValue;
|
||||
|
||||
// add worker
|
||||
myVsd.addWorker(this);
|
||||
@@ -41,32 +70,25 @@ namespace BCG
|
||||
public event FilterChangeDelegate onFilterChange;
|
||||
|
||||
#region IStreamFilter
|
||||
public bool filter(IBlueRow row)
|
||||
{
|
||||
DateTime now = DateTime.Now;
|
||||
DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
|
||||
public bool filter(IBlueRow row) {
|
||||
DateTime now = DateTime.Now;
|
||||
|
||||
if (now > timeStamp)
|
||||
{
|
||||
TimeSpan span = now - timeStamp;
|
||||
|
||||
if (span >= myRowTimeToLive)
|
||||
{
|
||||
DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
|
||||
if (now > row_time) {
|
||||
TimeSpan span = now - row_time;
|
||||
if (span >= _ttl) {
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
||||
return (true);
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region IWorker
|
||||
public void doWork()
|
||||
{
|
||||
public void doWork() {
|
||||
DateTime now = DateTime.Now;
|
||||
|
||||
if (now > myNextFilter)
|
||||
{
|
||||
if (now > myNextFilter) {
|
||||
// fire event
|
||||
onFilterChange();
|
||||
|
||||
@@ -76,8 +98,7 @@ namespace BCG
|
||||
#endregion
|
||||
|
||||
#region IDisposable
|
||||
public void Dispose()
|
||||
{
|
||||
public void Dispose() {
|
||||
// remove worker
|
||||
myVsd.removeWorker(this);
|
||||
}
|
||||
@@ -85,10 +106,20 @@ namespace BCG
|
||||
|
||||
#region state
|
||||
private ViewStreamDatabase myVsd;
|
||||
private string myRowTimeColumnName;
|
||||
private TimeSpan myRowTimeToLive;
|
||||
private TimeSpan myFilterFrequency;
|
||||
private DateTime myNextFilter;
|
||||
private string _time_col;
|
||||
private TimeSpan _ttl;
|
||||
private TimeSpan myFilterFrequency;
|
||||
private DateTime myNextFilter;
|
||||
#endregion
|
||||
}
|
||||
[DataStreamUserRespresentation("timefilter")]
|
||||
public class TimeBasedFilterStream : ViewStream {
|
||||
public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
|
||||
:
|
||||
this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
|
||||
public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
|
||||
TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
|
||||
mySetRealSource(new FilterStream(base_stream, filter));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,16 +4,12 @@ using BCG.Utilities;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class GroupByStream : MethodDrivenStream
|
||||
{
|
||||
private class MyFuncGroupByEvaluator : IGroupByEvaluator
|
||||
{
|
||||
namespace BCG {
|
||||
public class GroupByStream : MethodDrivenStream {
|
||||
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
||||
private string[] _groupby_cols;
|
||||
private Func<DataPacket, IBlueRow> _groupby_func;
|
||||
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
||||
{
|
||||
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
|
||||
_groupby_cols = groupby_cols.ToArray();
|
||||
_groupby_func = groupby_func;
|
||||
}
|
||||
@@ -29,7 +25,7 @@ namespace BCG
|
||||
/// Serves as a quick way to access all rows belonging to the same group
|
||||
/// </summary>
|
||||
private bool myIdGuard = true;
|
||||
private RowDictionaryTable myBookKeepingTable ;
|
||||
private RowDictionaryTable myBookKeepingTable;
|
||||
private IGroupByEvaluator myGroupByEvaluator;
|
||||
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
|
||||
//can map a schema to another
|
||||
@@ -37,112 +33,98 @@ namespace BCG
|
||||
private int myIdGenerator = 0;
|
||||
|
||||
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
||||
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
|
||||
{ }
|
||||
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
|
||||
|
||||
public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
|
||||
: this (stream, groupByEvaluator, true)
|
||||
{}
|
||||
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
|
||||
: this(stream, groupByEvaluator, true) { }
|
||||
|
||||
public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
|
||||
: base (stream)
|
||||
{
|
||||
public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
|
||||
: base(stream) {
|
||||
myGroupByEvaluator = groupbyEvaluator;
|
||||
myIdGuard = idGuard;
|
||||
}
|
||||
|
||||
protected override void myHandleFirstCall ()
|
||||
{
|
||||
protected override void myHandleFirstCall() {
|
||||
mySchema = myLastArrivedPacket.schema;
|
||||
myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
|
||||
myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows,
|
||||
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
|
||||
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
|
||||
myGroupByEvaluator.groupByColumns);
|
||||
GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
|
||||
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
|
||||
{
|
||||
IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
|
||||
rowGroup, new BlueTable (mySchema)));
|
||||
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
|
||||
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
|
||||
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
|
||||
rowGroup, new BlueTable(mySchema)));
|
||||
if (newRow != null)
|
||||
temp.addRow (newRow, myGetGroupString (rowGroup));
|
||||
temp.addRow(newRow, myGetGroupString(rowGroup));
|
||||
}
|
||||
myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
|
||||
myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adding need not to anything as deletion already takes care
|
||||
/// of addition
|
||||
/// </summary>
|
||||
protected override void myHandleIncrementalAdd() {}
|
||||
protected override void myHandleIncrementalAdd() { }
|
||||
|
||||
/// <summary>
|
||||
/// Both deletes and adds are handled here to make the intermediate results not
|
||||
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream
|
||||
/// because of the nature of it)
|
||||
/// </summary>
|
||||
protected override void myHandleIncrementalDelete()
|
||||
{
|
||||
protected override void myHandleIncrementalDelete() {
|
||||
myFireSwitch = false;
|
||||
RowDictionaryTable delGroups, addGroups;
|
||||
//form groups for deleted and added rows
|
||||
myInitIncrementalChangeData (out delGroups, out addGroups);
|
||||
ArrayList handledAddGroups = new ArrayList ();
|
||||
myInitIncrementalChangeData(out delGroups, out addGroups);
|
||||
ArrayList handledAddGroups = new ArrayList();
|
||||
string groupRepString;
|
||||
|
||||
foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
|
||||
{
|
||||
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
|
||||
// BlueEventLog.logAssert (delGroup.Count > 0);
|
||||
groupRepString = myGetGroupString (delGroup);
|
||||
IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
|
||||
new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
|
||||
handledAddGroups.Add (addGroup);
|
||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
|
||||
new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
|
||||
IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
|
||||
myFireSwitch |= myMergeRow (newRow, groupRepString);
|
||||
groupRepString = myGetGroupString(delGroup);
|
||||
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
|
||||
new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
|
||||
handledAddGroups.Add(addGroup);
|
||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
|
||||
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
|
||||
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
|
||||
myFireSwitch |= myMergeRow(newRow, groupRepString);
|
||||
}
|
||||
|
||||
foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
|
||||
{
|
||||
foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
|
||||
// BlueEventLog.logAssert (addGroup.Count > 0);
|
||||
if (!handledAddGroups.Contains (addGroup))
|
||||
{
|
||||
if (!handledAddGroups.Contains(addGroup)) {
|
||||
//this is the case when there are rows added to a group and none
|
||||
//have been removed, hence it 'escapes' from being dealt with in
|
||||
//the loop above
|
||||
|
||||
groupRepString = myGetGroupString (addGroup);
|
||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
|
||||
groupRepString = myGetGroupString(addGroup);
|
||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
|
||||
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
|
||||
IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup,
|
||||
new BlueTable (aggGroup.schema)));
|
||||
myFireSwitch |= myMergeRow (newRow, groupRepString);
|
||||
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
|
||||
new BlueTable(aggGroup.schema)));
|
||||
myFireSwitch |= myMergeRow(newRow, groupRepString);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool myMergeRow (IBlueRow newRow, string groupRepString)
|
||||
{
|
||||
IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
|
||||
private bool myMergeRow(IBlueRow newRow, string groupRepString) {
|
||||
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
|
||||
//BlueEventLog.logAssert (newRow != null || existingRow != null);
|
||||
if (existingRow == null && newRow == null)
|
||||
return false;
|
||||
else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
|
||||
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
|
||||
return false;
|
||||
else if (existingRow == null)
|
||||
{
|
||||
else if (existingRow == null) {
|
||||
//row added for the first time
|
||||
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
|
||||
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
|
||||
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
|
||||
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
|
||||
((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
|
||||
if (newRow != null)
|
||||
{
|
||||
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
|
||||
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
|
||||
} else {
|
||||
((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
|
||||
((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
|
||||
if (newRow != null) {
|
||||
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
|
||||
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -154,77 +136,68 @@ namespace BCG
|
||||
/// </summary>
|
||||
/// <param name="delGroups"></param>
|
||||
/// <param name="addGroups"></param>
|
||||
private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
|
||||
{
|
||||
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
|
||||
delGroups =
|
||||
new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
|
||||
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
|
||||
addGroups =
|
||||
new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
|
||||
new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
|
||||
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
|
||||
myBookKeepingTable.removeRow (deletedRow);
|
||||
myBookKeepingTable.removeRow(deletedRow);
|
||||
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
|
||||
myBookKeepingTable.addRow (addedRow);
|
||||
myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
|
||||
myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
|
||||
myBookKeepingTable.addRow(addedRow);
|
||||
myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
|
||||
myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
|
||||
}
|
||||
|
||||
protected override void myOnTableClear ()
|
||||
{
|
||||
protected override void myOnTableClear() {
|
||||
if (myBookKeepingTable.Count == 0)
|
||||
return; //ie there arent any rows, so no need to optimize the 'cleansing' process
|
||||
myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
|
||||
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
||||
(myLastFiredPacket.newRows as GroupByOutputTable).clear ();
|
||||
myBookKeepingTable.clear ();
|
||||
myGroupByEvaluator.clear ();
|
||||
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
|
||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
(myLastFiredPacket.newRows as GroupByOutputTable).clear();
|
||||
myBookKeepingTable.clear();
|
||||
myGroupByEvaluator.clear();
|
||||
}
|
||||
|
||||
private IBlueRow myEvaluate (DataPacket dp)
|
||||
{
|
||||
IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
|
||||
private IBlueRow myEvaluate(DataPacket dp) {
|
||||
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
|
||||
return evalRow == null ? null :
|
||||
myIdGuard ?
|
||||
BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
|
||||
BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
|
||||
}
|
||||
|
||||
private string myGetGroupString (IBlueRowCollection group)
|
||||
{
|
||||
return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
|
||||
private string myGetGroupString(IBlueRowCollection group) {
|
||||
return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber ()
|
||||
{
|
||||
protected override void myOnLastSubscriber() {
|
||||
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
|
||||
//gets resubscribed, we start the process of building data again, and treat the callback as the first
|
||||
//callback.
|
||||
myInit ();
|
||||
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
||||
myInit();
|
||||
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
||||
}
|
||||
|
||||
protected override void myInit ()
|
||||
{
|
||||
base.myInit ();
|
||||
if (myBookKeepingTable != null)
|
||||
{
|
||||
protected override void myInit() {
|
||||
base.myInit();
|
||||
if (myBookKeepingTable != null) {
|
||||
//to avoid calling init before the first callback
|
||||
myBookKeepingTable.clear ();
|
||||
myGroupByEvaluator.clear ();
|
||||
myBookKeepingTable.clear();
|
||||
myGroupByEvaluator.clear();
|
||||
}
|
||||
}
|
||||
|
||||
public static string getRepString (DataPacket dp, string[] colNames)
|
||||
{
|
||||
return BlueDataUtility.rowToString (getRepRow (dp), colNames);
|
||||
public static string getRepString(DataPacket dp, string[] colNames) {
|
||||
return BlueDataUtility.rowToString(getRepRow(dp), colNames);
|
||||
}
|
||||
|
||||
public static IBlueRow getRepRow (DataPacket dp)
|
||||
{
|
||||
public static IBlueRow getRepRow(DataPacket dp) {
|
||||
if (dp.newRows.Count > 0)
|
||||
return BlueDataUtility.extractRow (dp.newRows, 0);
|
||||
return BlueDataUtility.extractRow(dp.newRows, 0);
|
||||
else if (dp.addedRows.Count > 0)
|
||||
return BlueDataUtility.extractRow (dp.addedRows, 0);
|
||||
return BlueDataUtility.extractRow(dp.addedRows, 0);
|
||||
else if (dp.deletedRows.Count > 0)
|
||||
return BlueDataUtility.extractRow (dp.deletedRows, 0);
|
||||
return BlueDataUtility.extractRow(dp.deletedRows, 0);
|
||||
// BlueEventLog.logAssert (false, "Illegal Data Packet");
|
||||
return null;
|
||||
}
|
||||
@@ -237,22 +210,21 @@ namespace BCG
|
||||
/// <param name="newValue"></param>
|
||||
/// <param name="isAdd"></param>
|
||||
/// <returns></returns>
|
||||
public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
|
||||
{
|
||||
if (lastValue == null || lastValue.Equals (DBNull.Value))
|
||||
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
|
||||
if (lastValue == null || lastValue.Equals(DBNull.Value))
|
||||
return newValue;
|
||||
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
|
||||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
|
||||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
|
||||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
|
||||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
|
||||
Object retVal = null;
|
||||
if (newValue.GetType().Equals (typeof(int)))
|
||||
if (newValue.GetType().Equals(typeof(int)))
|
||||
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
|
||||
else if (newValue.GetType().Equals (typeof(Double)))
|
||||
else if (newValue.GetType().Equals(typeof(Double)))
|
||||
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
|
||||
else if (lastValue.GetType().Equals (typeof(Decimal)))
|
||||
else if (lastValue.GetType().Equals(typeof(Decimal)))
|
||||
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
|
||||
else{}
|
||||
// BlueEventLog.logAssert (false, "Illegal type");
|
||||
else { }
|
||||
// BlueEventLog.logAssert (false, "Illegal type");
|
||||
return retVal;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,47 +1,42 @@
|
||||
// using System;
|
||||
// using System.Collections;
|
||||
// using BCG.Utilities;
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// /// <summary>
|
||||
// /// Computes the max for a given column
|
||||
// /// </summary>
|
||||
// public class MaxGroupByEvaluator : GroupByEvaluator
|
||||
// {
|
||||
using BCG.Utilities;
|
||||
|
||||
// public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
|
||||
// {}
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Computes the max for a given column
|
||||
/// </summary>
|
||||
public class MaxGroupByEvaluator : GroupByEvaluator {
|
||||
|
||||
// public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
|
||||
// : base (colMapEpression, outputCol, groupByCols) {}
|
||||
public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
|
||||
|
||||
// public override IBlueRow evaluate (DataPacket dp)
|
||||
// {
|
||||
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
|
||||
// myOutputTable;
|
||||
// string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
|
||||
// IBlueRow currMaxRow = myOutputTable.getRow (hashString);
|
||||
// IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
|
||||
// if (maxRow != null)
|
||||
// {
|
||||
// BlueEventLog.logAssert (currMaxRow != null);
|
||||
// BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
|
||||
// if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
|
||||
// {
|
||||
// maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
|
||||
// if (maxRow == null)
|
||||
// myOutputTable.removeRow (hashString);
|
||||
// else
|
||||
// myOutputTable.addRow (maxRow, hashString);
|
||||
// }
|
||||
// }
|
||||
// maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
|
||||
// // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
|
||||
// // otherwise if maxRow > currMaxRow then its replaced
|
||||
// if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
|
||||
// myOutputTable.addRow (maxRow, hashString);
|
||||
// return myProject (myOutputTable.getRow (hashString));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
|
||||
myOutputTable;
|
||||
string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
||||
IBlueRow currMaxRow = myOutputTable.getRow(hashString);
|
||||
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
|
||||
if (maxRow != null) {
|
||||
bts.Log.assert(currMaxRow != null);
|
||||
bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
|
||||
if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
|
||||
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
|
||||
if (maxRow == null)
|
||||
myOutputTable.removeRow(hashString);
|
||||
else
|
||||
myOutputTable.addRow(maxRow, hashString);
|
||||
}
|
||||
}
|
||||
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
|
||||
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
|
||||
// otherwise if maxRow > currMaxRow then its replaced
|
||||
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
|
||||
myOutputTable.addRow(maxRow, hashString);
|
||||
return myProject(myOutputTable.getRow(hashString));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,15 @@
|
||||
// using System;
|
||||
using System;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class MaxStream : ViewStream
|
||||
// {
|
||||
// public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
|
||||
// {
|
||||
// }
|
||||
// public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
|
||||
// {
|
||||
// mySetRealSource (new GroupByStream (stream,
|
||||
// new MaxGroupByEvaluator (outputCol, inputCols)));
|
||||
// }
|
||||
namespace BCG {
|
||||
public class MaxStream : ViewStream {
|
||||
public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
|
||||
}
|
||||
public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
|
||||
mySetRealSource(new GroupByStream(stream,
|
||||
new MaxGroupByEvaluator(outputCol, inputCols)));
|
||||
}
|
||||
|
||||
// public MaxStream (ViewStream stream, string outputCol)
|
||||
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
|
||||
// {}
|
||||
// }
|
||||
// }
|
||||
public MaxStream(ViewStream stream, string outputCol)
|
||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
||||
}
|
||||
}
|
||||
@@ -1,40 +1,36 @@
|
||||
// using System;
|
||||
// using BCG.Utilities;
|
||||
using System;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class MinGroupByEvaluator : MaxGroupByEvaluator
|
||||
// {
|
||||
// public MinGroupByEvaluator (string outputCol, string[] groupByCols)
|
||||
// : base (outputCol, groupByCols) {}
|
||||
using BCG.Utilities;
|
||||
|
||||
// public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
|
||||
// base (colMapEpression, outputCol, groupByCols) {}
|
||||
namespace BCG {
|
||||
public class MinGroupByEvaluator : MaxGroupByEvaluator {
|
||||
public MinGroupByEvaluator(string outputCol, string[] groupByCols)
|
||||
: base(outputCol, groupByCols) { }
|
||||
|
||||
// public override IBlueRow evaluate (DataPacket dp)
|
||||
// {
|
||||
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
|
||||
// string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
|
||||
// IBlueRow currMinRow = myOutputTable.getRow (stringRep);
|
||||
// IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
|
||||
// if (minRow != null)
|
||||
// {
|
||||
// // BlueEventLog.logAssert (currMinRow != null);
|
||||
// // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
|
||||
// if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
|
||||
// {
|
||||
// minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
|
||||
// if (minRow == null)
|
||||
// myOutputTable.removeRow (stringRep);
|
||||
// else
|
||||
// myOutputTable.addRow (minRow, stringRep);
|
||||
// }
|
||||
// }
|
||||
// minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
|
||||
// if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
|
||||
// myOutputTable.addRow (minRow, stringRep);
|
||||
// return myOutputTable.getRow (stringRep);
|
||||
// }
|
||||
public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
|
||||
base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
// }
|
||||
// }
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
||||
IBlueRow currMinRow = myOutputTable.getRow(stringRep);
|
||||
IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
|
||||
if (minRow != null) {
|
||||
// BlueEventLog.logAssert (currMinRow != null);
|
||||
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
|
||||
if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
|
||||
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
|
||||
if (minRow == null)
|
||||
myOutputTable.removeRow(stringRep);
|
||||
else
|
||||
myOutputTable.addRow(minRow, stringRep);
|
||||
}
|
||||
}
|
||||
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
|
||||
if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
|
||||
myOutputTable.addRow(minRow, stringRep);
|
||||
return myOutputTable.getRow(stringRep);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,13 @@
|
||||
// using System;
|
||||
using System;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class MinStream : ViewStream
|
||||
// {
|
||||
// public MinStream (ViewStream stream, string outputCol, string[] inputCol)
|
||||
// {
|
||||
// mySetRealSource (new GroupByStream (stream,
|
||||
// new MinGroupByEvaluator (outputCol, inputCol)));
|
||||
// }
|
||||
namespace BCG {
|
||||
public class MinStream : ViewStream {
|
||||
public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
|
||||
mySetRealSource(new GroupByStream(stream,
|
||||
new MinGroupByEvaluator(outputCol, inputCol)));
|
||||
}
|
||||
|
||||
// public MinStream (ViewStream stream, string outputCol)
|
||||
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
|
||||
// {}
|
||||
// }
|
||||
// }
|
||||
public MinStream(ViewStream stream, string outputCol)
|
||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
||||
}
|
||||
}
|
||||
+17
-13
@@ -1,10 +1,8 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
using System.Diagnostics;
|
||||
|
||||
using bts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -14,7 +12,7 @@ namespace BCG
|
||||
private INonIncGroupByEvaluator _groupby_func;
|
||||
private BlueRowHasher _row_hasher;
|
||||
private string[] _groupby_cols;
|
||||
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||
private Stopwatch _timer = new();
|
||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||
|
||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
||||
@@ -33,19 +31,25 @@ namespace BCG
|
||||
{
|
||||
DataPacket dp = nv as DataPacket;
|
||||
///grouping together
|
||||
_timer.start();
|
||||
_timer.Restart();
|
||||
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
||||
_timer.stop();
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||
_timer.Stop();
|
||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
||||
if (groupElapsedMs >= 50) {
|
||||
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
||||
}
|
||||
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||
/// reducing
|
||||
_timer.start();
|
||||
_timer.Restart();
|
||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||
_timer.stop();
|
||||
_timer.Stop();
|
||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||
_n_total_fires++;
|
||||
if (_timer.lastIntervalTime > 50)
|
||||
if (applyElapsedMs >= 50)
|
||||
{
|
||||
_n_expensive_fires++;
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
|
||||
Log.DataFlow.info(
|
||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||
}
|
||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||
}
|
||||
|
||||
@@ -1,94 +1,82 @@
|
||||
// using System;
|
||||
using System;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// /// <summary>
|
||||
// /// Computes the sum for a given column
|
||||
// /// </summary>
|
||||
// public class SumGroupByEvaluator : GroupByEvaluator
|
||||
// {
|
||||
// public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
|
||||
// {}
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Computes the sum for a given column
|
||||
/// </summary>
|
||||
public class SumGroupByEvaluator : GroupByEvaluator {
|
||||
public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
|
||||
|
||||
// public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
|
||||
// : base (colMapEpression, outputCol, groupByCols) {}
|
||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
// public override IBlueRow evaluate (DataPacket dp)
|
||||
// {
|
||||
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
|
||||
// IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
|
||||
// return sumRow;
|
||||
// }
|
||||
// }
|
||||
IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
|
||||
return sumRow;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
|
||||
// {
|
||||
// private string[] myOutputCols, myGroupByCols;
|
||||
// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
|
||||
// {
|
||||
// myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
||||
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
||||
// }
|
||||
public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
|
||||
private string[] myOutputCols, myGroupByCols;
|
||||
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
|
||||
myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
||||
myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
||||
}
|
||||
|
||||
// public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
|
||||
// {
|
||||
// myOutputCols = output_columns;
|
||||
// myGroupByCols = groupby_columns;
|
||||
// }
|
||||
public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
|
||||
myOutputCols = output_columns;
|
||||
myGroupByCols = groupby_columns;
|
||||
}
|
||||
|
||||
// #region IGroupByEvaluator Members
|
||||
#region IGroupByEvaluator Members
|
||||
|
||||
// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
|
||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
|
||||
|
||||
// public string[] groupByColumns { get { return myGroupByCols; } }
|
||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
||||
|
||||
// public void clear() {/*stateless*/}
|
||||
public void clear() {/*stateless*/}
|
||||
|
||||
// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
|
||||
|
||||
// #endregion
|
||||
// }
|
||||
#endregion
|
||||
}
|
||||
|
||||
// public class SumGroupByEvaluator<T> : GroupByEvaluator
|
||||
// {
|
||||
// public SumGroupByEvaluator(string outputCol, string[] groupByCols)
|
||||
// : base(outputCol, groupByCols)
|
||||
// { }
|
||||
public class SumGroupByEvaluator<T> : GroupByEvaluator {
|
||||
public SumGroupByEvaluator(string outputCol, string[] groupByCols)
|
||||
: base(outputCol, groupByCols) { }
|
||||
|
||||
// public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
// : base(colMapEpression, outputCol, groupByCols) { }
|
||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
// public override IBlueRow evaluate(DataPacket dp)
|
||||
// {
|
||||
// myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
|
||||
// IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
|
||||
// return sumRow;
|
||||
// }
|
||||
// }
|
||||
IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
|
||||
return sumRow;
|
||||
}
|
||||
}
|
||||
|
||||
// public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
|
||||
// {
|
||||
// private string[] myOutputCols, myGroupByCols;
|
||||
// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
|
||||
// {
|
||||
// myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
||||
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
||||
// }
|
||||
public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
|
||||
private string[] myOutputCols, myGroupByCols;
|
||||
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
|
||||
myOutputCols = spaceSeperatedOutputCols.Split(' ');
|
||||
myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
|
||||
}
|
||||
|
||||
// #region IGroupByEvaluator Members
|
||||
#region IGroupByEvaluator Members
|
||||
|
||||
// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
|
||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
|
||||
|
||||
// public string[] groupByColumns { get { return myGroupByCols; } }
|
||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
||||
|
||||
// public void clear() {/*stateless*/}
|
||||
public void clear() {/*stateless*/}
|
||||
|
||||
// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
|
||||
|
||||
// #endregion
|
||||
// }
|
||||
#endregion
|
||||
}
|
||||
|
||||
// }
|
||||
}
|
||||
@@ -1,107 +1,89 @@
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class NullMapper : IStreamMapper
|
||||
// {
|
||||
// private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
|
||||
namespace BCG {
|
||||
public class NullMapper : IStreamMapper {
|
||||
private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
|
||||
|
||||
// public NullMapper(string[] valid_cols)
|
||||
// {
|
||||
// foreach (string c in valid_cols)
|
||||
// _valid_cols[c] = true;
|
||||
// _valid_cols["id"] = true;
|
||||
// }
|
||||
public NullMapper(string[] valid_cols) {
|
||||
foreach (string c in valid_cols)
|
||||
_valid_cols[c] = true;
|
||||
_valid_cols["id"] = true;
|
||||
}
|
||||
|
||||
// public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
|
||||
|
||||
// public IBlueRow map(IBlueRow row)
|
||||
// {
|
||||
// MutableBlueRow out_row = new MutableBlueRow(row);
|
||||
// foreach (string c in row.schema.columnNames)
|
||||
// if (!_valid_cols.ContainsKey(c))
|
||||
// out_row.setValue(c, DBNull.Value);
|
||||
// return out_row;
|
||||
// }
|
||||
// }
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
MutableBlueRow out_row = new MutableBlueRow(row);
|
||||
foreach (string c in row.schema.columnNames)
|
||||
if (!_valid_cols.ContainsKey(c))
|
||||
out_row.setValue(c, DBNull.Value);
|
||||
return out_row;
|
||||
}
|
||||
}
|
||||
|
||||
// public class SumStream : ViewStream
|
||||
// {
|
||||
// public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
|
||||
// {
|
||||
// string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
||||
// string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
||||
// mySetRealSource(new GroupByStream(stream,
|
||||
// outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
|
||||
// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
||||
// }
|
||||
public class SumStream : ViewStream {
|
||||
public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
|
||||
string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
||||
string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
||||
mySetRealSource(new GroupByStream(stream,
|
||||
outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
|
||||
new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
||||
}
|
||||
|
||||
// public SumStream(ViewStream stream, string[] output_columns)
|
||||
// {
|
||||
// ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
|
||||
// rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
|
||||
// rs = new ProjectStream(rs, "!fake_col");
|
||||
// mySetRealSource(rs);
|
||||
// }
|
||||
public SumStream(ViewStream stream, string[] output_columns) {
|
||||
ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
|
||||
rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
|
||||
rs = new ProjectStream(rs, "!fake_col");
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
|
||||
// public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
|
||||
// {
|
||||
// string[] output_columns = space_seperated_output_columns.Split(' ');
|
||||
// string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
|
||||
// mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
|
||||
// }
|
||||
public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
|
||||
string[] output_columns = space_seperated_output_columns.Split(' ');
|
||||
string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
|
||||
mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
|
||||
}
|
||||
|
||||
// public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
|
||||
// {
|
||||
// ViewStream rs = null;
|
||||
// groupby_columns = groupby_columns ?? new string[] {};
|
||||
// if (groupby_columns.Length == 0)
|
||||
// {
|
||||
// rs = new AddColumnsStream(stream, "fake_col=0");
|
||||
// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
|
||||
// rs = new ProjectStream(rs, "!fake_col");
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
|
||||
// }
|
||||
// if (force_null_into_other_columns)
|
||||
// {
|
||||
// List<string> valid_cols = new List<string>(output_columns);
|
||||
// valid_cols.AddRange(groupby_columns);
|
||||
// rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
|
||||
// }
|
||||
// mySetRealSource(rs);
|
||||
// }
|
||||
public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
|
||||
ViewStream rs = null;
|
||||
groupby_columns = groupby_columns ?? new string[] { };
|
||||
if (groupby_columns.Length == 0) {
|
||||
// rs = new AddColumnsStream(stream, "fake_col=0");
|
||||
// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
|
||||
// rs = new ProjectStream(rs, "!fake_col");
|
||||
} else {
|
||||
rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
|
||||
}
|
||||
if (force_null_into_other_columns) {
|
||||
List<string> valid_cols = new List<string>(output_columns);
|
||||
valid_cols.AddRange(groupby_columns);
|
||||
rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
|
||||
}
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
|
||||
// public SumStream (ViewStream stream, string outputCol)
|
||||
// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
|
||||
// {
|
||||
// }
|
||||
// }
|
||||
public SumStream(ViewStream stream, string outputCol)
|
||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
|
||||
}
|
||||
}
|
||||
|
||||
// public class IntSumStream : ViewStream
|
||||
// {
|
||||
// public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
|
||||
// {
|
||||
// string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
||||
// string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
||||
// mySetRealSource(new GroupByStream(stream,
|
||||
// outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
|
||||
// new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
||||
// }
|
||||
public class IntSumStream : ViewStream {
|
||||
public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
|
||||
string[] outputCols = spaceSeparatedOutputCols.Split(' ');
|
||||
string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
|
||||
mySetRealSource(new GroupByStream(stream,
|
||||
outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
|
||||
new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
|
||||
}
|
||||
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
// public class SumStream<T> : ViewStream
|
||||
// {
|
||||
// public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
|
||||
// {
|
||||
// string[] inputCols = spaceSeparatedInputCols.Split(' ');
|
||||
// mySetRealSource(new GroupByStream(stream,
|
||||
// new SumGroupByEvaluator<T>(outputCol, inputCols)));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
public class SumStream<T> : ViewStream {
|
||||
public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
|
||||
string[] inputCols = spaceSeparatedInputCols.Split(' ');
|
||||
mySetRealSource(new GroupByStream(stream,
|
||||
new SumGroupByEvaluator<T>(outputCol, inputCols)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
using BCG;
|
||||
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
using ZstdSharp.Unsafe;
|
||||
|
||||
namespace bts {
|
||||
public class RowDiffStream : ViewStream {
|
||||
private ViewStream _vs;
|
||||
private readonly string _order_col, _col_1, _col_2;
|
||||
private readonly int _diff;
|
||||
private IBlueTableSchema _out_schema;
|
||||
private readonly string _diff_col_name;
|
||||
public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
|
||||
_vs = vs;
|
||||
_order_col = order_col;
|
||||
_col_1 = col_1;
|
||||
_col_2 = col_2;
|
||||
_diff = diff;
|
||||
_diff_col_name = diff_col_name;
|
||||
}
|
||||
|
||||
private string _diff_col_name_perc {
|
||||
get {
|
||||
return $"{_diff_col_name}_percentage";
|
||||
}
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_vs.onChange += _handle_vs;
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
_vs.onChange -= _handle_vs;
|
||||
}
|
||||
|
||||
private void _handle_vs(object dp_obj) {
|
||||
var dp = dp_obj as DataPacket;
|
||||
IBlueRowCollection nrows = dp.newRows;
|
||||
_out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
|
||||
_out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
|
||||
List<IBlueRow> row_list = new List<IBlueRow>();
|
||||
foreach (IBlueRow r in nrows)
|
||||
row_list.Add(r);
|
||||
//OrderBy sorts in ascedning order
|
||||
var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
|
||||
var sorted_row_array = sorted_rows.ToArray();
|
||||
// reverse it to make it descending
|
||||
sorted_rows.Reverse();
|
||||
//sort rows so we can do the diff
|
||||
OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
|
||||
int i = 0;
|
||||
foreach (IBlueRow r in sorted_rows) {
|
||||
MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
|
||||
//So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
|
||||
// we assume x - y, and name variables like that
|
||||
double x = r.get<double>(_col_1);
|
||||
int y_row_idx = i + _diff;
|
||||
double diff = System.Double.NaN;
|
||||
double perc = System.Double.NaN;
|
||||
if (y_row_idx < sorted_rows.Count()) {
|
||||
var y = sorted_rows[y_row_idx].get<double>(_col_2);
|
||||
diff = x - y;
|
||||
perc = 100.0 * diff / y;
|
||||
}
|
||||
nr.setValue(_diff_col_name, diff);
|
||||
nr.setValue(_diff_col_name_perc, perc);
|
||||
rv.addRow(nr);
|
||||
i++;
|
||||
}
|
||||
|
||||
var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
|
||||
myFireEvent(dp_out);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,8 @@ namespace BCG
|
||||
string prependString = isLeft ? "left" : "right";
|
||||
return String.Concat (prependString, colName).GetHashCode ();
|
||||
}
|
||||
public override bool Equals (object obj) {return GetHashCode () == obj.GetHashCode ();}
|
||||
public override bool Equals (object? obj) =>
|
||||
GetHashCode() == obj?.GetHashCode();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -10,9 +10,9 @@ namespace BCG
|
||||
{
|
||||
public class JoinStream : ViewStream
|
||||
{
|
||||
protected InternalJoinStream myIjs;
|
||||
protected ProjectStream myPs;
|
||||
protected ColumnMap myLeftCm, myRightCm;
|
||||
protected InternalJoinStream? myIjs;
|
||||
protected ProjectStream? myPs;
|
||||
protected ColumnMap? myLeftCm, myRightCm;
|
||||
protected bool myIsProject = true;
|
||||
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
||||
|
||||
|
||||
+11
-13
@@ -1,10 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using bts;
|
||||
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -40,8 +40,7 @@ namespace BCG
|
||||
|
||||
public void _on_stream_cb(object dp_obj)
|
||||
{
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
DataPacket dp = (DataPacket)dp_obj;
|
||||
if (_schema == null)
|
||||
{
|
||||
@@ -67,13 +66,12 @@ namespace BCG
|
||||
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
||||
}
|
||||
}
|
||||
timer.stop();
|
||||
timer.Stop();
|
||||
_n_total_fires++;
|
||||
if (timer.lastIntervalTime > 50)
|
||||
{
|
||||
if (timer.ElapsedMilliseconds >= 50) {
|
||||
_n_expensive_fires++;
|
||||
string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
|
||||
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||
}
|
||||
if (on_new_data != null)
|
||||
on_new_data();
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using bts;
|
||||
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -114,8 +114,7 @@ namespace BCG
|
||||
if (_is_ready())
|
||||
{
|
||||
_construct_schema();
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
||||
List<object[]> output_rows = new List<object[]>();
|
||||
|
||||
@@ -138,13 +137,12 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
||||
timer.stop();
|
||||
timer.Stop();
|
||||
_n_total_fires++;
|
||||
if (timer.lastIntervalTime >= 250)
|
||||
{
|
||||
if (timer.ElapsedMilliseconds >= 250) {
|
||||
_n_expensive_fires++;
|
||||
string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
||||
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
|
||||
}
|
||||
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
||||
}
|
||||
@@ -154,15 +152,12 @@ namespace BCG
|
||||
{
|
||||
if (_output_mapper != null)
|
||||
{
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
IBlueRowCollection output_rows;
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
output_rows = _output_mapper.map_rows(unmapped_rows);
|
||||
timer.stop();
|
||||
if (timer.lastIntervalTime >= 50)
|
||||
{
|
||||
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||
timer.Stop();
|
||||
if (timer.ElapsedMilliseconds >= 50) {
|
||||
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
||||
}
|
||||
return output_rows;
|
||||
}
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using BCG.Utilities.General;
|
||||
using BCG.Utilities;
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
using Spark.Core.Collections;
|
||||
|
||||
namespace BCG
|
||||
@@ -226,7 +222,8 @@ namespace BCG
|
||||
|
||||
#region IDictionary Members
|
||||
|
||||
public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
|
||||
public void Add(object key, object? value) =>
|
||||
throw new NotSupportedException("IBlueRow is readonly");
|
||||
|
||||
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
|
||||
|
||||
@@ -256,10 +253,9 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object this[object key]
|
||||
{
|
||||
get { return key is string ? this[key.ToString()] : this[(int)key]; }
|
||||
set { throw new NotSupportedException("IBlueRow is readonly"); }
|
||||
public object? this[object key] {
|
||||
get => key is string str ? this[str] : this[(int)key];
|
||||
set => throw new NotSupportedException("IBlueRow is readonly");
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -294,7 +290,6 @@ namespace BCG
|
||||
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
||||
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
||||
private bool myManageUnmappedLeftRows = false;
|
||||
protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
|
||||
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
||||
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
||||
protected JoinProjectHelper myJoinProjectHelper;
|
||||
@@ -361,7 +356,6 @@ namespace BCG
|
||||
|
||||
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
||||
{
|
||||
myTimer.start();
|
||||
myInitSchema(dp.schema, isLeft);
|
||||
BlueTable dels = null, adds = null;
|
||||
if (myState != null)
|
||||
@@ -371,7 +365,6 @@ namespace BCG
|
||||
}
|
||||
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
||||
myAddRows(dp.addedRows, isLeft, myState, adds);
|
||||
myTimer.stop();
|
||||
if (myState != null)
|
||||
{
|
||||
myFireEvent(new DataPacket(myState, adds, dels));
|
||||
|
||||
@@ -91,7 +91,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Add (object key, object? value) =>
|
||||
throw new NotSupportedException ("The IBlueRow is readonly");
|
||||
|
||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
|
||||
|
||||
@@ -1,51 +1,64 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class LiteralStream : ViewStream
|
||||
{
|
||||
public LiteralStream(string tableName, string schema, string value)
|
||||
{
|
||||
string [] columns = schema.Split(' ');
|
||||
namespace BCG {
|
||||
public class LiteralStream : ViewStream {
|
||||
|
||||
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
||||
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
|
||||
string[] pairs = key_value_pairs.Split(' ');
|
||||
foreach (string p in pairs) {
|
||||
string[] kvp = p.Split(':');
|
||||
string col_name = kvp[0];
|
||||
string val = kvp[1];
|
||||
var tup = BlueDataUtility.guess_type(val);
|
||||
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
|
||||
rv.Add(Tuple.Create(ci, tup.Item2));
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
public LiteralStream(string tableName, string schema, string value) {
|
||||
string[] columns = schema.Split(' ');
|
||||
|
||||
|
||||
ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
|
||||
ci[0] = new ColumnInfo("id",0,typeof(int));
|
||||
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
|
||||
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
||||
int n = 1;
|
||||
foreach (string c in columns)
|
||||
{
|
||||
string [] columnAndType = c.Split('/');
|
||||
foreach (string c in columns) {
|
||||
string[] columnAndType = c.Split('/');
|
||||
if (columnAndType.Length == 1)
|
||||
ci[n] = new ColumnInfo(c,n,typeof(double));
|
||||
ci[n] = new ColumnInfo(c, n, typeof(double));
|
||||
else
|
||||
ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
|
||||
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
||||
n++;
|
||||
}
|
||||
|
||||
|
||||
BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
|
||||
string [] rows = value.Split(';');
|
||||
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
|
||||
string[] rows = value.Split(';');
|
||||
n = 0;
|
||||
foreach (string r in rows)
|
||||
try
|
||||
{
|
||||
if(r == string.Empty)
|
||||
try {
|
||||
if (r == string.Empty)
|
||||
continue;
|
||||
t.addRow(
|
||||
BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
|
||||
} catch (Exception e) {
|
||||
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
||||
}
|
||||
|
||||
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
||||
}
|
||||
|
||||
public LiteralStream(string schema, string value) : this ("literal",schema, value)
|
||||
{}
|
||||
public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
||||
|
||||
// public LiteralStream(string value) {
|
||||
|
||||
// }
|
||||
}
|
||||
}
|
||||
@@ -1,24 +1,24 @@
|
||||
using System;
|
||||
// using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
|
||||
{
|
||||
IBlueTableSchema mySchema;
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
|
||||
// {
|
||||
// IBlueTableSchema mySchema;
|
||||
|
||||
public IBlueRow map (IBlueRow row)
|
||||
{
|
||||
if (mySchema == null)
|
||||
{
|
||||
ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
|
||||
int i = 0;
|
||||
foreach (ColumnInfo ci in row.schema.columnInfo)
|
||||
cis[i++] = ci.colType == null ?
|
||||
new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
|
||||
new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
|
||||
mySchema = new BlueTableSchema (row.schema.tableName, cis);
|
||||
}
|
||||
return new BlueRow (mySchema, row);
|
||||
}
|
||||
}
|
||||
}
|
||||
// public IBlueRow map (IBlueRow row)
|
||||
// {
|
||||
// if (mySchema == null)
|
||||
// {
|
||||
// ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
|
||||
// int i = 0;
|
||||
// foreach (ColumnInfo ci in row.schema.columnInfo)
|
||||
// cis[i++] = ci.colType == null ?
|
||||
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
|
||||
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
|
||||
// mySchema = new BlueTableSchema (row.schema.tableName, cis);
|
||||
// }
|
||||
// return new BlueRow (mySchema, row);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
@@ -1,15 +1,10 @@
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Collections;
|
||||
|
||||
using Bts.DataStream;
|
||||
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
using mbts.config;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
using System;
|
||||
|
||||
namespace BCG {
|
||||
|
||||
|
||||
/// <summary>
|
||||
@@ -21,24 +16,45 @@ namespace BCG
|
||||
[DataStreamUserRespresentation (false)]
|
||||
public class MapStream : MethodDrivenStream
|
||||
{
|
||||
private HighPerformanceTimer t = new HighPerformanceTimer ();
|
||||
protected IStreamMapper myMapper;
|
||||
protected IChangeableStreamMapper myChangeableMapper = null;
|
||||
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
||||
|
||||
public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
|
||||
{
|
||||
if (myUseNewMap)
|
||||
mySetRealSource (new SuperMapStream (vs, mapper));
|
||||
else
|
||||
myMapper = mapper;
|
||||
public class FuncWrapper : IStreamMapper {
|
||||
private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func;
|
||||
Func<IBlueRow, IBlueRow> _row_map_func;
|
||||
|
||||
public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
|
||||
Func<IBlueRow, IBlueRow> row_map_func) {
|
||||
_schema_map_func = schema_map_func;
|
||||
_row_map_func = row_map_func;
|
||||
}
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
|
||||
return _schema_map_func(input_schema);
|
||||
}
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
return _row_map_func(row);
|
||||
}
|
||||
|
||||
}
|
||||
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
|
||||
{
|
||||
|
||||
public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
|
||||
if (myUseNewMap)
|
||||
mySetRealSource(new SuperMapStream(vs, mapper));
|
||||
else
|
||||
{
|
||||
myMapper = mapper;
|
||||
}
|
||||
|
||||
public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
|
||||
Func<IBlueRow, IBlueRow> row_map_func)
|
||||
:
|
||||
this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
|
||||
}
|
||||
|
||||
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
|
||||
if (myUseNewMap)
|
||||
mySetRealSource(new SuperMapStream(vs, mapper));
|
||||
else {
|
||||
mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
|
||||
myChangeableMapper = mapper;
|
||||
}
|
||||
@@ -47,74 +63,65 @@ namespace BCG
|
||||
/// <summary>
|
||||
/// Gets or sets the value of the mapper. Setting the value will result in re
|
||||
/// </summary>
|
||||
private void myMapperChanged ()
|
||||
{
|
||||
private void myMapperChanged() {
|
||||
if (myLastArrivedPacket != null)
|
||||
myChangeHandler (new DataPacket (myLastArrivedPacket.newRows,
|
||||
myChangeHandler(new DataPacket(myLastArrivedPacket.newRows,
|
||||
myLastArrivedPacket.newRows,
|
||||
myLastArrivedPacket.newRows));
|
||||
}
|
||||
|
||||
|
||||
protected override void myHandleFirstCall ()
|
||||
{
|
||||
protected override void myHandleFirstCall() {
|
||||
|
||||
IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
|
||||
myLastFiredPacket = new DataPacket ();
|
||||
myLastFiredPacket.newRows = new BlueTable (outputSchema);
|
||||
myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
|
||||
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
|
||||
myLastFiredPacket = new DataPacket();
|
||||
myLastFiredPacket.newRows = new BlueTable(outputSchema);
|
||||
myLastFiredPacket.deletedRows = new BlueTable(outputSchema);
|
||||
foreach (IBlueRow row in myLastArrivedPacket.newRows)
|
||||
((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
|
||||
((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row));
|
||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
|
||||
}
|
||||
|
||||
protected override void myHandleIncrementalAdd()
|
||||
{
|
||||
protected override void myHandleIncrementalAdd() {
|
||||
//t.start ();
|
||||
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
|
||||
foreach (IBlueRow row in myLastArrivedPacket.addedRows)
|
||||
{
|
||||
IBlueRow newRow = myMapper.map (row);
|
||||
(myLastFiredPacket.newRows as BlueTable).addRow (newRow);
|
||||
(myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
|
||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||
foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
|
||||
IBlueRow newRow = myMapper.map(row);
|
||||
(myLastFiredPacket.newRows as BlueTable).addRow(newRow);
|
||||
(myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
|
||||
}
|
||||
//t.stop ();
|
||||
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
||||
//System.Diagnostics.Trace.WriteLine (" ");
|
||||
}
|
||||
|
||||
protected override void myHandleIncrementalDelete()
|
||||
{
|
||||
protected override void myHandleIncrementalDelete() {
|
||||
//t.start ();
|
||||
myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
|
||||
foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
|
||||
{
|
||||
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
|
||||
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
|
||||
(myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
|
||||
(myLastFiredPacket.newRows as BlueTable).deleteRow (row);
|
||||
(myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
|
||||
(myLastFiredPacket.newRows as BlueTable).deleteRow(row);
|
||||
}
|
||||
//t.stop ();
|
||||
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
||||
}
|
||||
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
|
||||
protected override void myOnFirstSubscriber() {
|
||||
myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber ()
|
||||
{
|
||||
myInit ();
|
||||
protected override void myOnLastSubscriber() {
|
||||
myInit();
|
||||
if (myStream != null)
|
||||
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
||||
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
||||
}
|
||||
|
||||
protected override void myOnTableClear ()
|
||||
{
|
||||
protected override void myOnTableClear() {
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
||||
myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
using BCG.Utilities;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using mbts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// SuperMap gives the user the ability
|
||||
/// to map one row to one or more rows
|
||||
@@ -17,11 +18,9 @@ namespace BCG
|
||||
/// can chage itself and raise an event causing
|
||||
/// a remapping to happen
|
||||
/// </summary>
|
||||
public class SuperMapStream : ViewStream
|
||||
{
|
||||
public class SuperMapStream : ViewStream {
|
||||
#region super mapper that wraps around stream mapper
|
||||
private class MyMappingWrapper : ISuperStreamMapper
|
||||
{
|
||||
private class MyMappingWrapper : ISuperStreamMapper {
|
||||
private IStreamMapper myMapper;
|
||||
|
||||
public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
|
||||
@@ -32,8 +31,7 @@ namespace BCG
|
||||
|
||||
public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
|
||||
|
||||
public event MapperChangeDelegate onMapperChange
|
||||
{
|
||||
public event MapperChangeDelegate onMapperChange {
|
||||
add { }
|
||||
remove { }
|
||||
}
|
||||
@@ -42,21 +40,18 @@ namespace BCG
|
||||
}
|
||||
#endregion
|
||||
|
||||
private class MyFuncWrapper : ISuperStreamMapper
|
||||
{
|
||||
private class MyFuncWrapper : ISuperStreamMapper {
|
||||
private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
|
||||
private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
|
||||
public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
||||
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
||||
{
|
||||
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) {
|
||||
_get_schema = get_schema;
|
||||
_map_rows = map_rows;
|
||||
}
|
||||
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
|
||||
|
||||
public IBlueRow[] map(IBlueRow r)
|
||||
{
|
||||
public IBlueRow[] map(IBlueRow r) {
|
||||
IEnumerable<IBlueRow> rows = _map_rows(r);
|
||||
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
||||
}
|
||||
@@ -76,19 +71,15 @@ namespace BCG
|
||||
|
||||
public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
||||
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
||||
: this(vs, new MyFuncWrapper(get_schema, map_rows))
|
||||
{ }
|
||||
: this(vs, new MyFuncWrapper(get_schema, map_rows)) { }
|
||||
|
||||
public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
||||
: this(vs, s => out_schema, map_rows)
|
||||
{ }
|
||||
: this(vs, s => out_schema, map_rows) { }
|
||||
|
||||
public SuperMapStream(ViewStream vs, IStreamMapper mapper)
|
||||
: this(vs, new MyMappingWrapper(mapper))
|
||||
{ }
|
||||
: this(vs, new MyMappingWrapper(mapper)) { }
|
||||
|
||||
public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
|
||||
{
|
||||
public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) {
|
||||
myStream = vs;
|
||||
myMapper = mapper;
|
||||
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
|
||||
@@ -96,8 +87,7 @@ namespace BCG
|
||||
|
||||
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
protected override void myOnLastSubscriber() {
|
||||
myStream.onChange -= new ChangeDelegate(myHandleData);
|
||||
myRows = null;
|
||||
myBaseRows = null;
|
||||
@@ -105,8 +95,7 @@ namespace BCG
|
||||
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
|
||||
}
|
||||
|
||||
private void myHandleData(object newValue)
|
||||
{
|
||||
private void myHandleData(object newValue) {
|
||||
DataPacket idp = newValue as DataPacket;
|
||||
//HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
//timer.start();
|
||||
@@ -117,24 +106,19 @@ namespace BCG
|
||||
BlueTable dels = new BlueTable(myRows.schema),
|
||||
adds = new BlueTable(myRows.schema);
|
||||
|
||||
foreach (IBlueRow r in dp.deletedRows)
|
||||
{
|
||||
foreach (IBlueRow r in dp.deletedRows) {
|
||||
List<IBlueRow> mappedRows = myMap[(int)r["id"]];
|
||||
foreach (IBlueRow mr in mappedRows)
|
||||
{
|
||||
foreach (IBlueRow mr in mappedRows) {
|
||||
dels.addRow(mr);
|
||||
myRows.deleteRow(mr);
|
||||
}
|
||||
myMap.Remove((int)r["id"]);
|
||||
}
|
||||
foreach (IBlueRow r in dp.addedRows)
|
||||
{
|
||||
foreach (IBlueRow r in dp.addedRows) {
|
||||
IBlueRow[] mappedRows = myMapper.map(r);
|
||||
foreach (IBlueRow mr in mappedRows)
|
||||
{
|
||||
foreach (IBlueRow mr in mappedRows) {
|
||||
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
||||
if (!myMap.ContainsKey((int)r["id"]))
|
||||
{
|
||||
if (!myMap.ContainsKey((int)r["id"])) {
|
||||
myMap.Add((int)r["id"], new List<IBlueRow>());
|
||||
}
|
||||
myMap[(int)r["id"]].Add(idChangedMapRow);
|
||||
@@ -145,13 +129,12 @@ namespace BCG
|
||||
//timer.stop();
|
||||
// if (timer.lastIntervalTime > 20)
|
||||
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
|
||||
// myFireEvent(new DataPacket(myRows, adds, dels));
|
||||
myFireEvent(new DataPacket(myRows, adds, dels));
|
||||
}
|
||||
|
||||
private void myHandleMapChange()
|
||||
{
|
||||
private void myHandleMapChange() {
|
||||
if (myBaseRows != null)
|
||||
myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
|
||||
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using BCG;
|
||||
using System;
|
||||
using System.IO;
|
||||
using MongoDB.Bson;
|
||||
using BCG.Utilities.General;
|
||||
using System.Linq;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
|
||||
namespace bts {
|
||||
// listens to the underlying stream, and writes each seriialized packet in a
|
||||
// seperate file
|
||||
[DataStreamUserRespresentation("dp_recorder")]
|
||||
public class DataPacketRecorderStream : ViewStream {
|
||||
private ViewStream _base_stream;
|
||||
private readonly string _folder;
|
||||
StreamIOSerializationController _serializer = new StreamIOSerializationController();
|
||||
private UInt32 _n_updates = 0;
|
||||
public DataPacketRecorderStream(ViewStream base_stream, string folder) {
|
||||
_base_stream = base_stream;
|
||||
_folder = folder;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_base_stream.onChange += _handle_update;
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
_base_stream.onChange -= _handle_update;
|
||||
}
|
||||
|
||||
private void _handle_update(object dp_obj) {
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
byte[] ser_data = null;
|
||||
if (_n_updates == 0) {
|
||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||
ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||
} else {
|
||||
StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
|
||||
ser_data = _serializer.serialize(new StreamIOMessage(mra));
|
||||
}
|
||||
Log.assert(ser_data != null);
|
||||
_n_updates++;
|
||||
string file_name = $"{_folder}\\{_n_updates}.bin";
|
||||
_write_to_file(ser_data, file_name);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
private static void _write_to_file(byte[] data, string file_name) {
|
||||
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
|
||||
// Write the byte array to the file
|
||||
fs.Write(data, 0, data.Length);
|
||||
}
|
||||
}
|
||||
}
|
||||
[DataStreamUserRespresentation("dp_replay")]
|
||||
public class DataPacketReplayStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _folder_path;
|
||||
private readonly TimeSpan _fire_period;
|
||||
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
|
||||
private int _file_ptr = 0;
|
||||
private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
|
||||
private BlueTable _state = null;
|
||||
|
||||
public DataPacketReplayStream(string folder_path, string fire_period)
|
||||
:
|
||||
this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
|
||||
}
|
||||
|
||||
public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
|
||||
_vsd = vsd;
|
||||
_folder_path = folder_path;
|
||||
_fire_period = fire_period;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, _fire_period);
|
||||
_load_all_files();
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
if (_file_ptr >= _file_data.Count())
|
||||
// we can set _file_ptr to 0 and keep going as well
|
||||
return;
|
||||
var curr_d = _file_data[_file_ptr++];
|
||||
var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
|
||||
StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
|
||||
Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
|
||||
switch (msg.op) {
|
||||
case StreamIOMessage.StreamOp.CREATE: {
|
||||
_state = new BlueTable(msg.create_message.rows);
|
||||
break;
|
||||
}
|
||||
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
|
||||
var mra = msg.multiple_remove_add_message;
|
||||
foreach (IBlueRow r in mra.removed_rows)
|
||||
_state.deleteRow(r);
|
||||
foreach (IBlueRow r in mra.added_rows)
|
||||
_state.addRow(r);
|
||||
break;
|
||||
}
|
||||
}
|
||||
var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
private void _load_all_files() {
|
||||
Func<string, string> extract_file_name = (string full_path) => {
|
||||
string fn = full_path.Split("\\").Last();
|
||||
fn = fn.Split(".").First();
|
||||
return fn;
|
||||
|
||||
};
|
||||
string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
|
||||
|
||||
// Iterate over the .bin files and perform an action
|
||||
foreach (string file in bin_files) {
|
||||
byte[] file_data = File.ReadAllBytes(file);
|
||||
_file_data.Add(Tuple.Create(file, file_data));
|
||||
}
|
||||
_file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
|
||||
string fn1 = extract_file_name(t1.Item1);
|
||||
string fn2 = extract_file_name(t2.Item1);
|
||||
int fn_i1 = Int32.Parse(fn1);
|
||||
int fn_i2 = Int32.Parse(fn2);
|
||||
if (fn_i1 < fn_i2)
|
||||
return -1;
|
||||
else if (fn_i1 == fn_i2)
|
||||
return 0;
|
||||
else if (fn_i1 > fn_i2)
|
||||
return 1;
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
using System.Collections.Generic;
|
||||
|
||||
//using MongoDB.Driver;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace bts
|
||||
{
|
||||
public class PubSubTestingStream : ViewStream, IWorker
|
||||
{
|
||||
public class PayloadMessage
|
||||
{
|
||||
public int msg_id = 0;
|
||||
public string guid;
|
||||
public Int64 travel_epoch_ms = 0;
|
||||
public byte[] payload = null;
|
||||
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
|
||||
{
|
||||
msg_id = msg_id_args;
|
||||
travel_epoch_ms = send_epoch_args;
|
||||
payload = new byte[payload_size];
|
||||
for (int i = 0; i < payload_size; i++)
|
||||
{
|
||||
payload[i] = payload_byte;
|
||||
}
|
||||
}
|
||||
private PayloadMessage() { }
|
||||
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
|
||||
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
|
||||
{ }
|
||||
|
||||
public byte[] serialize()
|
||||
{
|
||||
var str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
bw.Write(msg_id);
|
||||
bw.Write(guid);
|
||||
bw.Write(travel_epoch_ms);
|
||||
bw.Write(payload.Length);
|
||||
bw.Write(payload);
|
||||
str.Position = 0;
|
||||
var bytes = str.ToArray();
|
||||
return bytes;
|
||||
}
|
||||
|
||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
||||
{
|
||||
var str = new MemoryStream(ser_bytes);
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
int msg_id = br.ReadInt32();
|
||||
string guid = br.ReadString();
|
||||
Int64 epoch = br.ReadInt64();
|
||||
int len = br.ReadInt32();
|
||||
var payload = br.ReadBytes(len);
|
||||
return new PayloadMessage()
|
||||
{
|
||||
msg_id = msg_id,
|
||||
guid = guid,
|
||||
travel_epoch_ms = epoch,
|
||||
payload = payload
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
//public void
|
||||
private readonly int _msg_size;
|
||||
private readonly TimeSpan _publish_period;
|
||||
private DateTime _last_publish_time = DateTime.MinValue;
|
||||
private PublisherCore _pub_core = new PublisherCore();
|
||||
private BlueTable _table;
|
||||
private int _n_pubs = 0;
|
||||
private readonly int _max_pubs;
|
||||
private int _n_updates = 0;
|
||||
private bool _continue_sub_thread = true;
|
||||
private ViewStreamDatabase _vsd;
|
||||
private bool _fire_dirty = false;
|
||||
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
||||
private readonly int _payload_sz;
|
||||
private const byte _payload_byte = (byte)'A';
|
||||
private readonly string _tag_name;
|
||||
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
||||
{
|
||||
_vsd = vsd;
|
||||
_payload_sz = payload_size;
|
||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("n_pubs", TypeCode.Int32),
|
||||
Tuple.Create("n_recvs", TypeCode.Int32),
|
||||
Tuple.Create("last_sent_id", TypeCode.Int32),
|
||||
Tuple.Create("last_recv_id", TypeCode.Int32),
|
||||
Tuple.Create("last_sent_guid", TypeCode.String),
|
||||
Tuple.Create("last_recv_guid", TypeCode.String),
|
||||
Tuple.Create("match", TypeCode.Boolean),
|
||||
Tuple.Create("echo_time", TypeCode.Int32));
|
||||
_table = new BlueTable(schema);
|
||||
_publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
|
||||
_msg_size = payload_size;
|
||||
_max_pubs = max_pubs;
|
||||
_tag_name = tag_name;
|
||||
}
|
||||
|
||||
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
||||
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
_vsd.addWorker(this);
|
||||
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
|
||||
th.IsBackground = true;
|
||||
th.Start();
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_vsd.removeWorker(this);
|
||||
_continue_sub_thread = false;
|
||||
}
|
||||
|
||||
void _log_updates(IEnumerable<PubSubServerMessage> updates)
|
||||
{
|
||||
var idx = 0;
|
||||
foreach (var msg in updates)
|
||||
{
|
||||
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
|
||||
private void _subscribe_thread_func()
|
||||
{
|
||||
// subscription is done in another thread
|
||||
SubscriberCore sub_core = new SubscriberCore();
|
||||
sub_core.subscribe(_tag_name);
|
||||
while (_continue_sub_thread)
|
||||
{
|
||||
// TODO recv_id
|
||||
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
||||
if (updates == null || updates.Count() == 0)
|
||||
continue;
|
||||
_log_updates(updates);
|
||||
_n_updates += updates.Count();
|
||||
if (updates.Count() > 0)
|
||||
{
|
||||
var server_msg = updates.Last();
|
||||
if (server_msg.data != null && server_msg.data.Length > 0)
|
||||
{
|
||||
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
||||
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
||||
}
|
||||
else
|
||||
{
|
||||
// why are we getting empty or null data ? (server is sending data regardless?)
|
||||
}
|
||||
_fire_dirty = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _publish_next_packet()
|
||||
{
|
||||
if (_last_sent_msg == null)
|
||||
{
|
||||
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
|
||||
{
|
||||
guid = Guid.NewGuid().ToString()
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
_last_sent_msg.msg_id++;
|
||||
_last_sent_msg.guid = Guid.NewGuid().ToString();
|
||||
_last_sent_msg.travel_epoch_ms = epoch_now();
|
||||
}
|
||||
var ser_bytes = _last_sent_msg.serialize();
|
||||
_n_pubs++;
|
||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
||||
_fire_dirty = true;
|
||||
}
|
||||
|
||||
private void _fire_dp()
|
||||
{
|
||||
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
|
||||
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
|
||||
object echo_time = DBNull.Value;
|
||||
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
|
||||
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
|
||||
if (_last_sent_msg != null && _last_recvd_msg != null)
|
||||
{
|
||||
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
|
||||
}
|
||||
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
|
||||
IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
|
||||
last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
|
||||
BlueTable table = new BlueTable(_table.schema);
|
||||
table.addRow(nr);
|
||||
var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
if (DateTime.Now - _last_publish_time > _publish_period)
|
||||
if (_n_pubs <= _max_pubs)
|
||||
{
|
||||
_publish_next_packet();
|
||||
_last_publish_time = DateTime.Now;
|
||||
}
|
||||
// publish
|
||||
// TODO: send id
|
||||
if (_fire_dirty)
|
||||
{
|
||||
_fire_dirty = false;
|
||||
_fire_dp();
|
||||
}
|
||||
}
|
||||
|
||||
public static long epoch_now()
|
||||
{
|
||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
using System.Collections.Generic;
|
||||
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using bts.api;
|
||||
|
||||
namespace bts
|
||||
{
|
||||
[DataStreamUserRespresentation("pss_echo")]
|
||||
public class PubsubEchoTestStream : ViewStream, IWorker
|
||||
{
|
||||
public class PayloadMessage
|
||||
{
|
||||
public readonly string msg_id;
|
||||
public readonly Int64 timestamp = 0;
|
||||
public readonly byte[] payload = null;
|
||||
|
||||
public static byte[] create_data(int payload_size, byte payload_byte)
|
||||
{
|
||||
var rv = new byte[payload_size];
|
||||
for (int i = 0; i < payload_size; i++)
|
||||
{
|
||||
rv[i] = payload_byte;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
|
||||
// {
|
||||
// guid = guid_;
|
||||
// timestamp = timestamp_;
|
||||
// payload = new byte[payload_size];
|
||||
// for (int i = 0; i < payload_size; i++)
|
||||
// {
|
||||
// payload[i] = payload_byte;
|
||||
// }
|
||||
// }
|
||||
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
|
||||
{
|
||||
msg_id = guid_;
|
||||
timestamp = timestamp_;
|
||||
payload = payload_;
|
||||
}
|
||||
|
||||
private PayloadMessage() { }
|
||||
public byte[] serialize()
|
||||
{
|
||||
var str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
bw.Write(msg_id);
|
||||
bw.Write(timestamp);
|
||||
bw.Write(payload.Length);
|
||||
bw.Write(payload);
|
||||
str.Position = 0;
|
||||
var bytes = str.ToArray();
|
||||
return bytes;
|
||||
}
|
||||
|
||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
||||
{
|
||||
var str = new MemoryStream(ser_bytes);
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
string guid = br.ReadString();
|
||||
Int64 timestamp_ = br.ReadInt64();
|
||||
int len = br.ReadInt32();
|
||||
var payload = br.ReadBytes(len);
|
||||
return new PayloadMessage(guid, timestamp_, payload);
|
||||
}
|
||||
}
|
||||
//public void
|
||||
private readonly int _msg_size;
|
||||
private readonly TimeSpan _publish_period;
|
||||
private DateTime _last_publish_time = DateTime.MinValue;
|
||||
private PublisherCore _pub_core;
|
||||
private ISubscriberCore _sub_core;
|
||||
private BlueTable _table;
|
||||
private int _n_pubs = 0;
|
||||
private readonly int _max_pubs;
|
||||
private int _n_updates = 0;
|
||||
private ViewStreamDatabase _vsd;
|
||||
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
||||
private const byte _payload_byte = (byte)'A';
|
||||
private readonly string _tag_name;
|
||||
private readonly int _payload_sz;
|
||||
private readonly IPubSubRouter _router;
|
||||
private enum IOState
|
||||
{
|
||||
SEND = 0,
|
||||
RECV = 1
|
||||
};
|
||||
IOState _state = IOState.SEND;
|
||||
|
||||
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
||||
{
|
||||
var route = new PubSubRoute(server_url, ".*");
|
||||
_router = new PubSubRouter(route);
|
||||
_vsd = vsd;
|
||||
_payload_sz = payload_size;
|
||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("sent_id", TypeCode.String),
|
||||
Tuple.Create("sent_time", TypeCode.Int64),
|
||||
Tuple.Create("recv_id", TypeCode.String),
|
||||
Tuple.Create("recv_time", TypeCode.Int64),
|
||||
Tuple.Create("match", TypeCode.Boolean),
|
||||
Tuple.Create("echo_time", TypeCode.Int32));
|
||||
_table = new BlueTable(schema);
|
||||
_publish_period = publish_period;
|
||||
_tag_name = tag_name;
|
||||
}
|
||||
|
||||
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
||||
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
_pub_core = new PublisherCore(_router);
|
||||
_sub_core = new SubscriberCore(_router);
|
||||
_sub_core.subscribe(_tag_name);
|
||||
_vsd.addWorker(this);
|
||||
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_vsd.removeWorker(this);
|
||||
_sub_core.unsubscribe(_tag_name);
|
||||
}
|
||||
|
||||
private void _fire_dp_after_send()
|
||||
{
|
||||
// as soon as the data is published, its reflected in the stream state
|
||||
Log.assert(_last_sent_msg != null);
|
||||
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
||||
DBNull.Value, DBNull.Value);
|
||||
_table.addRow(r);
|
||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
|
||||
private void _fire_dp_after_recv()
|
||||
{
|
||||
Log.assert(_last_recvd_msg != null);
|
||||
// update the last publish row
|
||||
int id = _table.Count - 1;
|
||||
IBlueRow last_row = _table[id];
|
||||
_table.deleteRow(last_row);
|
||||
var recv_ts = epoch_now();
|
||||
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
|
||||
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
|
||||
_table.addRow(nr);
|
||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
|
||||
private string _get_next_id() {
|
||||
return _table.Count.ToString();
|
||||
}
|
||||
|
||||
private string _get_next_id()
|
||||
{
|
||||
return _table.Count.ToString();
|
||||
}
|
||||
|
||||
private void _publish_next_packet()
|
||||
{
|
||||
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
||||
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
||||
var ser_bytes = _last_sent_msg.serialize();
|
||||
_n_pubs++;
|
||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
||||
_last_publish_time = DateTime.Now;
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
|
||||
{
|
||||
foreach (var u in updates)
|
||||
{
|
||||
var recv_msg = PayloadMessage.deserialize(u.data);
|
||||
if (recv_msg.msg_id == _last_sent_msg.msg_id)
|
||||
{
|
||||
return recv_msg;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
switch (_state)
|
||||
{
|
||||
case IOState.SEND:
|
||||
if (DateTime.Now - _last_publish_time >= _publish_period)
|
||||
{
|
||||
_publish_next_packet();
|
||||
_last_publish_time = DateTime.Now;
|
||||
_fire_dp_after_send();
|
||||
_state = IOState.RECV;
|
||||
return;
|
||||
}
|
||||
break;
|
||||
case IOState.RECV:
|
||||
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
||||
updates = updates.Where(u => u.tag == _tag_name);
|
||||
var recv_msg = find_packet(updates);
|
||||
if (recv_msg != null)
|
||||
{
|
||||
_last_recvd_msg = recv_msg;
|
||||
_fire_dp_after_recv();
|
||||
_state = IOState.SEND;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public static long epoch_now()
|
||||
{
|
||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using System;
|
||||
|
||||
public class SimpleTCPEchoStream : ViewStream, IWorker {
|
||||
private string _ip;
|
||||
private int _port;
|
||||
private SimpleTCPClient _tcp_client;
|
||||
private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("event", TypeCode.String));
|
||||
private BlueTable _table;
|
||||
private SyncQueue<string> _action_queue = new SyncQueue<string>();
|
||||
|
||||
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
|
||||
: this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
|
||||
|
||||
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
|
||||
_ip = ip;
|
||||
_port = port;
|
||||
_table = new BlueTable(_schema);
|
||||
_tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
ViewStreamDatabase.instance.addWorker(this);
|
||||
_tcp_client.register_listener((msg) => { _on_msg(msg); });
|
||||
_tcp_client.start();
|
||||
}
|
||||
|
||||
private void _on_msg(string msg) {
|
||||
_action_queue.enqueue(msg);
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
if (_action_queue.getCount() == 0)
|
||||
return;
|
||||
BlueTable added_rows = new BlueTable(_schema);
|
||||
var sz = _action_queue.getCount();
|
||||
for (int i = 0; i < sz; i++) {
|
||||
string msg = _action_queue.dequeue();
|
||||
if (msg == null)
|
||||
break;
|
||||
IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
|
||||
_table.addRow(nr);
|
||||
added_rows.addRow(nr);
|
||||
}
|
||||
if (added_rows.Count == 0)
|
||||
return;
|
||||
DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Net;
|
||||
|
||||
using BCG;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using MongoDB.Bson;
|
||||
using BCG.Utilities.General;
|
||||
using System.Linq;
|
||||
|
||||
|
||||
namespace bts {
|
||||
// this class, a little specialized, takes the first snapshot thats fired
|
||||
// from the base stream sorts them by a given column, and fires one row at a time
|
||||
public class SnapshotReplayStream : ViewStream, IWorker {
|
||||
|
||||
private ViewStreamDatabase _vsd;
|
||||
private ViewStream _snapshot_stream;
|
||||
private readonly string _sort_col;
|
||||
private readonly int _buffer_size;
|
||||
private TimeSpan _fire_period;
|
||||
private int _in_row_ptr = 0;
|
||||
private List<IBlueRow> _in_rows = new List<IBlueRow>();
|
||||
private CircularBufferTable _out_table;
|
||||
|
||||
|
||||
public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period)
|
||||
: this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { }
|
||||
|
||||
public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) {
|
||||
_vsd = vsd;
|
||||
_snapshot_stream = base_str;
|
||||
_sort_col = sort_col;
|
||||
_buffer_size = buffer_size;
|
||||
_fire_period = fire_period;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_snapshot_stream.onChange += _handle_stream_update;
|
||||
MasterWorker.getInstance(false).addWorker(this, _fire_period);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
base.myOnLastSubscriber();
|
||||
}
|
||||
|
||||
private void _handle_stream_update(object dp_obj) {
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
if (_in_rows.Count > 0)
|
||||
return;
|
||||
_out_table = new CircularBufferTable(dp.schema, _buffer_size);
|
||||
// _in_rows is filled up only once
|
||||
foreach (IBlueRow r in dp.newRows) {
|
||||
_in_rows.Add(r);
|
||||
}
|
||||
_in_rows.Sort((IBlueRow r1, IBlueRow r2) => {
|
||||
return BlueDataUtility.compare_rows(r1, r2, _sort_col);
|
||||
});
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
if (_in_row_ptr >= _in_rows.Count) {
|
||||
// we can loop around or not here
|
||||
return;
|
||||
}
|
||||
if (_in_rows.Count > 0) {
|
||||
// fire row
|
||||
IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
|
||||
_out_table.add_row(curr_row);
|
||||
DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
|
||||
myLastValue == null ? null : myLastValue as DataPacket, false);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,809 @@
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
|
||||
namespace BCG {
|
||||
|
||||
/// <summary>
|
||||
/// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
|
||||
/// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
|
||||
/// </summary>
|
||||
[DataStreamUserRespresentation("tests")]
|
||||
public class TesStream : ViewStream {
|
||||
static byte[] create_byte_array() {
|
||||
Random random = new Random();
|
||||
var sz = random.Next(1, 1000);// % 1000;
|
||||
|
||||
// Generate a random byte
|
||||
byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
|
||||
byte[] rv = new byte[sz];
|
||||
for (int i = 0; i < sz; i++) {
|
||||
rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
private IBlueTableSchema _schema;
|
||||
class TestResult {
|
||||
public string test_name = MethodBase.GetCurrentMethod()?.Name
|
||||
?? throw new InvalidOperationException("Failed to get test name from method");
|
||||
public bool test_result;
|
||||
}
|
||||
private List<Func<TestResult>> _tests = new
|
||||
List<Func<TestResult>>();
|
||||
|
||||
public TesStream() {
|
||||
_schema = BlueDataUtility.form_schema("Tests",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("test", TypeCode.String),
|
||||
Tuple.Create("result", TypeCode.Boolean),
|
||||
Tuple.Create("exception", TypeCode.String)
|
||||
);
|
||||
// register tests here
|
||||
_tests.Add(new Func<TestResult>(_run_sub_msg_test));
|
||||
_tests.Add(new Func<TestResult>(_run_pub_msg_test));
|
||||
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
|
||||
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
|
||||
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
|
||||
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
|
||||
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
|
||||
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
|
||||
//_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
|
||||
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
|
||||
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
|
||||
// _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
|
||||
// _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
|
||||
//_tests.Add(new Func<TestResult>(_cpp_data_test));
|
||||
//_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
|
||||
_tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
|
||||
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
BlueTable rows = new BlueTable(_schema);
|
||||
// u the tests here
|
||||
foreach (var test in _tests) {
|
||||
string ex_msg = String.Empty;
|
||||
TestResult tr = new TestResult() {
|
||||
test_name = test.Method.Name,
|
||||
test_result = false
|
||||
};
|
||||
try {
|
||||
tr = test();
|
||||
} catch (Exception ex) {
|
||||
ex_msg = ex.Message + "\n" + ex.StackTrace;
|
||||
}
|
||||
IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
|
||||
rows.addRow(r);
|
||||
}
|
||||
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
protected override void myOnLastSubscriber() {
|
||||
}
|
||||
|
||||
private TestResult _run_sub_msg_test() {
|
||||
bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
|
||||
connection = Guid.NewGuid().ToString(),
|
||||
subscribing_app = Guid.NewGuid().ToString(),
|
||||
tags = new string[] { "one", "two", "three" }
|
||||
};
|
||||
MemoryStream str = new MemoryStream();
|
||||
sub_msg.serialize_into_str(str);
|
||||
str.Position = 0;
|
||||
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
|
||||
sub_msg_.consruct_deserialize(str);
|
||||
bool res = sub_msg.Equals(sub_msg_);
|
||||
return new TestResult {
|
||||
test_name = "SubscribeMsg",
|
||||
test_result = res
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _run_pub_msg_test() {
|
||||
var pub_msg = new bts.PublishMsg() {
|
||||
connection = Guid.NewGuid().ToString(),
|
||||
publishing_app = Guid.NewGuid().ToString(),
|
||||
tag = Guid.NewGuid().ToString(),
|
||||
tag_type = "DS_",
|
||||
is_compressed = false,
|
||||
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
||||
data_to_publish = create_byte_array(),
|
||||
};
|
||||
MemoryStream str = new MemoryStream();
|
||||
pub_msg.serialize_into_str(str);
|
||||
str.Position = 0;
|
||||
var pub_msg_ = new bts.PublishMsg();
|
||||
pub_msg_.consruct_deserialize(str);
|
||||
bool res = pub_msg.Equals(pub_msg_);
|
||||
return new TestResult {
|
||||
test_name = "PublishMsg",
|
||||
test_result = res
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _run_pubsub_server_msg_test() {
|
||||
var msg = new bts.PubSubServerMessage() {
|
||||
tag = Guid.NewGuid().ToString(),
|
||||
tag_type = Guid.NewGuid().ToString(),
|
||||
data = create_byte_array()
|
||||
};
|
||||
MemoryStream str = new MemoryStream();
|
||||
msg.serialize_into_str(str);
|
||||
str.Position = 0;
|
||||
var msg_ = new bts.PubSubServerMessage();
|
||||
msg_.consruct_deserialize(str);
|
||||
bool res = msg.Equals(msg_);
|
||||
return new TestResult {
|
||||
test_name = "PubSubServerMessage",
|
||||
test_result = res
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _run_pubsub_client_msg_test() {
|
||||
PubSubClientMessage msg = new PubSubClientMessage() {
|
||||
msg_type = PubSubMessageType.SUBSCRIBE,
|
||||
sub_msg = new bts.SubscribeMsg() {
|
||||
connection = Guid.NewGuid().ToString(),
|
||||
subscribing_app = Guid.NewGuid().ToString(),
|
||||
tags = new string[] { "one", "two", "three" }
|
||||
}
|
||||
};
|
||||
var str = new MemoryStream();
|
||||
msg.serialize_into_str(str);
|
||||
str.Position = 0;
|
||||
var msg_ = new PubSubClientMessage();
|
||||
msg_.consruct_deserialize(str);
|
||||
var res = msg.Equals(msg_);
|
||||
return new TestResult {
|
||||
test_name = "PubSubClientMessage",
|
||||
test_result = res
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _run_columninfo_serialization_test() {
|
||||
Func<ColumnInfo, bool> test_ci = (ci) => {
|
||||
MemoryStream str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
StreamSerializationUtils.serialize_column_info(ci, ref bw);
|
||||
str.Position = 0;
|
||||
//var ci_txt = BlueDataUtility.to_string(ci);
|
||||
//var ci_ser =
|
||||
//bts.Log.Gen.info("ci: {}")
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
|
||||
return BlueDataUtility.compute_equality(ci, ci_prime);
|
||||
};
|
||||
int i = 0;
|
||||
foreach (Type t in BlueDataUtility.valid_column_types()) {
|
||||
ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
|
||||
i++, t);
|
||||
var res = test_ci(ci);
|
||||
if (!res) {
|
||||
return new TestResult {
|
||||
test_name = "ColumnInfoSerialization",
|
||||
test_result = false
|
||||
};
|
||||
}
|
||||
}
|
||||
return new TestResult {
|
||||
test_name = "ColumnInfoSerialization",
|
||||
test_result = true
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
private TestResult _run_schema_serialization_test() {
|
||||
var table_name = "test_table";
|
||||
var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
|
||||
IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
|
||||
MemoryStream str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
StreamSerializationUtils.serialize_schema(schema, ref bw);
|
||||
_write_to_file(str, @"c:\Temp\tests\schema.bin");
|
||||
var hex_txt = StreamSerializationUtils.to_hex(str);
|
||||
bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
|
||||
|
||||
str.Position = 0;
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
|
||||
var res = BlueDataUtility.compute_equality(schema, schema_prime);
|
||||
|
||||
return new TestResult {
|
||||
test_name = "SchemaSerialization",
|
||||
test_result = res
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
private TestResult _run_datetime_seriliaztion_test() {
|
||||
DateTime dt = DateTime.Now;
|
||||
DateTimeOffset dt_off = new DateTimeOffset(dt);
|
||||
long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
|
||||
DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
|
||||
DateTime dt_p = dt_off_pr.DateTime;
|
||||
var passes = dt.Equals(dt_p);
|
||||
return new TestResult {
|
||||
test_name = "_run_datetime_seriliaztion_test",
|
||||
test_result = false
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _run_full_stream_serialization_test() {
|
||||
// create message
|
||||
var schema = create_schema_with_all_types();
|
||||
BlueTable cr_rows = new BlueTable(schema);
|
||||
// 100 rows
|
||||
int n_rows = 100;
|
||||
for (int i = 0; i < n_rows; i++) {
|
||||
var nr = generate_random_row(schema, i);
|
||||
cr_rows.addRow(nr);
|
||||
}
|
||||
MemoryStream str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
||||
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
|
||||
UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
|
||||
ser.serialize(stream_io_msg, ref bw);
|
||||
str.Position = 0;
|
||||
BinaryReader br = new System.IO.BinaryReader(str);
|
||||
UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
|
||||
var stream_io_msg_prime = deser.deserialize(ref br);
|
||||
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
|
||||
var n_dels = 50;
|
||||
BlueTable del_rows = new BlueTable(cr_rows.schema);
|
||||
foreach (IBlueRow r in cr_rows) {
|
||||
del_rows.addRow(r);
|
||||
if (del_rows.Count == n_dels)
|
||||
break;
|
||||
}
|
||||
IBlueRowCollection added_rows = del_rows;
|
||||
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
||||
str = new MemoryStream();
|
||||
bw = new BinaryWriter(str);
|
||||
stream_io_msg = new bts.StreamIOMessage(mra_msg);
|
||||
ser.serialize(stream_io_msg, ref bw);
|
||||
bw.BaseStream.Position = 0;
|
||||
br = new BinaryReader(str);
|
||||
//deser = new UniversalStreamIODeserializer();
|
||||
stream_io_msg_prime = deser.deserialize(ref br);
|
||||
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
|
||||
return new TestResult {
|
||||
test_name = "UniversalSerialization",
|
||||
test_result = passes
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _run_full_stream_serialization_with_controller_test() {
|
||||
// create message
|
||||
var schema = create_schema_with_all_types();
|
||||
BlueTable cr_rows = new BlueTable(schema);
|
||||
// 100 rows
|
||||
int n_rows = 100;
|
||||
for (int i = 0; i < n_rows; i++) {
|
||||
var nr = generate_random_row(schema, i);
|
||||
cr_rows.addRow(nr);
|
||||
}
|
||||
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
||||
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
|
||||
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
|
||||
var ser_bytes = ser.serialize(stream_io_msg);
|
||||
IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
|
||||
var stream_io_msg_prime = deser.deserialize(ser_bytes);
|
||||
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
|
||||
//mult remove adds
|
||||
// remove all rows and add 50 more
|
||||
|
||||
BlueTable del_rows = new BlueTable(cr_rows);
|
||||
BlueTable added_rows = new BlueTable(cr_rows.schema);
|
||||
var n_adds = 50;
|
||||
for (int i = 0; i < n_adds; i++) {
|
||||
var nr = generate_random_row(schema, i);
|
||||
added_rows.addRow(nr);
|
||||
}
|
||||
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
||||
stream_io_msg = new bts.StreamIOMessage(mra_msg);
|
||||
ser_bytes = ser.serialize(stream_io_msg);
|
||||
var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
|
||||
Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
|
||||
stream_io_msg_prime = deser.deserialize(ser_bytes);
|
||||
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
|
||||
return new TestResult {
|
||||
test_name = "UniversalSerializationController",
|
||||
test_result = passes
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _row_serialization_tests_no_dbnulls() {
|
||||
// IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
||||
// new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
||||
// new Tuple<string, TypeCode>("i1", TypeCode.Int32),
|
||||
// new Tuple<string, TypeCode>("d1", TypeCode.Double),
|
||||
// new Tuple<string, TypeCode>("string_1", TypeCode.String));
|
||||
// bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||
// IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
|
||||
var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
|
||||
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||
MemoryStream str = new MemoryStream();
|
||||
IBlueRow r = generate_random_row(schema, 0, false);
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
row_ser.serialize_row(r, 0, ref bw);
|
||||
_write_to_file(str, @"C:\Temp\tests\row.bin");
|
||||
str.Position = 0;
|
||||
byte[] raw_byte_array = str.ToArray();
|
||||
var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
|
||||
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
||||
IBlueRow r_prime = des.deserialize_row(ref br);
|
||||
string row_str = r_prime.to_string();
|
||||
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
|
||||
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
||||
return new TestResult {
|
||||
test_name = "UniversalRowSerialization_NonNull",
|
||||
test_result = passes
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _deserialize_pub_msg_test() {
|
||||
var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
|
||||
PubSubClientMessage ps_client_msg = new PubSubClientMessage();
|
||||
ps_client_msg.consruct_deserialize(pub_msg_ser);
|
||||
bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
|
||||
return new TestResult {
|
||||
test_name = "_deserialize_pub_msg_test",
|
||||
test_result = passes
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _cpp_data_test() {
|
||||
//create and write schema
|
||||
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
||||
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
||||
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
|
||||
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
|
||||
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
|
||||
new Tuple<string, TypeCode>("string_1", TypeCode.String)
|
||||
);
|
||||
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||
UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
|
||||
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||
StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
||||
|
||||
IBlueTableSchema schema_p = null;
|
||||
var schema_file = @"c:\Temp\tests\schema.bin";
|
||||
var row_file = @"C:\Temp\tests\row.bin";
|
||||
var create_op_file = @"C:\Temp\tests\create_op.bin";
|
||||
var mra_op_file = @"C:\Temp\tests\mra_op.bin";
|
||||
IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
|
||||
|
||||
//serialize schema, write it to a file, read it back, deserialize it and match schemas
|
||||
byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
|
||||
_write_to_file(ser_schema_bytes, schema_file);
|
||||
|
||||
byte[] schema_file_bytes = _read_from_file(schema_file);
|
||||
schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
|
||||
bool passes = BlueDataUtility.compute_equality(schema, schema_p);
|
||||
|
||||
//serialize row, write it to a file, read it back, deserialize it and match schemas
|
||||
byte[] ser_row_bytes = row_ser.serialize_row(row);
|
||||
_write_to_file(ser_row_bytes, row_file);
|
||||
|
||||
byte[] row_file_bytes = _read_from_file(row_file);
|
||||
var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
|
||||
Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
|
||||
|
||||
IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
|
||||
passes = passes && BlueDataUtility.compute_equality(row, row_p);
|
||||
|
||||
{
|
||||
// serialize a create operation
|
||||
int n_rows = 100;
|
||||
BlueTable table = new BlueTable(schema);
|
||||
for (int i = 0; i < n_rows; i++) {
|
||||
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
|
||||
table.addRow(nr);
|
||||
}
|
||||
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
|
||||
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
|
||||
_write_to_file(create_ser_bytes, create_op_file);
|
||||
|
||||
// read it and make sure its right
|
||||
byte[] create_op_file_bytes = _read_from_file(create_op_file);
|
||||
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
||||
var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
|
||||
_write_to_file(create_ser_bytes, mra_op_file);
|
||||
|
||||
passes = passes && cr_msg.Equals(cr_msg_p);
|
||||
|
||||
}
|
||||
//multiple remove adds
|
||||
{
|
||||
// remove all rows and add them again
|
||||
int n_rows = 100;
|
||||
BlueTable table = new BlueTable(schema);
|
||||
for (int i = 0; i < n_rows; i++) {
|
||||
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
|
||||
table.addRow(nr);
|
||||
}
|
||||
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
|
||||
|
||||
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
|
||||
_write_to_file(mra_ser_bytes, mra_op_file);
|
||||
|
||||
// reasd it and make sure it matches
|
||||
byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
|
||||
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
||||
StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
|
||||
passes = passes && mra_msg.Equals(mra_msg_p);
|
||||
}
|
||||
return new TestResult {
|
||||
test_name = "_cpp_data_test",
|
||||
test_result = passes
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _write_stream_updates_to_files() {
|
||||
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
||||
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
||||
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
|
||||
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
|
||||
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
|
||||
new Tuple<string, TypeCode>("string_1", TypeCode.String)
|
||||
);
|
||||
var packets_path = @"C:\Temp\tests\stream_packets\";
|
||||
Func<long> now_epochs = () => {
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
// Convert to Unix time in milliseconds
|
||||
long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
|
||||
return unix_time_milliseconds;
|
||||
};
|
||||
Func<int, bool, string> curr_file_path = (id, create) => {
|
||||
//var epoch_ms = now_epochs();
|
||||
var file_path = $"{packets_path}{id}.bin";
|
||||
if (create)
|
||||
file_path = $"{packets_path}create_{id}.bin";
|
||||
return file_path;
|
||||
};
|
||||
// write 1000 stream updates
|
||||
{
|
||||
int init_sz = 100;
|
||||
BlueTable table = new BlueTable(schema);
|
||||
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||
// create table of 100 rows
|
||||
for (int i = 0; i < init_sz; i++) {
|
||||
IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
|
||||
table.addRow(nr);
|
||||
}
|
||||
// write create msg
|
||||
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
|
||||
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
|
||||
var create_file_path = curr_file_path(table.Count, true);
|
||||
_write_to_file(create_ser_bytes, create_file_path);
|
||||
|
||||
// add 1000 rows and write an io msg for each update and write it to a file
|
||||
int n_updates = 1000;
|
||||
int period_ms = 50;
|
||||
|
||||
// write a create
|
||||
for (int i = 0; i < n_updates; i++) {
|
||||
// addition
|
||||
BlueTable adds = new BlueTable(schema);
|
||||
int id = table.Count;
|
||||
IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
|
||||
adds.addRow(nr);
|
||||
BlueTable dels = new BlueTable(schema);
|
||||
table.addRow(nr);
|
||||
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
|
||||
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
|
||||
var mra_file_path = curr_file_path(id, false);
|
||||
_write_to_file(mra_ser_bytes, mra_file_path);
|
||||
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
|
||||
}
|
||||
// now start writing increments and add 1 row at a time
|
||||
}
|
||||
return new TestResult {
|
||||
test_name = "_write_updates_as_files",
|
||||
test_result = true
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
private TestResult _row_serialization_tests_with_dbnulls() {
|
||||
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
||||
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
||||
new Tuple<string, TypeCode>("i1", TypeCode.Int32),
|
||||
new Tuple<string, TypeCode>("d1", TypeCode.Double),
|
||||
new Tuple<string, TypeCode>("string_1", TypeCode.String));
|
||||
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||
IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
|
||||
MemoryStream str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
row_ser.serialize_row(r, 0, ref bw);
|
||||
str.Position = 0;
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
||||
IBlueRow r_prime = des.deserialize_row(ref br);
|
||||
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
||||
return new TestResult {
|
||||
test_name = "UniversalRowSerialization_WithDBNulls",
|
||||
test_result = passes
|
||||
};
|
||||
}
|
||||
|
||||
private TestResult _row_serialization_tests_all_types() {
|
||||
var schema = create_schema_with_all_types();
|
||||
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||
IBlueRow r = generate_random_row(schema, 0, false);
|
||||
MemoryStream str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
row_ser.serialize_row(r, 0, ref bw);
|
||||
_write_to_file(str, @"C:\Temp\tests\row.bin");
|
||||
str.Position = 0;
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
||||
IBlueRow r_prime = des.deserialize_row(ref br);
|
||||
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
||||
return new TestResult {
|
||||
test_name = "UniversalRowSerialization_AllTypes",
|
||||
test_result = passes
|
||||
};
|
||||
}
|
||||
|
||||
private static TestResult _run_pubsub_server_stream_tag_handler_test() {
|
||||
// create message
|
||||
var schema = create_schema_with_all_types();
|
||||
var tag = "DS_" + schema.tableName;
|
||||
BlueTable cr_rows = new BlueTable(schema);
|
||||
// 100 rows
|
||||
int n_rows = 100;
|
||||
for (int i = 0; i < n_rows; i++) {
|
||||
var nr = generate_random_row(schema, i);
|
||||
cr_rows.addRow(nr);
|
||||
}
|
||||
|
||||
PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
|
||||
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
||||
bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
|
||||
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
|
||||
byte[] msg_bytes = ser.serialize(str_io_msg);
|
||||
handler.update(tag, msg_bytes);
|
||||
|
||||
//now get it back for the handler
|
||||
byte[] content = handler.get_content(tag);
|
||||
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
||||
bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
|
||||
// test here
|
||||
var passes = str_io_msg.Equals(str_io_msg_p);
|
||||
// delete all rows
|
||||
BlueTable del_rows = new BlueTable(cr_rows);
|
||||
cr_rows.clear();
|
||||
// added 50 new rows
|
||||
BlueTable added_rows = new BlueTable(del_rows.schema);
|
||||
int new_state_sz = 50;
|
||||
for (int i = 0; i < new_state_sz; i++) {
|
||||
var nr = generate_random_row(schema, i);
|
||||
added_rows.addRow(nr);
|
||||
}
|
||||
cr_rows.addRows(added_rows);
|
||||
// the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
|
||||
// 1. generate op
|
||||
bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
||||
str_io_msg = new bts.StreamIOMessage(mra);
|
||||
// 2. serialize it into bytes
|
||||
msg_bytes = ser.serialize(str_io_msg);
|
||||
// 3. send it to handler
|
||||
handler.update(tag, msg_bytes);
|
||||
// 4. get the state from the handler
|
||||
byte[] content_p = handler.get_content(tag);
|
||||
// 5. deserialize the state into a stream op
|
||||
str_io_msg_p = deser.deserialize(content_p);
|
||||
// 6. check to make sure the state's rows are good
|
||||
passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
|
||||
BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
|
||||
return new TestResult {
|
||||
test_result = passes,
|
||||
test_name = "run_pubsub_server_stream_tag_handler_test"
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
private static TestResult _run_single_tag_data_test_clean() {
|
||||
// create message
|
||||
var test_name = MethodBase.GetCurrentMethod().Name;
|
||||
var schema = create_schema_with_all_types();
|
||||
var tag = "DS_" + schema.tableName;
|
||||
BlueTable table = new BlueTable(schema);
|
||||
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
|
||||
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
||||
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
|
||||
// create with 100 rows
|
||||
int n_rows = 100;
|
||||
bool is_succ = true;
|
||||
for (int i = 0; i < n_rows; i++) {
|
||||
var nr = generate_random_row(schema, i);
|
||||
// perform the same action SingletagData, and verify the states match
|
||||
PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
|
||||
table.addRow(nr);
|
||||
st_data.handle_publish(pub_msg);
|
||||
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
||||
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
||||
}
|
||||
// change all rows
|
||||
for (int i = 0; i < n_rows; i++) {
|
||||
var nr = generate_random_row(schema, i);
|
||||
var dr = table[i];
|
||||
table.deleteRow(dr);
|
||||
table.addRow(nr);
|
||||
PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
|
||||
st_data.handle_publish(pub_msg);
|
||||
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
||||
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
||||
}
|
||||
// delete all rows
|
||||
var aux_table = new BlueTable(table);
|
||||
foreach (IBlueRow r in aux_table) {
|
||||
table.deleteRow(r);
|
||||
PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
|
||||
st_data.handle_publish(pub_msg);
|
||||
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
||||
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
||||
}
|
||||
return new TestResult() {
|
||||
test_result = is_succ,
|
||||
test_name = test_name
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
#region Helper functions
|
||||
|
||||
private static void _write_to_file(byte[] ser_bytes, string file_name) {
|
||||
_write_to_file(new MemoryStream(ser_bytes), file_name);
|
||||
}
|
||||
|
||||
private static void _write_to_file(MemoryStream str, string file_name) {
|
||||
if (String.IsNullOrEmpty(file_name))
|
||||
return;
|
||||
byte[] str_bytes = str.ToArray();
|
||||
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
|
||||
// Write the byte array to the file
|
||||
fs.Write(str_bytes, 0, str_bytes.Length);
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] _read_from_file(string file_path) {
|
||||
var rv = File.ReadAllBytes(file_path);
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
|
||||
var content = st_data.get_current_content();
|
||||
var msg_p = deser.deserialize(content);
|
||||
Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
|
||||
return msg_p.create_message.rows;
|
||||
|
||||
}
|
||||
|
||||
public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
|
||||
ref IStreamIOMsgSerializer serializer) {
|
||||
Func<IBlueTableSchema> get_schema = () => {
|
||||
return (added_row != null ? added_row : removed_row).schema;
|
||||
};
|
||||
bts.StreamIOMessage str_io_msg = null;
|
||||
if (is_create) {
|
||||
Log.assert(added_row != null);
|
||||
var aux_new = new BlueTable(added_row.schema);
|
||||
aux_new.addRow(added_row);
|
||||
var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
|
||||
str_io_msg = new bts.StreamIOMessage(cr_msg);
|
||||
} else {
|
||||
var schema = get_schema();
|
||||
BlueTable added_table = new BlueTable(schema);
|
||||
if (added_row != null)
|
||||
added_table.addRow(added_row);
|
||||
BlueTable rem_table = new BlueTable(schema);
|
||||
if (removed_row != null)
|
||||
rem_table.addRow(removed_row);
|
||||
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
|
||||
new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
|
||||
str_io_msg = new bts.StreamIOMessage(mra_msg);
|
||||
}
|
||||
byte[] msg_bytes = serializer.serialize(str_io_msg);
|
||||
PublishMsg pub_msg = new PublishMsg() {
|
||||
connection = "test_connection",
|
||||
tag = $"DS_{get_schema()}",
|
||||
tag_type = "DS",
|
||||
data_to_publish = msg_bytes
|
||||
};
|
||||
return pub_msg;
|
||||
}
|
||||
|
||||
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
|
||||
excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
|
||||
var table_name = "test_table";
|
||||
BlueTableSchema schema = new BlueTableSchema();
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
foreach (Type t in BlueDataUtility.valid_column_types()) {
|
||||
if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
|
||||
continue;
|
||||
var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
|
||||
ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
|
||||
cis.Add(ci);
|
||||
}
|
||||
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
|
||||
schema = new BlueTableSchema(table_name, cis.ToArray());
|
||||
return schema;
|
||||
}
|
||||
|
||||
public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
|
||||
Random rnd = new Random();
|
||||
object[] row_vals = new object[schema.columnInfo.Length];
|
||||
foreach (ColumnInfo ci in schema.columnInfo) {
|
||||
if (ci.colName == "id") {
|
||||
row_vals[ci.colNo] = id;
|
||||
continue;
|
||||
}
|
||||
object v = get_random_object(ci.type_code());
|
||||
if (include_dbnulls && rnd.NextSingle() % 4 == 0)
|
||||
v = DBNull.Value;
|
||||
row_vals[ci.colNo] = v;
|
||||
}
|
||||
return new BlueRow(schema, row_vals);
|
||||
}
|
||||
|
||||
public static object get_random_object(TypeCode type_code) {
|
||||
Random random = new Random();
|
||||
switch (type_code) {
|
||||
case TypeCode.Boolean:
|
||||
return random.Next(2) == 0; // Random boolean value (true or false)
|
||||
case TypeCode.Byte:
|
||||
return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
|
||||
case TypeCode.Char:
|
||||
return (char)random.Next(65, 65 + 27); // Random char value
|
||||
case TypeCode.Decimal:
|
||||
Decimal dec = 100.123M;
|
||||
return dec;
|
||||
case TypeCode.DateTime:
|
||||
return DateTime.Now;
|
||||
case TypeCode.Double:
|
||||
return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
|
||||
case TypeCode.Int16:
|
||||
return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
|
||||
case TypeCode.Int32:
|
||||
return random.Next(int.MinValue, int.MaxValue); // Random int value
|
||||
case TypeCode.Int64:
|
||||
return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
|
||||
case TypeCode.SByte:
|
||||
return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
|
||||
case TypeCode.Single:
|
||||
return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
|
||||
case TypeCode.String:
|
||||
return Guid.NewGuid().ToString();
|
||||
case TypeCode.UInt16:
|
||||
return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
|
||||
case TypeCode.UInt32:
|
||||
return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
|
||||
case TypeCode.UInt64:
|
||||
byte[] ulongBuffer = new byte[8];
|
||||
random.NextBytes(ulongBuffer);
|
||||
return BitConverter.ToUInt64(ulongBuffer, 0);
|
||||
default:
|
||||
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace BCG {
|
||||
using System;
|
||||
|
||||
public class TickingTestStream : ViewStream {
|
||||
public TickingTestStream(int ticking_period_msecs)
|
||||
:
|
||||
this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
|
||||
|
||||
public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
|
||||
ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
|
||||
vs = new MathStream(vs, "x=id*123 y=x*x");
|
||||
mySetRealSource(vs);
|
||||
//vs.onChange += _on_str_update;
|
||||
}
|
||||
|
||||
// private void _on_str_update(object dp_obj) {
|
||||
// DataPacket dp = dp_obj as DataPacket;
|
||||
// IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
|
||||
// BlueTable state = new BlueTable(out_schema);
|
||||
// foreach (IBlueRow r in dp.newRows) {
|
||||
// MutableBlueRow nr = new MutableBlueRow(out_schema, r);
|
||||
// DateTime time = r.get_datetime("time");
|
||||
// var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
||||
// nr.setValue("time_txt", time_txt);
|
||||
// state.addRow(nr);
|
||||
// }
|
||||
// var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
|
||||
// myFireEvent(dp_out);
|
||||
// }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts;
|
||||
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Reactive.Subjects;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
|
||||
readonly string collectionName;
|
||||
readonly string databaseName;
|
||||
readonly string url;
|
||||
readonly object syncRoot = new();
|
||||
readonly CancellationTokenSource cts = new();
|
||||
readonly Subject<DataPacket> subject = new();
|
||||
|
||||
bool hasStarted = false;
|
||||
IBlueRowFactory<BsonDocument>? rowFactory;
|
||||
MongoClient? clientBacking;
|
||||
IMongoDatabase? databaseBacking;
|
||||
IMongoCollection<BsonDocument>? collectionBacking = null;
|
||||
BlueTableImmutable? table = null;
|
||||
|
||||
public MongoCollectionStream(string collectionName)
|
||||
: this(
|
||||
collectionName,
|
||||
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
|
||||
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
||||
}
|
||||
|
||||
public MongoCollectionStream(string collectionName, string databaseName)
|
||||
: this(
|
||||
collectionName,
|
||||
databaseName,
|
||||
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
||||
}
|
||||
|
||||
public MongoCollectionStream(string collectionName, string databaseName, string url)
|
||||
: this(collectionName, databaseName, url, null) { }
|
||||
|
||||
internal MongoCollectionStream(
|
||||
string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
|
||||
this.collectionName = collectionName;
|
||||
this.databaseName = databaseName;
|
||||
this.url = url;
|
||||
this.rowFactory = rowFactory;
|
||||
mySetRealSource(new ObservableViewStream(this));
|
||||
}
|
||||
|
||||
MongoClient client => clientBacking ??= new(url);
|
||||
IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
|
||||
IMongoCollection<BsonDocument>? collection {
|
||||
get => Volatile.Read(ref collectionBacking);
|
||||
set => Volatile.Write(ref collectionBacking, value);
|
||||
}
|
||||
|
||||
public IDisposable Subscribe(IObserver<DataPacket> observer) {
|
||||
lock (syncRoot) {
|
||||
if (hasStarted) {
|
||||
if (table is not null) {
|
||||
// The first update has already been fired. Seed observer w/ current state.
|
||||
observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
|
||||
}
|
||||
return subject.Subscribe(observer);
|
||||
}
|
||||
hasStarted = true;
|
||||
}
|
||||
var subscription = subject.Subscribe(observer);
|
||||
// Retrieve collection snapshot to bootstrap sequence
|
||||
initTable().ignore();
|
||||
// Watch changes
|
||||
Task.Run(watchChanges);
|
||||
return Disposable.Create(Dispose);
|
||||
|
||||
void Dispose() {
|
||||
Log.Gen.info("Disposing MongoDbChangeObservable subscription");
|
||||
subscription.Dispose();
|
||||
cts.Cancel();
|
||||
cts.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
|
||||
MongoSchemaAdaptor.InferSchema(docs);
|
||||
|
||||
async Task watchChanges() {
|
||||
Log.Gen.info(
|
||||
$"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}");
|
||||
while (collection is null) {
|
||||
await Task.Delay(100);
|
||||
}
|
||||
Log.notnull(collection);
|
||||
using var cursor = collection.Watch(new ChangeStreamOptions {
|
||||
FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
|
||||
});
|
||||
try {
|
||||
while (await cursor.MoveNextAsync(cts.Token)) {
|
||||
/// Ignore while waiting for init.
|
||||
lock (syncRoot) {
|
||||
/// Note: <see cref="rowFactory"/> and <see cref="table"/> are initialized together
|
||||
/// in the <see cref="initTable"/> fn since both depend on the schema, which is
|
||||
/// inferred from the first incoming BsonDocument(s).
|
||||
if (table is null || rowFactory is null) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
var bd = table.ToDataPacketBuilder();
|
||||
foreach (var ch in cursor.Current) {
|
||||
var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
|
||||
var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
|
||||
Log.Gen.info(
|
||||
$"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
|
||||
$"changes:\n\t\t{updDescJson}");
|
||||
switch (ch.OperationType) {
|
||||
case ChangeStreamOperationType.Insert: {
|
||||
bd.AddOrUpdate(Row());
|
||||
break;
|
||||
}
|
||||
case ChangeStreamOperationType.Update: {
|
||||
bd.Remove(OldRow().Id());
|
||||
bd.AddOrUpdate(Row());
|
||||
break;
|
||||
}
|
||||
case ChangeStreamOperationType.Replace: {
|
||||
bd.Remove(OldRow().Id());
|
||||
bd.AddOrUpdate(Row());
|
||||
break;
|
||||
}
|
||||
case ChangeStreamOperationType.Delete: {
|
||||
bd.Remove(OldRow().Id());
|
||||
break;
|
||||
}
|
||||
default:
|
||||
Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
|
||||
break;
|
||||
}
|
||||
|
||||
IBlueRow Row() =>
|
||||
rowFactory.GetBlueRow(ch.FullDocument);
|
||||
|
||||
IBlueRow OldRow() =>
|
||||
rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
|
||||
}
|
||||
var dp = bd.ToDataPacket();
|
||||
if (!dp.HasChanges()) {
|
||||
Log.Gen.warn(
|
||||
$"MongoCollectionStream. Ignoring update because it didn't change any rows.");
|
||||
continue;
|
||||
}
|
||||
lock (syncRoot) {
|
||||
table = (BlueTableImmutable)dp.newRows;
|
||||
subject.OnNext(dp);
|
||||
}
|
||||
}
|
||||
} catch (TaskCanceledException ex) {
|
||||
Log.Gen.warn(
|
||||
$"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
[MemberNotNull(nameof(collection))]
|
||||
async Task initTable() {
|
||||
collection = database.GetCollection<BsonDocument>(collectionName);
|
||||
var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
var docList = await cursor.ToListAsync();
|
||||
rowFactory ??= getRowFactory(docList);
|
||||
var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
|
||||
foreach (var doc in docList) {
|
||||
var row = rowFactory.GetBlueRow(doc);
|
||||
firstTable.AddOrUpdate(row);
|
||||
}
|
||||
var dp = firstTable.ToDataPacket();
|
||||
lock (syncRoot) {
|
||||
table = (BlueTableImmutable)dp.newRows;
|
||||
subject.OnNext(dp);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,128 +1,133 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Bson;
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
using ZstdSharp.Unsafe;
|
||||
using System.Runtime.CompilerServices;
|
||||
using mbts.spark;
|
||||
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
|
||||
namespace bts {
|
||||
// [DataStreamUserRespresentation("mongodb_sql")]
|
||||
// public class MongoDBSQLStream : ViewStream, IWorker {
|
||||
// private ViewStreamDatabase _vsd;
|
||||
// private readonly string _connection_string, _db_name, _collection_name;
|
||||
// private MongoClient _client;
|
||||
// // make this a sorted table
|
||||
// private BlueTable _table;
|
||||
|
||||
|
||||
namespace BCG {
|
||||
[DataStreamUserRespresentation("mongodb_sql")]
|
||||
public class MongoDBSQLStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _connection_string, _db_name, _collection_name;
|
||||
private MongoClient _client;
|
||||
private BlueTable _table;
|
||||
// // Access the database
|
||||
// private IMongoDatabase _database;
|
||||
// private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// // mongodb://192.168.1.80:27017
|
||||
|
||||
// Access the database
|
||||
private IMongoDatabase _database;
|
||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// mongodb://192.168.1.80:27017
|
||||
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||
// : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
|
||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
// }
|
||||
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
// bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
// $"db_name: {db_name}");
|
||||
// _connection_string = connection_string;
|
||||
// _db_name = db_name;
|
||||
// _collection_name = collection_name;
|
||||
// _vsd = vsd;
|
||||
// _client = new MongoClient(_connection_string);
|
||||
// _database = _client.GetDatabase(_db_name);
|
||||
// }
|
||||
|
||||
}
|
||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
$"db_name: {db_name}");
|
||||
_connection_string = connection_string;
|
||||
_db_name = db_name;
|
||||
_collection_name = collection_name;
|
||||
_vsd = vsd;
|
||||
_client = new MongoClient(_connection_string);
|
||||
_database = _client.GetDatabase(_db_name);
|
||||
}
|
||||
// protected override void myOnFirstSubscriber() {
|
||||
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||
// }
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
// protected override void myOnLastSubscriber() {
|
||||
// MasterWorker.getInstance(false).removeWorker(this);
|
||||
// }
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
// public void doWork() {
|
||||
// _read_mogodb();
|
||||
// }
|
||||
|
||||
public void doWork() {
|
||||
_read_mogodb();
|
||||
}
|
||||
// private void _read_mogodb() {
|
||||
// var timer = Stopwatch.StartNew();
|
||||
// // Access the collection
|
||||
// IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
|
||||
private void _read_mogodb() {
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
// // Fetch all documents from the collection
|
||||
// var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
// _table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// // Iterate through the documents and print values
|
||||
// foreach (var document in documents.ToEnumerable()) {
|
||||
// bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
// if (_table == null) {
|
||||
// _init_table(document);
|
||||
// }
|
||||
// _add_row(document);
|
||||
// }
|
||||
// timer.Stop();
|
||||
// Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
||||
// timer.Restart();
|
||||
// var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
// timer.Stop();
|
||||
// Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
||||
// if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
// return;
|
||||
// myFireEvent(dp);
|
||||
// }
|
||||
|
||||
// Fetch all documents from the collection
|
||||
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// Iterate through the documents and print values
|
||||
foreach (var document in documents.ToEnumerable()) {
|
||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
if (_table == null) {
|
||||
_init_table(document);
|
||||
}
|
||||
_add_row(document);
|
||||
}
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||
timer.start();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
}
|
||||
// private void _init_table(BsonDocument doc_row) {
|
||||
// bts.Log.assert(_table == null);
|
||||
// List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
// cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||
// foreach (var n in doc_row.Names) {
|
||||
// cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||
// }
|
||||
// IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||
// _table = new BlueTable(schema);
|
||||
// }
|
||||
|
||||
private void _init_table(BsonDocument doc_row) {
|
||||
bts.Log.assert(_table == null);
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||
foreach (var n in doc_row.Names) {
|
||||
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||
}
|
||||
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||
_table = new BlueTable(schema);
|
||||
}
|
||||
|
||||
private void _add_row(BsonDocument doc) {
|
||||
var key_val = doc["_id"];
|
||||
var key = key_val.ToString();
|
||||
int id;
|
||||
if (!_id_map.TryGetValue(key, out id)) {
|
||||
id = _id_map.Count;
|
||||
_id_map.Add(key, id);
|
||||
}
|
||||
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||
nr.setValue("id", id);
|
||||
foreach (var n in doc.Names) {
|
||||
var val = doc[n].ToString();
|
||||
nr.setValue(n, val);
|
||||
}
|
||||
_table.addRow(nr);
|
||||
}
|
||||
}
|
||||
// private void _add_row(BsonDocument doc) {
|
||||
// var key_val = doc["_id"];
|
||||
// var key = key_val.ToString();
|
||||
// int id;
|
||||
// if (!_id_map.TryGetValue(key, out id)) {
|
||||
// id = _id_map.Count;
|
||||
// _id_map.Add(key, id);
|
||||
// }
|
||||
// MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||
// nr.setValue("id", id);
|
||||
// foreach (var n in doc.Names) {
|
||||
// var val = doc[n].ToString();
|
||||
// nr.setValue(n, val);
|
||||
// }
|
||||
// _table.addRow(nr);
|
||||
// }
|
||||
// }
|
||||
|
||||
[DataStreamUserRespresentation("mongodb")]
|
||||
public class MongoDBStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _connection_string, _db_name, _collection_name;
|
||||
private readonly TimeSpan _poll_period;
|
||||
private MongoClient _client;
|
||||
private BlueTable _table;
|
||||
private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
|
||||
private BlockingQueue _work_queue = new BlockingQueue();
|
||||
|
||||
// Access the database
|
||||
private IMongoDatabase _database;
|
||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// mongodb://192.168.1.80:27017
|
||||
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name)
|
||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
|
||||
: this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
|
||||
|
||||
}
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
$"db_name: {db_name}");
|
||||
_connection_string = connection_string;
|
||||
@@ -130,24 +135,34 @@ namespace BCG {
|
||||
_collection_name = collection_name;
|
||||
_vsd = vsd;
|
||||
_client = new MongoClient(_connection_string);
|
||||
_poll_period = poll_period;
|
||||
_database = _client.GetDatabase(_db_name);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
_vsd.addWorker(this);
|
||||
//MasterWorker.getInstance(false).addWorker(this, _poll_period);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
_vsd.removeWorker(this);
|
||||
//MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
|
||||
private DateTime _last_poll_time = DateTime.MinValue;
|
||||
public void doWork() {
|
||||
_read_mogodb();
|
||||
var now = DateTime.Now;
|
||||
if (now - _last_poll_time > _poll_period) {
|
||||
ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
|
||||
_last_poll_time = DateTime.Now;
|
||||
}
|
||||
var dp = _table_queue.dequeue(0);
|
||||
if (dp != null)
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
private void _read_mogodb() {
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
|
||||
@@ -156,21 +171,22 @@ namespace BCG {
|
||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// Iterate through the documents and print values
|
||||
foreach (var document in documents.ToEnumerable()) {
|
||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
//bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
if (_table == null) {
|
||||
_init_table(document);
|
||||
}
|
||||
_add_doc(document);
|
||||
}
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||
timer.start();
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
|
||||
timer.Restart();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
_table_queue.enqueue(dp);
|
||||
//myFireEvent(dp);
|
||||
}
|
||||
|
||||
private void _init_table(BsonDocument doc_row) {
|
||||
@@ -209,4 +225,46 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
// [DataStreamUserRespresentation("mongodb_flat")]
|
||||
// public class MongoDBFlatStream : ViewStream {
|
||||
// public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
||||
// :
|
||||
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
||||
|
||||
// public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
||||
// ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||
// rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
||||
// rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
||||
// //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
||||
// mySetRealSource(rs);
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
// [DataStreamUserRespresentation("newsreplay")]
|
||||
// public class NewsReplayStream : ViewStream {
|
||||
// public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
||||
// int buffer_sz, string fire_period) :
|
||||
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
||||
|
||||
// public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
||||
// int buffer_sz, TimeSpan fire_period) {
|
||||
// ViewStream r;
|
||||
|
||||
// r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||
// r = new KeyValueToRelationalStream(r,
|
||||
// "doc_id",
|
||||
// "key",
|
||||
// "value");
|
||||
// r = new DateStandardizationStream(r,
|
||||
// "created_at",
|
||||
// "ISO8601");
|
||||
// r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
||||
|
||||
// mySetRealSource(r);
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using MongoDB.Bson;
|
||||
|
||||
using System;
|
||||
|
||||
public static class MongoDbExtensions {
|
||||
public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
|
||||
doc[elementName].AsBsonArray;
|
||||
|
||||
public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
|
||||
val.AsBsonArray : null;
|
||||
|
||||
public static string GetString(this BsonDocument doc, string elementName) =>
|
||||
doc[elementName].AsString;
|
||||
|
||||
public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsString ?
|
||||
val.AsString : null;
|
||||
|
||||
public static int GetInt32(this BsonDocument doc, string elementName) =>
|
||||
doc[elementName].AsInt32;
|
||||
|
||||
public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
|
||||
val.AsInt32 : null;
|
||||
|
||||
public static long GetInt64(this BsonDocument doc, string elementName) =>
|
||||
doc[elementName].AsInt64;
|
||||
|
||||
public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
|
||||
val.AsInt64 : null;
|
||||
|
||||
public static double GetDouble(this BsonDocument doc, string elementName) =>
|
||||
doc[elementName].AsDouble;
|
||||
|
||||
public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsDouble ?
|
||||
val.AsDouble : null;
|
||||
|
||||
public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
|
||||
doc[elementName].ToUniversalTime();
|
||||
|
||||
public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
|
||||
val.ToUniversalTime() : null;
|
||||
|
||||
public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
|
||||
doc[elementName].ToUniversalTime();
|
||||
|
||||
public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
|
||||
val.ToUniversalTime() : null;
|
||||
|
||||
public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
|
||||
doc[elementName].AsObjectId;
|
||||
|
||||
public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
|
||||
val.AsObjectId : null;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using bts;
|
||||
|
||||
using MongoDB.Bson;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
public static class MongoSchemaAdaptor {
|
||||
public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
|
||||
var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
|
||||
foreach (var doc in docs) {
|
||||
foreach (var el in doc) {
|
||||
if (schemaAdaptor.Contains(el.Name)) {
|
||||
continue;
|
||||
}
|
||||
switch (el.Value.BsonType) {
|
||||
case BsonType.Array:
|
||||
schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
|
||||
break;
|
||||
case BsonType.String:
|
||||
schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
|
||||
break;
|
||||
case BsonType.Int32:
|
||||
schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.Int64:
|
||||
schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.Double:
|
||||
schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.DateTime:
|
||||
schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.Timestamp:
|
||||
schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.ObjectId:
|
||||
schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
|
||||
break;
|
||||
case BsonType.Null:
|
||||
Log.Gen.warn(
|
||||
$"Ignoring column '{el.Name}' because at least one element is null");
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException(
|
||||
$"Unsupported BsonType '{el.Value.BsonType}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
return schemaAdaptor;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
|
||||
using ProtoBuf;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
public class ProjectData {
|
||||
// schema from the base stream
|
||||
public readonly IBlueTableSchema in_schema;
|
||||
// this is the projected schema
|
||||
public readonly IBlueTableSchema out_schema;
|
||||
// projected cols to the input cols, for cols who's names are changed
|
||||
public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
|
||||
public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
|
||||
IDictionary<string, string> mapped_to_orig_col_map_) {
|
||||
in_schema = in_schema_;
|
||||
out_schema = out_schema_;
|
||||
mapped_to_orig_col_map = mapped_to_orig_col_map_;
|
||||
}
|
||||
}
|
||||
|
||||
public class NewProjectRow : IBlueRow {
|
||||
#region IBlueRow members
|
||||
private ProjectData _project_data;
|
||||
private IBlueRow _row;
|
||||
public NewProjectRow(ProjectData project_data, IBlueRow row) {
|
||||
_project_data = project_data;
|
||||
_row = row;
|
||||
}
|
||||
public IBlueTableSchema schema { get { return _project_data.out_schema; } }
|
||||
#endregion
|
||||
|
||||
#region ICollection members
|
||||
public int Count { get { return schema.columnNames.Length; } }
|
||||
|
||||
public bool IsSynchronized { get { return false; } }
|
||||
|
||||
public object SyncRoot { get { return this; } }
|
||||
|
||||
public void CopyTo(Array array, int index) {
|
||||
int i = index;
|
||||
foreach (object o in this as IEnumerable)
|
||||
array.SetValue(o, i++);
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region IDictionary members
|
||||
public bool IsFixedSize { get { return true; } }
|
||||
|
||||
public bool IsReadOnly { get { return true; } }
|
||||
|
||||
public object this[object key] {
|
||||
get {
|
||||
if (key is string)
|
||||
return this[(string)key];
|
||||
return this[(int)key];
|
||||
}
|
||||
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
}
|
||||
|
||||
public object this[int index] {
|
||||
get {
|
||||
return this[schema.columnInfo[index].colName];
|
||||
}
|
||||
}
|
||||
|
||||
public object this[string col_name] {
|
||||
get {
|
||||
string mapped_to_col = null;
|
||||
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
|
||||
return _row[mapped_to_col];
|
||||
}
|
||||
return _row[col_name];
|
||||
}
|
||||
}
|
||||
|
||||
public ICollection Keys { get { return schema.columnNames; } }
|
||||
|
||||
public ICollection Values {
|
||||
get {
|
||||
object[] retVal = new object[schema.columnNames.Length];
|
||||
CopyTo(retVal, 0);
|
||||
return retVal;
|
||||
}
|
||||
}
|
||||
|
||||
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
|
||||
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
|
||||
public bool Contains(object key) { return schema[(string)key] != null; }
|
||||
|
||||
/// <summary>
|
||||
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
||||
/// iterate through the values, typecast this into an IEnumerable
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
||||
|
||||
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
#endregion
|
||||
|
||||
}
|
||||
|
||||
public class NewProjectStream : ViewStream, IStreamMapper {
|
||||
|
||||
private ViewStream _stream;
|
||||
private string _project_expression;
|
||||
private ProjectData _project_data = null;
|
||||
//public ProjectData
|
||||
public NewProjectStream(ViewStream vs, string project_expression) {
|
||||
_stream = vs;
|
||||
_project_expression = project_expression;
|
||||
_stream = new MapStream(vs, this);
|
||||
mySetRealSource(_stream);
|
||||
}
|
||||
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
|
||||
_project_data = _create_project_data(input_schema, _project_expression);
|
||||
return _project_data.out_schema;
|
||||
}
|
||||
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
return new NewProjectRow(_project_data, row);
|
||||
}
|
||||
|
||||
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
|
||||
string project_expression) {
|
||||
project_expression = project_expression.Trim();
|
||||
string[] project_toks = project_expression.Split(' ');
|
||||
HashSet<string> removed_cols = new HashSet<string>();
|
||||
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
|
||||
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
|
||||
foreach (string p_tok in project_toks) {
|
||||
if (p_tok.StartsWith("!")) {
|
||||
string removed_col = p_tok.Substring(1);
|
||||
removed_cols.Add(removed_col);
|
||||
} else if (p_tok.Contains(":")) {
|
||||
string[] map_toks = p_tok.Split(":");
|
||||
string orig_col = map_toks[0];
|
||||
string mapped_col = map_toks[1];
|
||||
mapped_to_orig_map.Add(mapped_col, orig_col);
|
||||
orig_to_mapped.Add(orig_col, mapped_col);
|
||||
} else {
|
||||
if (input_schema.columnNames.Contains(p_tok)) {
|
||||
string orig_col = p_tok;
|
||||
string mapped_col = orig_col;
|
||||
orig_to_mapped.Add(orig_col, mapped_col);
|
||||
}
|
||||
}
|
||||
}
|
||||
// make the new schema
|
||||
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
||||
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
||||
if (ci.colName == "id") {
|
||||
// if there's a hidden col, other columns will show by default, but if cols are being
|
||||
// projected epxlicitely, than others wil be hidden
|
||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||
new_cols.Add(nc);
|
||||
continue;
|
||||
}
|
||||
if (removed_cols.Contains(ci.colName)) {
|
||||
continue;
|
||||
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
||||
string mapped_name = orig_to_mapped[ci.colName];
|
||||
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
||||
new_cols.Add(nc);
|
||||
} else {
|
||||
if (ci.colName == "id" || removed_cols.Count > 0) {
|
||||
// if there's a hidden col, other columns will show by default, but if cols are being
|
||||
// projected epxlicitely, than others wil be hidden
|
||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||
new_cols.Add(nc);
|
||||
}
|
||||
}
|
||||
}
|
||||
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
||||
return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,17 @@
|
||||
namespace BCG {
|
||||
using mbts.spark;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
[DataStreamUserRespresentation("oldproject")]
|
||||
public class ProjectStream : ViewStream
|
||||
{
|
||||
private ProjectHelper myHelper;
|
||||
private string myColExp, myTableName;
|
||||
private ViewStream myVs;
|
||||
private IDictionary<string, string> myDic;
|
||||
private HighPerformanceTimer _timer = new();
|
||||
/// <summary>
|
||||
/// </summary>
|
||||
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
||||
@@ -70,10 +67,8 @@ namespace BCG {
|
||||
DataPacket dp = newValue as DataPacket;
|
||||
if (myHelper != null)
|
||||
{
|
||||
_timer.start();
|
||||
IBlueRowCollection dels, adds;
|
||||
BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
|
||||
_timer.stop();
|
||||
if (dels.Count + adds.Count > 0)
|
||||
myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
|
||||
new ProjectedTable (adds, myHelper),
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
using BCG;
|
||||
|
||||
namespace bts {
|
||||
public class SchemaGuesserStream : ViewStream {
|
||||
|
||||
private IBlueRowCollection _rows = null;
|
||||
private ViewStream _vs;
|
||||
public SchemaGuesserStream(ViewStream vs) {
|
||||
_vs = vs;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_vs.onChange += _on_data;
|
||||
}
|
||||
protected override void myOnLastSubscriber() {
|
||||
_vs.onChange -= _on_data;
|
||||
}
|
||||
|
||||
private void _on_data(object obj) {
|
||||
DataPacket dp = obj as DataPacket;
|
||||
if (_rows == null) {
|
||||
_rows = BlueDataUtility.get_typed_table(dp.newRows);
|
||||
}
|
||||
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
|
||||
myFireEvent(dp_out);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -69,18 +69,14 @@ public class SortRow : IComparable, IBlueRow {
|
||||
/// The key can either be the string corrsponding to the column name or
|
||||
/// the order of the column
|
||||
/// </summary>
|
||||
public object this[object key] {
|
||||
get {
|
||||
if (key is string)
|
||||
return this[(string)key];
|
||||
return this[(int)key];
|
||||
}
|
||||
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
public object? this[object key] {
|
||||
get => key is string str ? this[str] : this[(int)key];
|
||||
set => throw new NotSupportedException("The IBlueRow is readonly");
|
||||
}
|
||||
|
||||
public object this[int index] => row[index];
|
||||
public object? this[int index] => row[index];
|
||||
|
||||
public object this[string colName] => row[colName];
|
||||
public object? this[string colName] => row[colName];
|
||||
|
||||
public ICollection Keys => row.Keys;
|
||||
|
||||
|
||||
@@ -22,9 +22,12 @@ namespace BCG
|
||||
/// <param name="rows"></param>
|
||||
public void loadCollection (IBlueRowCollection rows)
|
||||
{
|
||||
myFireEvent (new DataPacket (rows, new BlueTable(rows),
|
||||
myLastValue != null ?
|
||||
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
|
||||
myFireEvent(new DataPacket(
|
||||
rows,
|
||||
new BlueTable(rows),
|
||||
myLastValue is DataPacket dp
|
||||
? new BlueTable(dp.newRows)
|
||||
: new BlueTable(rows.schema)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -123,7 +123,7 @@ namespace BCG
|
||||
public Guid subscription;
|
||||
}
|
||||
private ThreadSafeQueue<Update> _queued_updates;
|
||||
TargetDoWorkRelay _do_work_relay;
|
||||
TargetDoWorkRelay? _do_work_relay;
|
||||
Guid _current_source_subscription;
|
||||
Guid _current_target_subscription;
|
||||
int _source_thread;
|
||||
|
||||
@@ -3,24 +3,23 @@ using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
|
||||
/// </summary>
|
||||
public enum DateConversion
|
||||
{
|
||||
public enum DateConversion {
|
||||
ConvertDoubleToDateTime = 0,
|
||||
ChopTime,
|
||||
UTCToCurrent,
|
||||
ConvertStringToDateTime,
|
||||
ConvertDateTimeToMilliseconds,
|
||||
SecondsOfDay
|
||||
SecondsOfDay,
|
||||
ISO8601,
|
||||
EpochToDatetime
|
||||
}
|
||||
|
||||
[DataStreamUserRespresentation("dateStandardize")]
|
||||
public class DateStandardizationStream : ViewStream, IStreamMapper
|
||||
{
|
||||
public class DateStandardizationStream : ViewStream, IStreamMapper {
|
||||
private IBlueTableSchema mySchema;
|
||||
private DateConversion myConversion;
|
||||
private string myConversionColName;
|
||||
@@ -31,12 +30,13 @@ namespace BCG
|
||||
{DateConversion.UTCToCurrent , typeof(DateTime)},
|
||||
{DateConversion.ConvertStringToDateTime , typeof(DateTime)},
|
||||
{DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
|
||||
{DateConversion.SecondsOfDay , typeof(Double)}
|
||||
{DateConversion.SecondsOfDay , typeof(Double)},
|
||||
{DateConversion.ISO8601 , typeof(DateTime)},
|
||||
{DateConversion.EpochToDatetime , typeof(DateTime)}
|
||||
};
|
||||
|
||||
|
||||
private static DateConversion _parse(string s)
|
||||
{
|
||||
private static DateConversion _parse(string s) {
|
||||
DateConversion rv = DateConversion.ChopTime;
|
||||
Enum.TryParse<DateConversion>(s, false, out rv);
|
||||
return rv;
|
||||
@@ -45,11 +45,9 @@ namespace BCG
|
||||
/// for DSQA
|
||||
/// </summary>
|
||||
public DateStandardizationStream(ViewStream vs, string colName, string conversion)
|
||||
: this(vs, colName, _parse(conversion))
|
||||
{ }
|
||||
: this(vs, colName, _parse(conversion)) { }
|
||||
|
||||
public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion)
|
||||
{
|
||||
public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion) {
|
||||
myConversionColName = colName;
|
||||
myConversion = conversion;
|
||||
mySetRealSource(new MapStream(vs, this));
|
||||
@@ -57,24 +55,18 @@ namespace BCG
|
||||
|
||||
#region IStreamMapper Members
|
||||
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
|
||||
{
|
||||
if (mySchema == null)
|
||||
{
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||
if (mySchema == null) {
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
foreach (ColumnInfo c in inputSchema.columnInfo)
|
||||
{
|
||||
if (c.colName == myConversionColName)
|
||||
{
|
||||
foreach (ColumnInfo c in inputSchema.columnInfo) {
|
||||
if (c.colName == myConversionColName) {
|
||||
Type convType;
|
||||
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType))
|
||||
{
|
||||
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) {
|
||||
//BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
|
||||
convType = typeof(DateTime);
|
||||
}
|
||||
cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
|
||||
}
|
||||
else
|
||||
} else
|
||||
cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
|
||||
}
|
||||
mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
|
||||
@@ -86,56 +78,59 @@ namespace BCG
|
||||
|
||||
#region IDynamicStreamMapper Members
|
||||
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
|
||||
rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
|
||||
return rv;
|
||||
//return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
public static object mapDateTime(object objIn, DateConversion conv)
|
||||
{
|
||||
if (objIn == DBNull.Value || objIn == null)
|
||||
{
|
||||
public static object mapDateTime(object obj_in, DateConversion conv) {
|
||||
if (obj_in == DBNull.Value || obj_in == null) {
|
||||
return DBNull.Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (conv == DateConversion.ChopTime)
|
||||
{
|
||||
DateTime dtIn = (DateTime)objIn;
|
||||
return dtIn.Date;
|
||||
}
|
||||
else if (conv == DateConversion.ConvertDoubleToDateTime)
|
||||
{
|
||||
double dtIn = (double)objIn;
|
||||
return DateTime.FromOADate(dtIn);
|
||||
}
|
||||
else if (conv == DateConversion.UTCToCurrent)
|
||||
{
|
||||
DateTime dtIn = (DateTime)objIn;
|
||||
return dtIn.ToLocalTime();
|
||||
}
|
||||
else if (conv == DateConversion.ConvertStringToDateTime)
|
||||
{
|
||||
return DateTime.Parse((string)objIn);
|
||||
}
|
||||
else if (conv == DateConversion.ConvertDateTimeToMilliseconds)
|
||||
{
|
||||
DateTime dtIn = (DateTime)objIn;
|
||||
return dtIn.TimeOfDay.TotalMilliseconds;
|
||||
}
|
||||
else if (conv == DateConversion.SecondsOfDay)
|
||||
{
|
||||
DateTime dtIn = (DateTime)objIn;
|
||||
DateTime start_of_day = dtIn.Date;
|
||||
TimeSpan day_period = dtIn - start_of_day;
|
||||
return day_period.TotalSeconds;
|
||||
}
|
||||
else
|
||||
{
|
||||
bts.Log.Gen.info("incorrect type");
|
||||
return null;
|
||||
} else {
|
||||
switch (conv) {
|
||||
case DateConversion.ChopTime: {
|
||||
DateTime dtIn = (DateTime)obj_in;
|
||||
return dtIn.Date;
|
||||
}
|
||||
case DateConversion.ConvertDoubleToDateTime: {
|
||||
double dtIn = (double)obj_in;
|
||||
return DateTime.FromOADate(dtIn);
|
||||
}
|
||||
case DateConversion.UTCToCurrent: {
|
||||
DateTime dtIn = (DateTime)obj_in;
|
||||
return dtIn.ToLocalTime();
|
||||
}
|
||||
case DateConversion.ConvertStringToDateTime: {
|
||||
return DateTime.Parse((string)obj_in);
|
||||
}
|
||||
case DateConversion.ConvertDateTimeToMilliseconds: {
|
||||
DateTime dtIn = (DateTime)obj_in;
|
||||
return dtIn.TimeOfDay.TotalMilliseconds;
|
||||
}
|
||||
case DateConversion.SecondsOfDay: {
|
||||
DateTime dtIn = (DateTime)obj_in;
|
||||
DateTime start_of_day = dtIn.Date;
|
||||
TimeSpan day_period = dtIn - start_of_day;
|
||||
return day_period.TotalSeconds;
|
||||
}
|
||||
case DateConversion.ISO8601: {
|
||||
string iso8601Date = obj_in.ToString();
|
||||
return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind);
|
||||
}
|
||||
case DateConversion.EpochToDatetime: {
|
||||
double d = Convert.ToDouble(obj_in);
|
||||
long unix_time_seconds = (long)d;
|
||||
DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
return epoch.AddSeconds(unix_time_seconds);
|
||||
}
|
||||
default: {
|
||||
bts.Log.Gen.info("incorrect type");
|
||||
return DBNull.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,29 +1,71 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class MSecondStream : TimeUnitStream
|
||||
{
|
||||
public MSecondStream (int intervalInms, int size)
|
||||
: base (DateTime.Now, new TimeSpan (0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance)
|
||||
{}
|
||||
using BCG.Utilities.General;
|
||||
|
||||
protected override DateTime myRoundTime (DateTime dt) {return dt;}
|
||||
using bts;
|
||||
|
||||
namespace BCG {
|
||||
[DataStreamUserRespresentation("milliseconds")]
|
||||
public class MillisecondStream : ViewStream, IWorker {
|
||||
private DateTime _last_fire_time = DateTime.MinValue;
|
||||
private readonly TimeSpan _period;
|
||||
private readonly int _size;
|
||||
private readonly ViewStreamDatabase _vsd;
|
||||
private IBlueTableSchema _schema;
|
||||
private bts.CircularBufferTable _table;
|
||||
|
||||
public MillisecondStream(int interval_msecs, int size)
|
||||
: this(ViewStreamDatabase.instance, interval_msecs, size) {
|
||||
}
|
||||
|
||||
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
|
||||
_vsd = vsd;
|
||||
_period = TimeSpan.FromMilliseconds(interval_msecs);
|
||||
_size = size;
|
||||
_schema = BlueDataUtility.form_schema("Milliseconds",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("time", TypeCode.DateTime),
|
||||
Tuple.Create("display_time", TypeCode.String));
|
||||
_table = new CircularBufferTable(_schema, _size);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_vsd.addWorker(this);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
_vsd.removeWorker(this);
|
||||
}
|
||||
|
||||
private int _id = 0;
|
||||
public void doWork() {
|
||||
if (DateTime.Now - _last_fire_time >= _period) {
|
||||
_last_fire_time = DateTime.Now;
|
||||
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
|
||||
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
|
||||
_table.add_row(nr);
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
}
|
||||
}
|
||||
public class MSecondStream : TimeUnitStream {
|
||||
public MSecondStream(int intervalInms, int size)
|
||||
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
|
||||
|
||||
protected override DateTime myRoundTime(DateTime dt) { return dt; }
|
||||
}
|
||||
|
||||
public class DisplayableMillisecondStream : ViewStream, IStreamMapper
|
||||
{
|
||||
private IBlueTableSchema mySchema = new BlueTableSchema ("ms",
|
||||
new ColumnInfo ("id", 0, typeof(int)),
|
||||
new ColumnInfo ("time", 1, typeof(string)));
|
||||
public DisplayableMillisecondStream (int intervalInMs, int sz)
|
||||
{
|
||||
mySetRealSource (new MapStream (new MSecondStream (intervalInMs, sz), this));
|
||||
public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
|
||||
private IBlueTableSchema mySchema = new BlueTableSchema("ms",
|
||||
new ColumnInfo("id", 0, typeof(int)),
|
||||
new ColumnInfo("time", 1, typeof(string)));
|
||||
public DisplayableMillisecondStream(int intervalInMs, int sz) {
|
||||
mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
|
||||
}
|
||||
#region IStreamMapper Members
|
||||
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
|
||||
{
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||
return mySchema;
|
||||
}
|
||||
|
||||
@@ -31,15 +73,15 @@ namespace BCG
|
||||
|
||||
#region IDynamicStreamMapper Members
|
||||
|
||||
public IBlueRow map (IBlueRow row)
|
||||
{
|
||||
return new BlueRow (mySchema, row["id"], myToString ((DateTime)row["time"]));
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
DateTime time = row.get_datetime("time");
|
||||
var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
||||
return new BlueRow(mySchema, row["id"], time_txt);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private string myToString (DateTime dt)
|
||||
{
|
||||
private string myToString(DateTime dt) {
|
||||
string sep = ":";
|
||||
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
|
||||
}
|
||||
|
||||
@@ -1,49 +1,43 @@
|
||||
// using System;
|
||||
using System;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class TopStream : MethodDrivenStream
|
||||
// {
|
||||
// private int myTopCount;
|
||||
namespace BCG {
|
||||
public class TopStream : MethodDrivenStream {
|
||||
private int myTopCount;
|
||||
|
||||
// public TopStream (ViewStream stream, int topCount) : base (stream) {myTopCount = topCount;}
|
||||
public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
|
||||
|
||||
// public TopStream (ViewStream stream,
|
||||
// int topCount,
|
||||
// string sortExpression)
|
||||
// :
|
||||
// this (sortExpression == null?
|
||||
// stream :
|
||||
// new SortStream (stream, sortExpression), topCount)
|
||||
// {}
|
||||
public TopStream(ViewStream stream,
|
||||
int topCount,
|
||||
string sortExpression)
|
||||
:
|
||||
this(sortExpression == null ?
|
||||
stream :
|
||||
new SortStream(stream, sortExpression), topCount) { }
|
||||
|
||||
// protected override void myOnTableClear ()
|
||||
// {
|
||||
// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
// myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
|
||||
// myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
|
||||
// }
|
||||
protected override void myOnTableClear() {
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
}
|
||||
|
||||
// protected override void myHandleFirstCall ()
|
||||
// {
|
||||
// myLastFiredPacket = new DataPacket ();
|
||||
// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
|
||||
// myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
|
||||
// myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
|
||||
// }
|
||||
protected override void myHandleFirstCall() {
|
||||
myLastFiredPacket = new DataPacket();
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
|
||||
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
|
||||
myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
|
||||
}
|
||||
|
||||
// protected override void myHandleIncrementalAdd () {}
|
||||
protected override void myHandleIncrementalAdd() { }
|
||||
|
||||
// protected override void myHandleIncrementalDelete()
|
||||
// {
|
||||
// IBlueRowCollection oldRows = myLastFiredPacket.newRows;
|
||||
// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
|
||||
// BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
|
||||
// oldRows,
|
||||
// myLastFiredPacket.schema.columnNames,
|
||||
// out myLastFiredPacket.deletedRows,
|
||||
// out myLastFiredPacket.addedRows);
|
||||
// myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
protected override void myHandleIncrementalDelete() {
|
||||
IBlueRowCollection oldRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
|
||||
BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
|
||||
oldRows,
|
||||
myLastFiredPacket.schema.columnNames,
|
||||
out myLastFiredPacket.deletedRows,
|
||||
out myLastFiredPacket.addedRows);
|
||||
myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,38 +8,36 @@ namespace BCG {
|
||||
private ViewStream mySrc;
|
||||
private IBlueTableSchema mySchema = null;
|
||||
|
||||
public AlphabetizeColumnsStream (ViewStream src) {
|
||||
public AlphabetizeColumnsStream(ViewStream src) {
|
||||
mySrc = src;
|
||||
}
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
protected override void myOnFirstSubscriber() {
|
||||
mySrc.onChange += new ChangeDelegate(mySrc_onChange);
|
||||
}
|
||||
protected override void myOnLastSubscriber ()
|
||||
{
|
||||
mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
|
||||
protected override void myOnLastSubscriber() {
|
||||
mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
|
||||
}
|
||||
private void mySrc_onChange (object newValue) {
|
||||
private void mySrc_onChange(object newValue) {
|
||||
DataPacket dp = (DataPacket)newValue;
|
||||
if (mySchema == null) {
|
||||
SortedList alphabetical = new SortedList ();
|
||||
SortedList alphabetical = new SortedList();
|
||||
foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo)
|
||||
alphabetical.Add (cinfo.colName,cinfo);
|
||||
alphabetical.Add(cinfo.colName, cinfo);
|
||||
ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count];
|
||||
alphabetical.Values.CopyTo (cinfos,0);
|
||||
alphabetical.Values.CopyTo(cinfos, 0);
|
||||
for (int i = 0; i < cinfos.Length; ++i)
|
||||
cinfos[i] = new ColumnInfo (cinfos[i].colName,i,cinfos[i].colType,cinfos[i].colValue);
|
||||
mySchema = new BlueTableSchema (cinfos,dp.schema.tableName);
|
||||
cinfos[i] = new ColumnInfo(cinfos[i].colName, i, cinfos[i].colType);
|
||||
mySchema = new BlueTableSchema(cinfos, dp.schema.tableName);
|
||||
}
|
||||
BlueTable
|
||||
deleted = new BlueTable (mySchema),
|
||||
added = new BlueTable (mySchema),
|
||||
newrows = new BlueTable (mySchema);
|
||||
deleted.addRows (dp.deletedRows);
|
||||
added.addRows (dp.addedRows);
|
||||
newrows.addRows (dp.newRows);
|
||||
deleted = new BlueTable(mySchema),
|
||||
added = new BlueTable(mySchema),
|
||||
newrows = new BlueTable(mySchema);
|
||||
deleted.addRows(dp.deletedRows);
|
||||
added.addRows(dp.addedRows);
|
||||
newrows.addRows(dp.newRows);
|
||||
|
||||
myFireEvent (new DataPacket (newrows,added,deleted));
|
||||
myFireEvent(new DataPacket(newrows, added, deleted));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -183,7 +181,6 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class MutableAppendStream : ViewStream
|
||||
{
|
||||
HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||
private bool mySubscriptionActive = false;
|
||||
private MutableAppendTable myTable;
|
||||
private int myMovingIndex;
|
||||
@@ -272,7 +269,6 @@ namespace BCG
|
||||
private void myHandleStreamChanges(object newValue)
|
||||
{
|
||||
DataPacket dp = newValue as DataPacket;
|
||||
_timer.start();
|
||||
myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
|
||||
BlueTable adds = new BlueTable(myTable.schema),
|
||||
dels = new BlueTable(myTable.schema);
|
||||
@@ -287,7 +283,6 @@ namespace BCG
|
||||
IBlueRow nr = myTable.addRow(r, index);
|
||||
adds.addRow(nr);
|
||||
}
|
||||
_timer.stop();
|
||||
myFireEvent(new DataPacket(myTable, adds, dels));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,71 +1,61 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Publishes the rows in the form of a circular buffer, ie if the buffer size is n
|
||||
/// then the n'th row being puclished results in the deletion of the 0th row
|
||||
/// </summary>
|
||||
public class CircularPublisher
|
||||
{
|
||||
public class CircularPublisher {
|
||||
private CircularRowCollection myBuffer;
|
||||
private PubTableBroker myBroker;
|
||||
private int myBufferSize;
|
||||
public CircularPublisher (int bufferSize)
|
||||
{
|
||||
public CircularPublisher(int bufferSize) {
|
||||
myBufferSize = bufferSize;
|
||||
}
|
||||
|
||||
public CircularPublisher (int bufferSize, IBlueRowCollection rows)
|
||||
: this (bufferSize)
|
||||
{
|
||||
publishRows (rows);
|
||||
}
|
||||
|
||||
public void publishRow (IBlueRow nr)
|
||||
{
|
||||
if (myBuffer == null)
|
||||
{
|
||||
myBuffer = new CircularRowCollection (
|
||||
new BlueTableSchema (nr.schema.tableName, nr.schema.columnInfo),
|
||||
myBufferSize);
|
||||
myBroker = new PubTableBroker ();
|
||||
myBroker.createTable (myBuffer);
|
||||
}
|
||||
IBlueRow br = myBuffer.addRowWithReturn (nr);
|
||||
myBroker.makeBatchChanges (myBuffer,
|
||||
myReturnCollection (br),
|
||||
myReturnCollection (nr));
|
||||
public CircularPublisher(int bufferSize, IBlueRowCollection rows)
|
||||
: this(bufferSize) {
|
||||
publishRows(rows);
|
||||
}
|
||||
|
||||
public void publishRows (IBlueRowCollection nrs)
|
||||
{
|
||||
if (myBuffer == null)
|
||||
{
|
||||
myBuffer = new CircularRowCollection (
|
||||
new BlueTableSchema (BlueDataUtility.extractRow (nrs, 0).schema.tableName,
|
||||
BlueDataUtility.extractRow (nrs, 0).schema.columnInfo),
|
||||
public void publishRow(IBlueRow nr) {
|
||||
if (myBuffer == null) {
|
||||
myBuffer = new CircularRowCollection(
|
||||
new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo),
|
||||
myBufferSize);
|
||||
myBroker = new PubTableBroker ();
|
||||
myBroker.createTable (myBuffer);
|
||||
myBroker = new PubTableBroker();
|
||||
myBroker.create_table(myBuffer);
|
||||
}
|
||||
BlueTable deletes = new BlueTable (myBuffer.schema);
|
||||
foreach (IBlueRow nr in nrs)
|
||||
{
|
||||
IBlueRow dr = myBuffer.addRowWithReturn (nr);
|
||||
IBlueRow br = myBuffer.addRowWithReturn(nr);
|
||||
myBroker.make_batch_changes(myBuffer,
|
||||
myReturnCollection(br),
|
||||
myReturnCollection(nr));
|
||||
}
|
||||
|
||||
public void publishRows(IBlueRowCollection nrs) {
|
||||
if (myBuffer == null) {
|
||||
myBuffer = new CircularRowCollection(
|
||||
new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName,
|
||||
BlueDataUtility.extractRow(nrs, 0).schema.columnInfo),
|
||||
myBufferSize);
|
||||
myBroker = new PubTableBroker();
|
||||
myBroker.create_table(myBuffer);
|
||||
}
|
||||
BlueTable deletes = new BlueTable(myBuffer.schema);
|
||||
foreach (IBlueRow nr in nrs) {
|
||||
IBlueRow dr = myBuffer.addRowWithReturn(nr);
|
||||
if (dr != null)
|
||||
deletes.addRow (dr);
|
||||
deletes.addRow(dr);
|
||||
}
|
||||
myBroker.makeBatchChanges (myBuffer, deletes, nrs);
|
||||
myBroker.make_batch_changes(myBuffer, deletes, nrs);
|
||||
}
|
||||
|
||||
private IBlueRowCollection myReturnCollection (IBlueRow r)
|
||||
{
|
||||
private IBlueRowCollection myReturnCollection(IBlueRow r) {
|
||||
if (r == null)
|
||||
return new BlueTable (myBuffer.schema);
|
||||
BlueTable retVal = new BlueTable (r.schema);
|
||||
retVal.addRow (r);
|
||||
return new BlueTable(myBuffer.schema);
|
||||
BlueTable retVal = new BlueTable(r.schema);
|
||||
retVal.addRow(r);
|
||||
return retVal;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq.Expressions;
|
||||
using System.Xml.Serialization;
|
||||
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// This class will be used by the clients to publish tabular data to the pubsub
|
||||
/// </summary>
|
||||
public class PubTableBroker {
|
||||
protected PublisherCore _publisher_core;
|
||||
protected IDictionary<string, IPubTablePublisher> _publisher_map = new Dictionary<string, IPubTablePublisher>();
|
||||
protected Dictionary<string, IBlueTableSchema> _schema_map = new Dictionary<string, IBlueTableSchema>();
|
||||
|
||||
protected bool myWaitingForFirstOp = false;
|
||||
|
||||
public PubTableBroker()
|
||||
: this(new PublisherCore()) { }
|
||||
|
||||
public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; }
|
||||
|
||||
public PubTableBroker(IBlueRowCollection[] tablesToPublish)
|
||||
: this(new PublisherCore(), tablesToPublish) { }
|
||||
|
||||
public PubTableBroker(PublisherCore publisher, IBlueRowCollection[] tablesToPublish) {
|
||||
_publisher_core = publisher;
|
||||
|
||||
_publisher_core.beginTransaction();
|
||||
foreach (IBlueRowCollection table in tablesToPublish)
|
||||
createTable(table);
|
||||
_publisher_core.commitTransaction();
|
||||
}
|
||||
|
||||
public void beginTransaction() {
|
||||
myWaitingForFirstOp = true;
|
||||
}
|
||||
|
||||
public void commitTransaction() {
|
||||
bts.Log.Gen.info("PubTableBroker.commitTransaction");
|
||||
if (myWaitingForFirstOp)
|
||||
myWaitingForFirstOp = false;
|
||||
else
|
||||
_publisher_core.commitTransaction();
|
||||
}
|
||||
|
||||
public void rollbackTransaction() {
|
||||
bts.Log.Gen.info("PubTableBroker.rollbackTransaction");
|
||||
if (myWaitingForFirstOp)
|
||||
myWaitingForFirstOp = false;
|
||||
else
|
||||
_publisher_core.abortTransaction();
|
||||
}
|
||||
|
||||
public virtual void createTable(IBlueRowCollection newTable) {
|
||||
string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
|
||||
foreach (IBlueRow r in newTable) {
|
||||
log_msg += $"id: {r["id"]}";
|
||||
}
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(newTable.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = newTable.schema.tableName.ToLower();
|
||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
|
||||
table_publisher = _create_pubtable_publisher(name_key);
|
||||
_publisher_map.Add(name_key, table_publisher);
|
||||
_schema_map.Add(name_key, newTable.schema);
|
||||
}
|
||||
table_publisher.createTable(newTable);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
|
||||
/// will be thrown
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void addRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
||||
throw new ApplicationException("Row can not be added, as the table that has not been created yet");
|
||||
else
|
||||
table_publisher.addRow(r);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
|
||||
/// will be thrown
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void deleteRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
||||
throw new ApplicationException("Row can not be removed, as the table has not been created yet");
|
||||
else
|
||||
table_publisher.deleteRow(r);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
|
||||
/// will be thrown
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void changeRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
||||
throw new ApplicationException("Row can not be changed, as the table has not been created yet");
|
||||
else
|
||||
table_publisher.changeRow(r);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
|
||||
/// will be thrown
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void makeBatchChanges(IBlueRowCollection newRows,
|
||||
IBlueRowCollection deletedRows,
|
||||
IBlueRowCollection addedRows) {
|
||||
string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
|
||||
foreach (IBlueRow r in deletedRows)
|
||||
log_msg += $"id: {r["id"]} ";
|
||||
log_msg += "addedRows: ";
|
||||
foreach (IBlueRow r in addedRows)
|
||||
log_msg += $"id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(newRows.schema);
|
||||
_check_schema_consistency(deletedRows.schema);
|
||||
_check_schema_consistency(addedRows.schema);
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = newRows.schema.tableName.ToLower();
|
||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
||||
throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
|
||||
else
|
||||
table_publisher.batchRemoveAdd(deletedRows, addedRows);
|
||||
}
|
||||
|
||||
public virtual void deleteTableRows(string tableName) {
|
||||
myCheckTransaction();
|
||||
(_publisher_map[tableName.ToLower()] as IPubTablePublisher).createTable(
|
||||
new BlueTable(_schema_map[tableName.ToLower()] as IBlueTableSchema));
|
||||
}
|
||||
|
||||
public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
|
||||
|
||||
public void clearTable(string tableName) { _publisher_core.clearState(PubStreamPrefixer.prefix(tableName)); }
|
||||
|
||||
protected void myCheckTransaction() {
|
||||
if (myWaitingForFirstOp)
|
||||
_publisher_core.beginTransaction();
|
||||
myWaitingForFirstOp = false;
|
||||
}
|
||||
|
||||
protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
|
||||
bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
|
||||
return new PubTablePublisher(tag, _publisher_core);
|
||||
}
|
||||
|
||||
private void _check_schema_consistency(IBlueTableSchema schema_arg) {
|
||||
IBlueTableSchema schema;
|
||||
if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
|
||||
bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace BCG {
|
||||
// this for testing purposes mainly, its wraps a ViewPublisher around a stream,
|
||||
// and hence epublishes it out to the PS
|
||||
public class PublisherStream : ViewStream {
|
||||
private readonly ViewStream _main_stream;
|
||||
private readonly string _name;
|
||||
private ViewPublisher _vs_publisher;
|
||||
private readonly PubTableBroker _broker = new PubTableBroker();
|
||||
|
||||
public PublisherStream(ViewStream main_stream, string name) {
|
||||
_main_stream = main_stream;
|
||||
_name = name;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
|
||||
_main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
_main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
|
||||
_vs_publisher.Dispose();
|
||||
}
|
||||
|
||||
private void _handle_update(object o) {
|
||||
myFireEvent(o);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -7,48 +7,32 @@ using BCG.Utilities;
|
||||
namespace BCG {
|
||||
public class ViewPublisher : IDisposable {
|
||||
protected ViewPublisher() { }
|
||||
private readonly PubTableBroker _broker;
|
||||
protected readonly ViewStream _stream;
|
||||
private int _n_updates = 0;
|
||||
|
||||
/// If an name IsNullOrEmpty then it uses the name
|
||||
/// in the schema (be careful with that because this
|
||||
/// could lead to double publishes if the schema name
|
||||
/// is not uniquely chosen)
|
||||
public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
|
||||
myName = name;
|
||||
myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
|
||||
myBroker = br;
|
||||
myFirstTime = true;
|
||||
myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
|
||||
public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) {
|
||||
_stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
|
||||
_broker = broker;
|
||||
_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
||||
if (!String.IsNullOrEmpty(myName))
|
||||
myBroker.clearTable(myName);
|
||||
_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
|
||||
}
|
||||
|
||||
private void myChangeHandler(object o) {
|
||||
DataPacket dp = (DataPacket)o;
|
||||
if (myFirstTime) {
|
||||
myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName;
|
||||
myLastRowCount = dp.newRows.Count;
|
||||
myBroker.createTable(dp.newRows);
|
||||
myFirstTime = false;
|
||||
private void _handle_update(object o) {
|
||||
DataPacket dp = o as DataPacket;
|
||||
if (_n_updates == 0) {
|
||||
_broker.create_table(dp.newRows);
|
||||
_n_updates++;
|
||||
} else {
|
||||
if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) {
|
||||
myLastRowCount = dp.newRows.Count;
|
||||
myBroker.makeBatchChanges(dp.newRows, dp.deletedRows, dp.addedRows);
|
||||
}
|
||||
_broker.make_batch_changes(dp.newRows, dp.deletedRows, dp.addedRows);
|
||||
}
|
||||
}
|
||||
|
||||
public int getLastRowPublishedCount() { return myLastRowCount; }
|
||||
|
||||
public bool hasPublished() { return !myFirstTime; }
|
||||
|
||||
private int myLastRowCount = -1;
|
||||
private string myName;
|
||||
private PubTableBroker myBroker;
|
||||
protected ViewStream myView;
|
||||
private bool myFirstTime;
|
||||
}
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class ViewStreamPublisher
|
||||
{
|
||||
private IDynamicStreamMapper myMapper;
|
||||
private PubTableBroker myBroker;
|
||||
private IBlueRowCollection myOutputTable;
|
||||
private IBlueRowCollection myInputTable;
|
||||
|
||||
public ViewStreamPublisher (PubTableBroker broker)
|
||||
{
|
||||
myBroker = broker;
|
||||
}
|
||||
|
||||
public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper)
|
||||
: this (broker)
|
||||
{
|
||||
myMapper = mapper;
|
||||
}
|
||||
|
||||
public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper, ViewStream vs)
|
||||
: this (broker, mapper)
|
||||
{
|
||||
vs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChanges);
|
||||
}
|
||||
|
||||
private void myHandleChanges (object newValue)
|
||||
{
|
||||
DataPacket dp = newValue as DataPacket;
|
||||
if (myMapper == null)
|
||||
{
|
||||
if (myOutputTable == null)
|
||||
{
|
||||
myBroker.createTable (dp.newRows);
|
||||
}
|
||||
else
|
||||
{
|
||||
myBroker.makeBatchChanges (dp.newRows, dp.deletedRows, dp.addedRows);
|
||||
}
|
||||
myOutputTable = dp.newRows;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (myOutputTable != null)
|
||||
{
|
||||
BlueTable dRows = new BlueTable (myOutputTable.schema),
|
||||
aRows = new BlueTable (myOutputTable.schema);
|
||||
foreach (IBlueRow dRow in dp.deletedRows)
|
||||
{
|
||||
(myOutputTable as BlueTable).deleteRow ((int)dRow["id"]);
|
||||
dRows.addRow (myMapper.map (dRow));
|
||||
}
|
||||
foreach (IBlueRow aRow in dp.addedRows)
|
||||
{
|
||||
IBlueRow r = myMapper.map (aRow);
|
||||
(myOutputTable as BlueTable).addRow (r);
|
||||
aRows.addRow (r);
|
||||
}
|
||||
myBroker.makeBatchChanges (myOutputTable, dRows, aRows);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (dp.newRows.Count > 0)
|
||||
{
|
||||
myOutputTable = new BlueTable (
|
||||
myMapper.map (BlueDataUtility.extractRow (dp.newRows, 0)).schema);
|
||||
myHandleChanges (newValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void addRows (IBlueRowCollection aRows)
|
||||
{
|
||||
if (myInputTable != null)
|
||||
(myInputTable as BlueTable).addRows (aRows);
|
||||
else
|
||||
myInputTable = new BlueTable (aRows);
|
||||
myHandleChanges (new DataPacket(myInputTable, aRows, new BlueTable (aRows.schema)));
|
||||
}
|
||||
|
||||
public void deleteRows (IBlueRowCollection dRows)
|
||||
{
|
||||
// BlueEventLog.logAssert (myInputTable != null);
|
||||
(myInputTable as BlueTable).deleteRows (dRows);
|
||||
myHandleChanges (new DataPacket (myInputTable, new BlueTable (myInputTable.schema), dRows));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,8 @@ using System.Linq.Expressions;
|
||||
//using mbts.spark;
|
||||
using mbts;
|
||||
using BCG.Extensions;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace BCG {
|
||||
public static class BlueDataUtility {
|
||||
@@ -48,12 +49,6 @@ namespace BCG {
|
||||
rv.Add(r.get<T>(col));
|
||||
return rv;
|
||||
}
|
||||
public static IEnumerable<T> get_colvalues<T>(IBlueRowCollection rows, string col) {
|
||||
List<T> rv = new List<T>();
|
||||
foreach (IBlueRow r in rows)
|
||||
rv.Add(r.get<T>(col));
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
|
||||
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
|
||||
@@ -73,12 +68,17 @@ namespace BCG {
|
||||
// The values are returned as string values
|
||||
// doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
|
||||
public static IBlueRowCollection keyvalue_to_relational<T>(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
|
||||
HashSet<string> unique_cols = new HashSet<string>();
|
||||
HashSet<string> keys = new HashSet<string>();
|
||||
foreach (string col in get_column_values<string>(kv_rows, key_col)) {
|
||||
unique_cols.Add(col);
|
||||
keys.Add(col);
|
||||
}
|
||||
//unique_cols.Add(doc_col);
|
||||
var schema_cols = unique_cols.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
|
||||
if (keys.Contains("id")) {
|
||||
keys.Remove("id");
|
||||
}
|
||||
//unique_cols.Add(doc_col)
|
||||
// all the keys become cols
|
||||
var schema_cols = keys.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
|
||||
//schema of the final table needs doc and id
|
||||
schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
|
||||
schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
|
||||
var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
|
||||
@@ -90,6 +90,8 @@ namespace BCG {
|
||||
nr.setValue(doc_col, kv.Key);
|
||||
foreach (var r in kv.Value) {
|
||||
var col = r.get<string>(key_col);
|
||||
if (!keys.Contains(col))
|
||||
continue;
|
||||
var val = r.get<string>(val_col);
|
||||
nr.setValue(col, val);
|
||||
}
|
||||
@@ -117,6 +119,15 @@ namespace BCG {
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
|
||||
Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
|
||||
foreach (IBlueRow r in rows) {
|
||||
var id = r.get<int>("id");
|
||||
rv.Add(id, r);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
|
||||
foreach (Tuple<Type, string> c in cols)
|
||||
@@ -178,6 +189,35 @@ namespace BCG {
|
||||
|
||||
#endregion
|
||||
|
||||
#region json
|
||||
|
||||
public static List<JsonNode> json_to_json_node_list(string jsonString) {
|
||||
var jsonNodeList = new List<JsonNode>();
|
||||
bool noe = String.IsNullOrEmpty(jsonString);
|
||||
if (noe)
|
||||
return jsonNodeList;
|
||||
try {
|
||||
// Parse the JSON string
|
||||
JsonArray jsonArray = JsonNode.Parse(jsonString) as JsonArray;
|
||||
|
||||
if (jsonArray == null) {
|
||||
Console.WriteLine("The JSON string must represent a list.");
|
||||
return jsonNodeList;
|
||||
}
|
||||
|
||||
// Iterate through each JsonNode in the JsonArray
|
||||
foreach (JsonNode jsonNode in jsonArray) {
|
||||
jsonNodeList.Add(jsonNode);
|
||||
}
|
||||
} catch (JsonException ex) {
|
||||
Console.WriteLine($"Invalid JSON: {ex.Message}");
|
||||
}
|
||||
|
||||
return jsonNodeList;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width) {
|
||||
var rv = get_printed_rows(table, get_col_width);
|
||||
Console.WriteLine(rv);
|
||||
@@ -207,37 +247,98 @@ namespace BCG {
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
#region Schema types
|
||||
// these are the only types allowed in schemas
|
||||
private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
|
||||
{
|
||||
{ TypeCode.Boolean, typeof(bool) },
|
||||
{ TypeCode.Byte, typeof(byte) },
|
||||
{ TypeCode.Char, typeof(char) },
|
||||
{ TypeCode.DateTime, typeof(DateTime) },
|
||||
{ TypeCode.DBNull, typeof(DBNull) },
|
||||
{ TypeCode.Decimal, typeof(decimal) },
|
||||
{ TypeCode.Double, typeof(double) },
|
||||
{ TypeCode.Empty, null },
|
||||
{ TypeCode.Int16, typeof(short) },
|
||||
{ TypeCode.Int32, typeof(int) },
|
||||
{ TypeCode.Int64, typeof(long) },
|
||||
{ TypeCode.Object, typeof(object) },
|
||||
{ TypeCode.SByte, typeof(sbyte) },
|
||||
{ TypeCode.Single, typeof(Single) },
|
||||
{ TypeCode.String, typeof(string) },
|
||||
{ TypeCode.UInt16, typeof(UInt16) },
|
||||
{ TypeCode.UInt32, typeof(UInt32) },
|
||||
{ TypeCode.UInt64, typeof(UInt64) }
|
||||
{ TypeCode.Boolean, typeof(bool) }, //3
|
||||
{ TypeCode.Byte, typeof(byte) }, //6
|
||||
{ TypeCode.Char, typeof(char) }, //4
|
||||
// TODO
|
||||
{ TypeCode.DateTime, typeof(DateTime) },//16
|
||||
{ TypeCode.Decimal, typeof(decimal) }, //15
|
||||
{ TypeCode.Double, typeof(double) }, //14
|
||||
//{ TypeCode.Empty, null },
|
||||
{ TypeCode.Int16, typeof(short) }, //7
|
||||
{ TypeCode.Int32, typeof(int) }, //9
|
||||
{ TypeCode.Int64, typeof(long) }, //11
|
||||
{ TypeCode.SByte, typeof(sbyte) }, //5
|
||||
{ TypeCode.Single, typeof(Single) }, //13
|
||||
{ TypeCode.String, typeof(string) }, //18
|
||||
{ TypeCode.UInt16, typeof(UInt16) }, //8
|
||||
{ TypeCode.UInt32, typeof(UInt32) }, //10
|
||||
{ TypeCode.UInt64, typeof(UInt64) } //12
|
||||
};
|
||||
|
||||
public static IEnumerable<TypeCode> valid_column_typecodes() {
|
||||
return _typecode_to_type.Keys;
|
||||
}
|
||||
|
||||
public static IEnumerable<Type> valid_column_types() {
|
||||
return _typecode_to_type.Values;
|
||||
}
|
||||
|
||||
public static bool is_valid_col_type(Type t) {
|
||||
TypeCode type_code = get_typecode(t);
|
||||
return type_code != TypeCode.Empty;
|
||||
}
|
||||
|
||||
public static TypeCode get_typecode(Type t) {
|
||||
foreach (var kv in _typecode_to_type) {
|
||||
if (kv.Value == t)
|
||||
return kv.Key;
|
||||
}
|
||||
bts.Log.assert_d(false, $"type: {t} not found");
|
||||
//bts.Log.assert_d(false, $"type: {t} not found");
|
||||
return TypeCode.Empty;
|
||||
}
|
||||
|
||||
public static Type get_type(TypeCode type_code) {
|
||||
Type rv = null;
|
||||
if (_typecode_to_type.TryGetValue(type_code, out rv)) {
|
||||
return rv;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static TypeCode type_code(this ColumnInfo ci) {
|
||||
return get_typecode(ci.colType);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
public static Tuple<Type, object> guess_type(string value) {
|
||||
if (string.IsNullOrEmpty(value)) {
|
||||
return Tuple.Create(typeof(string), (object)value);
|
||||
}
|
||||
|
||||
int val = 0;
|
||||
// Try to parse as integer
|
||||
if (int.TryParse(value, out val)) {
|
||||
return Tuple.Create(typeof(int), (object)val);
|
||||
}
|
||||
|
||||
double dval = 0.0;
|
||||
// Try to parse as double
|
||||
if (double.TryParse(value, out dval)) {
|
||||
return Tuple.Create(typeof(double), (object)dval);
|
||||
}
|
||||
|
||||
DateTime dt;
|
||||
// Try to parse as DateTime
|
||||
if (DateTime.TryParse(value, out dt)) {
|
||||
return Tuple.Create(typeof(DateTime), (object)dt);
|
||||
}
|
||||
|
||||
bool bval;
|
||||
// Try to parse as boolean
|
||||
if (bool.TryParse(value, out bval)) {
|
||||
return Tuple.Create(typeof(bool), (object)bval);
|
||||
}
|
||||
|
||||
// If none of the above, return string type
|
||||
return Tuple.Create(typeof(string), (object)value);
|
||||
}
|
||||
|
||||
public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
foreach (Tuple<string, TypeCode> c in cols)
|
||||
@@ -247,67 +348,63 @@ namespace BCG {
|
||||
return new BlueTableSchema(table_name, cis.ToArray());
|
||||
}
|
||||
|
||||
public static TypeCode convert_column_type(IEnumerable<string> col_vals) {
|
||||
// bool
|
||||
bool success = true;
|
||||
foreach (var val in col_vals) {
|
||||
bool parsed_val;
|
||||
success = success && Boolean.TryParse(val, out parsed_val);
|
||||
if (!success)
|
||||
break;
|
||||
}
|
||||
if (success)
|
||||
return TypeCode.Boolean;
|
||||
// int
|
||||
success = true;
|
||||
foreach (var val in col_vals) {
|
||||
int parsed_val;
|
||||
success = success && Int32.TryParse(val, out parsed_val);
|
||||
if (!success)
|
||||
break;
|
||||
}
|
||||
if (success)
|
||||
return TypeCode.Int32;
|
||||
// try double
|
||||
success = true;
|
||||
foreach (var val in col_vals) {
|
||||
double parsed_val;
|
||||
success = success && Double.TryParse(val, out parsed_val);
|
||||
if (!success)
|
||||
break;
|
||||
}
|
||||
if (success)
|
||||
return TypeCode.Double;
|
||||
// try DateTime
|
||||
success = true;
|
||||
foreach (var val in col_vals) {
|
||||
DateTime parsed_val;
|
||||
success = success && DateTime.TryParse(val, out parsed_val);
|
||||
if (!success)
|
||||
break;
|
||||
}
|
||||
if (success)
|
||||
return TypeCode.DateTime;
|
||||
bts.Log.assert(false);
|
||||
return TypeCode.Empty;
|
||||
}
|
||||
|
||||
public static IBlueTableSchema to_typed_schema(IBlueRowCollection rows) {
|
||||
public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
|
||||
// just use one row to guess
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
foreach (var col in rows.schema.columnInfo) {
|
||||
var text_vals = BlueDataUtility.get_colvalues<string>(rows, col.colName);
|
||||
TypeCode type_code = convert_column_type(text_vals);
|
||||
cis.Add(new ColumnInfo(col.colName, cis.Count, _typecode_to_type[type_code]));
|
||||
foreach (IBlueRow r in rows) {
|
||||
foreach (var ci in r.schema.columnInfo) {
|
||||
if (ci.colType != typeof(System.String)) {
|
||||
cis.Add(ci);
|
||||
continue;
|
||||
}
|
||||
var (type, _) = guess_type(r[ci.colName] as string);
|
||||
cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
|
||||
}
|
||||
return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
|
||||
}
|
||||
return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
|
||||
return null;
|
||||
}
|
||||
|
||||
// public static IBlueRow convert_string_row(IBlueRow r, IBlueTableSchema out_schema) {
|
||||
// foreach (var ci in out_schema.columnInfo){
|
||||
// converts a row where all columns are string types to a schema where the best types are guesed
|
||||
public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
|
||||
IBlueTableSchema typed_schema = get_typed_schema(string_rows);
|
||||
var rv = new BlueTable(typed_schema);
|
||||
foreach (IBlueRow r in string_rows) {
|
||||
var typed_row = get_typed_row(r, typed_schema);
|
||||
rv.addRow(typed_row);
|
||||
}
|
||||
return rv;
|
||||
|
||||
// }
|
||||
// }
|
||||
static IBlueRow get_typed_row(IBlueRow string_row, IBlueTableSchema typed_schema) {
|
||||
var rv = new MutableBlueRow(typed_schema);
|
||||
foreach (var ci in typed_schema.columnInfo) {
|
||||
var typed_val = convert(ci.colType, string_row[ci.colName].ToString());
|
||||
rv.setValue(ci.colName, typed_val);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
static object convert(Type t, string value) {
|
||||
// Attempt to convert the string to the specified type
|
||||
try {
|
||||
return Convert.ChangeType(value, t);
|
||||
} catch (Exception) {
|
||||
if (t == typeof(double)) {
|
||||
// some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
|
||||
string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
|
||||
cleaned_val = cleaned_val.Replace('−', '-');
|
||||
double rv_d;
|
||||
if (Double.TryParse(cleaned_val, out rv_d))
|
||||
return rv_d;
|
||||
else
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
// Handle conversion failure gracefully
|
||||
// throw new ArgumentException("Failed to convert the string to the specified type.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
|
||||
@@ -443,12 +540,12 @@ namespace BCG {
|
||||
foreach (PropertyInfo pi in t.GetProperties())
|
||||
if (myIsActiveProperty(pi))
|
||||
colInfos.Add(new ColumnInfo(pi.Name, i++,
|
||||
ColumnInfo.isAllowedType(pi.PropertyType) ?
|
||||
is_valid_col_type(pi.PropertyType) ?
|
||||
pi.PropertyType : typeof(String)));
|
||||
foreach (FieldInfo fi in t.GetFields())
|
||||
if (myIsActiveField(fi))
|
||||
colInfos.Add(new ColumnInfo(fi.Name, i++,
|
||||
ColumnInfo.isAllowedType(fi.FieldType) ? fi.FieldType : typeof(String)));
|
||||
is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String)));
|
||||
|
||||
if (createIdField)
|
||||
colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
|
||||
@@ -473,12 +570,12 @@ namespace BCG {
|
||||
foreach (PropertyInfo pi in t.GetProperties())
|
||||
if (myIsActiveProperty(pi))
|
||||
rowArray[schema[pi.Name].colNo] =
|
||||
ColumnInfo.isAllowedType(pi.PropertyType) ?
|
||||
is_valid_col_type(pi.PropertyType) ?
|
||||
pi.GetValue(o, null) : pi.GetValue(o, null).ToString();
|
||||
foreach (FieldInfo fi in t.GetFields())
|
||||
if (myIsActiveField(fi))
|
||||
rowArray[schema[fi.Name].colNo] =
|
||||
ColumnInfo.isAllowedType(fi.FieldType) ?
|
||||
is_valid_col_type(fi.FieldType) ?
|
||||
fi.GetValue(o) : fi.GetValue(o).ToString();
|
||||
if (idVal != null)
|
||||
rowArray[schema["id"].colNo] = idVal;
|
||||
@@ -492,12 +589,12 @@ namespace BCG {
|
||||
foreach (PropertyInfo pi in t.GetProperties())
|
||||
if (myIsActiveProperty(pi))
|
||||
rowArray[schema[pi.Name].colNo] =
|
||||
ColumnInfo.isAllowedType(pi.PropertyType) ?
|
||||
is_valid_col_type(pi.PropertyType) ?
|
||||
pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null));
|
||||
foreach (FieldInfo fi in t.GetFields())
|
||||
if (myIsActiveField(fi))
|
||||
rowArray[schema[fi.Name].colNo] =
|
||||
ColumnInfo.isAllowedType(fi.FieldType) ?
|
||||
is_valid_col_type(fi.FieldType) ?
|
||||
fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o));
|
||||
if (idVal != null)
|
||||
rowArray[schema["id"].colNo] = idVal;
|
||||
@@ -686,15 +783,7 @@ namespace BCG {
|
||||
/// <param name="r1"></param>
|
||||
/// <param name="r2"></param>
|
||||
/// <returns></returns>
|
||||
public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) {
|
||||
return computeRowEquality(r1, r2, r1.schema.columnNames);
|
||||
}
|
||||
|
||||
public static bool computeRowEquality(IBlueRow r1,
|
||||
IBlueRow r2,
|
||||
params string[] colNames) {
|
||||
return compareRows(r1, r2, colNames) == 0 ? true : false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes a string equivelant for the row, using the columns provided
|
||||
@@ -732,7 +821,7 @@ namespace BCG {
|
||||
/// <param name="r2"></param>
|
||||
/// <param name="colNames"></param>
|
||||
/// <returns></returns>
|
||||
public static int compareRows(IBlueRow r1,
|
||||
public static int compare_rows(IBlueRow r1,
|
||||
IBlueRow r2,
|
||||
params string[] colNames) {
|
||||
int retVal = 0;
|
||||
@@ -883,7 +972,7 @@ namespace BCG {
|
||||
public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) {
|
||||
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
|
||||
foreach (IBlueRow r in rows)
|
||||
retVal = compareRows(r, retVal, cols) > 0 ? r : retVal;
|
||||
retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal;
|
||||
return retVal;
|
||||
}
|
||||
|
||||
@@ -896,7 +985,7 @@ namespace BCG {
|
||||
public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) {
|
||||
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
|
||||
foreach (IBlueRow r in rows)
|
||||
retVal = compareRows(r, retVal, cols) < 0 ? r : retVal;
|
||||
retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal;
|
||||
return retVal;
|
||||
}
|
||||
|
||||
@@ -975,8 +1064,58 @@ namespace BCG {
|
||||
out aRows);
|
||||
}
|
||||
|
||||
public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) {
|
||||
IBlueRowCollection d_rows, a_rows;
|
||||
compute_differences_fast(rows2, rows1, out d_rows, out a_rows);
|
||||
return d_rows.Count == 0 && a_rows.Count == 0;
|
||||
}
|
||||
|
||||
public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) {
|
||||
return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType);
|
||||
}
|
||||
|
||||
public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) {
|
||||
if (s1.tableName != s1.tableName)
|
||||
return false;
|
||||
if (s1.columnInfo.Length != s2.columnInfo.Length)
|
||||
return false;
|
||||
for (int i = 0; i < s1.columnInfo.Length; i++) {
|
||||
ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i];
|
||||
if (!compute_equality(ci1, ci2))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public static bool compute_equality(IBlueRow r1, IBlueRow r2) {
|
||||
return compute_row_equality(r1, r2, r1.schema.columnNames);
|
||||
}
|
||||
|
||||
public static bool compute_row_equality(IBlueRow r1,
|
||||
IBlueRow r2,
|
||||
params string[] col_names) {
|
||||
return compare_rows(r1, r2, col_names) == 0 ? true : false;
|
||||
}
|
||||
|
||||
public static string to_string(ColumnInfo ci) {
|
||||
return ci.colNo + ";" + ci.colName + ";" + ci.colType;
|
||||
}
|
||||
|
||||
public static string to_string(this IBlueRow r) {
|
||||
string rv = String.Empty;
|
||||
foreach (ColumnInfo c in r.schema.columnInfo) {
|
||||
rv += $"{c.colName}: {r[c.colName]}\n";
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
|
||||
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
||||
if (old_rows == null || old_rows.Count == 0) {
|
||||
out_d_rows = new BlueTable(new_rows.schema);
|
||||
out_a_rows = new BlueTable(new_rows);
|
||||
return;
|
||||
}
|
||||
int[] col_indices = new int[new_rows.schema.columnNames.Length];
|
||||
for (int i = 0; i < col_indices.Length; i++)
|
||||
col_indices[i] = i;
|
||||
@@ -985,6 +1124,11 @@ namespace BCG {
|
||||
|
||||
public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
|
||||
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
||||
if (old_rows == null || old_rows.Count == 0) {
|
||||
out_d_rows = new BlueTable(new_rows.schema);
|
||||
out_a_rows = new BlueTable(new_rows);
|
||||
return;
|
||||
}
|
||||
int[] col_indices = new int[] { new_rows.schema["id"].colNo };
|
||||
compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
|
||||
}
|
||||
@@ -1028,7 +1172,7 @@ namespace BCG {
|
||||
foreach (IBlueRow nr in c1)
|
||||
if (!BlueDataUtility.rowPresent(c2, nr))
|
||||
(aRows as BlueTable).addRow(nr);
|
||||
else if (BlueDataUtility.compareRows(nr, c2[(int)
|
||||
else if (BlueDataUtility.compare_rows(nr, c2[(int)
|
||||
nr["id"]], columns) != 0) {
|
||||
(aRows as BlueTable).addRow(nr);
|
||||
(dRows as BlueTable).addRow(c2[(int)nr["id"]]);
|
||||
@@ -1108,7 +1252,7 @@ namespace BCG {
|
||||
truelyAdds = new BlueTable(adds);
|
||||
foreach (IBlueRow r in dels) {
|
||||
if (adds.containsID((int)r["id"]) &&
|
||||
BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) {
|
||||
BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) {
|
||||
(truelyDels as BlueTable).deleteRow((int)r["id"]);
|
||||
(truelyAdds as BlueTable).deleteRow((int)r["id"]);
|
||||
}
|
||||
@@ -1484,6 +1628,13 @@ namespace BCG {
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
public static string id_string(IBlueRowCollection table) {
|
||||
string rv = String.Empty;
|
||||
foreach (IBlueRow r in table)
|
||||
rv += $"{r["id"]} ";
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
|
||||
BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
|
||||
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
|
||||
@@ -1632,6 +1783,18 @@ namespace BCG {
|
||||
#endregion
|
||||
}
|
||||
|
||||
public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
|
||||
if (rows == null || rows.Count() == 0)
|
||||
return null;
|
||||
IBlueRow max_row = rows.ElementAt(0);
|
||||
foreach (IBlueRow r in rows) {
|
||||
int c = compare_rows(r, max_row, cols_to_compare);
|
||||
if (c > 0)
|
||||
max_row = r;
|
||||
}
|
||||
return max_row;
|
||||
}
|
||||
|
||||
public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
|
||||
foreach (KeyValuePair<string, bool> p in sort_map) {
|
||||
int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
|
||||
@@ -2066,7 +2229,10 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) {
|
||||
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) =>
|
||||
createRowsFromCSVFileWithoutTypes(fileAddress, out var _);
|
||||
|
||||
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress, out IBlueTableSchema schema) {
|
||||
StreamReader sr = new StreamReader(fileAddress);
|
||||
string cols = sr.ReadLine();
|
||||
string[] columns = cols.Split(',');
|
||||
@@ -2074,11 +2240,11 @@ namespace BCG {
|
||||
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
||||
for (int i = 1; i < ci.Length; i++)
|
||||
ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
|
||||
IBlueTableSchema bts = new BlueTableSchema(ci, "csvTable");
|
||||
BlueTable result = new BlueTable(bts);
|
||||
schema = new BlueTableSchema(ci, "csvTable");
|
||||
BlueTable result = new BlueTable(schema);
|
||||
int id = 0;
|
||||
while (!sr.EndOfStream)
|
||||
result.addRow(createRowFromCSVFileRow(sr.ReadLine(), bts, id++));
|
||||
result.addRow(createRowFromCSVFileRow(sr.ReadLine(), schema, id++));
|
||||
sr.Close();
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1,45 +1,42 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
/// Allowed types
|
||||
/// </summary>
|
||||
public class BlueDataValidTypes
|
||||
{
|
||||
public static readonly Type BOOL = typeof(bool);
|
||||
public static readonly Type BYTE = typeof(byte);
|
||||
public static readonly Type CHAR = typeof(char);
|
||||
public static readonly Type DECIMAL = typeof(decimal);
|
||||
public static readonly Type DOUBLE = typeof(double);
|
||||
public static readonly Type SHORT = typeof(short);
|
||||
public static readonly Type INT = typeof(int);
|
||||
public static readonly Type LONG = typeof(long);
|
||||
public static readonly Type SBYTE = typeof(sbyte);
|
||||
//public static readonly Type FLOAT = typeof(float);
|
||||
public static readonly Type STRING = typeof(string);
|
||||
//public static readonly Type USHORT = typeof(ushort);
|
||||
public static readonly Type UINT = typeof(uint);
|
||||
public static readonly Type ULONG = typeof(ulong);
|
||||
public static readonly Type DATETIME = typeof(DateTime);
|
||||
public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
|
||||
public static readonly Type SINGLE = typeof(Single);
|
||||
public static readonly Type BYTEARRAY = typeof(byte[]);
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Allowed types
|
||||
/// </summary>
|
||||
public class BlueDataValidTypes {
|
||||
public static readonly Type BOOL = typeof(bool);
|
||||
public static readonly Type BYTE = typeof(byte);
|
||||
public static readonly Type CHAR = typeof(char);
|
||||
public static readonly Type DECIMAL = typeof(decimal);
|
||||
public static readonly Type DOUBLE = typeof(double);
|
||||
public static readonly Type SHORT = typeof(short);
|
||||
public static readonly Type INT = typeof(int);
|
||||
public static readonly Type LONG = typeof(long);
|
||||
public static readonly Type SBYTE = typeof(sbyte);
|
||||
//public static readonly Type FLOAT = typeof(float);
|
||||
public static readonly Type STRING = typeof(string);
|
||||
//public static readonly Type USHORT = typeof(ushort);
|
||||
public static readonly Type UINT = typeof(uint);
|
||||
public static readonly Type ULONG = typeof(ulong);
|
||||
public static readonly Type DATETIME = typeof(DateTime);
|
||||
public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
|
||||
public static readonly Type SINGLE = typeof(Single);
|
||||
public static readonly Type BYTEARRAY = typeof(byte[]);
|
||||
|
||||
public static int getValueTypeSize (Type t)
|
||||
{
|
||||
if (t == BOOL || t == BYTE || t == SBYTE)
|
||||
return 1;
|
||||
else if (t == SHORT || t == CHAR)
|
||||
return 2;
|
||||
else if (t == INT || t == UINT || t == SINGLE)
|
||||
return 4;
|
||||
else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
|
||||
return 8;
|
||||
else if (t == DECIMAL)
|
||||
return 16;
|
||||
else
|
||||
throw new ApplicationException("Unsupported Data Type");
|
||||
}
|
||||
}
|
||||
public static int getValueTypeSize(Type t) {
|
||||
if (t == BOOL || t == BYTE || t == SBYTE)
|
||||
return 1;
|
||||
else if (t == SHORT || t == CHAR)
|
||||
return 2;
|
||||
else if (t == INT || t == UINT || t == SINGLE)
|
||||
return 4;
|
||||
else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
|
||||
return 8;
|
||||
else if (t == DECIMAL)
|
||||
return 16;
|
||||
else
|
||||
throw new ApplicationException("Unsupported Data Type");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
#nullable enable
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
|
||||
public class BlueRowCollectionEqualityComparer : IEqualityComparer<IBlueRowCollection?> {
|
||||
readonly IEqualityComparer<IBlueRow> rowComparer = BlueRowEqualityComparer.Instance;
|
||||
|
||||
public static BlueRowCollectionEqualityComparer Instance { get; } = new();
|
||||
|
||||
public bool Equals(IBlueRowCollection? x, IBlueRowCollection? y) {
|
||||
if (x is null) {
|
||||
return y is null;
|
||||
}
|
||||
if (x.Count != y?.Count) {
|
||||
return false;
|
||||
}
|
||||
return x.schema.columnInfo.SequenceEqual(y.schema.columnInfo)
|
||||
&& x.Cast<IBlueRow>().OrderBy(r => r.Id()).SequenceEqual(
|
||||
y.Cast<IBlueRow>().OrderBy(r => r.Id()),
|
||||
rowComparer);
|
||||
}
|
||||
|
||||
public int GetHashCode([DisallowNull] IBlueRowCollection rows) {
|
||||
var hash = new HashCode();
|
||||
hash.Add(rows.schema.columnInfo.Length);
|
||||
hash.Add(rows.Count);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
|
||||
public class BlueRowEqualityComparer : IEqualityComparer<IBlueRow> {
|
||||
public static BlueRowEqualityComparer Instance { get; } = new();
|
||||
|
||||
public bool Equals(IBlueRow x, IBlueRow y) =>
|
||||
x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) &&
|
||||
x.Values.Cast<object>().SequenceEqual(y.Values.Cast<object>());
|
||||
|
||||
public int GetHashCode([DisallowNull] IBlueRow obj) {
|
||||
var hash = new HashCode();
|
||||
foreach (var v in obj.Values) {
|
||||
hash.Add(v?.GetHashCode() ?? 0);
|
||||
}
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
@@ -1,116 +1,117 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
// using System.Linq;
|
||||
// using System.Text;
|
||||
|
||||
/*
|
||||
* This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns.
|
||||
* if groupNo is zero, then the group is not stord in a lookup table at all.
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
namespace BCG
|
||||
{
|
||||
public class BlueRowLookupGroup
|
||||
{
|
||||
BlueTableSchema myEntrySchema;
|
||||
public IBlueTableSchema entrySchema
|
||||
{
|
||||
get { return myEntrySchema; }
|
||||
//cant' set this value, have to build it from the row's schema
|
||||
//TODORemove
|
||||
// /*
|
||||
// * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns.
|
||||
// * if groupNo is zero, then the group is not stord in a lookup table at all.
|
||||
// *
|
||||
// *
|
||||
// *
|
||||
// */
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class BlueRowLookupGroup
|
||||
// {
|
||||
// BlueTableSchema myEntrySchema;
|
||||
// public IBlueTableSchema entrySchema
|
||||
// {
|
||||
// get { return myEntrySchema; }
|
||||
// //cant' set this value, have to build it from the row's schema
|
||||
|
||||
}
|
||||
// }
|
||||
|
||||
//for faster access and such
|
||||
private int[] myGroupColNos;
|
||||
public int[] groupColNos
|
||||
{
|
||||
get { return myGroupColNos; }
|
||||
// //for faster access and such
|
||||
// private int[] myGroupColNos;
|
||||
// public int[] groupColNos
|
||||
// {
|
||||
// get { return myGroupColNos; }
|
||||
|
||||
}
|
||||
private string[] myGroupColNames;
|
||||
public string[] groupColNames
|
||||
{
|
||||
get { return myGroupColNames; }
|
||||
}
|
||||
// }
|
||||
// private string[] myGroupColNames;
|
||||
// public string[] groupColNames
|
||||
// {
|
||||
// get { return myGroupColNames; }
|
||||
// }
|
||||
|
||||
private short myGroupNumber;
|
||||
public short groupNumber
|
||||
{
|
||||
get { return myGroupNumber; }
|
||||
// private short myGroupNumber;
|
||||
// public short groupNumber
|
||||
// {
|
||||
// get { return myGroupNumber; }
|
||||
|
||||
}
|
||||
// }
|
||||
|
||||
/* how many bits are the ids into the table?
|
||||
* In c# everything is stored as integers, but when it goes across the wire
|
||||
* the data can be represented as a byte, a short, or an int
|
||||
*/
|
||||
private short myTableSize;
|
||||
public short tableSize
|
||||
{
|
||||
get { return myTableSize; }
|
||||
// /* how many bits are the ids into the table?
|
||||
// * In c# everything is stored as integers, but when it goes across the wire
|
||||
// * the data can be represented as a byte, a short, or an int
|
||||
// */
|
||||
// private short myTableSize;
|
||||
// public short tableSize
|
||||
// {
|
||||
// get { return myTableSize; }
|
||||
|
||||
}
|
||||
// }
|
||||
|
||||
//if the group number is 0, the group is not stored in tables for serialization
|
||||
public bool isStoredInTable
|
||||
{
|
||||
get { return myGroupNumber != 0; }
|
||||
// //if the group number is 0, the group is not stored in tables for serialization
|
||||
// public bool isStoredInTable
|
||||
// {
|
||||
// get { return myGroupNumber != 0; }
|
||||
|
||||
}
|
||||
public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
|
||||
{
|
||||
// }
|
||||
// public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
|
||||
// {
|
||||
|
||||
myGroupNumber = num;
|
||||
myTableSize = sizeInBits;
|
||||
myGroupColNames = cols ;
|
||||
}
|
||||
// myGroupNumber = num;
|
||||
// myTableSize = sizeInBits;
|
||||
// myGroupColNames = cols ;
|
||||
// }
|
||||
|
||||
/*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
|
||||
each of these entries in the lookup table. This function should be called by the LookupTableSchema
|
||||
during its constructor.
|
||||
// /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
|
||||
// each of these entries in the lookup table. This function should be called by the LookupTableSchema
|
||||
// during its constructor.
|
||||
|
||||
This function also caches the column numbers for this group, for quicker access*/
|
||||
public void buildEntrySchema(LookupTableSchema theSchema)
|
||||
{
|
||||
ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
|
||||
myGroupColNos = new int[myGroupColNames.Length];
|
||||
//the first column in our table is always the ID
|
||||
int idColumn = 0;
|
||||
columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
|
||||
//go through each of the columns
|
||||
for (int col = 1; col < columns.Length; ++col)
|
||||
{
|
||||
ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
|
||||
if (columnInInputTable.isConstant)
|
||||
columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
|
||||
else
|
||||
columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
|
||||
// This function also caches the column numbers for this group, for quicker access*/
|
||||
// public void buildEntrySchema(LookupTableSchema theSchema)
|
||||
// {
|
||||
// ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
|
||||
// myGroupColNos = new int[myGroupColNames.Length];
|
||||
// //the first column in our table is always the ID
|
||||
// int idColumn = 0;
|
||||
// columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
|
||||
// //go through each of the columns
|
||||
// for (int col = 1; col < columns.Length; ++col)
|
||||
// {
|
||||
// ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
|
||||
// if (columnInInputTable.isConstant)
|
||||
// columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
|
||||
// else
|
||||
// columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
|
||||
|
||||
myGroupColNos[col - 1] = columnInInputTable.colNo;
|
||||
}
|
||||
// myGroupColNos[col - 1] = columnInInputTable.colNo;
|
||||
// }
|
||||
|
||||
|
||||
|
||||
myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
|
||||
// myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
|
||||
|
||||
}
|
||||
// }
|
||||
|
||||
/*takes a row as input, and extracts the members of the group from the row */
|
||||
public object[] extractGroupFromRow(IBlueRow input)
|
||||
{
|
||||
int numCols = myGroupColNames.Length;
|
||||
object[] results = new object[numCols];
|
||||
// /*takes a row as input, and extracts the members of the group from the row */
|
||||
// public object[] extractGroupFromRow(IBlueRow input)
|
||||
// {
|
||||
// int numCols = myGroupColNames.Length;
|
||||
// object[] results = new object[numCols];
|
||||
|
||||
for (int col = 0; col < numCols; ++col)
|
||||
results[col] = input[myGroupColNos[col]];
|
||||
// for (int col = 0; col < numCols; ++col)
|
||||
// results[col] = input[myGroupColNos[col]];
|
||||
|
||||
return results;
|
||||
// return results;
|
||||
|
||||
}
|
||||
// }
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
// }
|
||||
// }
|
||||
@@ -1,8 +1,15 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.DataStream;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
//using BCG.Utilities;
|
||||
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG {
|
||||
@@ -10,7 +17,7 @@ namespace BCG {
|
||||
/// Anologue of the dotNet DataTable.
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class BlueTable : IBlueRowCollection {
|
||||
public class BlueTable : IBlueRowCollection, IDictionary<int, IBlueRow> {
|
||||
/// <summary>
|
||||
/// Hashtable indexed on "id" to speed up removes
|
||||
/// </summary>
|
||||
@@ -86,6 +93,12 @@ namespace BCG {
|
||||
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
return (IBlueRow)row;
|
||||
}
|
||||
set {
|
||||
if ((int)value["id"] != id) {
|
||||
throw new ArgumentException("id arg not equal to row's actual id", nameof(id));
|
||||
}
|
||||
changeRow(value);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -202,6 +215,8 @@ namespace BCG {
|
||||
|
||||
#endregion
|
||||
|
||||
public IDictionary<int, IBlueRow> AsDictionary() => this;
|
||||
|
||||
#region ICollection members
|
||||
public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
|
||||
{ return myIdIndex.Values.GetEnumerator(); }
|
||||
@@ -214,5 +229,55 @@ namespace BCG {
|
||||
|
||||
public virtual int Count { get { return myIdIndex.Count; } }
|
||||
#endregion
|
||||
|
||||
#region IDictionary<int, IBlueRow> members
|
||||
public void Add(int key, IBlueRow value) {
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
bool IDictionary<int, IBlueRow>.ContainsKey(int key) =>
|
||||
containsID(key);
|
||||
|
||||
bool IDictionary<int, IBlueRow>.Remove(int key) =>
|
||||
deleteRow(key);
|
||||
|
||||
bool IDictionary<int, IBlueRow>.TryGetValue(
|
||||
int key, [MaybeNullWhen(false)] out IBlueRow value) =>
|
||||
tryGetValue(key, out value);
|
||||
|
||||
void ICollection<KeyValuePair<int, IBlueRow>>.Add(KeyValuePair<int, IBlueRow> item) =>
|
||||
addRow(item.Value);
|
||||
|
||||
void ICollection<KeyValuePair<int, IBlueRow>>.Clear() =>
|
||||
clear();
|
||||
|
||||
bool ICollection<KeyValuePair<int, IBlueRow>>.Contains(KeyValuePair<int, IBlueRow> item) =>
|
||||
tryGetValue(item.Key, out var r)
|
||||
&& BlueRowEqualityComparer.Instance.Equals(r, item.Value);
|
||||
|
||||
void ICollection<KeyValuePair<int, IBlueRow>>.CopyTo(
|
||||
KeyValuePair<int, IBlueRow>[] array, int arrayIndex) {
|
||||
foreach (DictionaryEntry de in myIdIndex) {
|
||||
array[arrayIndex++] = new((int)de.Key, (IBlueRow)de.Value);
|
||||
}
|
||||
}
|
||||
|
||||
bool ICollection<KeyValuePair<int, IBlueRow>>.Remove(KeyValuePair<int, IBlueRow> item) =>
|
||||
deleteRow(item.Key);
|
||||
|
||||
IEnumerator<KeyValuePair<int, IBlueRow>> IEnumerable<KeyValuePair<int, IBlueRow>>.GetEnumerator() {
|
||||
foreach (DictionaryEntry de in myIdIndex) {
|
||||
yield return new((int)de.Key, (IBlueRow)de.Value);
|
||||
}
|
||||
}
|
||||
|
||||
ICollection<int> IDictionary<int, IBlueRow>.Keys =>
|
||||
GenericCollectionWrapper.Create<int>(myIdIndex.Keys);
|
||||
|
||||
ICollection<IBlueRow> IDictionary<int, IBlueRow>.Values =>
|
||||
GenericCollectionWrapper.Create<IBlueRow>(myIdIndex.Values);
|
||||
|
||||
bool ICollection<KeyValuePair<int, IBlueRow>>.IsReadOnly => false;
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
#nullable enable
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
// I would have named this ImmutableBlueTable, but that was already taken.
|
||||
public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
|
||||
readonly ImmutableDictionary<int, IBlueRow> rows;
|
||||
|
||||
public BlueTableImmutable(IBlueTableSchema schema)
|
||||
: this(schema, ImmutableDictionary<int, IBlueRow>.Empty) { }
|
||||
|
||||
public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, IBlueRow> rows) {
|
||||
this.schema = schema;
|
||||
this.rows = rows;
|
||||
}
|
||||
|
||||
public int Count => rows.Count;
|
||||
public bool IsSynchronized => false;
|
||||
public IBlueTableSchema schema { get; }
|
||||
public object SyncRoot => this;
|
||||
|
||||
public IBlueRow this[int id] => rows[id];
|
||||
|
||||
public bool containsID(int id) =>
|
||||
rows.ContainsKey(id);
|
||||
|
||||
public void CopyTo(Array array, int index) {
|
||||
foreach (var r in rows.Values) {
|
||||
array.SetValue(r, index++);
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerator GetEnumerator() =>
|
||||
rows.Values.GetEnumerator();
|
||||
|
||||
public DataPacketBuilder ToDataPacketBuilder() =>
|
||||
new(this);
|
||||
|
||||
public class DataPacketBuilder {
|
||||
readonly IBlueTableSchema schema;
|
||||
readonly ImmutableDictionary<int, IBlueRow>.Builder table;
|
||||
readonly ImmutableDictionary<int, IBlueRow>.Builder added =
|
||||
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
|
||||
readonly ImmutableDictionary<int, IBlueRow>.Builder removed =
|
||||
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
|
||||
|
||||
public DataPacketBuilder(BlueTableImmutable table) {
|
||||
schema = table.schema;
|
||||
this.table = table.rows.ToBuilder();
|
||||
}
|
||||
|
||||
public void AddOrUpdate(IBlueRow row, IEqualityComparer<IBlueRow>? comparer = null) {
|
||||
var id = row.Id();
|
||||
if (table.TryGetValue(id, out var orig)) {
|
||||
if (comparer?.Equals(row, orig) ?? false) {
|
||||
// No op: This row was already in the table (according to 'comparer').
|
||||
// Remove it from 'added' and 'removed'.
|
||||
added.Remove(id);
|
||||
removed.Remove(id);
|
||||
return;
|
||||
}
|
||||
// Change op: A row with this id was already in the table,
|
||||
// but its values have changed (according to 'comparer').
|
||||
removed[id] = orig;
|
||||
}
|
||||
added[id] = row;
|
||||
}
|
||||
|
||||
public void Remove(int id) {
|
||||
added.Remove(id);
|
||||
if (table.TryGetValue(id, out var orig)) {
|
||||
removed[id] = orig;
|
||||
}
|
||||
}
|
||||
|
||||
public DataPacket ToDataPacket(bool reset = true) {
|
||||
table.RemoveRange(removed.Keys);
|
||||
table.AddRange(added);
|
||||
var newRows = new BlueTableImmutable(schema, table.ToImmutable());
|
||||
var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
|
||||
var removedRows = new BlueTableImmutable(schema, removed.ToImmutable());
|
||||
if (reset) {
|
||||
added.Clear();
|
||||
removed.Clear();
|
||||
}
|
||||
return new(newRows, addedRows, removedRows);
|
||||
}
|
||||
|
||||
public BlueTableImmutable ToImmutable() =>
|
||||
new(schema, table.ToImmutable());
|
||||
}
|
||||
}
|
||||
@@ -8,8 +8,7 @@ using System.Linq;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
|
||||
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
|
||||
@@ -17,112 +16,105 @@ namespace BCG
|
||||
/// the respective data, sorted according to the ColumnOrder.
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class BlueTableSchema : IBlueTableSchema
|
||||
{
|
||||
public class BlueTableSchemaSerializationHelper
|
||||
{
|
||||
public static object parse(Type t, string data)
|
||||
{
|
||||
if (t == BlueDataValidTypes.BOOL)
|
||||
return Boolean.Parse(data);
|
||||
else if (t == BlueDataValidTypes.BYTE)
|
||||
return Byte.Parse(data);
|
||||
else if (t == BlueDataValidTypes.CHAR)
|
||||
return char.Parse(data);
|
||||
else if (t == BlueDataValidTypes.DATETIME)
|
||||
return DateTime.Parse(data);
|
||||
else if (t == BlueDataValidTypes.DOUBLE)
|
||||
return Double.Parse(data);
|
||||
else if (t == BlueDataValidTypes.DECIMAL)
|
||||
return Decimal.Parse(data);
|
||||
else if (t == BlueDataValidTypes.INT)
|
||||
return int.Parse(data);
|
||||
else if (t == BlueDataValidTypes.LONG)
|
||||
return long.Parse(data);
|
||||
else if (t == BlueDataValidTypes.SBYTE)
|
||||
return SByte.Parse(data);
|
||||
else if (t == BlueDataValidTypes.STRING)
|
||||
return data;
|
||||
else if (t == BlueDataValidTypes.UINT)
|
||||
return uint.Parse(data);
|
||||
else if (t == BlueDataValidTypes.ULONG)
|
||||
return ulong.Parse(data);
|
||||
else if (t == BlueDataValidTypes.SINGLE)
|
||||
return Single.Parse(data);
|
||||
else
|
||||
throw new ApplicationException("Can not be parsed");
|
||||
}
|
||||
public static void serialize(BinaryWriter bw, Type t, object o)
|
||||
{
|
||||
if (t == BlueDataValidTypes.BOOL)
|
||||
bw.Write((bool)o);
|
||||
else if (t == BlueDataValidTypes.BYTE)
|
||||
bw.Write((byte)o);
|
||||
else if (t == BlueDataValidTypes.CHAR)
|
||||
bw.Write((char)o);
|
||||
else if (t == BlueDataValidTypes.DATETIME)
|
||||
bw.Write(((DateTime)o).Ticks);
|
||||
else if (t == BlueDataValidTypes.DOUBLE)
|
||||
bw.Write((double)o);
|
||||
else if (t == BlueDataValidTypes.DECIMAL)
|
||||
bw.Write((decimal)o);
|
||||
else if (t == BlueDataValidTypes.INT)
|
||||
bw.Write((int)o);
|
||||
else if (t == BlueDataValidTypes.LONG)
|
||||
bw.Write((long)o);
|
||||
else if (t == BlueDataValidTypes.SBYTE)
|
||||
bw.Write((sbyte)o);
|
||||
else if (t == BlueDataValidTypes.STRING)
|
||||
bw.Write(o as string);
|
||||
else if (t == BlueDataValidTypes.SINGLE)
|
||||
bw.Write((Single)o);
|
||||
else if (t == BlueDataValidTypes.BYTEARRAY)
|
||||
{
|
||||
byte[] bs = o as byte[];
|
||||
bw.Write(bs.Length);
|
||||
bw.Write(bs);
|
||||
}
|
||||
else if (t == BlueDataValidTypes.UINT)
|
||||
bw.Write((uint)o);
|
||||
else if (t == BlueDataValidTypes.ULONG)
|
||||
bw.Write((ulong)o);
|
||||
else
|
||||
throw new ApplicationException("Unsupported type");
|
||||
}
|
||||
public class BlueTableSchema : IBlueTableSchema {
|
||||
public class BlueTableSchemaSerializationHelper {
|
||||
// public static object parse(Type t, string data) {
|
||||
// if (t == BlueDataValidTypes.BOOL)
|
||||
// return Boolean.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.BYTE)
|
||||
// return Byte.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.CHAR)
|
||||
// return char.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.DATETIME)
|
||||
// return DateTime.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||
// return Double.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||
// return Decimal.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.INT)
|
||||
// return int.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.LONG)
|
||||
// return long.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.SBYTE)
|
||||
// return SByte.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.STRING)
|
||||
// return data;
|
||||
// else if (t == BlueDataValidTypes.UINT)
|
||||
// return uint.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.ULONG)
|
||||
// return ulong.Parse(data);
|
||||
// else if (t == BlueDataValidTypes.SINGLE)
|
||||
// return Single.Parse(data);
|
||||
// else
|
||||
// throw new ApplicationException("Can not be parsed");
|
||||
// }
|
||||
// public static void serialize(BinaryWriter bw, Type t, object o) {
|
||||
// if (t == BlueDataValidTypes.BOOL)
|
||||
// bw.Write((bool)o);
|
||||
// else if (t == BlueDataValidTypes.BYTE)
|
||||
// bw.Write((byte)o);
|
||||
// else if (t == BlueDataValidTypes.CHAR)
|
||||
// bw.Write((char)o);
|
||||
// else if (t == BlueDataValidTypes.DATETIME)
|
||||
// bw.Write(((DateTime)o).Ticks);
|
||||
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||
// bw.Write((double)o);
|
||||
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||
// bw.Write((decimal)o);
|
||||
// else if (t == BlueDataValidTypes.INT)
|
||||
// bw.Write((int)o);
|
||||
// else if (t == BlueDataValidTypes.LONG)
|
||||
// bw.Write((long)o);
|
||||
// else if (t == BlueDataValidTypes.SBYTE)
|
||||
// bw.Write((sbyte)o);
|
||||
// else if (t == BlueDataValidTypes.STRING)
|
||||
// bw.Write(o as string);
|
||||
// else if (t == BlueDataValidTypes.SINGLE)
|
||||
// bw.Write((Single)o);
|
||||
// else if (t == BlueDataValidTypes.BYTEARRAY) {
|
||||
// byte[] bs = o as byte[];
|
||||
// bw.Write(bs.Length);
|
||||
// bw.Write(bs);
|
||||
// } else if (t == BlueDataValidTypes.UINT)
|
||||
// bw.Write((uint)o);
|
||||
// else if (t == BlueDataValidTypes.ULONG)
|
||||
// bw.Write((ulong)o);
|
||||
// else
|
||||
// throw new ApplicationException("Unsupported type");
|
||||
// }
|
||||
|
||||
public static object deserialize(BinaryReader br, Type t)
|
||||
{
|
||||
if (t == BlueDataValidTypes.BOOL)
|
||||
return br.ReadBoolean();
|
||||
else if (t == BlueDataValidTypes.BYTE)
|
||||
return br.ReadByte();
|
||||
else if (t == BlueDataValidTypes.CHAR)
|
||||
return br.ReadChar();
|
||||
else if (t == BlueDataValidTypes.DATETIME)
|
||||
return new DateTime(br.ReadInt64());
|
||||
else if (t == BlueDataValidTypes.DOUBLE)
|
||||
return br.ReadDouble();
|
||||
else if (t == BlueDataValidTypes.DECIMAL)
|
||||
return br.ReadDecimal();
|
||||
else if (t == BlueDataValidTypes.INT)
|
||||
return br.ReadInt32();
|
||||
else if (t == BlueDataValidTypes.LONG)
|
||||
return br.ReadInt64();
|
||||
else if (t == BlueDataValidTypes.SBYTE)
|
||||
return br.ReadSByte();
|
||||
else if (t == BlueDataValidTypes.STRING)
|
||||
return br.ReadString();
|
||||
else if (t == BlueDataValidTypes.BYTEARRAY)
|
||||
return br.ReadBytes(br.ReadInt32());
|
||||
else if (t == BlueDataValidTypes.UINT)
|
||||
return br.ReadUInt32();
|
||||
else if (t == BlueDataValidTypes.ULONG)
|
||||
return br.ReadUInt64();
|
||||
else if (t == BlueDataValidTypes.SINGLE)
|
||||
return br.ReadSingle();
|
||||
else
|
||||
throw new ApplicationException("Unsupported type");
|
||||
}
|
||||
// public static object deserialize(BinaryReader br, Type t) {
|
||||
// if (t == BlueDataValidTypes.BOOL)
|
||||
// return br.ReadBoolean();
|
||||
// else if (t == BlueDataValidTypes.BYTE)
|
||||
// return br.ReadByte();
|
||||
// else if (t == BlueDataValidTypes.CHAR)
|
||||
// return br.ReadChar();
|
||||
// else if (t == BlueDataValidTypes.DATETIME)
|
||||
// return new DateTime(br.ReadInt64());
|
||||
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||
// return br.ReadDouble();
|
||||
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||
// return br.ReadDecimal();
|
||||
// else if (t == BlueDataValidTypes.INT)
|
||||
// return br.ReadInt32();
|
||||
// else if (t == BlueDataValidTypes.LONG)
|
||||
// return br.ReadInt64();
|
||||
// else if (t == BlueDataValidTypes.SBYTE)
|
||||
// return br.ReadSByte();
|
||||
// else if (t == BlueDataValidTypes.STRING)
|
||||
// return br.ReadString();
|
||||
// else if (t == BlueDataValidTypes.BYTEARRAY)
|
||||
// return br.ReadBytes(br.ReadInt32());
|
||||
// else if (t == BlueDataValidTypes.UINT)
|
||||
// return br.ReadUInt32();
|
||||
// else if (t == BlueDataValidTypes.ULONG)
|
||||
// return br.ReadUInt64();
|
||||
// else if (t == BlueDataValidTypes.SINGLE)
|
||||
// return br.ReadSingle();
|
||||
// else
|
||||
// throw new ApplicationException("Unsupported type");
|
||||
// }
|
||||
}
|
||||
|
||||
protected Hashtable myIndexByColName = new Hashtable();
|
||||
@@ -132,29 +124,24 @@ namespace BCG
|
||||
|
||||
public BlueTableSchema() { }
|
||||
|
||||
public BlueTableSchema(DataTable table)
|
||||
{
|
||||
public BlueTableSchema(DataTable table) {
|
||||
myIndexByColOrder = new ColumnInfo[table.Columns.Count];
|
||||
myTableName = table.TableName;
|
||||
foreach (DataColumn col in table.Columns)
|
||||
{
|
||||
foreach (DataColumn col in table.Columns) {
|
||||
ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
|
||||
myIndexByColName[ci.colName] = ci;
|
||||
myIndexByColOrder[ci.colNo] = ci;
|
||||
}
|
||||
}
|
||||
|
||||
public BlueTableSchema(ColumnInfo[] cis, string tableName)
|
||||
{
|
||||
public BlueTableSchema(ColumnInfo[] cis, string tableName) {
|
||||
myConstruct(cis, tableName);
|
||||
}
|
||||
|
||||
private void myConstruct(ColumnInfo[] cis, string tableName)
|
||||
{
|
||||
private void myConstruct(ColumnInfo[] cis, string tableName) {
|
||||
myIndexByColOrder = new ColumnInfo[cis.Length];
|
||||
myTableName = tableName;
|
||||
foreach (ColumnInfo ci in cis)
|
||||
{
|
||||
foreach (ColumnInfo ci in cis) {
|
||||
myIndexByColOrder[ci.colNo] = ci;
|
||||
myIndexByColName[ci.colName] = ci;
|
||||
}
|
||||
@@ -169,15 +156,13 @@ namespace BCG
|
||||
/// </summary>
|
||||
/// <param name="tableName">the table name</param>
|
||||
/// <param name="fromType">the plain old data type whose fields will be mirrored in the schema</param>
|
||||
public BlueTableSchema(string tableName, Type fromType)
|
||||
{
|
||||
public BlueTableSchema(string tableName, Type fromType) {
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
int i = 0;
|
||||
// we sort the field name so that we have a deterministic order given
|
||||
// the same type.
|
||||
bool foundId = false;
|
||||
foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; }))
|
||||
{
|
||||
foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; })) {
|
||||
if (fi.Name.ToLower().Equals("id"))
|
||||
foundId = true;
|
||||
cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType));
|
||||
@@ -188,32 +173,25 @@ namespace BCG
|
||||
myConstruct(cis.ToArray(), tableName);
|
||||
}
|
||||
|
||||
public ColumnInfo this[string colName]
|
||||
{
|
||||
get
|
||||
{
|
||||
public ColumnInfo this[string colName] {
|
||||
get {
|
||||
object retVal = myIndexByColName[colName];
|
||||
return retVal != null ? retVal as ColumnInfo : null;
|
||||
}
|
||||
}
|
||||
|
||||
public ColumnInfo this[int colNo]
|
||||
{
|
||||
public ColumnInfo this[int colNo] {
|
||||
get { return myIndexByColOrder[colNo]; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The returned Array is sorted according to the Column No
|
||||
/// </summary>
|
||||
public string[] columnNames
|
||||
{
|
||||
get
|
||||
{
|
||||
if (myTableColNames == null)
|
||||
{
|
||||
public string[] columnNames {
|
||||
get {
|
||||
if (myTableColNames == null) {
|
||||
myTableColNames = new string[myIndexByColName.Count];
|
||||
foreach (DictionaryEntry de in myIndexByColName)
|
||||
{
|
||||
foreach (DictionaryEntry de in myIndexByColName) {
|
||||
ColumnInfo temp = (ColumnInfo)de.Value;
|
||||
myTableColNames[temp.colNo] = temp.colName;
|
||||
}
|
||||
@@ -225,8 +203,7 @@ namespace BCG
|
||||
/// <summary>
|
||||
/// The returned Array is sorted according to the Column No
|
||||
/// </summary>
|
||||
public ColumnInfo[] columnInfo
|
||||
{
|
||||
public ColumnInfo[] columnInfo {
|
||||
get { return myIndexByColOrder; }
|
||||
}
|
||||
|
||||
@@ -234,49 +211,30 @@ namespace BCG
|
||||
|
||||
|
||||
#region ISerializedType methods
|
||||
public void serialize(BinaryWriter bw)
|
||||
{
|
||||
public void serialize(BinaryWriter bw) {
|
||||
bw.Write(columnNames.Length);
|
||||
bw.Write(tableName);
|
||||
foreach (ColumnInfo ci in columnInfo)
|
||||
{
|
||||
foreach (ColumnInfo ci in columnInfo) {
|
||||
bw.Write(ci.colName);
|
||||
bw.Write(ci.colType.ToString());
|
||||
if (ci.isConstant)
|
||||
{
|
||||
bw.Write(true);
|
||||
BlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
|
||||
}
|
||||
else
|
||||
{
|
||||
bw.Write(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void deserialze(BinaryReader br)
|
||||
{
|
||||
public void deserialze(BinaryReader br) {
|
||||
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
|
||||
string name = br.ReadString();
|
||||
for (int i = 0; i < cis.Length; i++)
|
||||
{
|
||||
for (int i = 0; i < cis.Length; i++) {
|
||||
string colName = br.ReadString();
|
||||
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
|
||||
bool isConstant = br.ReadBoolean();
|
||||
if (isConstant)
|
||||
{
|
||||
cis[i] = new ColumnInfo(colName, i, t, BlueTableSchemaSerializationHelper.deserialize(br, t));
|
||||
}
|
||||
else
|
||||
{
|
||||
cis[i] = new ColumnInfo(colName, i, t);
|
||||
}
|
||||
cis[i] = new ColumnInfo(colName, i, t);
|
||||
}
|
||||
myConstruct(cis, name);
|
||||
}
|
||||
|
||||
public int getSerializedSize()
|
||||
{
|
||||
public int getSerializedSize() {
|
||||
throw new NotImplementedException("Feature for provided constant values to" +
|
||||
"non-primitive types in schema, not provided as of now");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
#nullable enable
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.collections;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
public class BlueTableSchemaAdaptor<T> : IMutableBlueRowFactory<T> {
|
||||
readonly string name;
|
||||
readonly List<ColumnAdaptor> columns = new();
|
||||
BlueTableSchema? schema;
|
||||
|
||||
public BlueTableSchemaAdaptor(string name) =>
|
||||
this.name = name;
|
||||
|
||||
public IBlueTableSchema Schema =>
|
||||
schema ??= new BlueTableSchema(columns.WrapSelectorRO(c => c.Info).ToArray(), name);
|
||||
|
||||
public bool Contains(string name) =>
|
||||
columns.Any(c => c.Info.colName == name);
|
||||
|
||||
public void AddColumn(string name, Func<T, int> getValue) =>
|
||||
AddColumn(name, r => getValue(r), typeof(int));
|
||||
|
||||
public void AddColumn(string name, Func<T, int?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int));
|
||||
|
||||
public void AddColumn(string name, Func<T, long> getValue) =>
|
||||
AddColumn(name, r => getValue(r), typeof(long));
|
||||
|
||||
public void AddColumn(string name, Func<T, long?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long));
|
||||
|
||||
public void AddColumn(string name, Func<T, double> getValue) =>
|
||||
AddColumn(name, r => getValue(r), typeof(double));
|
||||
|
||||
public void AddColumn(string name, Func<T, double?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double));
|
||||
|
||||
public void AddColumn(string name, Func<T, DateTime> getValue) =>
|
||||
AddColumn(name, r => getValue(r), typeof(DateTime));
|
||||
|
||||
public void AddColumn(string name, Func<T, DateTime?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime));
|
||||
|
||||
public void AddColumn(string name, Func<T, string?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string));
|
||||
|
||||
public void AddColumn(string name, Func<T, object?> getValue, Type dataType) {
|
||||
if (!ColumnInfo.isAllowedType(dataType)) {
|
||||
throw new ArgumentException(
|
||||
$"'{dataType}' is not a supported column type", nameof(dataType));
|
||||
}
|
||||
// Invalidate cached schema (set it to null)
|
||||
schema = null;
|
||||
columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r)));
|
||||
}
|
||||
|
||||
public IBlueRow GetBlueRow(T item) =>
|
||||
GetMutableBlueRow(item);
|
||||
|
||||
public MutableBlueRow GetMutableBlueRow(T item) {
|
||||
var mbr = new MutableBlueRow(Schema);
|
||||
foreach (var col in columns) {
|
||||
mbr.setValue(col.Info.colNo, col.GetValue(item));
|
||||
}
|
||||
return mbr;
|
||||
}
|
||||
|
||||
class ColumnAdaptor {
|
||||
readonly Func<T, object?> getValue;
|
||||
|
||||
public ColumnAdaptor(ColumnInfo info, Func<T, object?> getValue) {
|
||||
Info = info;
|
||||
this.getValue = getValue;
|
||||
}
|
||||
|
||||
public ColumnInfo Info { get; }
|
||||
|
||||
public object? GetValue(T row) =>
|
||||
getValue(row);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Linq;
|
||||
|
||||
using BCG;
|
||||
|
||||
namespace bts {
|
||||
public class CircularBufferTable : IBlueRowCollection {
|
||||
private BlueTable _table;
|
||||
private List<IBlueRow> _ordered_list = new List<IBlueRow>();
|
||||
private int _buffer_sz;
|
||||
public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) {
|
||||
_table = new BlueTable(schema);
|
||||
_buffer_sz = buffer_sz;
|
||||
}
|
||||
|
||||
// returns the row that might need to be deleted to keep
|
||||
// the circular buffer intact
|
||||
public IBlueRow add_row(IBlueRow row) {
|
||||
_ordered_list.Add(row);
|
||||
_table.addRow(row);
|
||||
if (_ordered_list.Count > _buffer_sz) {
|
||||
IBlueRow rv = _ordered_list.ElementAt(0);
|
||||
_ordered_list.RemoveAt(0);
|
||||
_table.deleteRow(rv);
|
||||
return rv;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public IBlueRow this[int id] {
|
||||
get {
|
||||
return _table[id];
|
||||
}
|
||||
}
|
||||
public IBlueTableSchema schema {
|
||||
get {
|
||||
return _table.schema;
|
||||
}
|
||||
}
|
||||
public bool containsID(int id) {
|
||||
return _table.containsID(id);
|
||||
}
|
||||
|
||||
public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); }
|
||||
public int Count { get { return _table.Count; } }
|
||||
public bool IsSynchronized { get { return false; } }
|
||||
public object SyncRoot { get { return this; } }
|
||||
public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); }
|
||||
}
|
||||
}
|
||||
@@ -1,56 +1,32 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class ColumnInfo : IComparable
|
||||
{
|
||||
public static bool isAllowedType (Type t)
|
||||
{
|
||||
return
|
||||
t.Equals (typeof (System.Int32)) ||
|
||||
t.Equals (typeof (System.Int64)) ||
|
||||
t.Equals (typeof (System.Double)) ||
|
||||
t.Equals (typeof (System.String)) ||
|
||||
t.Equals (typeof (System.DateTime)) ? true :
|
||||
false;
|
||||
}
|
||||
public class ColumnInfo : IComparable {
|
||||
|
||||
//the default column info does not cach values in a lookup table, therefore it is in group 0
|
||||
public ColumnInfo(string colName, int colNo, Type colType)
|
||||
: this(colName, colNo, colType, 0)
|
||||
{
|
||||
|
||||
|
||||
}
|
||||
|
||||
public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
|
||||
{
|
||||
public ColumnInfo(string colName, int colNo, Type colType, object colValue = null) {
|
||||
myColName = colName;
|
||||
myColNo = colNo;
|
||||
myColType = colType;
|
||||
myLookupTableGroup = colGroup;
|
||||
myColValue = null;
|
||||
myColValue = colValue;
|
||||
}
|
||||
|
||||
|
||||
public ColumnInfo (string colName, int colNo, object colValue)
|
||||
: this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;}
|
||||
|
||||
public ColumnInfo (string colName, int colNo, Type colType, object colValue)
|
||||
: this (colName, colNo, colType)
|
||||
{
|
||||
|
||||
myColValue = colValue == null ? DBNull.Value : colValue;
|
||||
|
||||
public ColumnInfo(string col_name, int col_no, TypeCode type_code)
|
||||
: this(col_name, col_no, BlueDataUtility.get_type(type_code)) {
|
||||
}
|
||||
|
||||
public static bool isAllowedType(Type type) =>
|
||||
type == typeof(int) ||
|
||||
type == typeof(long) ||
|
||||
type == typeof(double) ||
|
||||
type == typeof(string) ||
|
||||
type == typeof(DateTime);
|
||||
|
||||
public int CompareTo (object obj)
|
||||
{
|
||||
public int CompareTo(object obj) {
|
||||
int otherColNo = ((ColumnInfo)(obj)).colNo;
|
||||
if (myColNo < otherColNo)
|
||||
return -1;
|
||||
@@ -59,37 +35,30 @@ namespace BCG
|
||||
return 0;
|
||||
}
|
||||
|
||||
//public bool isCalculated{get{return myExpression != null;}}
|
||||
public override bool Equals(object obj) {
|
||||
if (obj is ColumnInfo) {
|
||||
ColumnInfo other = obj as ColumnInfo;
|
||||
return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public override string ToString() {
|
||||
return colName + " " + colNo + " " + colType;
|
||||
}
|
||||
|
||||
|
||||
// for backwards compatibility to allow build
|
||||
public bool isConstant {get{return myColValue != null;}}
|
||||
|
||||
private int myColNo;
|
||||
private string myColName;
|
||||
private Type myColType;
|
||||
//private string myExpression;
|
||||
private object myColValue;
|
||||
|
||||
//no table will ever have more than 65,536 columns
|
||||
//if it does, we are in serious trouble
|
||||
private short myLookupTableGroup;
|
||||
|
||||
public int colNo{get{return myColNo;}}
|
||||
public string colName {get{return myColName;}}
|
||||
public Type colType {get{return myColType;}}
|
||||
|
||||
|
||||
//this value corresponds to how the data in the column is serialized.
|
||||
//if the group number is 0, the values are not stored in a table
|
||||
//if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
|
||||
//are stored together in a lookup table
|
||||
|
||||
|
||||
public short lookupTableGroup
|
||||
{
|
||||
get { return myLookupTableGroup; }
|
||||
set { myLookupTableGroup = value; }
|
||||
}
|
||||
//public string expression {get{return myExpression;}}
|
||||
public int colNo { get { return myColNo; } }
|
||||
public string colName { get { return myColName; } }
|
||||
public Type colType { get { return myColType; } }
|
||||
public object colValue {get{return myColValue;}}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
#nullable enable
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
|
||||
public class DataPacketEqualityComparer : IEqualityComparer<DataPacket?> {
|
||||
readonly IEqualityComparer<IBlueRowCollection> rowCollectionComparer;
|
||||
|
||||
public DataPacketEqualityComparer(IEqualityComparer<IBlueRowCollection>? rowCollectionComparer = null) =>
|
||||
this.rowCollectionComparer = rowCollectionComparer ?? BlueRowCollectionEqualityComparer.Instance;
|
||||
|
||||
public static DataPacketEqualityComparer Instance { get; } = new();
|
||||
|
||||
public bool Equals(DataPacket? x, DataPacket? y) {
|
||||
if (x is null) {
|
||||
return y is null;
|
||||
}
|
||||
return y is not null
|
||||
&& x.schema.columnInfo.SequenceEqual(y.schema.columnInfo)
|
||||
&& rowCollectionComparer.Equals(x.addedRows, y.addedRows)
|
||||
&& rowCollectionComparer.Equals(x.deletedRows, y.deletedRows)
|
||||
&& rowCollectionComparer.Equals(x.newRows, y.newRows);
|
||||
}
|
||||
|
||||
public int GetHashCode([DisallowNull] DataPacket obj) {
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.schema.columnNames.Length);
|
||||
hash.Add(obj.addedRows.Count);
|
||||
hash.Add(obj.deletedRows.Count);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
Loaded 100 of 788 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user