Compare commits
225
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
11e6d25001 | ||
|
|
83d66ae9d7 | ||
|
|
e5e74b7b42 | ||
|
|
6ce2fb2cc5 | ||
|
|
9d63a004ec | ||
|
|
ccd53d71ea | ||
|
|
74a0235227 | ||
|
|
42bb101b43 | ||
|
|
c53561c6ef | ||
|
|
3701a70bd9 | ||
|
|
df56de3dd2 | ||
|
|
c9a418ce4b | ||
|
|
aaad404507 | ||
|
|
45acaa20b7 | ||
|
|
96a08500fb | ||
|
|
510e7b0050 | ||
|
|
2ab76c753e | ||
|
|
a02c6b77d4 | ||
|
|
6a289a5561 | ||
|
|
fddd310a03 | ||
|
|
d2a2d5fb37 | ||
|
|
4b76442a8f | ||
|
|
09b863f2ab | ||
|
|
dd221aba10 | ||
|
|
5ef862b026 | ||
|
|
cc43e9d3a4 | ||
|
|
b6eb727bc9 | ||
|
|
8ace69ab74 | ||
|
|
dc88b4ee10 | ||
|
|
dee32dead4 | ||
|
|
b2e022505f | ||
|
|
ac9b3ec365 | ||
|
|
ed89da7f38 | ||
|
|
a334d9be88 | ||
|
|
aee56569d1 | ||
|
|
7fdd3f6b1b | ||
|
|
3e1949d94f | ||
|
|
4cb3dbc313 | ||
|
|
0268bd476f | ||
|
|
2511b0d966 | ||
|
|
92456649df | ||
|
|
ec32078dd5 | ||
|
|
6bcce832ac | ||
|
|
85529b2232 | ||
|
|
c3cb8995bf | ||
|
|
ab2f0f975e | ||
|
|
2e133ea182 | ||
|
|
30615f3429 | ||
|
|
61371804a2 | ||
|
|
02d82e39bc | ||
|
|
50188e6688 | ||
|
|
5137785f7f | ||
|
|
def021612c | ||
|
|
217c5be131 | ||
|
|
a75441cd07 | ||
|
|
78eb096d6f | ||
|
|
2b66f46db1 | ||
|
|
ae3029d9ed | ||
|
|
d4a0603933 | ||
|
|
db6e96cf00 | ||
|
|
371db4afd2 | ||
|
|
08484a0954 | ||
|
|
39db44ed63 | ||
|
|
b4dca8e23f | ||
|
|
b4bf06d17a | ||
|
|
8c17705f02 | ||
|
|
551f26e5f3 | ||
|
|
f4adf35e49 | ||
|
|
90128863d9 | ||
|
|
300831c97a | ||
|
|
f3f6a82d9a | ||
|
|
7c8a41b2b7 | ||
|
|
408114c1f6 | ||
|
|
5c9be0bfb9 | ||
|
|
c0f970ab89 | ||
|
|
25e1804286 | ||
|
|
3c84dc48f6 | ||
|
|
9ae20f0c4d | ||
|
|
e8a72e7bd9 | ||
|
|
e99e2d817a | ||
|
|
7f4468c812 | ||
|
|
6206cbc425 | ||
|
|
21e50745e2 | ||
|
|
990e21e387 | ||
|
|
357be39db3 | ||
|
|
68e04cb3f5 | ||
|
|
b4a5d5c832 | ||
|
|
242fd758f4 | ||
|
|
3c9755f0dd | ||
|
|
61ed847d7d | ||
|
|
fc9094fea8 | ||
|
|
e36c0efc4b | ||
|
|
4158a14cff | ||
|
|
201a45cba0 | ||
|
|
ab31b9b253 | ||
|
|
780ddc79a7 | ||
|
|
d5a2adaed6 | ||
|
|
008ab3f068 | ||
|
|
7d926f588d | ||
|
|
707cd07876 | ||
|
|
f7ecb9090c | ||
|
|
a19e8a7315 | ||
|
|
cf189ecf81 | ||
|
|
4405b7adcb | ||
|
|
e51ee7b7b9 | ||
|
|
dfaa74f426 | ||
|
|
1d2ce59009 | ||
|
|
3602cc05bb | ||
|
|
ac88a81ac6 | ||
|
|
01fa8e0c63 | ||
|
|
59547e0f37 | ||
|
|
3ce100a9cf | ||
|
|
6513fb0cd5 | ||
|
|
fd0fb1f760 | ||
|
|
5cf7eacd82 | ||
|
|
087d8ab8c9 | ||
|
|
b69b7b3350 | ||
|
|
5620f33524 | ||
|
|
f3e33fa390 | ||
|
|
3b631e0d5f | ||
|
|
f93d28ee76 | ||
|
|
23d7d3775d | ||
|
|
4940771166 | ||
|
|
c1967a8f6a | ||
|
|
8c797ee76b | ||
|
|
ce80957d07 | ||
|
|
c60018b59e | ||
|
|
3a27af87d1 | ||
|
|
068c1c6973 | ||
|
|
645a58c97d | ||
|
|
b7f455b29a | ||
|
|
4a2735839e | ||
|
|
d2e9fe8237 | ||
|
|
cfc60766d8 | ||
|
|
b03fec7f79 | ||
|
|
fb8e7c5b81 | ||
|
|
e44601d1b1 | ||
|
|
07bade53c0 | ||
|
|
fac0a9c25b | ||
|
|
1d1de3dd18 | ||
|
|
cfcbf004ba | ||
|
|
78c7dadbbe | ||
|
|
7b5a6031af | ||
|
|
c13a7782c8 | ||
|
|
45f02391d1 | ||
|
|
a3ca06addc | ||
|
|
32869e54ec | ||
|
|
ca73220a2e | ||
|
|
9fc3c2618e | ||
|
|
e72e0c0a30 | ||
|
|
a5bfc75278 | ||
|
|
261a184d25 | ||
|
|
54e96a06cb | ||
|
|
17e60df32b | ||
|
|
c99d5fe24c | ||
|
|
6049826e90 | ||
|
|
672149dd89 | ||
|
|
8c99147f04 | ||
|
|
4d4b0b7ae3 | ||
|
|
1a7955c1b1 | ||
|
|
f8fc2a1d96 | ||
|
|
9a7b3cb77c | ||
|
|
8a3e4edb4d | ||
|
|
e59183bb1e | ||
|
|
dd4d4e8dc6 | ||
|
|
f1744450ca | ||
|
|
3a7827a75b | ||
|
|
8d22e91c0d | ||
|
|
c425145fc9 | ||
|
|
4855935ae0 | ||
|
|
65f412ec9f | ||
|
|
40681c6093 | ||
|
|
ab2d36e523 | ||
|
|
b1ab3a66d2 | ||
|
|
5e7e7e8ef4 | ||
|
|
9291d30ae9 | ||
|
|
549708fcc2 | ||
|
|
1d5551f301 | ||
|
|
283db0f9ea | ||
|
|
9fb04b2db9 | ||
|
|
ceca96e10a | ||
|
|
fc48cc64af | ||
|
|
015d846941 | ||
|
|
e39108bbda | ||
|
|
6240d89a08 | ||
|
|
c76700e5e5 | ||
|
|
a8a0e8dea0 | ||
|
|
d65a2d6ed6 | ||
|
|
1824d5f9a4 | ||
|
|
d3eb29087b | ||
|
|
d6fae238f9 | ||
|
|
44b46033e3 | ||
|
|
5741ec9ab2 | ||
|
|
596975b309 | ||
|
|
9050318ceb | ||
|
|
7ee6933789 | ||
|
|
306c572b53 | ||
|
|
63306173b1 | ||
|
|
df1b45ca3e | ||
|
|
c31a1f1bf3 | ||
|
|
e95997ea17 | ||
|
|
12a070fd8d | ||
|
|
e0459c54fd | ||
|
|
15262a5a95 | ||
|
|
90ec64d25c | ||
|
|
061481419e | ||
|
|
8035255868 | ||
|
|
7feaf869dc | ||
|
|
7104e90fc9 | ||
|
|
52c1e500e6 | ||
|
|
07274f897a | ||
|
|
f626e53415 | ||
|
|
5a7b2965be | ||
|
|
0aab1628bc | ||
|
|
b9649e1d8e | ||
|
|
3bc9bf2207 | ||
|
|
888c06262b | ||
|
|
03e80dabcb | ||
|
|
54184d0875 | ||
|
|
90bcf5a455 | ||
|
|
d32d5ef40c | ||
|
|
2e0e0658c6 | ||
|
|
bbdb50a41e | ||
|
|
90705cac83 | ||
|
|
b91cdd6d91 |
No files matched your search
@@ -87,6 +87,9 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
|
|||||||
# Pcap
|
# Pcap
|
||||||
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
||||||
|
|
||||||
|
# Databento (deps)
|
||||||
|
RUN apt-get update && apt-get -y install libzstd-dev
|
||||||
|
|
||||||
# Set up default executables (update-alternatives)
|
# Set up default executables (update-alternatives)
|
||||||
RUN \
|
RUN \
|
||||||
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
||||||
|
|||||||
@@ -25,7 +25,9 @@
|
|||||||
"-p",
|
"-p",
|
||||||
"8002:8002",
|
"8002:8002",
|
||||||
"-p",
|
"-p",
|
||||||
"8003:8003"
|
"8003:8003",
|
||||||
|
"-p",
|
||||||
|
"12345:12345"
|
||||||
],
|
],
|
||||||
"mounts": [
|
"mounts": [
|
||||||
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
|
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
|
||||||
@@ -45,4 +47,4 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
*.cmd text=auto eol=crlf
|
||||||
|
*.cs text=auto eol=crlf
|
||||||
|
*.sh text=auto eol=lf
|
||||||
+3
-1
@@ -8,7 +8,9 @@ cmake-build-debug/
|
|||||||
.idea/
|
.idea/
|
||||||
.dir-locals.el
|
.dir-locals.el
|
||||||
*.pyc
|
*.pyc
|
||||||
**.vscode
|
**.vscode/*
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/tasks.json
|
||||||
.vs
|
.vs
|
||||||
obj/
|
obj/
|
||||||
*.csproj.user
|
*.csproj.user
|
||||||
|
|||||||
Vendored
+158
@@ -0,0 +1,158 @@
|
|||||||
|
{
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": ".NET Core Attach",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "attach"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "DataStreamPublisher",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"preLaunchTask": "Build DataStream (Debug)",
|
||||||
|
"program": "",
|
||||||
|
"linux": {
|
||||||
|
"program": "DataStreamPublisher",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"program": "DataStreamPublisher.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "(gdb) Launch",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
|
"args": ["run"],
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
|
"environment": [],
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "DataStreamQueryAnalyzer",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"program": "DataStreamQueryAnalyzer.exe",
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0-windows",
|
||||||
|
"env": {
|
||||||
|
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "PubSubServer",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"preLaunchTask": "Build DataStream (Debug)",
|
||||||
|
"program": "",
|
||||||
|
"linux": {
|
||||||
|
"program": "PubSubServer",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"program": "PubSubServer.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Spark",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"preLaunchTask": "Build Spark (Debug)",
|
||||||
|
"program": "Spark.exe",
|
||||||
|
"cwd": "build/Debug/bin/net7.0-windows",
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Spark (Parameterized)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"preLaunchTask": "Build Spark (Debug)",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"program": "Spark.exe",
|
||||||
|
"args": [
|
||||||
|
"--sparkd",
|
||||||
|
"${input:sparkConfigServer}"
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0-windows",
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Sparkd",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"preLaunchTask": "Build Sparkd (Debug | Clang)",
|
||||||
|
"program": "",
|
||||||
|
"linux": {
|
||||||
|
"program": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"cwd": "build/sparkd-debug-clang/server",
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"id": "dataStreamPublisherConfig",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "Publisher Configuration",
|
||||||
|
"options": [
|
||||||
|
"MarketDataPublisher",
|
||||||
|
"SimpleEchoClient",
|
||||||
|
"SimpleTestDataPublisher",
|
||||||
|
"TestDataPublisher",
|
||||||
|
],
|
||||||
|
"default": "MarketDataPublisher",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "pubSubServerConfig",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "PubSub Configuration",
|
||||||
|
"options": [
|
||||||
|
"PubSubServer",
|
||||||
|
"SimpleEchoClient",
|
||||||
|
],
|
||||||
|
"default": "PubSubServer",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkConfigServer",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Config Server URL",
|
||||||
|
"default": "http://localhost:8000/config",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
Vendored
+828
@@ -0,0 +1,828 @@
|
|||||||
|
{
|
||||||
|
"version": "2.0.0",
|
||||||
|
"tasks": [
|
||||||
|
{
|
||||||
|
"label": "Build Api Server",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_api_server.sh",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_api_server.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"${input:buildConfiguration}",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Api Server (Debug)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_api_server.sh",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_api_server.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Debug",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Api Server (Release)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_api_server.cmd",
|
||||||
|
},
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_api_server.sh",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Release",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build DataStream",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_data_stream.sh",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_data_stream.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"${input:buildConfiguration}",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build DataStream (Debug)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_data_stream.sh",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_data_stream.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Debug",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build DataStream (Release)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_data_stream.sh",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_data_stream.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Release",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Spark",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_spark.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"${input:buildConfiguration}",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Spark (Debug)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_spark.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Debug",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Spark (Release)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_spark.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Release",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Sparkd",
|
||||||
|
"group": "build",
|
||||||
|
"type": "shell",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_sparkd.sh",
|
||||||
|
"args": [
|
||||||
|
"--preset",
|
||||||
|
"${input:sparkdCmakePreset}",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Sparkd (Debug)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "shell",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_sparkd.sh",
|
||||||
|
"args": [
|
||||||
|
"--build-type",
|
||||||
|
"Debug",
|
||||||
|
"--compiler",
|
||||||
|
"${input:sparkdCompiler}"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Sparkd (Debug | Gcc)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "shell",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_sparkd.sh",
|
||||||
|
"args": [
|
||||||
|
"--build-type",
|
||||||
|
"Debug",
|
||||||
|
"--compiler",
|
||||||
|
"gcc"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Sparkd (Debug | Clang)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "shell",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_sparkd.sh",
|
||||||
|
"args": [
|
||||||
|
"--build-type",
|
||||||
|
"Debug",
|
||||||
|
"--compiler",
|
||||||
|
"clang"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Sparkd (Release)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "shell",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_sparkd.sh",
|
||||||
|
"args": [
|
||||||
|
"--build-type",
|
||||||
|
"Release",
|
||||||
|
"--compiler",
|
||||||
|
"${input:sparkdCompiler}"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Sparkd (Release | Gcc)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "shell",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_sparkd.sh",
|
||||||
|
"args": [
|
||||||
|
"--build-type",
|
||||||
|
"Release",
|
||||||
|
"--compiler",
|
||||||
|
"gcc"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build Sparkd (Release | Clang)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "shell",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_sparkd.sh",
|
||||||
|
"args": [
|
||||||
|
"--build-type",
|
||||||
|
"Release",
|
||||||
|
"--compiler",
|
||||||
|
"clang"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Edit DataStream Config",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "editor",
|
||||||
|
"args": [
|
||||||
|
"${env:HOME}/.local/share/DataStream/conf/spark.conf"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "vim",
|
||||||
|
"args": [
|
||||||
|
"${env:APPDATA}/bts/DataStream/conf/spark.conf"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"presentation": {
|
||||||
|
"reveal": "always",
|
||||||
|
"focus": true,
|
||||||
|
"panel": "shared",
|
||||||
|
"showReuseMessage": false,
|
||||||
|
"close": true,
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "wrench"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Edit Spark Config",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "vim",
|
||||||
|
"args": [
|
||||||
|
"${env:APPDATA}/bts/conf/spark.conf"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"presentation": {
|
||||||
|
"reveal": "always",
|
||||||
|
"focus": true,
|
||||||
|
"panel": "dedicated",
|
||||||
|
"showReuseMessage": false,
|
||||||
|
"close": true,
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "wrench"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run DataStreamQueryAnalyzer",
|
||||||
|
"dependsOn": "Build DataStream",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "DataStreamQueryAnalyzer.exe",
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run DataStreamQueryAnalyzer (Debug)",
|
||||||
|
"dependsOn": "Build DataStream (Debug)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "DataStreamQueryAnalyzer.exe",
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run DataStreamQueryAnalyzer (Release)",
|
||||||
|
"dependsOn": "Build DataStream (Release)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "DataStreamQueryAnalyzer.exe",
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run DataStreamPublisher",
|
||||||
|
"dependsOn": "Build DataStream",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "DataStreamPublisher",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "DataStreamPublisher.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run DataStreamPublisher (Debug)",
|
||||||
|
"dependsOn": "Build DataStream (Debug)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "DataStreamPublisher",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "DataStreamPublisher.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run DataStreamPublisher (Release)",
|
||||||
|
"dependsOn": "Build DataStream (Release)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "DataStreamPublisher",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "DataStreamPublisher.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run PubSubServer",
|
||||||
|
"dependsOn": "Build DataStream",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "PubSubServer",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "PubSubServer.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run PubSubServer (Debug)",
|
||||||
|
"dependsOn": "Build DataStream (Debug)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "PubSubServer",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "PubSubServer.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run PubSubServer (Debug | Skip Build)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "PubSubServer",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "PubSubServer.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run PubSubServer (Release)",
|
||||||
|
"dependsOn": "Build DataStream (Release)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "PubSubServer",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "PubSubServer.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run PubSubServer (Release | Skip Build)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "PubSubServer",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "PubSubServer.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Spark",
|
||||||
|
"dependsOn": "Build Spark",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "Spark.exe",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Spark (Debug)",
|
||||||
|
"dependsOn": "Build Spark (Debug)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "Spark.exe",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Spark (Release)",
|
||||||
|
"dependsOn": "Build Spark (Release)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "Spark.exe",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Spark Config Query",
|
||||||
|
"dependsOn": "Build Spark",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "Spark.Console",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "Spark.Console.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"config",
|
||||||
|
"--query",
|
||||||
|
"${input:sparkConfigQuery}",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Sparkd",
|
||||||
|
"dependsOn": "Build Sparkd",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Sparkd Config Query",
|
||||||
|
"dependsOn": "Build Sparkd",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"config",
|
||||||
|
"--query",
|
||||||
|
"${input:sparkdConfigQuery}",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/logging/show_last_run.cmd",
|
||||||
|
"args": [
|
||||||
|
"DataStream",
|
||||||
|
"DataStreamQueryAnalyzer.exe",
|
||||||
|
"'Starting DataStreamQueryAnalyzer'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"presentation": {
|
||||||
|
"reveal": "always",
|
||||||
|
"focus": true,
|
||||||
|
"panel": "shared",
|
||||||
|
"showReuseMessage": false,
|
||||||
|
"close": true,
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "wrench"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "View PubSubServer Logs (Last Run)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/logging/show_last_run.sh",
|
||||||
|
"args": [
|
||||||
|
"${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
|
||||||
|
"Starting PubSubServer"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/logging/show_last_run.cmd",
|
||||||
|
"args": [
|
||||||
|
"DataStream",
|
||||||
|
"PubSubServer.exe",
|
||||||
|
"Starting PubSubServer"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"presentation": {
|
||||||
|
"reveal": "always",
|
||||||
|
"focus": true,
|
||||||
|
"panel": "shared",
|
||||||
|
"showReuseMessage": false,
|
||||||
|
"close": true,
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "wrench"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"id": "buildConfiguration",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "Configuration",
|
||||||
|
"options": [
|
||||||
|
"Debug",
|
||||||
|
"Release",
|
||||||
|
],
|
||||||
|
"default": "Debug",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkdCmakePreset",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "Configuration",
|
||||||
|
"options": [
|
||||||
|
"sparkd-debug-clang",
|
||||||
|
"sparkd-debug-gcc",
|
||||||
|
"sparkd-release-clang",
|
||||||
|
"sparkd-release-gcc",
|
||||||
|
],
|
||||||
|
"default": "sparkd-debug-clang",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkdCompiler",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "Configuration",
|
||||||
|
"options": [
|
||||||
|
"clang",
|
||||||
|
"gcc",
|
||||||
|
],
|
||||||
|
"default": "sparkd-debug-clang",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "dataStreamPublisherConfig",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "Publisher Configuration",
|
||||||
|
"options": [
|
||||||
|
"MarketDataPublisher",
|
||||||
|
"SimpleEchoClient",
|
||||||
|
"SimpleTestDataPublisher",
|
||||||
|
"TestDataPublisher",
|
||||||
|
],
|
||||||
|
"default": "MarketDataPublisher",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "pubSubServerConfig",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "PubSub Configuration",
|
||||||
|
"options": [
|
||||||
|
"PubSubServer",
|
||||||
|
"SimpleEchoClient",
|
||||||
|
],
|
||||||
|
"default": "PubSubServer",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkConfigQuery",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Config Type(s)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkdConfigQuery",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Config Type(s)"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
+14
-13
@@ -14,12 +14,17 @@ set(
|
|||||||
CACHE
|
CACHE
|
||||||
STRING
|
STRING
|
||||||
"External dependencies root folder (should contain folders: lib, bin, include")
|
"External dependencies root folder (should contain folders: lib, bin, include")
|
||||||
|
|
||||||
|
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
|
||||||
|
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
|
||||||
|
# api server, etc.).
|
||||||
|
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
|
||||||
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
|
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
|
||||||
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
|
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
|
||||||
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
||||||
set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
|
|
||||||
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
||||||
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
||||||
|
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
|
||||||
if (WIN32)
|
if (WIN32)
|
||||||
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
|
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
|
||||||
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
|
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
|
||||||
@@ -31,13 +36,13 @@ endif()
|
|||||||
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
||||||
link_directories(${SPARK_LIB})
|
link_directories(${SPARK_LIB})
|
||||||
|
|
||||||
if(BUILD_API_SERVER)
|
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
||||||
add_compile_definitions(SPARK_ENABLE_API_SERVER)
|
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
||||||
|
if(DEFINED APP_DATA_DIR_OVERRIDE)
|
||||||
|
add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
|
||||||
endif()
|
endif()
|
||||||
|
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
|
|
||||||
add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
|
|
||||||
# subdirs
|
# subdirs
|
||||||
add_subdirectory(src/lib)
|
add_subdirectory(src/lib)
|
||||||
else()
|
else()
|
||||||
@@ -47,13 +52,7 @@ else()
|
|||||||
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
||||||
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
||||||
|
|
||||||
if(BUILD_API_SERVER)
|
if(BUILD_SPARKD)
|
||||||
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
|
|
||||||
elseif(BUILD_SPARKD)
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
|
|
||||||
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
|
|
||||||
|
|
||||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||||
# read by their code). We override some below.
|
# read by their code). We override some below.
|
||||||
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
||||||
@@ -133,6 +132,8 @@ else()
|
|||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
||||||
|
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
|
||||||
|
|
||||||
COMMAND
|
COMMAND
|
||||||
ln ARGS -sf
|
ln ARGS -sf
|
||||||
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
||||||
|
|||||||
+113
-62
@@ -15,11 +15,13 @@
|
|||||||
"rhs": "Windows"
|
"rhs": "Windows"
|
||||||
},
|
},
|
||||||
"generator": "Visual Studio 17 2022",
|
"generator": "Visual Studio 17 2022",
|
||||||
"binaryDir": "${sourceDir}/build",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"environment": {
|
"environment": {
|
||||||
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
||||||
},
|
},
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
|
"APP_DATA_DIR": "${dollar}{APPDATA}/bts",
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
|
||||||
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||||
@@ -30,7 +32,12 @@
|
|||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
|
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
||||||
|
"EXPORT_MD_CONSUMER": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
|
"SHARED_MEMORY_PREFIX": "spark"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -46,6 +53,8 @@
|
|||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"platform": "",
|
"platform": "",
|
||||||
|
"APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
|
||||||
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -62,7 +71,8 @@
|
|||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
}
|
},
|
||||||
|
"SHARED_MEMORY_PREFIX": "sparkd"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -95,98 +105,72 @@
|
|||||||
{
|
{
|
||||||
"name": "sparkd-debug-clang",
|
"name": "sparkd-debug-clang",
|
||||||
"inherits": "sparkd-debug",
|
"inherits": "sparkd-debug",
|
||||||
"displayName": "Debug (clang)",
|
"displayName": "Sparkd (Debug | Clang)",
|
||||||
"description": "Debug build compiled w/ clang",
|
"description": "Debug build compiled w/ clang",
|
||||||
"binaryDir": "${sourceDir}/build/clang/debug",
|
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "clang",
|
"CMAKE_C_COMPILER": "clang",
|
||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-debug-gcc",
|
"name": "sparkd-debug-gcc",
|
||||||
"inherits": "sparkd-debug",
|
"inherits": "sparkd-debug",
|
||||||
"displayName": "Debug (gcc)",
|
"displayName": "Sparkd (Debug | Gcc)",
|
||||||
"description": "Debug build compiled w/ gcc",
|
"description": "Debug build compiled w/ gcc",
|
||||||
"binaryDir": "${sourceDir}/build/gcc/debug",
|
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "gcc",
|
"CMAKE_C_COMPILER": "gcc",
|
||||||
"CMAKE_CXX_COMPILER": "g++",
|
"CMAKE_CXX_COMPILER": "g++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-clang",
|
"name": "sparkd-release-clang",
|
||||||
"inherits": "sparkd-release",
|
"inherits": "sparkd-release",
|
||||||
"displayName": "Release (clang)",
|
"displayName": "Sparkd (Release | Clang)",
|
||||||
"description": "Release build compiled w/ clang",
|
"description": "Release build compiled w/ clang",
|
||||||
"binaryDir": "${sourceDir}/build/clang/release",
|
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "clang",
|
"CMAKE_C_COMPILER": "clang",
|
||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-gcc",
|
"name": "sparkd-release-gcc",
|
||||||
"inherits": "sparkd-release",
|
"inherits": "sparkd-release",
|
||||||
"displayName": "Release (gcc)",
|
"displayName": "Sparkd (Release | Gcc)",
|
||||||
"description": "Release build compiled w/ gcc",
|
"description": "Release build compiled w/ gcc",
|
||||||
"binaryDir": "${sourceDir}/build/gcc/release",
|
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_C_COMPILER": "gcc",
|
"CMAKE_C_COMPILER": "gcc",
|
||||||
"CMAKE_CXX_COMPILER": "g++",
|
"CMAKE_CXX_COMPILER": "g++",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "spark-api-server-base",
|
|
||||||
"hidden": true,
|
|
||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
|
||||||
"cacheVariables": {
|
|
||||||
"BUILD_API_SERVER": {
|
|
||||||
"type": "bool",
|
|
||||||
"value": "ON"
|
|
||||||
},
|
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows",
|
"name": "spark-api-server-windows",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"displayName": "Spark Api Server",
|
"displayName": "Spark Api Server (Windows)",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"BUILD_API_SERVER": {
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
||||||
"type": "bool",
|
"SHARED_MEMORY_PREFIX": "spark_api_server"
|
||||||
"value": "ON"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-linux",
|
"name": "spark-api-server-linux",
|
||||||
"inherits": "sparkd",
|
"inherits": "sparkd",
|
||||||
"displayName": "Spark Api Server",
|
"displayName": "Spark Api Server (Linux)",
|
||||||
"generator": "Ninja Multi-Config",
|
"generator": "Ninja Multi-Config",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
},
|
},
|
||||||
"BUILD_API_SERVER": {
|
|
||||||
"type": "bool",
|
|
||||||
"value": "ON"
|
|
||||||
},
|
|
||||||
"BUILD_SPARKD": {
|
"BUILD_SPARKD": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
@@ -197,7 +181,11 @@
|
|||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
|
"EXPORT_MD_CONSUMER": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "ON"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -220,13 +208,62 @@
|
|||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite",
|
"name": "data-stream-suite-windows",
|
||||||
|
"condition": {
|
||||||
|
"type": "equals",
|
||||||
|
"lhs": "${hostSystemName}",
|
||||||
|
"rhs": "Windows"
|
||||||
|
},
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"displayName": "Data Stream Suite",
|
"displayName": "Data Stream Suite (Windows)",
|
||||||
"description": "Data Stream utils ported over from Edge",
|
"description": "Data Stream utils ported over from Edge",
|
||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
|
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
||||||
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
||||||
|
"SHARED_MEMORY_PREFIX": "data_stream"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"name": "data-stream-suite-linux",
|
||||||
|
"displayName": "Data Stream Suite (Linux)",
|
||||||
|
"description": "Data Stream utils ported over from Edge",
|
||||||
|
"condition": {
|
||||||
|
"type": "equals",
|
||||||
|
"lhs": "${hostSystemName}",
|
||||||
|
"rhs": "Linux"
|
||||||
|
},
|
||||||
|
"inherits": "sparkd",
|
||||||
|
"generator": "Ninja Multi-Config",
|
||||||
|
"cacheVariables": {
|
||||||
|
"APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
||||||
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "OFF"
|
||||||
|
},
|
||||||
|
"BUILD_SDK": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "OFF"
|
||||||
|
},
|
||||||
|
"BUILD_SPARKD": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "OFF"
|
||||||
|
},
|
||||||
|
"CMAKE_C_COMPILER": "clang",
|
||||||
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
||||||
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
|
"EXPORT_MD_CONSUMER": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
|
"SHARED_MEMORY_PREFIX": "data_stream"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
@@ -244,7 +281,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-debug",
|
"name": "spark-debug",
|
||||||
"displayName": "Debug",
|
"displayName": "Spark (Debug)",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"configurePreset": "spark",
|
"configurePreset": "spark",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
@@ -252,7 +289,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-release",
|
"name": "spark-release",
|
||||||
"displayName": "Release",
|
"displayName": "Spark (Release)",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"configurePreset": "spark",
|
"configurePreset": "spark",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
@@ -280,28 +317,28 @@
|
|||||||
{
|
{
|
||||||
"name": "sparkd-debug-clang",
|
"name": "sparkd-debug-clang",
|
||||||
"configurePreset": "sparkd-debug-clang",
|
"configurePreset": "sparkd-debug-clang",
|
||||||
"displayName": "Debug (clang)",
|
"displayName": "Sparkd (Debug | Clang)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-debug-gcc",
|
"name": "sparkd-debug-gcc",
|
||||||
"configurePreset": "sparkd-debug-gcc",
|
"configurePreset": "sparkd-debug-gcc",
|
||||||
"displayName": "Debug (gcc)",
|
"displayName": "Sparkd (Debug | Gcc)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-clang",
|
"name": "sparkd-release-clang",
|
||||||
"configurePreset": "sparkd-release-clang",
|
"configurePreset": "sparkd-release-clang",
|
||||||
"displayName": "Release (clang)",
|
"displayName": "Sparkd (Release | Clang)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-release-gcc",
|
"name": "sparkd-release-gcc",
|
||||||
"configurePreset": "sparkd-release-gcc",
|
"configurePreset": "sparkd-release-gcc",
|
||||||
"displayName": "Release (gcc)",
|
"displayName": "Sparkd (Release | Gcc)",
|
||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -318,14 +355,14 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows-debug",
|
"name": "spark-api-server-windows-debug",
|
||||||
"displayName": "Spark Api Server (Debug)",
|
"displayName": "Spark Api Server (Windows | Debug)",
|
||||||
"inherits": "spark-api-server-windows",
|
"inherits": "spark-api-server-windows",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-windows-release",
|
"name": "spark-api-server-windows-release",
|
||||||
"displayName": "Spark Api Server (Debug)",
|
"displayName": "Spark Api Server (Windows | Release)",
|
||||||
"inherits": "spark-api-server-windows",
|
"inherits": "spark-api-server-windows",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
},
|
},
|
||||||
@@ -344,14 +381,14 @@
|
|||||||
{
|
{
|
||||||
"name": "spark-api-server-linux-debug",
|
"name": "spark-api-server-linux-debug",
|
||||||
"inherits": "spark-api-server-linux",
|
"inherits": "spark-api-server-linux",
|
||||||
"displayName": "Spark Api Server (Debug)",
|
"displayName": "Spark Api Server (Linux | Debug)",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "spark-api-server-linux-release",
|
"name": "spark-api-server-linux-release",
|
||||||
"inherits": "spark-api-server-linux",
|
"inherits": "spark-api-server-linux",
|
||||||
"displayName": "Spark Api Server (Release)",
|
"displayName": "Spark Api Server (Linux | Release)",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -371,16 +408,30 @@
|
|||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite-debug",
|
"name": "data-stream-suite-windows-debug",
|
||||||
"displayName": "DataStreamSuite (Debug)",
|
"displayName": "Data Stream Suite (Windows | Debug)",
|
||||||
"configurePreset": "data-stream-suite",
|
"configurePreset": "data-stream-suite-windows",
|
||||||
"configuration": "Debug"
|
"configuration": "Debug"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "data-stream-suite-release",
|
"name": "data-stream-suite-windows-release",
|
||||||
"displayName": "DataStreamSuite (Release)",
|
"displayName": "Data Stream Suite (Windows | Release)",
|
||||||
"configurePreset": "data-stream-suite",
|
"configurePreset": "data-stream-suite-windows",
|
||||||
|
"configuration": "Release"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"name": "data-stream-suite-linux-debug",
|
||||||
|
"displayName": "Data Stream Suite (Linux | Debug)",
|
||||||
|
"configurePreset": "data-stream-suite-linux",
|
||||||
|
"configuration": "Debug"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"name": "data-stream-suite-linux-release",
|
||||||
|
"displayName": "Data Stream Suite (Linux | Release)",
|
||||||
|
"configurePreset": "data-stream-suite-linux",
|
||||||
"configuration": "Release"
|
"configuration": "Release"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
Submodule
+1
Submodule docker/linux added at 56c61231f3.
Submodule
+1
Submodule docker/windows added at de7d40c09d.
+1
-1
Submodule externals updated: cae5fd20da...8069cc893f.
File diff suppressed because it is too large.
Load diff
@@ -1,41 +1,22 @@
|
|||||||
using mbts.spark;
|
using System.Diagnostics;
|
||||||
|
|
||||||
using System;
|
namespace BCG.Utilities.General;
|
||||||
using System.Runtime;
|
|
||||||
|
|
||||||
namespace BCG.Utilities.General
|
/// <summary>
|
||||||
{
|
/// Summary description for Crono.
|
||||||
/// <summary>
|
/// </summary>
|
||||||
/// Summary description for Crono.
|
public class Crono {
|
||||||
/// </summary>
|
readonly Stopwatch myTimer = new();
|
||||||
public class Crono
|
|
||||||
{
|
|
||||||
public Crono()
|
|
||||||
{
|
|
||||||
myTimer = new HighPerformanceTimer();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void start()
|
public void start() =>
|
||||||
{myTimer.start();}
|
myTimer.Start();
|
||||||
|
|
||||||
public void stop()
|
public void stop() =>
|
||||||
{myTimer.stop();}
|
myTimer.Stop();
|
||||||
|
|
||||||
public TimeSpan time
|
public TimeSpan time =>
|
||||||
{
|
myTimer.Elapsed;
|
||||||
get
|
|
||||||
{
|
|
||||||
return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public double miliseconds
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{return myTimer.totalMilliseconds;}
|
|
||||||
}
|
|
||||||
|
|
||||||
private HighPerformanceTimer myTimer;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
public double miliseconds =>
|
||||||
|
myTimer.Elapsed.TotalMilliseconds;
|
||||||
}
|
}
|
||||||
@@ -1,192 +1,192 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
public enum RegistryRoot { HKCU, HKLM }
|
public enum RegistryRoot { HKCU, HKLM }
|
||||||
|
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class BlueRegistry
|
public class BlueRegistry
|
||||||
{
|
{
|
||||||
public const string BlueSubKey = @"Software\Blue Capital Group\";
|
public const string BlueSubKey = @"Software\Blue Capital Group\";
|
||||||
public static readonly string ApplicationPrefix;
|
public static readonly string ApplicationPrefix;
|
||||||
protected string myFullPath;
|
protected string myFullPath;
|
||||||
|
|
||||||
public BlueRegistry(string subPath)
|
public BlueRegistry(string subPath)
|
||||||
{
|
{
|
||||||
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
|
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
public BlueRegistry(BlueRegistry parent, string subPath)
|
public BlueRegistry(BlueRegistry parent, string subPath)
|
||||||
{
|
{
|
||||||
myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
|
myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
static BlueRegistry()
|
static BlueRegistry()
|
||||||
{
|
{
|
||||||
string app = "Applications";
|
string app = "Applications";
|
||||||
|
|
||||||
Assembly ass = Assembly.GetEntryAssembly();
|
Assembly ass = Assembly.GetEntryAssembly();
|
||||||
if (ass != null)
|
if (ass != null)
|
||||||
{
|
{
|
||||||
app = ass.GetName().Name;
|
app = ass.GetName().Name;
|
||||||
|
|
||||||
object[] attrs = ass.GetCustomAttributes(
|
object[] attrs = ass.GetCustomAttributes(
|
||||||
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
|
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
|
||||||
|
|
||||||
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
|
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
|
||||||
{
|
{
|
||||||
app = ((AssemblyTitleAttribute)attrs[0]).Title;
|
app = ((AssemblyTitleAttribute)attrs[0]).Title;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
|
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
|
||||||
}
|
}
|
||||||
|
|
||||||
public object getValue(string name)
|
public object getValue(string name)
|
||||||
{
|
{
|
||||||
return getValue(RegistryRoot.HKCU, name);
|
return getValue(RegistryRoot.HKCU, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
public object getValue(string name, object defaultValue)
|
public object getValue(string name, object defaultValue)
|
||||||
{
|
{
|
||||||
return getValue(RegistryRoot.HKCU, name, defaultValue);
|
return getValue(RegistryRoot.HKCU, name, defaultValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setValue(string name, object value)
|
public void setValue(string name, object value)
|
||||||
{
|
{
|
||||||
setValue(RegistryRoot.HKCU, name, value);
|
setValue(RegistryRoot.HKCU, name, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteValue(string name)
|
public void deleteValue(string name)
|
||||||
{
|
{
|
||||||
deleteValue(RegistryRoot.HKCU, name);
|
deleteValue(RegistryRoot.HKCU, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
private RegistryKey myGetRegistryKey(RegistryRoot root)
|
private RegistryKey myGetRegistryKey(RegistryRoot root)
|
||||||
{
|
{
|
||||||
if (root == RegistryRoot.HKCU)
|
if (root == RegistryRoot.HKCU)
|
||||||
return Registry.CurrentUser.CreateSubKey(myFullPath);
|
return Registry.CurrentUser.CreateSubKey(myFullPath);
|
||||||
else
|
else
|
||||||
return Registry.LocalMachine.CreateSubKey(myFullPath);
|
return Registry.LocalMachine.CreateSubKey(myFullPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
#region Generic Methods
|
#region Generic Methods
|
||||||
internal virtual void setValue(RegistryRoot root, string name, object value)
|
public virtual void setValue(RegistryRoot root, string name, object value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
rk.SetValue(name, value);
|
rk.SetValue(name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual object getValue(RegistryRoot root, string name)
|
internal virtual object getValue(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return rk.GetValue(name);
|
return rk.GetValue(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual void deleteValue(RegistryRoot root, string name)
|
internal virtual void deleteValue(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
rk.DeleteValue(name);
|
rk.DeleteValue(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
|
public virtual object getValue(RegistryRoot root, string name, object defaultValue)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return rk.GetValue(name, defaultValue);
|
return rk.GetValue(name, defaultValue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Double
|
#region Double
|
||||||
internal virtual void setDouble(RegistryRoot root, string name, double value)
|
internal virtual void setDouble(RegistryRoot root, string name, double value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
DoubleRegistryHandler.setValue(rk, name, value);
|
DoubleRegistryHandler.setValue(rk, name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual double getDouble(RegistryRoot root, string name)
|
internal virtual double getDouble(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return DoubleRegistryHandler.getValue(rk, name);
|
return DoubleRegistryHandler.getValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Enum
|
#region Enum
|
||||||
internal virtual void setEnum(RegistryRoot root, string name, Enum value)
|
internal virtual void setEnum(RegistryRoot root, string name, Enum value)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
EnumRegistryHandler.setValue(rk, name, value);
|
EnumRegistryHandler.setValue(rk, name, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
|
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return EnumRegistryHandler.getValue(rk, name, enumType);
|
return EnumRegistryHandler.getValue(rk, name, enumType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Array
|
#region Array
|
||||||
internal virtual void setArray(RegistryRoot root, string name, Array array)
|
internal virtual void setArray(RegistryRoot root, string name, Array array)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
ArrayRegistryHandler.setValue(rk, name, array);
|
ArrayRegistryHandler.setValue(rk, name, array);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
|
internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return ArrayRegistryHandler.getValue(rk, name, elementType);
|
return ArrayRegistryHandler.getValue(rk, name, elementType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region SortedList
|
#region SortedList
|
||||||
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
|
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
SortedListRegistryHandler.setValue(rk, name, slist);
|
SortedListRegistryHandler.setValue(rk, name, slist);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual SortedList getSortedList(RegistryRoot root, string name)
|
internal virtual SortedList getSortedList(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
return SortedListRegistryHandler.getValue(rk, name);
|
return SortedListRegistryHandler.getValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal virtual void deleteSortedList(RegistryRoot root, string name)
|
internal virtual void deleteSortedList(RegistryRoot root, string name)
|
||||||
{
|
{
|
||||||
using (RegistryKey rk = myGetRegistryKey(root))
|
using (RegistryKey rk = myGetRegistryKey(root))
|
||||||
{
|
{
|
||||||
SortedListRegistryHandler.deleteValue(rk, name);
|
SortedListRegistryHandler.deleteValue(rk, name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,10 +1,9 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0-windows</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<UseWindowsForms>true</UseWindowsForms>
|
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,26 @@
|
|||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Reactive.Linq;
|
||||||
|
|
||||||
|
public static class DataStreamExtensions {
|
||||||
|
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
||||||
|
source as ViewStream ?? new ObservableViewStream(source);
|
||||||
|
|
||||||
|
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
||||||
|
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
||||||
|
|
||||||
|
IDisposable Subscribe(IObserver<DataPacket> observer) {
|
||||||
|
source.onChange += OnChange;
|
||||||
|
return Disposable.Create(() => source.onChange -= OnChange);
|
||||||
|
|
||||||
|
void OnChange(object? newValue) =>
|
||||||
|
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
|
||||||
|
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
|
||||||
|
public class PandasScriptStream : ViewStream {
|
||||||
|
public PandasScriptStream(ViewStream vs, string file)
|
||||||
|
: this(vs, file, null) { }
|
||||||
|
|
||||||
|
public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
|
||||||
|
: this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
|
public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
|
||||||
|
mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
|
||||||
|
|
||||||
|
string readFile() {
|
||||||
|
try {
|
||||||
|
return File.ReadAllText(file);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
Log.Gen.error($"PandasScriptStream error reading file: {ex}");
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,62 +1,130 @@
|
|||||||
// using System;
|
namespace Bts.DataStream;
|
||||||
// using System.Diagnostics;
|
|
||||||
|
using BCG;
|
||||||
// using BCG;
|
using BCG.Extensions;
|
||||||
// using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
|
using bts;
|
||||||
// namespace bts {
|
using bts.collections;
|
||||||
|
|
||||||
// public class PandasStream : ViewStream, IWorker {
|
using Bts.DataStreamCore;
|
||||||
// //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
// public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
|
|
||||||
|
using System;
|
||||||
// }
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics;
|
||||||
// public PandasStream(ViewStream vs, string pandas_code) :
|
using System.IO;
|
||||||
// this(vs, ViewStreamDatabase.instance, pandas_code) {
|
using System.Linq;
|
||||||
// }
|
using System.Runtime.CompilerServices;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
public class PandasStream : ViewStream, IWorker {
|
||||||
|
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
||||||
// // string python_code = @"
|
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
||||||
// // import winsound
|
readonly string pythonExe;
|
||||||
// // frequency = 2500 # Set frequency (in Hz)
|
readonly ViewStreamDatabase vsd;
|
||||||
// // duration = 1000 # Set duration (in milliseconds)
|
readonly CSVSaverStream inputCsvStream;
|
||||||
// // winsound.Beep(frequency, duration)
|
readonly Lazy<string> pythonCode;
|
||||||
// // ";
|
IDictionary<int, IBlueRow>? lastRowMap = null;
|
||||||
// string python_code = @"
|
int inputVersion = 0;
|
||||||
// file_path = \"C:\\Temp\\test2.txt\\" # r prefix for raw string to avoid escape characters
|
int outputVersion = 0;
|
||||||
// with open(file_path, \"w\") as f:
|
DateTime lastFireTime = DateTime.MinValue;
|
||||||
// f.write(\"Hello, world!\n\")
|
TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
|
||||||
// ";
|
IDisposable? dispose;
|
||||||
|
|
||||||
// execute_python_code(python_code);
|
public PandasStream(ViewStream vs, string dataFrameExpression)
|
||||||
// }
|
: this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
// static void execute_python_code(string python_code) {
|
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
|
||||||
// ProcessStartInfo start = new ProcessStartInfo();
|
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
|
||||||
// start.FileName = "python"; // Python executable
|
this.pythonExe = pythonExe;
|
||||||
// start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
|
this.vsd = vsd;
|
||||||
// start.UseShellExecute = false;
|
// This should fire an update when it writes to the output file
|
||||||
// start.RedirectStandardOutput = true;
|
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
|
||||||
|
pythonCode = new(getPythonCode);
|
||||||
// using (Process process = Process.Start(start)) {
|
|
||||||
// using (System.IO.StreamReader reader = process.StandardOutput) {
|
string getPythonCode() {
|
||||||
// string result = reader.ReadToEnd();
|
var imports =
|
||||||
// Console.Write(result);
|
@"import numpy as np
|
||||||
// }
|
import pandas as pd
|
||||||
// }
|
import sys
|
||||||
// }
|
";
|
||||||
|
var bd = new StringBuilder(imports);
|
||||||
// protected override void myOnLastSubscriber() {
|
bd.AppendLine($"df=pd.read_csv('{inputFile}')");
|
||||||
// }
|
bd.AppendLine(dataFrameExpression);
|
||||||
|
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
|
||||||
// public void doWork() {
|
return bd.ToString();
|
||||||
|
}
|
||||||
// }
|
}
|
||||||
|
|
||||||
// }
|
protected override void myOnFirstSubscriber() {
|
||||||
// }
|
vsd.addWorker(this);
|
||||||
|
inputCsvStream.onChange += onInputChanged;
|
||||||
|
this.dispose = Disposable.Create(dispose);
|
||||||
|
|
||||||
|
void onInputChanged(object newValue) =>
|
||||||
|
inputVersion++;
|
||||||
|
|
||||||
|
void dispose() {
|
||||||
|
inputCsvStream.onChange -= onInputChanged;
|
||||||
|
vsd.removeWorker(this);
|
||||||
|
deleteFile(inputFile);
|
||||||
|
deleteFile(outputFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
|
||||||
|
if (!File.Exists(file)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
File.Delete(file);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() =>
|
||||||
|
Disposable.dispose(ref dispose);
|
||||||
|
|
||||||
|
static void executePythonCode(string pythonCode, string? pythonExe = null) {
|
||||||
|
pythonExe ??= "python";
|
||||||
|
var start = new ProcessStartInfo {
|
||||||
|
FileName = pythonExe,
|
||||||
|
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
|
||||||
|
UseShellExecute = false,
|
||||||
|
RedirectStandardOutput = true
|
||||||
|
};
|
||||||
|
using var process = Process.Start(start)
|
||||||
|
?? throw new InvalidOperationException("Failed to launch python");
|
||||||
|
process.WaitForExit();
|
||||||
|
}
|
||||||
|
|
||||||
|
void IWorker.doWork() {
|
||||||
|
if (outputVersion == inputVersion) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
var now = DateTime.Now;
|
||||||
|
var elapsedSinceLastFire = now - lastFireTime;
|
||||||
|
if (elapsedSinceLastFire < fireThrottle) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
outputVersion = inputVersion;
|
||||||
|
executePythonCode(pythonCode.Value, pythonExe);
|
||||||
|
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
|
||||||
|
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
|
||||||
|
var comp = BlueRowEqualityComparer.Instance;
|
||||||
|
var deletedRows = new BlueTable(schema);
|
||||||
|
var addedRows = new BlueTable(schema);
|
||||||
|
if (lastRowMap is not null) {
|
||||||
|
deletedRows.SetItems(lastRowMap.Where(oldRow =>
|
||||||
|
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
|
||||||
|
addedRows.SetItems(rowMap.Where(newRow =>
|
||||||
|
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
|
||||||
|
}
|
||||||
|
lastRowMap = rowMap;
|
||||||
|
lastFireTime = DateTime.Now;
|
||||||
|
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -39,8 +39,8 @@ namespace BCG {
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
||||||
var server_address = new BlutoSubAddress(_ip, _port);
|
var server_rci = new RemoteConnectionInfo(_ip, _port);
|
||||||
bts.AsyncTCPConnector_APM.connect_to(server_address, this);
|
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
|
||||||
|
readonly IObservable<DataPacket> source;
|
||||||
|
IDisposable? subscription;
|
||||||
|
|
||||||
|
public ObservableViewStream(IObservable<DataPacket> source) =>
|
||||||
|
this.source = source;
|
||||||
|
|
||||||
|
public void Dispose() {
|
||||||
|
GC.SuppressFinalize(this);
|
||||||
|
if (subscription is not null) {
|
||||||
|
subscription.Dispose();
|
||||||
|
subscription = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
Log.isnull(subscription);
|
||||||
|
subscription = source.Subscribe(myFireEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
Log.notnull(subscription);
|
||||||
|
subscription.Dispose();
|
||||||
|
subscription = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
|
||||||
|
source.Subscribe(observer);
|
||||||
|
}
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
namespace BCG
|
|
||||||
{
|
|
||||||
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
|
||||||
[DataStreamUserRespresentation("add_to_cloud")]
|
|
||||||
public class AddToCloudStream : ViewStream
|
|
||||||
{
|
|
||||||
private readonly string _name, _stream_exp;
|
|
||||||
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
|
|
||||||
public AddToCloudStream(string name, params string[] lines)
|
|
||||||
: this(name, 0, lines)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
public AddToCloudStream(string name, int persist, params string[] lines)
|
|
||||||
{
|
|
||||||
_name = name;
|
|
||||||
_stream_exp = getExpression(lines);
|
|
||||||
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
|
|
||||||
mySetRealSource(new TableStream("cloud_streams"));
|
|
||||||
}
|
|
||||||
|
|
||||||
static string getExpression(params string[] lines) {
|
|
||||||
for (int i = 0; i < lines.Length; i++) {
|
|
||||||
// unescape quotes
|
|
||||||
lines[i] = lines[i].Replace("\\\"", "\"");
|
|
||||||
}
|
|
||||||
return string.Join("\n", lines);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,31 +1,22 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
using mbts.config;
|
|
||||||
using bts.utils;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using Bts.DataStreamExpressions;
|
using Bts.DataStreamExpressions;
|
||||||
using System.Collections.Generic;
|
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
using bts;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
// listens to the requests posted on "cloud_streams" request table, and
|
// listens to the requests posted on "cloud_streams" request table, and
|
||||||
// if unfulfilled starts a ViewPubsliher
|
// if unfulfilled starts a ViewPubsliher
|
||||||
public class CloudStreamManager : IWorker
|
public class RemoteQueryManager : IWorker {
|
||||||
{
|
public class StreamPubUnit {
|
||||||
public class StreamPubUnit
|
|
||||||
{
|
|
||||||
public string name;
|
public string name;
|
||||||
public string stream_expression;
|
public string stream_expression;
|
||||||
public ViewStream stream;
|
public ViewStream stream;
|
||||||
public bool persist;
|
public bool persist;
|
||||||
public ViewPublisher view_publisher;
|
public ViewPublisher view_publisher;
|
||||||
|
|
||||||
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
|
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
|
||||||
{
|
|
||||||
name = name_;
|
name = name_;
|
||||||
stream_expression = stream_expression_;
|
stream_expression = stream_expression_;
|
||||||
stream = stream_;
|
stream = stream_;
|
||||||
@@ -33,7 +24,7 @@ namespace BCG
|
|||||||
view_publisher = view_publisher_;
|
view_publisher = view_publisher_;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
static ViewStream build_stream(string stream_expression)
|
static ViewStream? build_stream(string stream_expression)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@@ -41,9 +32,7 @@ namespace BCG
|
|||||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||||
return rv;
|
return rv;
|
||||||
|
|
||||||
}
|
} catch (Exception ex) {
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -52,34 +41,37 @@ namespace BCG
|
|||||||
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
||||||
private ViewStream _stream = new TableStream("cloud_streams");
|
private ViewStream _stream = new TableStream("cloud_streams");
|
||||||
private PubTableBroker _broker = new PubTableBroker();
|
private PubTableBroker _broker = new PubTableBroker();
|
||||||
|
private ManagedSubscriber _sub = ManagedSubscriber.instance;
|
||||||
|
|
||||||
public CloudStreamManager()
|
|
||||||
{
|
public RemoteQueryManager() {
|
||||||
_stream.onChange += _on_str;
|
_stream.onChange += _on_str;
|
||||||
|
_sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
|
||||||
|
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
|
||||||
|
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
private void _on_str(object obj)
|
private void _on_str(object obj)
|
||||||
{
|
{
|
||||||
DataPacket dp = obj as DataPacket;
|
if (obj is not DataPacket dp) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
foreach (IBlueRow r in dp.newRows)
|
foreach (IBlueRow r in dp.newRows)
|
||||||
{
|
{
|
||||||
string name = r.get<string>("name");
|
string name = r.get<string>("name");
|
||||||
bool persist = r.get<bool>("persist");
|
bool persist = r.get<bool>("persist");
|
||||||
string stream_exp = r.get<string>("stream_exp");
|
string stream_exp = r.get<string>("stream_exp");
|
||||||
if (!_stream_pub_map.ContainsKey(name))
|
if (!_stream_pub_map.ContainsKey(name)) {
|
||||||
{
|
_spawn_remote_stream(name, stream_exp, persist);
|
||||||
_assign(name, stream_exp, persist);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _assign(string name, string stream_expression, bool persist)
|
private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
|
||||||
{
|
|
||||||
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
||||||
ViewStream stream = build_stream(stream_expression);
|
ViewStream stream = build_stream(stream_expression);
|
||||||
_expose_to_ws(name, stream);
|
_expose_to_ws(name, stream);
|
||||||
if (persist)
|
if (persist) {
|
||||||
{
|
|
||||||
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
||||||
}
|
}
|
||||||
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
||||||
@@ -87,14 +79,12 @@ namespace BCG
|
|||||||
_stream_pub_map.Add(name, u);
|
_stream_pub_map.Add(name, u);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _expose_to_ws(string name, ViewStream stream)
|
private void _expose_to_ws(string name, ViewStream stream) {
|
||||||
{
|
|
||||||
// TODO: john
|
// TODO: john
|
||||||
}
|
}
|
||||||
|
|
||||||
public void doWork()
|
public void doWork() {
|
||||||
{
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace BCG {
|
||||||
|
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
||||||
|
[DataStreamUserRespresentation("remoteQuery")]
|
||||||
|
public class RemoteQueryStream : ViewStream {
|
||||||
|
private readonly string _name, _stream_exp;
|
||||||
|
private PublisherCore _publisher_core = new PublisherCore();
|
||||||
|
private ViewStream _source_str;
|
||||||
|
|
||||||
|
public RemoteQueryStream(params string[] lines) {
|
||||||
|
_stream_exp = get_expression(lines);
|
||||||
|
_name = Guid.NewGuid().ToString();
|
||||||
|
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
|
||||||
|
//_name.Replace("-", "_");
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
|
||||||
|
byte[] rem_req_bytes = rem_req.serialize();
|
||||||
|
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
|
||||||
|
_source_str = new TableStream(_name);
|
||||||
|
_source_str.onChange += new ChangeDelegate(_handle_data);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
//maybe there should be a way to turn remote requests off ?
|
||||||
|
_source_str.onChange -= new ChangeDelegate(_handle_data);
|
||||||
|
}
|
||||||
|
|
||||||
|
static string get_expression(params string[] lines) {
|
||||||
|
for (int i = 0; i < lines.Length; i++) {
|
||||||
|
// unescape quotes
|
||||||
|
lines[i] = lines[i].Replace("\\\"", "\"");
|
||||||
|
}
|
||||||
|
return string.Join("\n", lines);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _handle_data(object o) {
|
||||||
|
myFireEvent(o);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
namespace bts {
|
||||||
|
using System.IO;
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System.IO;
|
||||||
|
using System.Threading.Tasks.Dataflow;
|
||||||
|
|
||||||
|
public class RemoteRequest {
|
||||||
|
public readonly string stream_expression;
|
||||||
|
public readonly string published_table_name;
|
||||||
|
public RemoteRequest(string stream_expression_, string published_table_name_) {
|
||||||
|
stream_expression = stream_expression_;
|
||||||
|
published_table_name = published_table_name_;
|
||||||
|
}
|
||||||
|
public byte[] serialize() {
|
||||||
|
var str = new MemoryStream();
|
||||||
|
var bw = new BinaryWriter(str);
|
||||||
|
bw.write_string_cpp(stream_expression);
|
||||||
|
bw.write_string_cpp(published_table_name);
|
||||||
|
return str.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
|
||||||
|
BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
|
||||||
|
string stream_expression_ = br.read_string_cpp();
|
||||||
|
string published_table_name_ = br.read_string_cpp();
|
||||||
|
return new RemoteRequest(stream_expression_, published_table_name_);
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,59 +1,54 @@
|
|||||||
using BCG;
|
// using BCG;
|
||||||
|
|
||||||
using System.Linq;
|
// using bts;
|
||||||
|
|
||||||
namespace BCG
|
// using System.Linq;
|
||||||
{
|
|
||||||
public class StreamOffloadRequestor
|
|
||||||
{
|
|
||||||
public class OffloadOp
|
|
||||||
{
|
|
||||||
string name;
|
|
||||||
string stream_exp;
|
|
||||||
}
|
|
||||||
private IBlueTableSchema _schema;
|
|
||||||
private IBlueRowCollection _rows;
|
|
||||||
private PubTableBroker _broker = new PubTableBroker();
|
|
||||||
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
|
||||||
public StreamOffloadRequestor()
|
|
||||||
{
|
|
||||||
_schema = new BlueTableSchema("cloud_streams",
|
|
||||||
new ColumnInfo("id", 0, typeof(int)),
|
|
||||||
new ColumnInfo("name", 1, typeof(string)),
|
|
||||||
new ColumnInfo("persist", 2, typeof(bool)),
|
|
||||||
new ColumnInfo("stream_exp", 3, typeof(string)));
|
|
||||||
_existing_offloads_vs.onChange += _on_str;
|
|
||||||
}
|
|
||||||
private void _on_str(object obj)
|
|
||||||
{
|
|
||||||
DataPacket dp = obj as DataPacket;
|
|
||||||
_rows = dp.newRows;
|
|
||||||
// this is needed so the broker knows the state of the table
|
|
||||||
_broker.createTable(_rows);
|
|
||||||
}
|
|
||||||
public void add_to_cloud(string name, string stream_exp, bool persist)
|
|
||||||
{
|
|
||||||
// check if it exists
|
|
||||||
if (_rows == null)
|
|
||||||
{
|
|
||||||
BlueTable table = new BlueTable(_schema);
|
|
||||||
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
|
||||||
table.addRow(nr);
|
|
||||||
_broker.createTable(table);
|
|
||||||
// create table
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (exists(name))
|
|
||||||
return;
|
|
||||||
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
|
||||||
_broker.addRow(nr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bool exists(string name)
|
// namespace BCG {
|
||||||
{
|
// // used on the client
|
||||||
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
// public class StreamOffloadRequestor {
|
||||||
}
|
// public class OffloadOp {
|
||||||
}
|
// string name;
|
||||||
}
|
// string stream_exp;
|
||||||
|
// }
|
||||||
|
// private IBlueTableSchema _schema;
|
||||||
|
// private IBlueRowCollection _rows;
|
||||||
|
// private PubTableBroker _broker = new PubTableBroker();
|
||||||
|
// ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
||||||
|
// public StreamOffloadRequestor() {
|
||||||
|
// _schema = new BlueTableSchema("cloud_streams",
|
||||||
|
// new ColumnInfo("id", 0, typeof(int)),
|
||||||
|
// new ColumnInfo("name", 1, typeof(string)),
|
||||||
|
// new ColumnInfo("persist", 2, typeof(bool)),
|
||||||
|
// new ColumnInfo("stream_exp", 3, typeof(string)));
|
||||||
|
// _existing_offloads_vs.onChange += _on_str;
|
||||||
|
// }
|
||||||
|
// private void _on_str(object obj) {
|
||||||
|
// DataPacket dp = obj as DataPacket;
|
||||||
|
// _rows = dp.newRows;
|
||||||
|
// // this is needed so the broker knows the state of the table
|
||||||
|
// _broker.create_table(_rows);
|
||||||
|
// }
|
||||||
|
// public void add_to_cloud(string name, string stream_exp, bool persist) {
|
||||||
|
// // check if it exists
|
||||||
|
// if (_rows == null) {
|
||||||
|
// BlueTable table = new BlueTable(_schema);
|
||||||
|
// IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
||||||
|
// table.addRow(nr);
|
||||||
|
// _broker.create_table(table);
|
||||||
|
// // create table
|
||||||
|
// } else {
|
||||||
|
// if (exists(name))
|
||||||
|
// return;
|
||||||
|
// IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
||||||
|
// // TODO: use make_batch_changes instead
|
||||||
|
// Log.assert(false, "Fix addRow");
|
||||||
|
// //_broker.addRow(nr);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// bool exists(string name) {
|
||||||
|
// return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,17 +1,14 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
internal class AddColumnsRow : IBlueRow {
|
||||||
internal class AddColumnsRow : IBlueRow
|
|
||||||
{
|
|
||||||
private IBlueTableSchema mySchema;
|
private IBlueTableSchema mySchema;
|
||||||
private IBlueRow myRow;
|
private IBlueRow myRow;
|
||||||
//private object[] myCachedValues;
|
//private object[] myCachedValues;
|
||||||
|
|
||||||
#region Constructors
|
#region Constructors
|
||||||
public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
|
public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
|
||||||
{
|
|
||||||
mySchema = schema;
|
mySchema = schema;
|
||||||
myRow = row;
|
myRow = row;
|
||||||
//myCachedValues = new object[schema.columnNames.Length];
|
//myCachedValues = new object[schema.columnNames.Length];
|
||||||
@@ -19,85 +16,78 @@ namespace BCG
|
|||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IBlueRow members
|
#region IBlueRow members
|
||||||
public IBlueTableSchema schema{get{return mySchema;}}
|
public IBlueTableSchema schema { get { return mySchema; } }
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region ICollection members
|
#region ICollection members
|
||||||
public int Count {get{return mySchema.columnNames.Length;}}
|
public int Count { get { return mySchema.columnNames.Length; } }
|
||||||
|
|
||||||
public bool IsSynchronized {get{return false;}}
|
public bool IsSynchronized { get { return false; } }
|
||||||
|
|
||||||
public object SyncRoot {get{return this;}}
|
public object SyncRoot { get { return this; } }
|
||||||
|
|
||||||
public void CopyTo (Array array, int index)
|
public void CopyTo(Array array, int index) {
|
||||||
{
|
|
||||||
int i = index;
|
int i = index;
|
||||||
foreach (object o in this as IEnumerable)
|
foreach (object o in this as IEnumerable)
|
||||||
array.SetValue (o, i++);
|
array.SetValue(o, i++);
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IDictionary members
|
#region IDictionary members
|
||||||
public bool IsFixedSize {get{return true;}}
|
public bool IsFixedSize { get { return true; } }
|
||||||
|
|
||||||
public bool IsReadOnly {get{return true;}}
|
public bool IsReadOnly { get { return true; } }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The key can either be the string corrsponding to the column name or
|
/// The key can either be the string corrsponding to the column name or
|
||||||
/// the order of the column
|
/// the order of the column
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public object this [object key]
|
public object this[object key] {
|
||||||
{
|
get {
|
||||||
get
|
|
||||||
{
|
|
||||||
if (key is string)
|
if (key is string)
|
||||||
return this[(string)key];
|
return this[(string)key];
|
||||||
return this[(int)key];
|
return this[(int)key];
|
||||||
}
|
}
|
||||||
set {throw new NotSupportedException ("The IBlueRow is readonly");}
|
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
}
|
}
|
||||||
|
|
||||||
public object this[int index]
|
public object this[int index] {
|
||||||
{
|
get {
|
||||||
get
|
|
||||||
{
|
|
||||||
//if (myCachedValues[index] == null)
|
//if (myCachedValues[index] == null)
|
||||||
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
|
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
|
||||||
//return myCachedValues[index];
|
//return myCachedValues[index];
|
||||||
return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
|
return mySchema[index].isConstant ? DBNull.Value : myRow[index];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public object this[string colName] {get{return this[mySchema[colName].colNo];}}
|
public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
|
||||||
|
|
||||||
public ICollection Keys{get{return mySchema.columnNames;}}
|
public ICollection Keys { get { return mySchema.columnNames; } }
|
||||||
|
|
||||||
public ICollection Values
|
public ICollection Values {
|
||||||
{
|
get {
|
||||||
get
|
|
||||||
{
|
|
||||||
object[] retVal = new object[mySchema.columnNames.Length];
|
object[] retVal = new object[mySchema.columnNames.Length];
|
||||||
CopyTo (retVal, 0);
|
CopyTo(retVal, 0);
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
public bool Contains (object key){return mySchema[(string)key] != null;}
|
public bool Contains(object key) { return mySchema[(string)key] != null; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
||||||
/// iterate through the values, typecast this into an IEnumerable
|
/// iterate through the values, typecast this into an IEnumerable
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
|
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
|
|
||||||
|
|
||||||
public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
||||||
|
|
||||||
|
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,34 @@
|
|||||||
namespace BCG;
|
using System;
|
||||||
|
|
||||||
public class AddColumnsStream : MapStream {
|
namespace BCG {
|
||||||
public AddColumnsStream(ViewStream vs, string columnDef)
|
public class AddColumnsStream : ViewStream {
|
||||||
: base(vs, new ColumnAdder(columnDef)) { }
|
public static Tuple<string, Type, object> parse_exp(string exp) {
|
||||||
}
|
string[] toks = exp.Split(':');
|
||||||
|
string col_name = toks[0];
|
||||||
|
string col_value = toks[1];
|
||||||
|
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
|
||||||
|
return Tuple.Create(col_name, tup.Item1, tup.Item2);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Tuple<string, Type, object> _new_col;
|
||||||
|
private IBlueTableSchema _out_schema;
|
||||||
|
|
||||||
|
public AddColumnsStream(ViewStream vs, string col_def) {
|
||||||
|
_new_col = parse_exp(col_def);
|
||||||
|
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
|
||||||
|
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
|
||||||
|
if (_out_schema == null)
|
||||||
|
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
|
||||||
|
return _out_schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
private IBlueRow _map(IBlueRow r) {
|
||||||
|
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
|
||||||
|
rv.setValue(_new_col.Item1, _new_col.Item3);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,23 +1,24 @@
|
|||||||
|
using bts;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Provides a stream that with distinct rows with respect to the columns
|
/// Provides a stream that with distinct rows with respect to the columns
|
||||||
/// the user specifies in the argument
|
/// the user specifies in the argument
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[DataStreamUserRespresentation ("distinct")]
|
[DataStreamUserRespresentation("distinct")]
|
||||||
public class DistinctRowStream : ViewStream
|
public class DistinctRowStream : ViewStream {
|
||||||
{
|
public DistinctRowStream(ViewStream stream, params string[] cols) {
|
||||||
public DistinctRowStream (ViewStream stream, params string[] cols)
|
var g_by_eval = new DistinctRowEvaluator(cols);
|
||||||
{
|
ViewStream r = new GroupByStream(stream, g_by_eval);
|
||||||
ArrayList groupByCols = new ArrayList ();
|
var proj_exp = String.Empty;
|
||||||
foreach (string col in cols)
|
foreach (var c in cols)
|
||||||
foreach (string c in col.Split (' '))
|
proj_exp += $"{c} ";
|
||||||
groupByCols.Add (c);
|
proj_exp = proj_exp.Trim();
|
||||||
mySetRealSource (new GroupByStream (stream,
|
r = new NewProjectStream(r, proj_exp);
|
||||||
new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
|
mySetRealSource(r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
// using System;
|
|
||||||
// using System.Collections.Generic;
|
|
||||||
// using System.Text;
|
|
||||||
// using System.IO;
|
|
||||||
|
|
||||||
// using System.Runtime.Serialization.Formatters.Binary;
|
|
||||||
|
|
||||||
// namespace BCG
|
|
||||||
// {
|
|
||||||
// public class DumpLoaderStream : ViewStream, ISideEffectStream
|
|
||||||
// {
|
|
||||||
// //wtf
|
|
||||||
// //public class Exception : System.Exception
|
|
||||||
// //{public Exception(string msg) : base(msg) { }}
|
|
||||||
|
|
||||||
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
|
|
||||||
// {
|
|
||||||
// mySetRealSource(
|
|
||||||
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
|
||||||
// ViewStreamDatabase.instance));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public DumpLoaderStream(string fileName)
|
|
||||||
// {
|
|
||||||
// if (fileName == null)
|
|
||||||
// throw new LoaderStream.Exception("File name cannot be null");
|
|
||||||
|
|
||||||
// for (int i = 0; i < 5; i++)
|
|
||||||
// try
|
|
||||||
// {
|
|
||||||
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
|
||||||
// {
|
|
||||||
// IClientDeserializer deser = new ClientUniversalDeserializer();
|
|
||||||
// byte[] the_ops;
|
|
||||||
// Array the_rows;
|
|
||||||
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
|
|
||||||
|
|
||||||
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
|
|
||||||
// }
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// catch (IOException)
|
|
||||||
// {
|
|
||||||
// System.Threading.Thread.Sleep(50);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,48 +1,25 @@
|
|||||||
using System;
|
using BCG;
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Text;
|
|
||||||
using System.IO;
|
|
||||||
|
|
||||||
using System.Runtime.Serialization.Formatters.Binary;
|
namespace bts {
|
||||||
|
// This is the other side of the SaverStream, its "loads" the stream and fires once
|
||||||
namespace BCG
|
|
||||||
{
|
|
||||||
[DataStreamUserRespresentation("loader")]
|
[DataStreamUserRespresentation("loader")]
|
||||||
public class LoaderStream : ViewStream, ISideEffectStream
|
public class LoaderStream : ViewStream {
|
||||||
{
|
|
||||||
public class Exception : System.Exception
|
|
||||||
{public Exception(string msg) : base(msg) { }}
|
|
||||||
|
|
||||||
public LoaderStream(string fileName, int secondsBetweenPolls)
|
private string _filename;
|
||||||
{
|
private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
|
||||||
mySetRealSource(
|
|
||||||
new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
|
||||||
ViewStreamDatabase.instance));
|
public LoaderStream(string filename) {
|
||||||
|
_filename = filename;
|
||||||
}
|
}
|
||||||
|
|
||||||
public LoaderStream(string fileName)
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
|
||||||
if (fileName == null)
|
StreamIOMessage msg = _deser.deserialize(ser_msg);
|
||||||
throw new LoaderStream.Exception("File name cannot be null");
|
Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
|
||||||
|
IBlueRowCollection rows = msg.create_message.rows;
|
||||||
for (int i = 0; i < 5; i++)
|
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
|
||||||
try
|
myFireEvent(dp);
|
||||||
{
|
|
||||||
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
|
||||||
{
|
|
||||||
BinaryFormatter bf = new BinaryFormatter();
|
|
||||||
#pragma warning disable SYSLIB0011
|
|
||||||
BlueTable bt = (BlueTable)bf.Deserialize(fs);
|
|
||||||
#pragma warning restore SYSLIB0011
|
|
||||||
mySetRealSource(new BlueRowCollectionStaticStream(bt));
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
catch (IOException)
|
|
||||||
{
|
|
||||||
System.Threading.Thread.Sleep(50);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,9 +5,11 @@ using System.Text;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
|
using BCG;
|
||||||
|
|
||||||
namespace BCG {
|
namespace bts {
|
||||||
// Writes all the udpates to a stream in a file.
|
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
|
||||||
|
// which just writes one update, as a create)
|
||||||
// File naming:
|
// File naming:
|
||||||
// token_epoch_yyyyMMdd_HHmmss
|
// token_epoch_yyyyMMdd_HHmmss
|
||||||
// File location: directory(passed)\token\
|
// File location: directory(passed)\token\
|
||||||
@@ -24,7 +26,7 @@ namespace BCG {
|
|||||||
private ViewStream _source_stream;
|
private ViewStream _source_stream;
|
||||||
private string _token;
|
private string _token;
|
||||||
private FileStream _out_fs;
|
private FileStream _out_fs;
|
||||||
private IBlueDataSerializer _serializer;
|
private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
|
||||||
private string _file_name = String.Empty;
|
private string _file_name = String.Empty;
|
||||||
private long _rollover_size;
|
private long _rollover_size;
|
||||||
private readonly string _directory_path;
|
private readonly string _directory_path;
|
||||||
@@ -93,20 +95,19 @@ namespace BCG {
|
|||||||
void _on_change(object new_value) {
|
void _on_change(object new_value) {
|
||||||
DataPacket dp = new_value as DataPacket;
|
DataPacket dp = new_value as DataPacket;
|
||||||
byte[] serialized_update = null;
|
byte[] serialized_update = null;
|
||||||
bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
|
bool first_write = _serializer == null || _n_updates_in_current_file == 0;
|
||||||
_serializer = _serializer ?? new UniversalSerializer(dp.schema);
|
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
||||||
if (_write_create) {
|
if (first_write) {
|
||||||
// new files always starts with create
|
// first file always starts with create
|
||||||
_serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
||||||
} else
|
} else
|
||||||
_serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
||||||
|
|
||||||
BinaryWriter bw = new BinaryWriter(_out_fs);
|
BinaryWriter bw = new BinaryWriter(_out_fs);
|
||||||
long utc_epoch = epoch_now();
|
long utc_epoch = epoch_now();
|
||||||
|
|
||||||
//time, length of data, data
|
//time, length of data, data
|
||||||
bw.Write(utc_epoch);
|
//bw.Write(utc_epoch);
|
||||||
bw.Write(serialized_update.Length);
|
//bw.Write(serialized_update.Length);
|
||||||
bw.Write(serialized_update);
|
bw.Write(serialized_update);
|
||||||
_out_fs.Flush();
|
_out_fs.Flush();
|
||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
|
|||||||
@@ -1,190 +0,0 @@
|
|||||||
// using System;
|
|
||||||
// using System.Collections.Generic;
|
|
||||||
// using System.Linq;
|
|
||||||
// using System.Text;
|
|
||||||
// using System.IO;
|
|
||||||
// using System.Runtime.Serialization.Formatters.Binary;
|
|
||||||
// using BCG.Utilities.General;
|
|
||||||
// using bts.collections;
|
|
||||||
// using System.Reflection.Metadata;
|
|
||||||
|
|
||||||
|
|
||||||
// namespace BCG {
|
|
||||||
|
|
||||||
// public class PlaybackStream : ViewStream, IWorker {
|
|
||||||
// private readonly PlaybackType _type;
|
|
||||||
// private BlueTable _current_table;
|
|
||||||
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
|
||||||
// // these are the packets already downloaded from the current file
|
|
||||||
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
|
|
||||||
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
|
|
||||||
// private IEnumerable<string> _file_names;
|
|
||||||
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
|
|
||||||
// private DateTime _playback_t0, _playback_t1;
|
|
||||||
// private bool _all_files_read = false;
|
|
||||||
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
|
|
||||||
|
|
||||||
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
|
|
||||||
// _type = PlaybackType.Streaming;
|
|
||||||
// _playback_t0 = PersistorStream.parse_datetime(datetime_start);
|
|
||||||
// _playback_t1 = PersistorStream.parse_datetime(datetime_end);
|
|
||||||
// string token_directory_name = $"{directory_name}\\{token}";
|
|
||||||
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
|
|
||||||
// List<string> files_in_range = new List<string>();
|
|
||||||
// if (Directory.Exists(token_directory_name)) {
|
|
||||||
// _file_names =
|
|
||||||
// Directory.GetFiles(token_directory_name).Where(f => {
|
|
||||||
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
|
|
||||||
// return file_time > _playback_t0 && file_time < _playback_t1;
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
|
||||||
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
|
|
||||||
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
|
|
||||||
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
|
|
||||||
// if (snapshot_packet != null) {
|
|
||||||
// foreach (PlaybackPacket p in _snapshot_data.packets) {
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
|
||||||
// byte op = ops[i];
|
|
||||||
// if (op == StreamOpCode.create) {
|
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
// if (p.idx == snapshot_packet.idx) {
|
|
||||||
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
|
||||||
// private void _load_packets_from_current_file() {
|
|
||||||
// int curr_idx = _current_fs.Item1;
|
|
||||||
// FileStream curr_fs = _current_fs.Item2;
|
|
||||||
// _packets = new List<PlaybackPacket>();
|
|
||||||
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private bool current_file_all_replayed {
|
|
||||||
// get {
|
|
||||||
// bool rv = true;
|
|
||||||
// _packets.ForEach(p => rv = rv && p.replayed);
|
|
||||||
// return rv;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
|
||||||
// var fs = _current_fs.Item2;
|
|
||||||
// if (fs != null) {
|
|
||||||
// fs.Close();
|
|
||||||
// }
|
|
||||||
// ViewStreamDatabase.instance.removeWorker(this);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// // public event PlaybackStreamProgress onProgress;
|
|
||||||
// // public event Action onFileExhausted;
|
|
||||||
// private PlaybackPacket _locate_packet_to_replay() {
|
|
||||||
// foreach (var p in _packets) {
|
|
||||||
// // returns the first non replayed packet
|
|
||||||
// if (!p.replayed)
|
|
||||||
// return p;
|
|
||||||
// }
|
|
||||||
// //else
|
|
||||||
// {
|
|
||||||
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
|
|
||||||
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
|
|
||||||
// if (onProgress != null)
|
|
||||||
// onProgress(myFileReader.Position, myFileReader.Length);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// #region IWorker Members
|
|
||||||
|
|
||||||
// public void doWork() {
|
|
||||||
// if (_all_files_read)
|
|
||||||
// return;
|
|
||||||
// if (current_file_all_replayed) {
|
|
||||||
// _load_next_file();
|
|
||||||
// }
|
|
||||||
// PlaybackPacket curr_packet = _locate_packet_to_replay();
|
|
||||||
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// // what a stupid interface!
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
|
||||||
// _fire_packet(ops, row_array);
|
|
||||||
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
|
||||||
// } else {
|
|
||||||
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
|
|
||||||
// // this is the time we need to fire (replay) this packet
|
|
||||||
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
|
|
||||||
// long now_epoch = PersistorStream.epoch_now();
|
|
||||||
// // fire when we are past the
|
|
||||||
// if (now_epoch >= time_to_fire) {
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
|
||||||
// _fire_packet(ops, row_array);
|
|
||||||
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _fire_packet(byte[] ops, Array row_array) {
|
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
|
||||||
// byte op = ops[i];
|
|
||||||
// if (op == StreamOpCode.create) {
|
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// //_current_table.addRows(full_table);
|
|
||||||
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
|
||||||
// (myLastValue as DataPacket).newRows));
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _load_next_file() {
|
|
||||||
// var new_idx = _current_fs.Item1 + 1;
|
|
||||||
// if (new_idx >= _file_names.Count()) {
|
|
||||||
// _all_files_read = true;
|
|
||||||
// bts.Log.Gen.info("All files read");
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
// var prev_fs = _current_fs.Item2;
|
|
||||||
// if (prev_fs != null)
|
|
||||||
// prev_fs.Close();
|
|
||||||
// var new_filename = _file_names.ElementAt(new_idx);
|
|
||||||
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
|
|
||||||
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
|
|
||||||
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
|
|
||||||
// _load_packets_from_current_file();
|
|
||||||
// }
|
|
||||||
|
|
||||||
// #endregion
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.IO;
|
|
||||||
|
|
||||||
namespace BCG
|
|
||||||
{
|
|
||||||
public class RecorderStream : ViewStream
|
|
||||||
{
|
|
||||||
private ViewStream mySource;
|
|
||||||
private DateTime myStartTime;
|
|
||||||
private FileStream myOutFile;
|
|
||||||
private string myFileName;
|
|
||||||
private IBlueDataSerializer mySerializer;
|
|
||||||
|
|
||||||
public RecorderStream(ViewStream src, string recordFile)
|
|
||||||
{
|
|
||||||
mySource = src;
|
|
||||||
myFileName = recordFile;
|
|
||||||
|
|
||||||
myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
|
||||||
{
|
|
||||||
myStartTime = DateTime.Now;
|
|
||||||
mySource.onChange += new ChangeDelegate(mySource_onChange);
|
|
||||||
myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
|
||||||
{
|
|
||||||
mySource.onChange -= mySource_onChange;
|
|
||||||
myOutFile.Close();
|
|
||||||
}
|
|
||||||
|
|
||||||
void mySource_onChange(object newValue)
|
|
||||||
{
|
|
||||||
DataPacket dp = (DataPacket)newValue;
|
|
||||||
|
|
||||||
BlueTable toWrite = new BlueTable(dp.schema);
|
|
||||||
|
|
||||||
if (mySerializer == null)
|
|
||||||
{
|
|
||||||
mySerializer = new UniversalSerializer(dp.schema);
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in dp.newRows)
|
|
||||||
toWrite.addRow(r);
|
|
||||||
|
|
||||||
byte[] result;
|
|
||||||
|
|
||||||
if (myLastValue == null)
|
|
||||||
mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
|
|
||||||
else
|
|
||||||
mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
|
|
||||||
|
|
||||||
BinaryWriter bw = new BinaryWriter(myOutFile);
|
|
||||||
//write down what time this happened
|
|
||||||
bw.Write((DateTime.Now - myStartTime).TotalSeconds);
|
|
||||||
bw.Write(result.Length);
|
|
||||||
bw.Write(result);
|
|
||||||
myOutFile.Flush();
|
|
||||||
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,64 +1,37 @@
|
|||||||
// using System;
|
using System;
|
||||||
// using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
// using System.Text;
|
using System.Text;
|
||||||
// using System.IO;
|
using System.IO;
|
||||||
|
using BCG;
|
||||||
|
|
||||||
// using System.Runtime.Serialization.Formatters.Binary;
|
namespace bts {
|
||||||
|
// this saves the updates as a Create on the same file, each update calls -save, which
|
||||||
|
// reqrites the whole data to the packet. The other side of this stream is the Loader
|
||||||
|
[DataStreamUserRespresentation("saver")]
|
||||||
|
public class SaverStream : ViewStream {
|
||||||
|
private ViewStream _stream;
|
||||||
|
private string _filename;
|
||||||
|
private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
|
||||||
|
|
||||||
// namespace BCG
|
public SaverStream(ViewStream vs, string filename) {
|
||||||
// {
|
_stream = vs;
|
||||||
// [DataStreamUserRespresentation("saver")]
|
_filename = filename;
|
||||||
// public class SaverStream : ViewStream, ISideEffectStream
|
}
|
||||||
// {
|
|
||||||
// private ViewStream myStream;
|
|
||||||
// private string myFileName;
|
|
||||||
|
|
||||||
// public class Exception : System.Exception
|
protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
|
||||||
// {
|
|
||||||
// public Exception(string msg) : base(msg) { }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public SaverStream(ViewStream vs, string fileName)
|
void _handle_changes(object dp_obj) {
|
||||||
// {
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
// if (fileName == null)
|
_save(dp);
|
||||||
// throw new SaverStream.Exception("File name cannot be null");
|
myFireEvent(dp_obj);
|
||||||
// myStream = vs;
|
}
|
||||||
// myFileName = fileName;
|
|
||||||
|
|
||||||
// }
|
protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber()
|
protected void _save(DataPacket dp) {
|
||||||
// { myStream.onChange += new ChangeDelegate(myHandleChanges); }
|
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||||
|
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||||
// void myHandleChanges(object newValue)
|
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
||||||
// {
|
}
|
||||||
// DataPacket dp = newValue as DataPacket;
|
}
|
||||||
// mySave(dp);
|
}
|
||||||
// myFireEvent(newValue);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber()
|
|
||||||
// { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
|
|
||||||
|
|
||||||
// protected void mySave(DataPacket dp)
|
|
||||||
// {
|
|
||||||
// for (int i = 0; i < 5 ; i++)
|
|
||||||
// try
|
|
||||||
// {
|
|
||||||
// using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
|
|
||||||
// {
|
|
||||||
// BinaryFormatter bf = new BinaryFormatter();
|
|
||||||
// BlueTable t = new BlueTable(dp.schema);
|
|
||||||
// foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
|
|
||||||
// t.addRow(new BlueRow(r));
|
|
||||||
// bf.Serialize(fs, t);
|
|
||||||
// }
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// catch (IOException)
|
|
||||||
// {
|
|
||||||
// System.Threading.Thread.Sleep(5);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections.Generic;
|
||||||
using System.Linq;
|
// using System.Linq;
|
||||||
using System.Text;
|
// using System.Text;
|
||||||
using System.IO;
|
// using System.IO;
|
||||||
using System.Runtime.Serialization.Formatters.Binary;
|
// using System.Runtime.Serialization.Formatters.Binary;
|
||||||
using BCG.Utilities.General;
|
// using BCG.Utilities.General;
|
||||||
using bts.collections;
|
// using bts.collections;
|
||||||
using System.Reflection.Metadata;
|
// using System.Reflection.Metadata;
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
@@ -15,115 +15,115 @@ namespace BCG {
|
|||||||
FirstOfDay,
|
FirstOfDay,
|
||||||
LastOfDay
|
LastOfDay
|
||||||
};
|
};
|
||||||
private BlueTable _current_table;
|
// private BlueTable _current_table;
|
||||||
private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||||
public DateTime _snapshot_time;
|
// public DateTime _snapshot_time;
|
||||||
public string _snapshot_file;
|
// public string _snapshot_file;
|
||||||
public IEnumerable<PlaybackPacket> _packets;
|
// public IEnumerable<PlaybackPacket> _packets;
|
||||||
public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
|
// public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
|
||||||
DateTime dt;
|
// DateTime dt;
|
||||||
var parsed = DateTime.TryParse(snapshot_date, out dt);
|
// var parsed = DateTime.TryParse(snapshot_date, out dt);
|
||||||
dt = dt.Date;
|
// dt = dt.Date;
|
||||||
bts.Log.assert(parsed);
|
// bts.Log.assert(parsed);
|
||||||
SpecificSnapshot specific_snapshot_enum;
|
// SpecificSnapshot specific_snapshot_enum;
|
||||||
parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
|
// parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
|
||||||
bts.Log.assert(parsed);
|
// bts.Log.assert(parsed);
|
||||||
string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
|
// string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
|
||||||
ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
|
// ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
|
||||||
mySetRealSource(rs);
|
// mySetRealSource(rs);
|
||||||
}
|
// }
|
||||||
|
|
||||||
public SnapshotStream(string snapshot_dt, string directory_name, string token) {
|
// public SnapshotStream(string snapshot_dt, string directory_name, string token) {
|
||||||
_snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
|
// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
|
||||||
string token_directory_name = $"{directory_name}\\{token}";
|
// string token_directory_name = $"{directory_name}\\{token}";
|
||||||
_snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
|
// _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
|
||||||
}
|
// }
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
// protected override void myOnFirstSubscriber() {
|
||||||
FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
|
// FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
|
||||||
_packets = StreamPersistorUtils.load_packets_from_file(fs);
|
// _packets = StreamPersistorUtils.load_packets_from_file(fs);
|
||||||
PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
|
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
|
||||||
if (snapshot_packet != null) {
|
// if (snapshot_packet != null) {
|
||||||
foreach (PlaybackPacket p in _packets) {
|
// foreach (PlaybackPacket p in _packets) {
|
||||||
byte[] ops;
|
// byte[] ops;
|
||||||
Array row_array;
|
// Array row_array;
|
||||||
_deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
||||||
for (int i = 0; i < ops.Length; i++) {
|
// for (int i = 0; i < ops.Length; i++) {
|
||||||
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
byte op = ops[i];
|
// byte op = ops[i];
|
||||||
if (op == StreamOpCode.create) {
|
// if (op == StreamOpCode.create) {
|
||||||
IBlueRowCollection full_table = rows_args[0];
|
// IBlueRowCollection full_table = rows_args[0];
|
||||||
_current_table = new BlueTable(full_table);
|
// _current_table = new BlueTable(full_table);
|
||||||
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
} else {
|
// } else {
|
||||||
bts.Log.assert(false, $"Oprator: {op} is not expected");
|
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
}
|
// }
|
||||||
if (p.idx == snapshot_packet.idx) {
|
// if (p.idx == snapshot_packet.idx) {
|
||||||
myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
||||||
break;
|
// break;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
private void _absorb_packet(byte[] ops, Array row_array) {
|
// private void _absorb_packet(byte[] ops, Array row_array) {
|
||||||
for (int i = 0; i < ops.Length; i++) {
|
// for (int i = 0; i < ops.Length; i++) {
|
||||||
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
byte op = ops[i];
|
// byte op = ops[i];
|
||||||
if (op == StreamOpCode.create) {
|
// if (op == StreamOpCode.create) {
|
||||||
IBlueRowCollection full_table = rows_args[0];
|
// IBlueRowCollection full_table = rows_args[0];
|
||||||
_current_table = new BlueTable(full_table);
|
// _current_table = new BlueTable(full_table);
|
||||||
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
} else {
|
// } else {
|
||||||
bts.Log.assert(false, $"Operator: {op} is not expected");
|
// bts.Log.assert(false, $"Operator: {op} is not expected");
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
// protected override void myOnLastSubscriber() {
|
||||||
}
|
// }
|
||||||
|
|
||||||
// public event SnapshotStream.Progress onProgress;
|
// // public event SnapshotStream.Progress onProgress;
|
||||||
// public event Action onFileExhausted;
|
// // public event Action onFileExhausted;
|
||||||
private PlaybackPacket _locate_packet_to_replay() {
|
// private PlaybackPacket _locate_packet_to_replay() {
|
||||||
foreach (var p in _packets) {
|
// foreach (var p in _packets) {
|
||||||
// returns the first non replayed packet
|
// // returns the first non replayed packet
|
||||||
if (!p.replayed)
|
// if (!p.replayed)
|
||||||
return p;
|
// return p;
|
||||||
}
|
// }
|
||||||
return null;
|
// return null;
|
||||||
}
|
// }
|
||||||
|
|
||||||
private void _fire_packet(byte[] ops, Array row_array) {
|
// private void _fire_packet(byte[] ops, Array row_array) {
|
||||||
for (int i = 0; i < ops.Length; i++) {
|
// for (int i = 0; i < ops.Length; i++) {
|
||||||
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
byte op = ops[i];
|
// byte op = ops[i];
|
||||||
if (op == StreamOpCode.create) {
|
// if (op == StreamOpCode.create) {
|
||||||
IBlueRowCollection full_table = rows_args[0];
|
// IBlueRowCollection full_table = rows_args[0];
|
||||||
_current_table = new BlueTable(full_table);
|
// _current_table = new BlueTable(full_table);
|
||||||
//_current_table.addRows(full_table);
|
// //_current_table.addRows(full_table);
|
||||||
myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
||||||
(myLastValue as DataPacket).newRows));
|
// (myLastValue as DataPacket).newRows));
|
||||||
} else if (op == StreamOpCode.multipleRemoveAdd) {
|
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
myFireEvent(new DataPacket(_current_table, adds, dels));
|
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
||||||
} else {
|
// } else {
|
||||||
bts.Log.assert(false, $"Oprator: {op} is not expected");
|
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
}
|
// }
|
||||||
|
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
|
|||||||
@@ -1,16 +1,10 @@
|
|||||||
using BCG;
|
|
||||||
using System;
|
|
||||||
using System.Data;
|
|
||||||
using System.Collections;
|
|
||||||
using BCG.Utilities;
|
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
|
|
||||||
using mbts;
|
|
||||||
using mbts.spark;
|
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
namespace BCG.StreamWorld.Addition
|
namespace BCG.StreamWorld.Addition
|
||||||
{
|
{
|
||||||
[DataStreamUserRespresentation (false)]
|
[DataStreamUserRespresentation (false)]
|
||||||
@@ -151,7 +145,6 @@ namespace BCG
|
|||||||
private string myFilterString;
|
private string myFilterString;
|
||||||
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
||||||
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
|
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
|
||||||
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
|
||||||
|
|
||||||
#region constructors
|
#region constructors
|
||||||
public delegate bool OnFilter(IBlueRow r);
|
public delegate bool OnFilter(IBlueRow r);
|
||||||
@@ -226,7 +219,6 @@ namespace BCG
|
|||||||
|
|
||||||
private void myHandleCallbacks(object newValue)
|
private void myHandleCallbacks(object newValue)
|
||||||
{
|
{
|
||||||
_timer.start();
|
|
||||||
DataPacket retVal;
|
DataPacket retVal;
|
||||||
DataPacket arrivedPacket = newValue as DataPacket;
|
DataPacket arrivedPacket = newValue as DataPacket;
|
||||||
myLastReceivedRows = arrivedPacket.newRows;
|
myLastReceivedRows = arrivedPacket.newRows;
|
||||||
@@ -255,7 +247,6 @@ namespace BCG
|
|||||||
if (myFilter.filter(r))
|
if (myFilter.filter(r))
|
||||||
myLastFiredRows.addRow(r);
|
myLastFiredRows.addRow(r);
|
||||||
}
|
}
|
||||||
_timer.stop();
|
|
||||||
myFireEvent(retVal);
|
myFireEvent(retVal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,37 +1,66 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Text.RegularExpressions;
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
|
||||||
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
|
|
||||||
{
|
public static TimeSpan parse_duration(string input) {
|
||||||
|
int days = 0, hours = 0, minutes = 0, seconds = 0;
|
||||||
|
|
||||||
|
// Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
|
||||||
|
Regex regex = new Regex(@"(\d+)([dhms])");
|
||||||
|
MatchCollection matches = regex.Matches(input);
|
||||||
|
|
||||||
|
foreach (Match match in matches) {
|
||||||
|
int value = int.Parse(match.Groups[1].Value);
|
||||||
|
string unit = match.Groups[2].Value;
|
||||||
|
|
||||||
|
switch (unit) {
|
||||||
|
case "d":
|
||||||
|
days = value;
|
||||||
|
break;
|
||||||
|
case "h":
|
||||||
|
hours = value;
|
||||||
|
break;
|
||||||
|
case "m":
|
||||||
|
minutes = value;
|
||||||
|
break;
|
||||||
|
case "s":
|
||||||
|
seconds = value;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new TimeSpan(days, hours, minutes, seconds);
|
||||||
|
}
|
||||||
|
|
||||||
#region constructors
|
#region constructors
|
||||||
|
|
||||||
|
public TimeBasedFilter(string col_name, string row_time_to_live)
|
||||||
|
: this(col_name, parse_duration(row_time_to_live)) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
|
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
|
||||||
: this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
|
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
|
||||||
{
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
|
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
|
||||||
: this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
|
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
|
||||||
{
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
||||||
: this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
|
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
|
||||||
{
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
|
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
|
||||||
{
|
myVsd = vsd;
|
||||||
myVsd = vsd;
|
_time_col = rowTimeColumnName;
|
||||||
myRowTimeColumnName = rowTimeColumnName;
|
_ttl = rowTimeToLive;
|
||||||
myRowTimeToLive = rowTimeToLive;
|
myFilterFrequency = filterFrequency;
|
||||||
myFilterFrequency = filterFrequency;
|
myNextFilter = DateTime.MinValue;
|
||||||
myNextFilter = DateTime.MinValue;
|
|
||||||
|
|
||||||
// add worker
|
// add worker
|
||||||
myVsd.addWorker(this);
|
myVsd.addWorker(this);
|
||||||
@@ -41,32 +70,25 @@ namespace BCG
|
|||||||
public event FilterChangeDelegate onFilterChange;
|
public event FilterChangeDelegate onFilterChange;
|
||||||
|
|
||||||
#region IStreamFilter
|
#region IStreamFilter
|
||||||
public bool filter(IBlueRow row)
|
public bool filter(IBlueRow row) {
|
||||||
{
|
DateTime now = DateTime.Now;
|
||||||
DateTime now = DateTime.Now;
|
|
||||||
DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
|
|
||||||
|
|
||||||
if (now > timeStamp)
|
DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
|
||||||
{
|
if (now > row_time) {
|
||||||
TimeSpan span = now - timeStamp;
|
TimeSpan span = now - row_time;
|
||||||
|
if (span >= _ttl) {
|
||||||
if (span >= myRowTimeToLive)
|
|
||||||
{
|
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return (true);
|
return (true);
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IWorker
|
#region IWorker
|
||||||
public void doWork()
|
public void doWork() {
|
||||||
{
|
|
||||||
DateTime now = DateTime.Now;
|
DateTime now = DateTime.Now;
|
||||||
|
|
||||||
if (now > myNextFilter)
|
if (now > myNextFilter) {
|
||||||
{
|
|
||||||
// fire event
|
// fire event
|
||||||
onFilterChange();
|
onFilterChange();
|
||||||
|
|
||||||
@@ -76,8 +98,7 @@ namespace BCG
|
|||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region IDisposable
|
#region IDisposable
|
||||||
public void Dispose()
|
public void Dispose() {
|
||||||
{
|
|
||||||
// remove worker
|
// remove worker
|
||||||
myVsd.removeWorker(this);
|
myVsd.removeWorker(this);
|
||||||
}
|
}
|
||||||
@@ -85,10 +106,20 @@ namespace BCG
|
|||||||
|
|
||||||
#region state
|
#region state
|
||||||
private ViewStreamDatabase myVsd;
|
private ViewStreamDatabase myVsd;
|
||||||
private string myRowTimeColumnName;
|
private string _time_col;
|
||||||
private TimeSpan myRowTimeToLive;
|
private TimeSpan _ttl;
|
||||||
private TimeSpan myFilterFrequency;
|
private TimeSpan myFilterFrequency;
|
||||||
private DateTime myNextFilter;
|
private DateTime myNextFilter;
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
|
[DataStreamUserRespresentation("timefilter")]
|
||||||
|
public class TimeBasedFilterStream : ViewStream {
|
||||||
|
public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
|
||||||
|
:
|
||||||
|
this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
|
||||||
|
public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
|
||||||
|
TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
|
||||||
|
mySetRealSource(new FilterStream(base_stream, filter));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -4,18 +4,14 @@ using BCG.Utilities;
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
public class GroupByStream : MethodDrivenStream {
|
||||||
public class GroupByStream : MethodDrivenStream
|
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
||||||
{
|
|
||||||
private class MyFuncGroupByEvaluator : IGroupByEvaluator
|
|
||||||
{
|
|
||||||
private string[] _groupby_cols;
|
private string[] _groupby_cols;
|
||||||
private Func<DataPacket, IBlueRow> _groupby_func;
|
private Func<DataPacket, IBlueRow> _groupby_func;
|
||||||
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
|
||||||
{
|
|
||||||
_groupby_cols = groupby_cols.ToArray();
|
_groupby_cols = groupby_cols.ToArray();
|
||||||
_groupby_func = groupby_func;
|
_groupby_func = groupby_func;
|
||||||
}
|
}
|
||||||
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
|
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
|
||||||
|
|
||||||
@@ -29,7 +25,7 @@ namespace BCG
|
|||||||
/// Serves as a quick way to access all rows belonging to the same group
|
/// Serves as a quick way to access all rows belonging to the same group
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private bool myIdGuard = true;
|
private bool myIdGuard = true;
|
||||||
private RowDictionaryTable myBookKeepingTable ;
|
private RowDictionaryTable myBookKeepingTable;
|
||||||
private IGroupByEvaluator myGroupByEvaluator;
|
private IGroupByEvaluator myGroupByEvaluator;
|
||||||
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
|
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
|
||||||
//can map a schema to another
|
//can map a schema to another
|
||||||
@@ -37,112 +33,98 @@ namespace BCG
|
|||||||
private int myIdGenerator = 0;
|
private int myIdGenerator = 0;
|
||||||
|
|
||||||
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
||||||
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
|
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
|
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
|
||||||
: this (stream, groupByEvaluator, true)
|
: this(stream, groupByEvaluator, true) { }
|
||||||
{}
|
|
||||||
|
|
||||||
public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
|
public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
|
||||||
: base (stream)
|
: base(stream) {
|
||||||
{
|
|
||||||
myGroupByEvaluator = groupbyEvaluator;
|
myGroupByEvaluator = groupbyEvaluator;
|
||||||
myIdGuard = idGuard;
|
myIdGuard = idGuard;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleFirstCall ()
|
protected override void myHandleFirstCall() {
|
||||||
{
|
|
||||||
mySchema = myLastArrivedPacket.schema;
|
mySchema = myLastArrivedPacket.schema;
|
||||||
myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
|
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
|
||||||
myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows,
|
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
|
||||||
myGroupByEvaluator.groupByColumns);
|
myGroupByEvaluator.groupByColumns);
|
||||||
GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
|
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
|
||||||
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
|
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
|
||||||
{
|
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
|
||||||
IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
|
rowGroup, new BlueTable(mySchema)));
|
||||||
rowGroup, new BlueTable (mySchema)));
|
|
||||||
if (newRow != null)
|
if (newRow != null)
|
||||||
temp.addRow (newRow, myGetGroupString (rowGroup));
|
temp.addRow(newRow, myGetGroupString(rowGroup));
|
||||||
}
|
}
|
||||||
myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
|
myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adding need not to anything as deletion already takes care
|
/// Adding need not to anything as deletion already takes care
|
||||||
/// of addition
|
/// of addition
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected override void myHandleIncrementalAdd() {}
|
protected override void myHandleIncrementalAdd() { }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Both deletes and adds are handled here to make the intermediate results not
|
/// Both deletes and adds are handled here to make the intermediate results not
|
||||||
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream
|
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream
|
||||||
/// because of the nature of it)
|
/// because of the nature of it)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected override void myHandleIncrementalDelete()
|
protected override void myHandleIncrementalDelete() {
|
||||||
{
|
|
||||||
myFireSwitch = false;
|
myFireSwitch = false;
|
||||||
RowDictionaryTable delGroups, addGroups;
|
RowDictionaryTable delGroups, addGroups;
|
||||||
//form groups for deleted and added rows
|
//form groups for deleted and added rows
|
||||||
myInitIncrementalChangeData (out delGroups, out addGroups);
|
myInitIncrementalChangeData(out delGroups, out addGroups);
|
||||||
ArrayList handledAddGroups = new ArrayList ();
|
ArrayList handledAddGroups = new ArrayList();
|
||||||
string groupRepString;
|
string groupRepString;
|
||||||
|
|
||||||
foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
|
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
|
||||||
{
|
|
||||||
// BlueEventLog.logAssert (delGroup.Count > 0);
|
// BlueEventLog.logAssert (delGroup.Count > 0);
|
||||||
groupRepString = myGetGroupString (delGroup);
|
groupRepString = myGetGroupString(delGroup);
|
||||||
IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
|
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
|
||||||
new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
|
new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
|
||||||
handledAddGroups.Add (addGroup);
|
handledAddGroups.Add(addGroup);
|
||||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
|
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
|
||||||
new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
|
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
|
||||||
IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
|
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
|
||||||
myFireSwitch |= myMergeRow (newRow, groupRepString);
|
myFireSwitch |= myMergeRow(newRow, groupRepString);
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
|
foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
|
||||||
{
|
|
||||||
// BlueEventLog.logAssert (addGroup.Count > 0);
|
// BlueEventLog.logAssert (addGroup.Count > 0);
|
||||||
if (!handledAddGroups.Contains (addGroup))
|
if (!handledAddGroups.Contains(addGroup)) {
|
||||||
{
|
|
||||||
//this is the case when there are rows added to a group and none
|
//this is the case when there are rows added to a group and none
|
||||||
//have been removed, hence it 'escapes' from being dealt with in
|
//have been removed, hence it 'escapes' from being dealt with in
|
||||||
//the loop above
|
//the loop above
|
||||||
|
|
||||||
groupRepString = myGetGroupString (addGroup);
|
groupRepString = myGetGroupString(addGroup);
|
||||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
|
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
|
||||||
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
|
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
|
||||||
IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup,
|
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
|
||||||
new BlueTable (aggGroup.schema)));
|
new BlueTable(aggGroup.schema)));
|
||||||
myFireSwitch |= myMergeRow (newRow, groupRepString);
|
myFireSwitch |= myMergeRow(newRow, groupRepString);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool myMergeRow (IBlueRow newRow, string groupRepString)
|
private bool myMergeRow(IBlueRow newRow, string groupRepString) {
|
||||||
{
|
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
|
||||||
IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
|
|
||||||
//BlueEventLog.logAssert (newRow != null || existingRow != null);
|
//BlueEventLog.logAssert (newRow != null || existingRow != null);
|
||||||
if (existingRow == null && newRow == null)
|
if (existingRow == null && newRow == null)
|
||||||
return false;
|
return false;
|
||||||
else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
|
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
|
||||||
return false;
|
return false;
|
||||||
else if (existingRow == null)
|
else if (existingRow == null) {
|
||||||
{
|
|
||||||
//row added for the first time
|
//row added for the first time
|
||||||
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
|
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
|
||||||
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
|
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
|
||||||
return true;
|
return true;
|
||||||
}
|
} else {
|
||||||
else
|
((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
|
||||||
{
|
((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
|
||||||
((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
|
if (newRow != null) {
|
||||||
((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
|
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
|
||||||
if (newRow != null)
|
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
|
||||||
{
|
|
||||||
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
|
|
||||||
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
|
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -154,77 +136,68 @@ namespace BCG
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="delGroups"></param>
|
/// <param name="delGroups"></param>
|
||||||
/// <param name="addGroups"></param>
|
/// <param name="addGroups"></param>
|
||||||
private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
|
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
|
||||||
{
|
|
||||||
delGroups =
|
delGroups =
|
||||||
new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
|
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
|
||||||
addGroups =
|
addGroups =
|
||||||
new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
|
new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
|
||||||
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
|
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
|
||||||
myBookKeepingTable.removeRow (deletedRow);
|
myBookKeepingTable.removeRow(deletedRow);
|
||||||
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
|
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
|
||||||
myBookKeepingTable.addRow (addedRow);
|
myBookKeepingTable.addRow(addedRow);
|
||||||
myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
|
myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
|
||||||
myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
|
myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnTableClear ()
|
protected override void myOnTableClear() {
|
||||||
{
|
|
||||||
if (myBookKeepingTable.Count == 0)
|
if (myBookKeepingTable.Count == 0)
|
||||||
return; //ie there arent any rows, so no need to optimize the 'cleansing' process
|
return; //ie there arent any rows, so no need to optimize the 'cleansing' process
|
||||||
myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
|
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
|
||||||
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||||
(myLastFiredPacket.newRows as GroupByOutputTable).clear ();
|
(myLastFiredPacket.newRows as GroupByOutputTable).clear();
|
||||||
myBookKeepingTable.clear ();
|
myBookKeepingTable.clear();
|
||||||
myGroupByEvaluator.clear ();
|
myGroupByEvaluator.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRow myEvaluate (DataPacket dp)
|
private IBlueRow myEvaluate(DataPacket dp) {
|
||||||
{
|
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
|
||||||
IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
|
return evalRow == null ? null :
|
||||||
return evalRow == null ? null :
|
|
||||||
myIdGuard ?
|
myIdGuard ?
|
||||||
BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
|
BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
|
||||||
}
|
}
|
||||||
|
|
||||||
private string myGetGroupString (IBlueRowCollection group)
|
private string myGetGroupString(IBlueRowCollection group) {
|
||||||
{
|
return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
|
||||||
return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
|
||||||
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
|
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
|
||||||
//gets resubscribed, we start the process of building data again, and treat the callback as the first
|
//gets resubscribed, we start the process of building data again, and treat the callback as the first
|
||||||
//callback.
|
//callback.
|
||||||
myInit ();
|
myInit();
|
||||||
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myInit ()
|
protected override void myInit() {
|
||||||
{
|
base.myInit();
|
||||||
base.myInit ();
|
if (myBookKeepingTable != null) {
|
||||||
if (myBookKeepingTable != null)
|
|
||||||
{
|
|
||||||
//to avoid calling init before the first callback
|
//to avoid calling init before the first callback
|
||||||
myBookKeepingTable.clear ();
|
myBookKeepingTable.clear();
|
||||||
myGroupByEvaluator.clear ();
|
myGroupByEvaluator.clear();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static string getRepString (DataPacket dp, string[] colNames)
|
public static string getRepString(DataPacket dp, string[] colNames) {
|
||||||
{
|
return BlueDataUtility.rowToString(getRepRow(dp), colNames);
|
||||||
return BlueDataUtility.rowToString (getRepRow (dp), colNames);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static IBlueRow getRepRow (DataPacket dp)
|
public static IBlueRow getRepRow(DataPacket dp) {
|
||||||
{
|
|
||||||
if (dp.newRows.Count > 0)
|
if (dp.newRows.Count > 0)
|
||||||
return BlueDataUtility.extractRow (dp.newRows, 0);
|
return BlueDataUtility.extractRow(dp.newRows, 0);
|
||||||
else if (dp.addedRows.Count > 0)
|
else if (dp.addedRows.Count > 0)
|
||||||
return BlueDataUtility.extractRow (dp.addedRows, 0);
|
return BlueDataUtility.extractRow(dp.addedRows, 0);
|
||||||
else if (dp.deletedRows.Count > 0)
|
else if (dp.deletedRows.Count > 0)
|
||||||
return BlueDataUtility.extractRow (dp.deletedRows, 0);
|
return BlueDataUtility.extractRow(dp.deletedRows, 0);
|
||||||
// BlueEventLog.logAssert (false, "Illegal Data Packet");
|
// BlueEventLog.logAssert (false, "Illegal Data Packet");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -237,22 +210,21 @@ namespace BCG
|
|||||||
/// <param name="newValue"></param>
|
/// <param name="newValue"></param>
|
||||||
/// <param name="isAdd"></param>
|
/// <param name="isAdd"></param>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
|
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
|
||||||
{
|
if (lastValue == null || lastValue.Equals(DBNull.Value))
|
||||||
if (lastValue == null || lastValue.Equals (DBNull.Value))
|
|
||||||
return newValue;
|
return newValue;
|
||||||
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
|
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
|
||||||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
|
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
|
||||||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
|
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
|
||||||
Object retVal = null;
|
Object retVal = null;
|
||||||
if (newValue.GetType().Equals (typeof(int)))
|
if (newValue.GetType().Equals(typeof(int)))
|
||||||
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
|
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
|
||||||
else if (newValue.GetType().Equals (typeof(Double)))
|
else if (newValue.GetType().Equals(typeof(Double)))
|
||||||
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
|
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
|
||||||
else if (lastValue.GetType().Equals (typeof(Decimal)))
|
else if (lastValue.GetType().Equals(typeof(Decimal)))
|
||||||
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
|
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
|
||||||
else{}
|
else { }
|
||||||
// BlueEventLog.logAssert (false, "Illegal type");
|
// BlueEventLog.logAssert (false, "Illegal type");
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -22,8 +22,8 @@ namespace BCG {
|
|||||||
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
|
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
|
||||||
if (maxRow != null) {
|
if (maxRow != null) {
|
||||||
bts.Log.assert(currMaxRow != null);
|
bts.Log.assert(currMaxRow != null);
|
||||||
bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
|
bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
|
||||||
if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
|
if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
|
||||||
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
|
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
|
||||||
if (maxRow == null)
|
if (maxRow == null)
|
||||||
myOutputTable.removeRow(hashString);
|
myOutputTable.removeRow(hashString);
|
||||||
@@ -34,7 +34,7 @@ namespace BCG {
|
|||||||
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
|
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
|
||||||
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
|
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
|
||||||
// otherwise if maxRow > currMaxRow then its replaced
|
// otherwise if maxRow > currMaxRow then its replaced
|
||||||
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
|
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
|
||||||
myOutputTable.addRow(maxRow, hashString);
|
myOutputTable.addRow(maxRow, hashString);
|
||||||
return myProject(myOutputTable.getRow(hashString));
|
return myProject(myOutputTable.getRow(hashString));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ namespace BCG {
|
|||||||
if (minRow != null) {
|
if (minRow != null) {
|
||||||
// BlueEventLog.logAssert (currMinRow != null);
|
// BlueEventLog.logAssert (currMinRow != null);
|
||||||
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
|
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
|
||||||
if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
|
if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
|
||||||
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
|
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
|
||||||
if (minRow == null)
|
if (minRow == null)
|
||||||
myOutputTable.removeRow(stringRep);
|
myOutputTable.removeRow(stringRep);
|
||||||
@@ -27,7 +27,7 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
|
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
|
||||||
if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
|
if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
|
||||||
myOutputTable.addRow(minRow, stringRep);
|
myOutputTable.addRow(minRow, stringRep);
|
||||||
return myOutputTable.getRow(stringRep);
|
return myOutputTable.getRow(stringRep);
|
||||||
}
|
}
|
||||||
|
|||||||
+18
-14
@@ -1,10 +1,8 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Diagnostics;
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
using bts;
|
||||||
using mbts;
|
|
||||||
using mbts.spark;
|
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -14,7 +12,7 @@ namespace BCG
|
|||||||
private INonIncGroupByEvaluator _groupby_func;
|
private INonIncGroupByEvaluator _groupby_func;
|
||||||
private BlueRowHasher _row_hasher;
|
private BlueRowHasher _row_hasher;
|
||||||
private string[] _groupby_cols;
|
private string[] _groupby_cols;
|
||||||
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
private Stopwatch _timer = new();
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
|
|
||||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
||||||
@@ -33,21 +31,27 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
DataPacket dp = nv as DataPacket;
|
DataPacket dp = nv as DataPacket;
|
||||||
///grouping together
|
///grouping together
|
||||||
_timer.start();
|
_timer.Restart();
|
||||||
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
||||||
_timer.stop();
|
_timer.Stop();
|
||||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
|
if (groupElapsedMs >= 50) {
|
||||||
|
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
||||||
|
}
|
||||||
|
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||||
/// reducing
|
/// reducing
|
||||||
_timer.start();
|
_timer.Restart();
|
||||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||||
_timer.stop();
|
_timer.Stop();
|
||||||
|
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (_timer.lastIntervalTime > 50)
|
if (applyElapsedMs >= 50)
|
||||||
{
|
{
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
|
Log.DataFlow.info(
|
||||||
|
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||||
}
|
}
|
||||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -27,7 +27,8 @@ namespace BCG
|
|||||||
string prependString = isLeft ? "left" : "right";
|
string prependString = isLeft ? "left" : "right";
|
||||||
return String.Concat (prependString, colName).GetHashCode ();
|
return String.Concat (prependString, colName).GetHashCode ();
|
||||||
}
|
}
|
||||||
public override bool Equals (object obj) {return GetHashCode () == obj.GetHashCode ();}
|
public override bool Equals (object? obj) =>
|
||||||
|
GetHashCode() == obj?.GetHashCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
public class JoinStream : ViewStream
|
public class JoinStream : ViewStream
|
||||||
{
|
{
|
||||||
protected InternalJoinStream myIjs;
|
protected InternalJoinStream? myIjs;
|
||||||
protected ProjectStream myPs;
|
protected ProjectStream? myPs;
|
||||||
protected ColumnMap myLeftCm, myRightCm;
|
protected ColumnMap? myLeftCm, myRightCm;
|
||||||
protected bool myIsProject = true;
|
protected bool myIsProject = true;
|
||||||
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
||||||
|
|
||||||
|
|||||||
+11
-13
@@ -1,10 +1,10 @@
|
|||||||
using System;
|
using bts;
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using mbts;
|
using mbts;
|
||||||
using mbts.spark;
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -40,8 +40,7 @@ namespace BCG
|
|||||||
|
|
||||||
public void _on_stream_cb(object dp_obj)
|
public void _on_stream_cb(object dp_obj)
|
||||||
{
|
{
|
||||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
var timer = Stopwatch.StartNew();
|
||||||
timer.start();
|
|
||||||
DataPacket dp = (DataPacket)dp_obj;
|
DataPacket dp = (DataPacket)dp_obj;
|
||||||
if (_schema == null)
|
if (_schema == null)
|
||||||
{
|
{
|
||||||
@@ -67,13 +66,12 @@ namespace BCG
|
|||||||
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
timer.stop();
|
timer.Stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (timer.lastIntervalTime > 50)
|
if (timer.ElapsedMilliseconds >= 50) {
|
||||||
{
|
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
|
||||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||||
}
|
}
|
||||||
if (on_new_data != null)
|
if (on_new_data != null)
|
||||||
on_new_data();
|
on_new_data();
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
using System;
|
using bts;
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using mbts;
|
using mbts;
|
||||||
using mbts.spark;
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -114,8 +114,7 @@ namespace BCG
|
|||||||
if (_is_ready())
|
if (_is_ready())
|
||||||
{
|
{
|
||||||
_construct_schema();
|
_construct_schema();
|
||||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
var timer = Stopwatch.StartNew();
|
||||||
timer.start();
|
|
||||||
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
||||||
List<object[]> output_rows = new List<object[]>();
|
List<object[]> output_rows = new List<object[]>();
|
||||||
|
|
||||||
@@ -138,13 +137,12 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
||||||
timer.stop();
|
timer.Stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (timer.lastIntervalTime >= 250)
|
if (timer.ElapsedMilliseconds >= 250) {
|
||||||
{
|
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
||||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
|
||||||
}
|
}
|
||||||
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
@@ -154,15 +152,12 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
{
|
{
|
||||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
|
||||||
IBlueRowCollection output_rows;
|
IBlueRowCollection output_rows;
|
||||||
timer.start();
|
var timer = Stopwatch.StartNew();
|
||||||
output_rows = _output_mapper.map_rows(unmapped_rows);
|
output_rows = _output_mapper.map_rows(unmapped_rows);
|
||||||
timer.stop();
|
timer.Stop();
|
||||||
if (timer.lastIntervalTime >= 50)
|
if (timer.ElapsedMilliseconds >= 50) {
|
||||||
{
|
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
||||||
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
|
|
||||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
|
||||||
}
|
}
|
||||||
return output_rows;
|
return output_rows;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,7 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Text;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
using BCG.Utilities;
|
|
||||||
using mbts;
|
using mbts;
|
||||||
using mbts.spark;
|
|
||||||
using Spark.Core.Collections;
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
@@ -226,7 +222,8 @@ namespace BCG
|
|||||||
|
|
||||||
#region IDictionary Members
|
#region IDictionary Members
|
||||||
|
|
||||||
public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
|
public void Add(object key, object? value) =>
|
||||||
|
throw new NotSupportedException("IBlueRow is readonly");
|
||||||
|
|
||||||
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
|
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
|
||||||
|
|
||||||
@@ -256,10 +253,9 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public object this[object key]
|
public object? this[object key] {
|
||||||
{
|
get => key is string str ? this[str] : this[(int)key];
|
||||||
get { return key is string ? this[key.ToString()] : this[(int)key]; }
|
set => throw new NotSupportedException("IBlueRow is readonly");
|
||||||
set { throw new NotSupportedException("IBlueRow is readonly"); }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
@@ -294,7 +290,6 @@ namespace BCG
|
|||||||
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
||||||
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
||||||
private bool myManageUnmappedLeftRows = false;
|
private bool myManageUnmappedLeftRows = false;
|
||||||
protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
|
|
||||||
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
||||||
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
||||||
protected JoinProjectHelper myJoinProjectHelper;
|
protected JoinProjectHelper myJoinProjectHelper;
|
||||||
@@ -361,7 +356,6 @@ namespace BCG
|
|||||||
|
|
||||||
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
||||||
{
|
{
|
||||||
myTimer.start();
|
|
||||||
myInitSchema(dp.schema, isLeft);
|
myInitSchema(dp.schema, isLeft);
|
||||||
BlueTable dels = null, adds = null;
|
BlueTable dels = null, adds = null;
|
||||||
if (myState != null)
|
if (myState != null)
|
||||||
@@ -371,7 +365,6 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
||||||
myAddRows(dp.addedRows, isLeft, myState, adds);
|
myAddRows(dp.addedRows, isLeft, myState, adds);
|
||||||
myTimer.stop();
|
|
||||||
if (myState != null)
|
if (myState != null)
|
||||||
{
|
{
|
||||||
myFireEvent(new DataPacket(myState, adds, dels));
|
myFireEvent(new DataPacket(myState, adds, dels));
|
||||||
|
|||||||
@@ -91,7 +91,8 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
public void Add (object key, object? value) =>
|
||||||
|
throw new NotSupportedException ("The IBlueRow is readonly");
|
||||||
|
|
||||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||||
|
|
||||||
|
|||||||
@@ -1,51 +1,64 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
public class LiteralStream : ViewStream {
|
||||||
public class LiteralStream : ViewStream
|
|
||||||
{
|
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
||||||
public LiteralStream(string tableName, string schema, string value)
|
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
|
||||||
{
|
string[] pairs = key_value_pairs.Split(' ');
|
||||||
string [] columns = schema.Split(' ');
|
foreach (string p in pairs) {
|
||||||
|
string[] kvp = p.Split(':');
|
||||||
|
string col_name = kvp[0];
|
||||||
|
string val = kvp[1];
|
||||||
|
var tup = BlueDataUtility.guess_type(val);
|
||||||
|
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
|
||||||
|
rv.Add(Tuple.Create(ci, tup.Item2));
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
public LiteralStream(string tableName, string schema, string value) {
|
||||||
|
string[] columns = schema.Split(' ');
|
||||||
|
|
||||||
|
|
||||||
ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
|
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
|
||||||
ci[0] = new ColumnInfo("id",0,typeof(int));
|
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
||||||
int n = 1;
|
int n = 1;
|
||||||
foreach (string c in columns)
|
foreach (string c in columns) {
|
||||||
{
|
string[] columnAndType = c.Split('/');
|
||||||
string [] columnAndType = c.Split('/');
|
|
||||||
if (columnAndType.Length == 1)
|
if (columnAndType.Length == 1)
|
||||||
ci[n] = new ColumnInfo(c,n,typeof(double));
|
ci[n] = new ColumnInfo(c, n, typeof(double));
|
||||||
else
|
else
|
||||||
ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
|
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
||||||
n++;
|
n++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
|
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
|
||||||
string [] rows = value.Split(';');
|
string[] rows = value.Split(';');
|
||||||
n = 0;
|
n = 0;
|
||||||
foreach (string r in rows)
|
foreach (string r in rows)
|
||||||
try
|
try {
|
||||||
{
|
if (r == string.Empty)
|
||||||
if(r == string.Empty)
|
|
||||||
continue;
|
continue;
|
||||||
t.addRow(
|
t.addRow(
|
||||||
BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));
|
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
|
||||||
}
|
} catch (Exception e) {
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
||||||
}
|
}
|
||||||
|
|
||||||
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
||||||
}
|
}
|
||||||
|
|
||||||
public LiteralStream(string schema, string value) : this ("literal",schema, value)
|
public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
||||||
{}
|
|
||||||
|
// public LiteralStream(string value) {
|
||||||
|
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,24 +1,24 @@
|
|||||||
using System;
|
// using System;
|
||||||
|
|
||||||
namespace BCG
|
// namespace BCG
|
||||||
{
|
// {
|
||||||
public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
|
// public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
|
||||||
{
|
// {
|
||||||
IBlueTableSchema mySchema;
|
// IBlueTableSchema mySchema;
|
||||||
|
|
||||||
public IBlueRow map (IBlueRow row)
|
// public IBlueRow map (IBlueRow row)
|
||||||
{
|
// {
|
||||||
if (mySchema == null)
|
// if (mySchema == null)
|
||||||
{
|
// {
|
||||||
ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
|
// ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
|
||||||
int i = 0;
|
// int i = 0;
|
||||||
foreach (ColumnInfo ci in row.schema.columnInfo)
|
// foreach (ColumnInfo ci in row.schema.columnInfo)
|
||||||
cis[i++] = ci.colType == null ?
|
// cis[i++] = ci.colType == null ?
|
||||||
new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
|
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
|
||||||
new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
|
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
|
||||||
mySchema = new BlueTableSchema (row.schema.tableName, cis);
|
// mySchema = new BlueTableSchema (row.schema.tableName, cis);
|
||||||
}
|
// }
|
||||||
return new BlueRow (mySchema, row);
|
// return new BlueRow (mySchema, row);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -1,15 +1,10 @@
|
|||||||
using System;
|
|
||||||
using System.Data;
|
|
||||||
using System.Collections;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
|
|
||||||
using mbts;
|
|
||||||
using mbts.spark;
|
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
|
|
||||||
namespace BCG
|
using System;
|
||||||
{
|
|
||||||
|
namespace BCG {
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -21,24 +16,45 @@ namespace BCG
|
|||||||
[DataStreamUserRespresentation (false)]
|
[DataStreamUserRespresentation (false)]
|
||||||
public class MapStream : MethodDrivenStream
|
public class MapStream : MethodDrivenStream
|
||||||
{
|
{
|
||||||
private HighPerformanceTimer t = new HighPerformanceTimer ();
|
|
||||||
protected IStreamMapper myMapper;
|
protected IStreamMapper myMapper;
|
||||||
protected IChangeableStreamMapper myChangeableMapper = null;
|
protected IChangeableStreamMapper myChangeableMapper = null;
|
||||||
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
||||||
|
|
||||||
public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
|
public class FuncWrapper : IStreamMapper {
|
||||||
{
|
private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func;
|
||||||
if (myUseNewMap)
|
Func<IBlueRow, IBlueRow> _row_map_func;
|
||||||
mySetRealSource (new SuperMapStream (vs, mapper));
|
|
||||||
else
|
public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
|
||||||
myMapper = mapper;
|
Func<IBlueRow, IBlueRow> row_map_func) {
|
||||||
|
_schema_map_func = schema_map_func;
|
||||||
|
_row_map_func = row_map_func;
|
||||||
|
}
|
||||||
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
|
||||||
|
return _schema_map_func(input_schema);
|
||||||
|
}
|
||||||
|
public IBlueRow map(IBlueRow row) {
|
||||||
|
return _row_map_func(row);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
|
|
||||||
{
|
public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
|
||||||
if (myUseNewMap)
|
if (myUseNewMap)
|
||||||
mySetRealSource(new SuperMapStream(vs, mapper));
|
mySetRealSource(new SuperMapStream(vs, mapper));
|
||||||
else
|
else
|
||||||
{
|
myMapper = mapper;
|
||||||
|
}
|
||||||
|
|
||||||
|
public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
|
||||||
|
Func<IBlueRow, IBlueRow> row_map_func)
|
||||||
|
:
|
||||||
|
this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
|
||||||
|
if (myUseNewMap)
|
||||||
|
mySetRealSource(new SuperMapStream(vs, mapper));
|
||||||
|
else {
|
||||||
mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
|
mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
|
||||||
myChangeableMapper = mapper;
|
myChangeableMapper = mapper;
|
||||||
}
|
}
|
||||||
@@ -47,77 +63,68 @@ namespace BCG
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the value of the mapper. Setting the value will result in re
|
/// Gets or sets the value of the mapper. Setting the value will result in re
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private void myMapperChanged ()
|
private void myMapperChanged() {
|
||||||
{
|
|
||||||
if (myLastArrivedPacket != null)
|
if (myLastArrivedPacket != null)
|
||||||
myChangeHandler (new DataPacket (myLastArrivedPacket.newRows,
|
myChangeHandler(new DataPacket(myLastArrivedPacket.newRows,
|
||||||
myLastArrivedPacket.newRows,
|
myLastArrivedPacket.newRows,
|
||||||
myLastArrivedPacket.newRows));
|
myLastArrivedPacket.newRows));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
protected override void myHandleFirstCall ()
|
protected override void myHandleFirstCall() {
|
||||||
{
|
|
||||||
|
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
|
||||||
IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
|
myLastFiredPacket = new DataPacket();
|
||||||
myLastFiredPacket = new DataPacket ();
|
myLastFiredPacket.newRows = new BlueTable(outputSchema);
|
||||||
myLastFiredPacket.newRows = new BlueTable (outputSchema);
|
myLastFiredPacket.deletedRows = new BlueTable(outputSchema);
|
||||||
myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
|
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.newRows)
|
foreach (IBlueRow row in myLastArrivedPacket.newRows)
|
||||||
((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
|
((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row));
|
||||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
|
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleIncrementalAdd()
|
protected override void myHandleIncrementalAdd() {
|
||||||
{
|
|
||||||
//t.start ();
|
//t.start ();
|
||||||
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
|
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.addedRows)
|
foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
|
||||||
{
|
IBlueRow newRow = myMapper.map(row);
|
||||||
IBlueRow newRow = myMapper.map (row);
|
(myLastFiredPacket.newRows as BlueTable).addRow(newRow);
|
||||||
(myLastFiredPacket.newRows as BlueTable).addRow (newRow);
|
(myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
|
||||||
(myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
|
|
||||||
}
|
}
|
||||||
//t.stop ();
|
//t.stop ();
|
||||||
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
||||||
//System.Diagnostics.Trace.WriteLine (" ");
|
//System.Diagnostics.Trace.WriteLine (" ");
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleIncrementalDelete()
|
protected override void myHandleIncrementalDelete() {
|
||||||
{
|
|
||||||
//t.start ();
|
//t.start ();
|
||||||
myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
|
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
|
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
|
||||||
{
|
|
||||||
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
|
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
|
||||||
(myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
|
(myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
|
||||||
(myLastFiredPacket.newRows as BlueTable).deleteRow (row);
|
(myLastFiredPacket.newRows as BlueTable).deleteRow(row);
|
||||||
}
|
}
|
||||||
//t.stop ();
|
//t.stop ();
|
||||||
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
|
||||||
{
|
protected override void myOnFirstSubscriber() {
|
||||||
myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
|
myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
myInit();
|
||||||
myInit ();
|
|
||||||
if (myStream != null)
|
if (myStream != null)
|
||||||
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnTableClear ()
|
protected override void myOnTableClear() {
|
||||||
{
|
|
||||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||||
myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||||
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
|
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
using mbts;
|
using mbts;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// SuperMap gives the user the ability
|
/// SuperMap gives the user the ability
|
||||||
/// to map one row to one or more rows
|
/// to map one row to one or more rows
|
||||||
@@ -17,13 +18,11 @@ namespace BCG
|
|||||||
/// can chage itself and raise an event causing
|
/// can chage itself and raise an event causing
|
||||||
/// a remapping to happen
|
/// a remapping to happen
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class SuperMapStream : ViewStream
|
public class SuperMapStream : ViewStream {
|
||||||
{
|
|
||||||
#region super mapper that wraps around stream mapper
|
#region super mapper that wraps around stream mapper
|
||||||
private class MyMappingWrapper : ISuperStreamMapper
|
private class MyMappingWrapper : ISuperStreamMapper {
|
||||||
{
|
|
||||||
private IStreamMapper myMapper;
|
private IStreamMapper myMapper;
|
||||||
|
|
||||||
public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
|
public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
|
||||||
|
|
||||||
#region ISuperStreamMapper Members
|
#region ISuperStreamMapper Members
|
||||||
@@ -32,8 +31,7 @@ namespace BCG
|
|||||||
|
|
||||||
public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
|
public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
|
||||||
|
|
||||||
public event MapperChangeDelegate onMapperChange
|
public event MapperChangeDelegate onMapperChange {
|
||||||
{
|
|
||||||
add { }
|
add { }
|
||||||
remove { }
|
remove { }
|
||||||
}
|
}
|
||||||
@@ -42,21 +40,18 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
private class MyFuncWrapper : ISuperStreamMapper
|
private class MyFuncWrapper : ISuperStreamMapper {
|
||||||
{
|
|
||||||
private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
|
private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
|
||||||
private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
|
private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
|
||||||
public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
||||||
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) {
|
||||||
{
|
|
||||||
_get_schema = get_schema;
|
_get_schema = get_schema;
|
||||||
_map_rows = map_rows;
|
_map_rows = map_rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
|
||||||
|
|
||||||
public IBlueRow[] map(IBlueRow r)
|
public IBlueRow[] map(IBlueRow r) {
|
||||||
{
|
|
||||||
IEnumerable<IBlueRow> rows = _map_rows(r);
|
IEnumerable<IBlueRow> rows = _map_rows(r);
|
||||||
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
||||||
}
|
}
|
||||||
@@ -76,19 +71,15 @@ namespace BCG
|
|||||||
|
|
||||||
public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
|
||||||
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
||||||
: this(vs, new MyFuncWrapper(get_schema, map_rows))
|
: this(vs, new MyFuncWrapper(get_schema, map_rows)) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
|
||||||
: this(vs, s => out_schema, map_rows)
|
: this(vs, s => out_schema, map_rows) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public SuperMapStream(ViewStream vs, IStreamMapper mapper)
|
public SuperMapStream(ViewStream vs, IStreamMapper mapper)
|
||||||
: this(vs, new MyMappingWrapper(mapper))
|
: this(vs, new MyMappingWrapper(mapper)) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
|
public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) {
|
||||||
{
|
|
||||||
myStream = vs;
|
myStream = vs;
|
||||||
myMapper = mapper;
|
myMapper = mapper;
|
||||||
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
|
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
|
||||||
@@ -96,8 +87,7 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
|
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
|
||||||
myStream.onChange -= new ChangeDelegate(myHandleData);
|
myStream.onChange -= new ChangeDelegate(myHandleData);
|
||||||
myRows = null;
|
myRows = null;
|
||||||
myBaseRows = null;
|
myBaseRows = null;
|
||||||
@@ -105,8 +95,7 @@ namespace BCG
|
|||||||
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
|
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleData(object newValue)
|
private void myHandleData(object newValue) {
|
||||||
{
|
|
||||||
DataPacket idp = newValue as DataPacket;
|
DataPacket idp = newValue as DataPacket;
|
||||||
//HighPerformanceTimer timer = new HighPerformanceTimer();
|
//HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
//timer.start();
|
//timer.start();
|
||||||
@@ -117,24 +106,19 @@ namespace BCG
|
|||||||
BlueTable dels = new BlueTable(myRows.schema),
|
BlueTable dels = new BlueTable(myRows.schema),
|
||||||
adds = new BlueTable(myRows.schema);
|
adds = new BlueTable(myRows.schema);
|
||||||
|
|
||||||
foreach (IBlueRow r in dp.deletedRows)
|
foreach (IBlueRow r in dp.deletedRows) {
|
||||||
{
|
|
||||||
List<IBlueRow> mappedRows = myMap[(int)r["id"]];
|
List<IBlueRow> mappedRows = myMap[(int)r["id"]];
|
||||||
foreach (IBlueRow mr in mappedRows)
|
foreach (IBlueRow mr in mappedRows) {
|
||||||
{
|
|
||||||
dels.addRow(mr);
|
dels.addRow(mr);
|
||||||
myRows.deleteRow(mr);
|
myRows.deleteRow(mr);
|
||||||
}
|
}
|
||||||
myMap.Remove((int)r["id"]);
|
myMap.Remove((int)r["id"]);
|
||||||
}
|
}
|
||||||
foreach (IBlueRow r in dp.addedRows)
|
foreach (IBlueRow r in dp.addedRows) {
|
||||||
{
|
|
||||||
IBlueRow[] mappedRows = myMapper.map(r);
|
IBlueRow[] mappedRows = myMapper.map(r);
|
||||||
foreach (IBlueRow mr in mappedRows)
|
foreach (IBlueRow mr in mappedRows) {
|
||||||
{
|
|
||||||
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
||||||
if (!myMap.ContainsKey((int)r["id"]))
|
if (!myMap.ContainsKey((int)r["id"])) {
|
||||||
{
|
|
||||||
myMap.Add((int)r["id"], new List<IBlueRow>());
|
myMap.Add((int)r["id"], new List<IBlueRow>());
|
||||||
}
|
}
|
||||||
myMap[(int)r["id"]].Add(idChangedMapRow);
|
myMap[(int)r["id"]].Add(idChangedMapRow);
|
||||||
@@ -145,13 +129,12 @@ namespace BCG
|
|||||||
//timer.stop();
|
//timer.stop();
|
||||||
// if (timer.lastIntervalTime > 20)
|
// if (timer.lastIntervalTime > 20)
|
||||||
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
|
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
|
||||||
// myFireEvent(new DataPacket(myRows, adds, dels));
|
myFireEvent(new DataPacket(myRows, adds, dels));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleMapChange()
|
private void myHandleMapChange() {
|
||||||
{
|
|
||||||
if (myBaseRows != null)
|
if (myBaseRows != null)
|
||||||
myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
|
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
using BCG;
|
||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using MongoDB.Bson;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Runtime.Intrinsics.X86;
|
||||||
|
|
||||||
|
|
||||||
|
namespace bts {
|
||||||
|
// listens to the underlying stream, and writes each seriialized packet in a
|
||||||
|
// seperate file
|
||||||
|
[DataStreamUserRespresentation("dp_recorder")]
|
||||||
|
public class DataPacketRecorderStream : ViewStream {
|
||||||
|
private ViewStream _base_stream;
|
||||||
|
private readonly string _folder;
|
||||||
|
StreamIOSerializationController _serializer = new StreamIOSerializationController();
|
||||||
|
private UInt32 _n_updates = 0;
|
||||||
|
public DataPacketRecorderStream(ViewStream base_stream, string folder) {
|
||||||
|
_base_stream = base_stream;
|
||||||
|
_folder = folder;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_base_stream.onChange += _handle_update;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_base_stream.onChange -= _handle_update;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _handle_update(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
byte[] ser_data = null;
|
||||||
|
if (_n_updates == 0) {
|
||||||
|
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||||
|
ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||||
|
} else {
|
||||||
|
StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
|
||||||
|
ser_data = _serializer.serialize(new StreamIOMessage(mra));
|
||||||
|
}
|
||||||
|
Log.assert(ser_data != null);
|
||||||
|
_n_updates++;
|
||||||
|
string file_name = $"{_folder}\\{_n_updates}.bin";
|
||||||
|
_write_to_file(ser_data, file_name);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void _write_to_file(byte[] data, string file_name) {
|
||||||
|
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
|
||||||
|
// Write the byte array to the file
|
||||||
|
fs.Write(data, 0, data.Length);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
[DataStreamUserRespresentation("dp_replay")]
|
||||||
|
public class DataPacketReplayStream : ViewStream, IWorker {
|
||||||
|
private ViewStreamDatabase _vsd;
|
||||||
|
private readonly string _folder_path;
|
||||||
|
private readonly TimeSpan _fire_period;
|
||||||
|
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
|
||||||
|
private int _file_ptr = 0;
|
||||||
|
private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
|
||||||
|
private BlueTable _state = null;
|
||||||
|
|
||||||
|
public DataPacketReplayStream(string folder_path, string fire_period)
|
||||||
|
:
|
||||||
|
this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
|
||||||
|
_vsd = vsd;
|
||||||
|
_folder_path = folder_path;
|
||||||
|
_fire_period = fire_period;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
MasterWorker.getInstance(false).addWorker(this, _fire_period);
|
||||||
|
_load_all_files();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
MasterWorker.getInstance(false).removeWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
if (_file_ptr >= _file_data.Count())
|
||||||
|
// we can set _file_ptr to 0 and keep going as well
|
||||||
|
return;
|
||||||
|
var curr_d = _file_data[_file_ptr++];
|
||||||
|
var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
|
||||||
|
StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
|
||||||
|
Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
|
||||||
|
switch (msg.op) {
|
||||||
|
case StreamIOMessage.StreamOp.CREATE: {
|
||||||
|
_state = new BlueTable(msg.create_message.rows);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
|
||||||
|
var mra = msg.multiple_remove_add_message;
|
||||||
|
foreach (IBlueRow r in mra.removed_rows)
|
||||||
|
_state.deleteRow(r);
|
||||||
|
foreach (IBlueRow r in mra.added_rows)
|
||||||
|
_state.addRow(r);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _load_all_files() {
|
||||||
|
Func<string, string> extract_file_name = (string full_path) => {
|
||||||
|
string fn = full_path.Split("\\").Last();
|
||||||
|
fn = fn.Split(".").First();
|
||||||
|
return fn;
|
||||||
|
|
||||||
|
};
|
||||||
|
string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
|
||||||
|
|
||||||
|
// Iterate over the .bin files and perform an action
|
||||||
|
foreach (string file in bin_files) {
|
||||||
|
byte[] file_data = File.ReadAllBytes(file);
|
||||||
|
_file_data.Add(Tuple.Create(file, file_data));
|
||||||
|
}
|
||||||
|
_file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
|
||||||
|
string fn1 = extract_file_name(t1.Item1);
|
||||||
|
string fn2 = extract_file_name(t2.Item1);
|
||||||
|
int fn_i1 = Int32.Parse(fn1);
|
||||||
|
int fn_i2 = Int32.Parse(fn2);
|
||||||
|
if (fn_i1 < fn_i2)
|
||||||
|
return -1;
|
||||||
|
else if (fn_i1 == fn_i2)
|
||||||
|
return 0;
|
||||||
|
else if (fn_i1 > fn_i2)
|
||||||
|
return 1;
|
||||||
|
return 0;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
using BCG;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
//using MongoDB.Driver;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace bts
|
||||||
|
{
|
||||||
|
public class PubSubTestingStream : ViewStream, IWorker
|
||||||
|
{
|
||||||
|
public class PayloadMessage
|
||||||
|
{
|
||||||
|
public int msg_id = 0;
|
||||||
|
public string guid;
|
||||||
|
public Int64 travel_epoch_ms = 0;
|
||||||
|
public byte[] payload = null;
|
||||||
|
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
|
||||||
|
{
|
||||||
|
msg_id = msg_id_args;
|
||||||
|
travel_epoch_ms = send_epoch_args;
|
||||||
|
payload = new byte[payload_size];
|
||||||
|
for (int i = 0; i < payload_size; i++)
|
||||||
|
{
|
||||||
|
payload[i] = payload_byte;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private PayloadMessage() { }
|
||||||
|
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
|
||||||
|
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
|
||||||
|
{ }
|
||||||
|
|
||||||
|
public byte[] serialize()
|
||||||
|
{
|
||||||
|
var str = new MemoryStream();
|
||||||
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
|
bw.Write(msg_id);
|
||||||
|
bw.Write(guid);
|
||||||
|
bw.Write(travel_epoch_ms);
|
||||||
|
bw.Write(payload.Length);
|
||||||
|
bw.Write(payload);
|
||||||
|
str.Position = 0;
|
||||||
|
var bytes = str.ToArray();
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static PayloadMessage deserialize(byte[] ser_bytes)
|
||||||
|
{
|
||||||
|
var str = new MemoryStream(ser_bytes);
|
||||||
|
BinaryReader br = new BinaryReader(str);
|
||||||
|
int msg_id = br.ReadInt32();
|
||||||
|
string guid = br.ReadString();
|
||||||
|
Int64 epoch = br.ReadInt64();
|
||||||
|
int len = br.ReadInt32();
|
||||||
|
var payload = br.ReadBytes(len);
|
||||||
|
return new PayloadMessage()
|
||||||
|
{
|
||||||
|
msg_id = msg_id,
|
||||||
|
guid = guid,
|
||||||
|
travel_epoch_ms = epoch,
|
||||||
|
payload = payload
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
//public void
|
||||||
|
private readonly int _msg_size;
|
||||||
|
private readonly TimeSpan _publish_period;
|
||||||
|
private DateTime _last_publish_time = DateTime.MinValue;
|
||||||
|
private PublisherCore _pub_core = new PublisherCore();
|
||||||
|
private BlueTable _table;
|
||||||
|
private int _n_pubs = 0;
|
||||||
|
private readonly int _max_pubs;
|
||||||
|
private int _n_updates = 0;
|
||||||
|
private bool _continue_sub_thread = true;
|
||||||
|
private ViewStreamDatabase _vsd;
|
||||||
|
private bool _fire_dirty = false;
|
||||||
|
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
||||||
|
private readonly int _payload_sz;
|
||||||
|
private const byte _payload_byte = (byte)'A';
|
||||||
|
private readonly string _tag_name;
|
||||||
|
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
||||||
|
{
|
||||||
|
_vsd = vsd;
|
||||||
|
_payload_sz = payload_size;
|
||||||
|
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||||
|
Tuple.Create("id", TypeCode.Int32),
|
||||||
|
Tuple.Create("n_pubs", TypeCode.Int32),
|
||||||
|
Tuple.Create("n_recvs", TypeCode.Int32),
|
||||||
|
Tuple.Create("last_sent_id", TypeCode.Int32),
|
||||||
|
Tuple.Create("last_recv_id", TypeCode.Int32),
|
||||||
|
Tuple.Create("last_sent_guid", TypeCode.String),
|
||||||
|
Tuple.Create("last_recv_guid", TypeCode.String),
|
||||||
|
Tuple.Create("match", TypeCode.Boolean),
|
||||||
|
Tuple.Create("echo_time", TypeCode.Int32));
|
||||||
|
_table = new BlueTable(schema);
|
||||||
|
_publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
|
||||||
|
_msg_size = payload_size;
|
||||||
|
_max_pubs = max_pubs;
|
||||||
|
_tag_name = tag_name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
||||||
|
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber()
|
||||||
|
{
|
||||||
|
_vsd.addWorker(this);
|
||||||
|
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
|
||||||
|
th.IsBackground = true;
|
||||||
|
th.Start();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber()
|
||||||
|
{
|
||||||
|
_vsd.removeWorker(this);
|
||||||
|
_continue_sub_thread = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void _log_updates(IEnumerable<PubSubServerMessage> updates)
|
||||||
|
{
|
||||||
|
var idx = 0;
|
||||||
|
foreach (var msg in updates)
|
||||||
|
{
|
||||||
|
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _subscribe_thread_func()
|
||||||
|
{
|
||||||
|
// subscription is done in another thread
|
||||||
|
SubscriberCore sub_core = new SubscriberCore();
|
||||||
|
sub_core.subscribe(_tag_name);
|
||||||
|
while (_continue_sub_thread)
|
||||||
|
{
|
||||||
|
// TODO recv_id
|
||||||
|
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
||||||
|
if (updates == null || updates.Count() == 0)
|
||||||
|
continue;
|
||||||
|
_log_updates(updates);
|
||||||
|
_n_updates += updates.Count();
|
||||||
|
if (updates.Count() > 0)
|
||||||
|
{
|
||||||
|
var server_msg = updates.Last();
|
||||||
|
if (server_msg.data != null && server_msg.data.Length > 0)
|
||||||
|
{
|
||||||
|
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
||||||
|
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// why are we getting empty or null data ? (server is sending data regardless?)
|
||||||
|
}
|
||||||
|
_fire_dirty = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _publish_next_packet()
|
||||||
|
{
|
||||||
|
if (_last_sent_msg == null)
|
||||||
|
{
|
||||||
|
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
|
||||||
|
{
|
||||||
|
guid = Guid.NewGuid().ToString()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_last_sent_msg.msg_id++;
|
||||||
|
_last_sent_msg.guid = Guid.NewGuid().ToString();
|
||||||
|
_last_sent_msg.travel_epoch_ms = epoch_now();
|
||||||
|
}
|
||||||
|
var ser_bytes = _last_sent_msg.serialize();
|
||||||
|
_n_pubs++;
|
||||||
|
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
||||||
|
_fire_dirty = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _fire_dp()
|
||||||
|
{
|
||||||
|
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
|
||||||
|
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
|
||||||
|
object echo_time = DBNull.Value;
|
||||||
|
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
|
||||||
|
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
|
||||||
|
if (_last_sent_msg != null && _last_recvd_msg != null)
|
||||||
|
{
|
||||||
|
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
|
||||||
|
}
|
||||||
|
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
|
||||||
|
IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
|
||||||
|
last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
|
||||||
|
BlueTable table = new BlueTable(_table.schema);
|
||||||
|
table.addRow(nr);
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork()
|
||||||
|
{
|
||||||
|
if (DateTime.Now - _last_publish_time > _publish_period)
|
||||||
|
if (_n_pubs <= _max_pubs)
|
||||||
|
{
|
||||||
|
_publish_next_packet();
|
||||||
|
_last_publish_time = DateTime.Now;
|
||||||
|
}
|
||||||
|
// publish
|
||||||
|
// TODO: send id
|
||||||
|
if (_fire_dirty)
|
||||||
|
{
|
||||||
|
_fire_dirty = false;
|
||||||
|
_fire_dp();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static long epoch_now()
|
||||||
|
{
|
||||||
|
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
using BCG;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
using bts.api;
|
||||||
|
|
||||||
|
namespace bts
|
||||||
|
{
|
||||||
|
[DataStreamUserRespresentation("pss_echo")]
|
||||||
|
public class PubsubEchoTestStream : ViewStream, IWorker
|
||||||
|
{
|
||||||
|
public class PayloadMessage
|
||||||
|
{
|
||||||
|
public readonly string msg_id;
|
||||||
|
public readonly Int64 timestamp = 0;
|
||||||
|
public readonly byte[] payload = null;
|
||||||
|
|
||||||
|
public static byte[] create_data(int payload_size, byte payload_byte)
|
||||||
|
{
|
||||||
|
var rv = new byte[payload_size];
|
||||||
|
for (int i = 0; i < payload_size; i++)
|
||||||
|
{
|
||||||
|
rv[i] = payload_byte;
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
|
||||||
|
// {
|
||||||
|
// guid = guid_;
|
||||||
|
// timestamp = timestamp_;
|
||||||
|
// payload = new byte[payload_size];
|
||||||
|
// for (int i = 0; i < payload_size; i++)
|
||||||
|
// {
|
||||||
|
// payload[i] = payload_byte;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
|
||||||
|
{
|
||||||
|
msg_id = guid_;
|
||||||
|
timestamp = timestamp_;
|
||||||
|
payload = payload_;
|
||||||
|
}
|
||||||
|
|
||||||
|
private PayloadMessage() { }
|
||||||
|
public byte[] serialize()
|
||||||
|
{
|
||||||
|
var str = new MemoryStream();
|
||||||
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
|
bw.Write(msg_id);
|
||||||
|
bw.Write(timestamp);
|
||||||
|
bw.Write(payload.Length);
|
||||||
|
bw.Write(payload);
|
||||||
|
str.Position = 0;
|
||||||
|
var bytes = str.ToArray();
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static PayloadMessage deserialize(byte[] ser_bytes)
|
||||||
|
{
|
||||||
|
var str = new MemoryStream(ser_bytes);
|
||||||
|
BinaryReader br = new BinaryReader(str);
|
||||||
|
string guid = br.ReadString();
|
||||||
|
Int64 timestamp_ = br.ReadInt64();
|
||||||
|
int len = br.ReadInt32();
|
||||||
|
var payload = br.ReadBytes(len);
|
||||||
|
return new PayloadMessage(guid, timestamp_, payload);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//public void
|
||||||
|
private readonly int _msg_size;
|
||||||
|
private readonly TimeSpan _publish_period;
|
||||||
|
private DateTime _last_publish_time = DateTime.MinValue;
|
||||||
|
private PublisherCore _pub_core;
|
||||||
|
private ISubscriberCore _sub_core;
|
||||||
|
private BlueTable _table;
|
||||||
|
private int _n_pubs = 0;
|
||||||
|
private readonly int _max_pubs;
|
||||||
|
private int _n_updates = 0;
|
||||||
|
private ViewStreamDatabase _vsd;
|
||||||
|
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
||||||
|
private const byte _payload_byte = (byte)'A';
|
||||||
|
private readonly string _tag_name;
|
||||||
|
private readonly int _payload_sz;
|
||||||
|
private readonly IPubSubRouter _router;
|
||||||
|
private enum IOState
|
||||||
|
{
|
||||||
|
SEND = 0,
|
||||||
|
RECV = 1
|
||||||
|
};
|
||||||
|
IOState _state = IOState.SEND;
|
||||||
|
|
||||||
|
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
||||||
|
{
|
||||||
|
var route = new PubSubRoute(server_url, ".*");
|
||||||
|
_router = new PubSubRouter(route);
|
||||||
|
_vsd = vsd;
|
||||||
|
_payload_sz = payload_size;
|
||||||
|
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||||
|
Tuple.Create("id", TypeCode.Int32),
|
||||||
|
Tuple.Create("sent_id", TypeCode.String),
|
||||||
|
Tuple.Create("sent_time", TypeCode.Int64),
|
||||||
|
Tuple.Create("recv_id", TypeCode.String),
|
||||||
|
Tuple.Create("recv_time", TypeCode.Int64),
|
||||||
|
Tuple.Create("match", TypeCode.Boolean),
|
||||||
|
Tuple.Create("echo_time", TypeCode.Int32));
|
||||||
|
_table = new BlueTable(schema);
|
||||||
|
_publish_period = publish_period;
|
||||||
|
_tag_name = tag_name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
||||||
|
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber()
|
||||||
|
{
|
||||||
|
_pub_core = new PublisherCore(_router);
|
||||||
|
_sub_core = new SubscriberCore(_router);
|
||||||
|
_sub_core.subscribe(_tag_name);
|
||||||
|
_vsd.addWorker(this);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber()
|
||||||
|
{
|
||||||
|
_vsd.removeWorker(this);
|
||||||
|
_sub_core.unsubscribe(_tag_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _fire_dp_after_send()
|
||||||
|
{
|
||||||
|
// as soon as the data is published, its reflected in the stream state
|
||||||
|
Log.assert(_last_sent_msg != null);
|
||||||
|
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
||||||
|
DBNull.Value, DBNull.Value);
|
||||||
|
_table.addRow(r);
|
||||||
|
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _fire_dp_after_recv()
|
||||||
|
{
|
||||||
|
Log.assert(_last_recvd_msg != null);
|
||||||
|
// update the last publish row
|
||||||
|
int id = _table.Count - 1;
|
||||||
|
IBlueRow last_row = _table[id];
|
||||||
|
_table.deleteRow(last_row);
|
||||||
|
var recv_ts = epoch_now();
|
||||||
|
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
|
||||||
|
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
|
||||||
|
_table.addRow(nr);
|
||||||
|
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string _get_next_id() {
|
||||||
|
return _table.Count.ToString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private string _get_next_id()
|
||||||
|
{
|
||||||
|
return _table.Count.ToString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _publish_next_packet()
|
||||||
|
{
|
||||||
|
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
||||||
|
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
||||||
|
var ser_bytes = _last_sent_msg.serialize();
|
||||||
|
_n_pubs++;
|
||||||
|
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
||||||
|
_last_publish_time = DateTime.Now;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork()
|
||||||
|
{
|
||||||
|
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
|
||||||
|
{
|
||||||
|
foreach (var u in updates)
|
||||||
|
{
|
||||||
|
var recv_msg = PayloadMessage.deserialize(u.data);
|
||||||
|
if (recv_msg.msg_id == _last_sent_msg.msg_id)
|
||||||
|
{
|
||||||
|
return recv_msg;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
switch (_state)
|
||||||
|
{
|
||||||
|
case IOState.SEND:
|
||||||
|
if (DateTime.Now - _last_publish_time >= _publish_period)
|
||||||
|
{
|
||||||
|
_publish_next_packet();
|
||||||
|
_last_publish_time = DateTime.Now;
|
||||||
|
_fire_dp_after_send();
|
||||||
|
_state = IOState.RECV;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case IOState.RECV:
|
||||||
|
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
||||||
|
updates = updates.Where(u => u.tag == _tag_name);
|
||||||
|
var recv_msg = find_packet(updates);
|
||||||
|
if (recv_msg != null)
|
||||||
|
{
|
||||||
|
_last_recvd_msg = recv_msg;
|
||||||
|
_fire_dp_after_recv();
|
||||||
|
_state = IOState.SEND;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static long epoch_now()
|
||||||
|
{
|
||||||
|
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
namespace bts {
|
||||||
|
using BCG;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class SimpleTCPEchoStream : ViewStream, IWorker {
|
||||||
|
private string _ip;
|
||||||
|
private int _port;
|
||||||
|
private SimpleTCPClient _tcp_client;
|
||||||
|
private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
|
||||||
|
Tuple.Create("id", TypeCode.Int32),
|
||||||
|
Tuple.Create("event", TypeCode.String));
|
||||||
|
private BlueTable _table;
|
||||||
|
private SyncQueue<string> _action_queue = new SyncQueue<string>();
|
||||||
|
|
||||||
|
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
|
||||||
|
: this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
|
||||||
|
|
||||||
|
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
|
||||||
|
_ip = ip;
|
||||||
|
_port = port;
|
||||||
|
_table = new BlueTable(_schema);
|
||||||
|
_tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
ViewStreamDatabase.instance.addWorker(this);
|
||||||
|
_tcp_client.register_listener((msg) => { _on_msg(msg); });
|
||||||
|
_tcp_client.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _on_msg(string msg) {
|
||||||
|
_action_queue.enqueue(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
if (_action_queue.getCount() == 0)
|
||||||
|
return;
|
||||||
|
BlueTable added_rows = new BlueTable(_schema);
|
||||||
|
var sz = _action_queue.getCount();
|
||||||
|
for (int i = 0; i < sz; i++) {
|
||||||
|
string msg = _action_queue.dequeue();
|
||||||
|
if (msg == null)
|
||||||
|
break;
|
||||||
|
IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
|
||||||
|
_table.addRow(nr);
|
||||||
|
added_rows.addRow(nr);
|
||||||
|
}
|
||||||
|
if (added_rows.Count == 0)
|
||||||
|
return;
|
||||||
|
DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
using System.Net.Sockets;
|
||||||
|
using System.Net;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using MongoDB.Bson;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
|
||||||
|
namespace bts {
|
||||||
|
// this class, a little specialized, takes the first snapshot thats fired
|
||||||
|
// from the base stream sorts them by a given column, and fires one row at a time
|
||||||
|
public class SnapshotReplayStream : ViewStream, IWorker {
|
||||||
|
|
||||||
|
private ViewStreamDatabase _vsd;
|
||||||
|
private ViewStream _snapshot_stream;
|
||||||
|
private readonly string _sort_col;
|
||||||
|
private readonly int _buffer_size;
|
||||||
|
private TimeSpan _fire_period;
|
||||||
|
private int _in_row_ptr = 0;
|
||||||
|
private List<IBlueRow> _in_rows = new List<IBlueRow>();
|
||||||
|
private CircularBufferTable _out_table;
|
||||||
|
|
||||||
|
|
||||||
|
public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period)
|
||||||
|
: this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { }
|
||||||
|
|
||||||
|
public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) {
|
||||||
|
_vsd = vsd;
|
||||||
|
_snapshot_stream = base_str;
|
||||||
|
_sort_col = sort_col;
|
||||||
|
_buffer_size = buffer_size;
|
||||||
|
_fire_period = fire_period;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_snapshot_stream.onChange += _handle_stream_update;
|
||||||
|
MasterWorker.getInstance(false).addWorker(this, _fire_period);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
base.myOnLastSubscriber();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _handle_stream_update(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
if (_in_rows.Count > 0)
|
||||||
|
return;
|
||||||
|
_out_table = new CircularBufferTable(dp.schema, _buffer_size);
|
||||||
|
// _in_rows is filled up only once
|
||||||
|
foreach (IBlueRow r in dp.newRows) {
|
||||||
|
_in_rows.Add(r);
|
||||||
|
}
|
||||||
|
_in_rows.Sort((IBlueRow r1, IBlueRow r2) => {
|
||||||
|
return BlueDataUtility.compare_rows(r1, r2, _sort_col);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
if (_in_row_ptr >= _in_rows.Count) {
|
||||||
|
// we can loop around or not here
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (_in_rows.Count > 0) {
|
||||||
|
// fire row
|
||||||
|
IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
|
||||||
|
_out_table.add_row(curr_row);
|
||||||
|
DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
|
||||||
|
myLastValue == null ? null : myLastValue as DataPacket, false);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,809 @@
|
|||||||
|
using bts;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Reflection;
|
||||||
|
|
||||||
|
namespace BCG {
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
|
||||||
|
/// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
|
||||||
|
/// </summary>
|
||||||
|
[DataStreamUserRespresentation("tests")]
|
||||||
|
public class TesStream : ViewStream {
|
||||||
|
static byte[] create_byte_array() {
|
||||||
|
Random random = new Random();
|
||||||
|
var sz = random.Next(1, 1000);// % 1000;
|
||||||
|
|
||||||
|
// Generate a random byte
|
||||||
|
byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
|
||||||
|
byte[] rv = new byte[sz];
|
||||||
|
for (int i = 0; i < sz; i++) {
|
||||||
|
rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
class TestResult {
|
||||||
|
public string test_name = MethodBase.GetCurrentMethod()?.Name
|
||||||
|
?? throw new InvalidOperationException("Failed to get test name from method");
|
||||||
|
public bool test_result;
|
||||||
|
}
|
||||||
|
private List<Func<TestResult>> _tests = new
|
||||||
|
List<Func<TestResult>>();
|
||||||
|
|
||||||
|
public TesStream() {
|
||||||
|
_schema = BlueDataUtility.form_schema("Tests",
|
||||||
|
Tuple.Create("id", TypeCode.Int32),
|
||||||
|
Tuple.Create("test", TypeCode.String),
|
||||||
|
Tuple.Create("result", TypeCode.Boolean),
|
||||||
|
Tuple.Create("exception", TypeCode.String)
|
||||||
|
);
|
||||||
|
// register tests here
|
||||||
|
_tests.Add(new Func<TestResult>(_run_sub_msg_test));
|
||||||
|
_tests.Add(new Func<TestResult>(_run_pub_msg_test));
|
||||||
|
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
|
||||||
|
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
|
||||||
|
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
|
||||||
|
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
|
||||||
|
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
|
||||||
|
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
|
||||||
|
//_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
|
||||||
|
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
|
||||||
|
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
|
||||||
|
// _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
|
||||||
|
// _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
|
||||||
|
//_tests.Add(new Func<TestResult>(_cpp_data_test));
|
||||||
|
//_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
|
||||||
|
_tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
BlueTable rows = new BlueTable(_schema);
|
||||||
|
// u the tests here
|
||||||
|
foreach (var test in _tests) {
|
||||||
|
string ex_msg = String.Empty;
|
||||||
|
TestResult tr = new TestResult() {
|
||||||
|
test_name = test.Method.Name,
|
||||||
|
test_result = false
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
tr = test();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex_msg = ex.Message + "\n" + ex.StackTrace;
|
||||||
|
}
|
||||||
|
IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
|
||||||
|
rows.addRow(r);
|
||||||
|
}
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _run_sub_msg_test() {
|
||||||
|
bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
|
||||||
|
connection = Guid.NewGuid().ToString(),
|
||||||
|
subscribing_app = Guid.NewGuid().ToString(),
|
||||||
|
tags = new string[] { "one", "two", "three" }
|
||||||
|
};
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
sub_msg.serialize_into_str(str);
|
||||||
|
str.Position = 0;
|
||||||
|
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
|
||||||
|
sub_msg_.consruct_deserialize(str);
|
||||||
|
bool res = sub_msg.Equals(sub_msg_);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "SubscribeMsg",
|
||||||
|
test_result = res
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _run_pub_msg_test() {
|
||||||
|
var pub_msg = new bts.PublishMsg() {
|
||||||
|
connection = Guid.NewGuid().ToString(),
|
||||||
|
publishing_app = Guid.NewGuid().ToString(),
|
||||||
|
tag = Guid.NewGuid().ToString(),
|
||||||
|
tag_type = "DS_",
|
||||||
|
is_compressed = false,
|
||||||
|
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
||||||
|
data_to_publish = create_byte_array(),
|
||||||
|
};
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
pub_msg.serialize_into_str(str);
|
||||||
|
str.Position = 0;
|
||||||
|
var pub_msg_ = new bts.PublishMsg();
|
||||||
|
pub_msg_.consruct_deserialize(str);
|
||||||
|
bool res = pub_msg.Equals(pub_msg_);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "PublishMsg",
|
||||||
|
test_result = res
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _run_pubsub_server_msg_test() {
|
||||||
|
var msg = new bts.PubSubServerMessage() {
|
||||||
|
tag = Guid.NewGuid().ToString(),
|
||||||
|
tag_type = Guid.NewGuid().ToString(),
|
||||||
|
data = create_byte_array()
|
||||||
|
};
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
msg.serialize_into_str(str);
|
||||||
|
str.Position = 0;
|
||||||
|
var msg_ = new bts.PubSubServerMessage();
|
||||||
|
msg_.consruct_deserialize(str);
|
||||||
|
bool res = msg.Equals(msg_);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "PubSubServerMessage",
|
||||||
|
test_result = res
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _run_pubsub_client_msg_test() {
|
||||||
|
PubSubClientMessage msg = new PubSubClientMessage() {
|
||||||
|
msg_type = PubSubMessageType.SUBSCRIBE,
|
||||||
|
sub_msg = new bts.SubscribeMsg() {
|
||||||
|
connection = Guid.NewGuid().ToString(),
|
||||||
|
subscribing_app = Guid.NewGuid().ToString(),
|
||||||
|
tags = new string[] { "one", "two", "three" }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
var str = new MemoryStream();
|
||||||
|
msg.serialize_into_str(str);
|
||||||
|
str.Position = 0;
|
||||||
|
var msg_ = new PubSubClientMessage();
|
||||||
|
msg_.consruct_deserialize(str);
|
||||||
|
var res = msg.Equals(msg_);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "PubSubClientMessage",
|
||||||
|
test_result = res
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _run_columninfo_serialization_test() {
|
||||||
|
Func<ColumnInfo, bool> test_ci = (ci) => {
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
|
StreamSerializationUtils.serialize_column_info(ci, ref bw);
|
||||||
|
str.Position = 0;
|
||||||
|
//var ci_txt = BlueDataUtility.to_string(ci);
|
||||||
|
//var ci_ser =
|
||||||
|
//bts.Log.Gen.info("ci: {}")
|
||||||
|
BinaryReader br = new BinaryReader(str);
|
||||||
|
var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
|
||||||
|
return BlueDataUtility.compute_equality(ci, ci_prime);
|
||||||
|
};
|
||||||
|
int i = 0;
|
||||||
|
foreach (Type t in BlueDataUtility.valid_column_types()) {
|
||||||
|
ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
|
||||||
|
i++, t);
|
||||||
|
var res = test_ci(ci);
|
||||||
|
if (!res) {
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "ColumnInfoSerialization",
|
||||||
|
test_result = false
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "ColumnInfoSerialization",
|
||||||
|
test_result = true
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private TestResult _run_schema_serialization_test() {
|
||||||
|
var table_name = "test_table";
|
||||||
|
var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
|
||||||
|
IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
|
StreamSerializationUtils.serialize_schema(schema, ref bw);
|
||||||
|
_write_to_file(str, @"c:\Temp\tests\schema.bin");
|
||||||
|
var hex_txt = StreamSerializationUtils.to_hex(str);
|
||||||
|
bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
|
||||||
|
|
||||||
|
str.Position = 0;
|
||||||
|
BinaryReader br = new BinaryReader(str);
|
||||||
|
var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
|
||||||
|
var res = BlueDataUtility.compute_equality(schema, schema_prime);
|
||||||
|
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "SchemaSerialization",
|
||||||
|
test_result = res
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private TestResult _run_datetime_seriliaztion_test() {
|
||||||
|
DateTime dt = DateTime.Now;
|
||||||
|
DateTimeOffset dt_off = new DateTimeOffset(dt);
|
||||||
|
long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
|
||||||
|
DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
|
||||||
|
DateTime dt_p = dt_off_pr.DateTime;
|
||||||
|
var passes = dt.Equals(dt_p);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "_run_datetime_seriliaztion_test",
|
||||||
|
test_result = false
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _run_full_stream_serialization_test() {
|
||||||
|
// create message
|
||||||
|
var schema = create_schema_with_all_types();
|
||||||
|
BlueTable cr_rows = new BlueTable(schema);
|
||||||
|
// 100 rows
|
||||||
|
int n_rows = 100;
|
||||||
|
for (int i = 0; i < n_rows; i++) {
|
||||||
|
var nr = generate_random_row(schema, i);
|
||||||
|
cr_rows.addRow(nr);
|
||||||
|
}
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
|
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
||||||
|
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
|
||||||
|
UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
|
||||||
|
ser.serialize(stream_io_msg, ref bw);
|
||||||
|
str.Position = 0;
|
||||||
|
BinaryReader br = new System.IO.BinaryReader(str);
|
||||||
|
UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
|
||||||
|
var stream_io_msg_prime = deser.deserialize(ref br);
|
||||||
|
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
|
||||||
|
var n_dels = 50;
|
||||||
|
BlueTable del_rows = new BlueTable(cr_rows.schema);
|
||||||
|
foreach (IBlueRow r in cr_rows) {
|
||||||
|
del_rows.addRow(r);
|
||||||
|
if (del_rows.Count == n_dels)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
IBlueRowCollection added_rows = del_rows;
|
||||||
|
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
||||||
|
str = new MemoryStream();
|
||||||
|
bw = new BinaryWriter(str);
|
||||||
|
stream_io_msg = new bts.StreamIOMessage(mra_msg);
|
||||||
|
ser.serialize(stream_io_msg, ref bw);
|
||||||
|
bw.BaseStream.Position = 0;
|
||||||
|
br = new BinaryReader(str);
|
||||||
|
//deser = new UniversalStreamIODeserializer();
|
||||||
|
stream_io_msg_prime = deser.deserialize(ref br);
|
||||||
|
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "UniversalSerialization",
|
||||||
|
test_result = passes
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _run_full_stream_serialization_with_controller_test() {
|
||||||
|
// create message
|
||||||
|
var schema = create_schema_with_all_types();
|
||||||
|
BlueTable cr_rows = new BlueTable(schema);
|
||||||
|
// 100 rows
|
||||||
|
int n_rows = 100;
|
||||||
|
for (int i = 0; i < n_rows; i++) {
|
||||||
|
var nr = generate_random_row(schema, i);
|
||||||
|
cr_rows.addRow(nr);
|
||||||
|
}
|
||||||
|
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
||||||
|
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
|
||||||
|
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
|
||||||
|
var ser_bytes = ser.serialize(stream_io_msg);
|
||||||
|
IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
|
||||||
|
var stream_io_msg_prime = deser.deserialize(ser_bytes);
|
||||||
|
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
|
||||||
|
//mult remove adds
|
||||||
|
// remove all rows and add 50 more
|
||||||
|
|
||||||
|
BlueTable del_rows = new BlueTable(cr_rows);
|
||||||
|
BlueTable added_rows = new BlueTable(cr_rows.schema);
|
||||||
|
var n_adds = 50;
|
||||||
|
for (int i = 0; i < n_adds; i++) {
|
||||||
|
var nr = generate_random_row(schema, i);
|
||||||
|
added_rows.addRow(nr);
|
||||||
|
}
|
||||||
|
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
||||||
|
stream_io_msg = new bts.StreamIOMessage(mra_msg);
|
||||||
|
ser_bytes = ser.serialize(stream_io_msg);
|
||||||
|
var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
|
||||||
|
Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
|
||||||
|
stream_io_msg_prime = deser.deserialize(ser_bytes);
|
||||||
|
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "UniversalSerializationController",
|
||||||
|
test_result = passes
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _row_serialization_tests_no_dbnulls() {
|
||||||
|
// IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
||||||
|
// new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
||||||
|
// new Tuple<string, TypeCode>("i1", TypeCode.Int32),
|
||||||
|
// new Tuple<string, TypeCode>("d1", TypeCode.Double),
|
||||||
|
// new Tuple<string, TypeCode>("string_1", TypeCode.String));
|
||||||
|
// bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||||
|
// IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
|
||||||
|
var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
|
||||||
|
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
IBlueRow r = generate_random_row(schema, 0, false);
|
||||||
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
|
row_ser.serialize_row(r, 0, ref bw);
|
||||||
|
_write_to_file(str, @"C:\Temp\tests\row.bin");
|
||||||
|
str.Position = 0;
|
||||||
|
byte[] raw_byte_array = str.ToArray();
|
||||||
|
var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
|
||||||
|
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
|
||||||
|
BinaryReader br = new BinaryReader(str);
|
||||||
|
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
||||||
|
IBlueRow r_prime = des.deserialize_row(ref br);
|
||||||
|
string row_str = r_prime.to_string();
|
||||||
|
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
|
||||||
|
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "UniversalRowSerialization_NonNull",
|
||||||
|
test_result = passes
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _deserialize_pub_msg_test() {
|
||||||
|
var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
|
||||||
|
PubSubClientMessage ps_client_msg = new PubSubClientMessage();
|
||||||
|
ps_client_msg.consruct_deserialize(pub_msg_ser);
|
||||||
|
bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "_deserialize_pub_msg_test",
|
||||||
|
test_result = passes
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _cpp_data_test() {
|
||||||
|
//create and write schema
|
||||||
|
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
||||||
|
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
||||||
|
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
|
||||||
|
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
|
||||||
|
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
|
||||||
|
new Tuple<string, TypeCode>("string_1", TypeCode.String)
|
||||||
|
);
|
||||||
|
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||||
|
UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
|
||||||
|
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||||
|
StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
||||||
|
|
||||||
|
IBlueTableSchema schema_p = null;
|
||||||
|
var schema_file = @"c:\Temp\tests\schema.bin";
|
||||||
|
var row_file = @"C:\Temp\tests\row.bin";
|
||||||
|
var create_op_file = @"C:\Temp\tests\create_op.bin";
|
||||||
|
var mra_op_file = @"C:\Temp\tests\mra_op.bin";
|
||||||
|
IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
|
||||||
|
|
||||||
|
//serialize schema, write it to a file, read it back, deserialize it and match schemas
|
||||||
|
byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
|
||||||
|
_write_to_file(ser_schema_bytes, schema_file);
|
||||||
|
|
||||||
|
byte[] schema_file_bytes = _read_from_file(schema_file);
|
||||||
|
schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
|
||||||
|
bool passes = BlueDataUtility.compute_equality(schema, schema_p);
|
||||||
|
|
||||||
|
//serialize row, write it to a file, read it back, deserialize it and match schemas
|
||||||
|
byte[] ser_row_bytes = row_ser.serialize_row(row);
|
||||||
|
_write_to_file(ser_row_bytes, row_file);
|
||||||
|
|
||||||
|
byte[] row_file_bytes = _read_from_file(row_file);
|
||||||
|
var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
|
||||||
|
Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
|
||||||
|
|
||||||
|
IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
|
||||||
|
passes = passes && BlueDataUtility.compute_equality(row, row_p);
|
||||||
|
|
||||||
|
{
|
||||||
|
// serialize a create operation
|
||||||
|
int n_rows = 100;
|
||||||
|
BlueTable table = new BlueTable(schema);
|
||||||
|
for (int i = 0; i < n_rows; i++) {
|
||||||
|
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
|
||||||
|
table.addRow(nr);
|
||||||
|
}
|
||||||
|
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
|
||||||
|
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||||
|
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
|
||||||
|
_write_to_file(create_ser_bytes, create_op_file);
|
||||||
|
|
||||||
|
// read it and make sure its right
|
||||||
|
byte[] create_op_file_bytes = _read_from_file(create_op_file);
|
||||||
|
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
||||||
|
var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
|
||||||
|
_write_to_file(create_ser_bytes, mra_op_file);
|
||||||
|
|
||||||
|
passes = passes && cr_msg.Equals(cr_msg_p);
|
||||||
|
|
||||||
|
}
|
||||||
|
//multiple remove adds
|
||||||
|
{
|
||||||
|
// remove all rows and add them again
|
||||||
|
int n_rows = 100;
|
||||||
|
BlueTable table = new BlueTable(schema);
|
||||||
|
for (int i = 0; i < n_rows; i++) {
|
||||||
|
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
|
||||||
|
table.addRow(nr);
|
||||||
|
}
|
||||||
|
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
|
||||||
|
|
||||||
|
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||||
|
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
|
||||||
|
_write_to_file(mra_ser_bytes, mra_op_file);
|
||||||
|
|
||||||
|
// reasd it and make sure it matches
|
||||||
|
byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
|
||||||
|
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
||||||
|
StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
|
||||||
|
passes = passes && mra_msg.Equals(mra_msg_p);
|
||||||
|
}
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "_cpp_data_test",
|
||||||
|
test_result = passes
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _write_stream_updates_to_files() {
|
||||||
|
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
||||||
|
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
||||||
|
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
|
||||||
|
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
|
||||||
|
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
|
||||||
|
new Tuple<string, TypeCode>("string_1", TypeCode.String)
|
||||||
|
);
|
||||||
|
var packets_path = @"C:\Temp\tests\stream_packets\";
|
||||||
|
Func<long> now_epochs = () => {
|
||||||
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||||
|
// Convert to Unix time in milliseconds
|
||||||
|
long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
|
||||||
|
return unix_time_milliseconds;
|
||||||
|
};
|
||||||
|
Func<int, bool, string> curr_file_path = (id, create) => {
|
||||||
|
//var epoch_ms = now_epochs();
|
||||||
|
var file_path = $"{packets_path}{id}.bin";
|
||||||
|
if (create)
|
||||||
|
file_path = $"{packets_path}create_{id}.bin";
|
||||||
|
return file_path;
|
||||||
|
};
|
||||||
|
// write 1000 stream updates
|
||||||
|
{
|
||||||
|
int init_sz = 100;
|
||||||
|
BlueTable table = new BlueTable(schema);
|
||||||
|
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||||
|
// create table of 100 rows
|
||||||
|
for (int i = 0; i < init_sz; i++) {
|
||||||
|
IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
|
||||||
|
table.addRow(nr);
|
||||||
|
}
|
||||||
|
// write create msg
|
||||||
|
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
|
||||||
|
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
|
||||||
|
var create_file_path = curr_file_path(table.Count, true);
|
||||||
|
_write_to_file(create_ser_bytes, create_file_path);
|
||||||
|
|
||||||
|
// add 1000 rows and write an io msg for each update and write it to a file
|
||||||
|
int n_updates = 1000;
|
||||||
|
int period_ms = 50;
|
||||||
|
|
||||||
|
// write a create
|
||||||
|
for (int i = 0; i < n_updates; i++) {
|
||||||
|
// addition
|
||||||
|
BlueTable adds = new BlueTable(schema);
|
||||||
|
int id = table.Count;
|
||||||
|
IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
|
||||||
|
adds.addRow(nr);
|
||||||
|
BlueTable dels = new BlueTable(schema);
|
||||||
|
table.addRow(nr);
|
||||||
|
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
|
||||||
|
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
|
||||||
|
var mra_file_path = curr_file_path(id, false);
|
||||||
|
_write_to_file(mra_ser_bytes, mra_file_path);
|
||||||
|
|
||||||
|
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
|
||||||
|
}
|
||||||
|
// now start writing increments and add 1 row at a time
|
||||||
|
}
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "_write_updates_as_files",
|
||||||
|
test_result = true
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private TestResult _row_serialization_tests_with_dbnulls() {
|
||||||
|
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
||||||
|
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
||||||
|
new Tuple<string, TypeCode>("i1", TypeCode.Int32),
|
||||||
|
new Tuple<string, TypeCode>("d1", TypeCode.Double),
|
||||||
|
new Tuple<string, TypeCode>("string_1", TypeCode.String));
|
||||||
|
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||||
|
IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
|
row_ser.serialize_row(r, 0, ref bw);
|
||||||
|
str.Position = 0;
|
||||||
|
BinaryReader br = new BinaryReader(str);
|
||||||
|
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
||||||
|
IBlueRow r_prime = des.deserialize_row(ref br);
|
||||||
|
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "UniversalRowSerialization_WithDBNulls",
|
||||||
|
test_result = passes
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestResult _row_serialization_tests_all_types() {
|
||||||
|
var schema = create_schema_with_all_types();
|
||||||
|
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
||||||
|
IBlueRow r = generate_random_row(schema, 0, false);
|
||||||
|
MemoryStream str = new MemoryStream();
|
||||||
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
|
row_ser.serialize_row(r, 0, ref bw);
|
||||||
|
_write_to_file(str, @"C:\Temp\tests\row.bin");
|
||||||
|
str.Position = 0;
|
||||||
|
BinaryReader br = new BinaryReader(str);
|
||||||
|
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
||||||
|
IBlueRow r_prime = des.deserialize_row(ref br);
|
||||||
|
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
||||||
|
return new TestResult {
|
||||||
|
test_name = "UniversalRowSerialization_AllTypes",
|
||||||
|
test_result = passes
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TestResult _run_pubsub_server_stream_tag_handler_test() {
|
||||||
|
// create message
|
||||||
|
var schema = create_schema_with_all_types();
|
||||||
|
var tag = "DS_" + schema.tableName;
|
||||||
|
BlueTable cr_rows = new BlueTable(schema);
|
||||||
|
// 100 rows
|
||||||
|
int n_rows = 100;
|
||||||
|
for (int i = 0; i < n_rows; i++) {
|
||||||
|
var nr = generate_random_row(schema, i);
|
||||||
|
cr_rows.addRow(nr);
|
||||||
|
}
|
||||||
|
|
||||||
|
PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
|
||||||
|
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
||||||
|
bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
|
||||||
|
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
|
||||||
|
byte[] msg_bytes = ser.serialize(str_io_msg);
|
||||||
|
handler.update(tag, msg_bytes);
|
||||||
|
|
||||||
|
//now get it back for the handler
|
||||||
|
byte[] content = handler.get_content(tag);
|
||||||
|
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
||||||
|
bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
|
||||||
|
// test here
|
||||||
|
var passes = str_io_msg.Equals(str_io_msg_p);
|
||||||
|
// delete all rows
|
||||||
|
BlueTable del_rows = new BlueTable(cr_rows);
|
||||||
|
cr_rows.clear();
|
||||||
|
// added 50 new rows
|
||||||
|
BlueTable added_rows = new BlueTable(del_rows.schema);
|
||||||
|
int new_state_sz = 50;
|
||||||
|
for (int i = 0; i < new_state_sz; i++) {
|
||||||
|
var nr = generate_random_row(schema, i);
|
||||||
|
added_rows.addRow(nr);
|
||||||
|
}
|
||||||
|
cr_rows.addRows(added_rows);
|
||||||
|
// the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
|
||||||
|
// 1. generate op
|
||||||
|
bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
||||||
|
str_io_msg = new bts.StreamIOMessage(mra);
|
||||||
|
// 2. serialize it into bytes
|
||||||
|
msg_bytes = ser.serialize(str_io_msg);
|
||||||
|
// 3. send it to handler
|
||||||
|
handler.update(tag, msg_bytes);
|
||||||
|
// 4. get the state from the handler
|
||||||
|
byte[] content_p = handler.get_content(tag);
|
||||||
|
// 5. deserialize the state into a stream op
|
||||||
|
str_io_msg_p = deser.deserialize(content_p);
|
||||||
|
// 6. check to make sure the state's rows are good
|
||||||
|
passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
|
||||||
|
BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
|
||||||
|
return new TestResult {
|
||||||
|
test_result = passes,
|
||||||
|
test_name = "run_pubsub_server_stream_tag_handler_test"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static TestResult _run_single_tag_data_test_clean() {
|
||||||
|
// create message
|
||||||
|
var test_name = MethodBase.GetCurrentMethod().Name;
|
||||||
|
var schema = create_schema_with_all_types();
|
||||||
|
var tag = "DS_" + schema.tableName;
|
||||||
|
BlueTable table = new BlueTable(schema);
|
||||||
|
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
|
||||||
|
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
||||||
|
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
|
||||||
|
// create with 100 rows
|
||||||
|
int n_rows = 100;
|
||||||
|
bool is_succ = true;
|
||||||
|
for (int i = 0; i < n_rows; i++) {
|
||||||
|
var nr = generate_random_row(schema, i);
|
||||||
|
// perform the same action SingletagData, and verify the states match
|
||||||
|
PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
|
||||||
|
table.addRow(nr);
|
||||||
|
st_data.handle_publish(pub_msg);
|
||||||
|
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
||||||
|
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
||||||
|
}
|
||||||
|
// change all rows
|
||||||
|
for (int i = 0; i < n_rows; i++) {
|
||||||
|
var nr = generate_random_row(schema, i);
|
||||||
|
var dr = table[i];
|
||||||
|
table.deleteRow(dr);
|
||||||
|
table.addRow(nr);
|
||||||
|
PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
|
||||||
|
st_data.handle_publish(pub_msg);
|
||||||
|
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
||||||
|
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
||||||
|
}
|
||||||
|
// delete all rows
|
||||||
|
var aux_table = new BlueTable(table);
|
||||||
|
foreach (IBlueRow r in aux_table) {
|
||||||
|
table.deleteRow(r);
|
||||||
|
PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
|
||||||
|
st_data.handle_publish(pub_msg);
|
||||||
|
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
||||||
|
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
||||||
|
}
|
||||||
|
return new TestResult() {
|
||||||
|
test_result = is_succ,
|
||||||
|
test_name = test_name
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#region Helper functions
|
||||||
|
|
||||||
|
private static void _write_to_file(byte[] ser_bytes, string file_name) {
|
||||||
|
_write_to_file(new MemoryStream(ser_bytes), file_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void _write_to_file(MemoryStream str, string file_name) {
|
||||||
|
if (String.IsNullOrEmpty(file_name))
|
||||||
|
return;
|
||||||
|
byte[] str_bytes = str.ToArray();
|
||||||
|
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
|
||||||
|
// Write the byte array to the file
|
||||||
|
fs.Write(str_bytes, 0, str_bytes.Length);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] _read_from_file(string file_path) {
|
||||||
|
var rv = File.ReadAllBytes(file_path);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
|
||||||
|
var content = st_data.get_current_content();
|
||||||
|
var msg_p = deser.deserialize(content);
|
||||||
|
Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
|
||||||
|
return msg_p.create_message.rows;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
|
||||||
|
ref IStreamIOMsgSerializer serializer) {
|
||||||
|
Func<IBlueTableSchema> get_schema = () => {
|
||||||
|
return (added_row != null ? added_row : removed_row).schema;
|
||||||
|
};
|
||||||
|
bts.StreamIOMessage str_io_msg = null;
|
||||||
|
if (is_create) {
|
||||||
|
Log.assert(added_row != null);
|
||||||
|
var aux_new = new BlueTable(added_row.schema);
|
||||||
|
aux_new.addRow(added_row);
|
||||||
|
var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
|
||||||
|
str_io_msg = new bts.StreamIOMessage(cr_msg);
|
||||||
|
} else {
|
||||||
|
var schema = get_schema();
|
||||||
|
BlueTable added_table = new BlueTable(schema);
|
||||||
|
if (added_row != null)
|
||||||
|
added_table.addRow(added_row);
|
||||||
|
BlueTable rem_table = new BlueTable(schema);
|
||||||
|
if (removed_row != null)
|
||||||
|
rem_table.addRow(removed_row);
|
||||||
|
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
|
||||||
|
new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
|
||||||
|
str_io_msg = new bts.StreamIOMessage(mra_msg);
|
||||||
|
}
|
||||||
|
byte[] msg_bytes = serializer.serialize(str_io_msg);
|
||||||
|
PublishMsg pub_msg = new PublishMsg() {
|
||||||
|
connection = "test_connection",
|
||||||
|
tag = $"DS_{get_schema()}",
|
||||||
|
tag_type = "DS",
|
||||||
|
data_to_publish = msg_bytes
|
||||||
|
};
|
||||||
|
return pub_msg;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
|
||||||
|
excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
|
||||||
|
var table_name = "test_table";
|
||||||
|
BlueTableSchema schema = new BlueTableSchema();
|
||||||
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
|
foreach (Type t in BlueDataUtility.valid_column_types()) {
|
||||||
|
if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
|
||||||
|
continue;
|
||||||
|
var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
|
||||||
|
ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
|
||||||
|
cis.Add(ci);
|
||||||
|
}
|
||||||
|
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
|
||||||
|
schema = new BlueTableSchema(table_name, cis.ToArray());
|
||||||
|
return schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
|
||||||
|
Random rnd = new Random();
|
||||||
|
object[] row_vals = new object[schema.columnInfo.Length];
|
||||||
|
foreach (ColumnInfo ci in schema.columnInfo) {
|
||||||
|
if (ci.colName == "id") {
|
||||||
|
row_vals[ci.colNo] = id;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
object v = get_random_object(ci.type_code());
|
||||||
|
if (include_dbnulls && rnd.NextSingle() % 4 == 0)
|
||||||
|
v = DBNull.Value;
|
||||||
|
row_vals[ci.colNo] = v;
|
||||||
|
}
|
||||||
|
return new BlueRow(schema, row_vals);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static object get_random_object(TypeCode type_code) {
|
||||||
|
Random random = new Random();
|
||||||
|
switch (type_code) {
|
||||||
|
case TypeCode.Boolean:
|
||||||
|
return random.Next(2) == 0; // Random boolean value (true or false)
|
||||||
|
case TypeCode.Byte:
|
||||||
|
return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
|
||||||
|
case TypeCode.Char:
|
||||||
|
return (char)random.Next(65, 65 + 27); // Random char value
|
||||||
|
case TypeCode.Decimal:
|
||||||
|
Decimal dec = 100.123M;
|
||||||
|
return dec;
|
||||||
|
case TypeCode.DateTime:
|
||||||
|
return DateTime.Now;
|
||||||
|
case TypeCode.Double:
|
||||||
|
return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
|
||||||
|
case TypeCode.Int16:
|
||||||
|
return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
|
||||||
|
case TypeCode.Int32:
|
||||||
|
return random.Next(int.MinValue, int.MaxValue); // Random int value
|
||||||
|
case TypeCode.Int64:
|
||||||
|
return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
|
||||||
|
case TypeCode.SByte:
|
||||||
|
return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
|
||||||
|
case TypeCode.Single:
|
||||||
|
return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
|
||||||
|
case TypeCode.String:
|
||||||
|
return Guid.NewGuid().ToString();
|
||||||
|
case TypeCode.UInt16:
|
||||||
|
return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
|
||||||
|
case TypeCode.UInt32:
|
||||||
|
return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
|
||||||
|
case TypeCode.UInt64:
|
||||||
|
byte[] ulongBuffer = new byte[8];
|
||||||
|
random.NextBytes(ulongBuffer);
|
||||||
|
return BitConverter.ToUInt64(ulongBuffer, 0);
|
||||||
|
default:
|
||||||
|
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
namespace BCG {
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class TickingTestStream : ViewStream {
|
||||||
|
public TickingTestStream(int ticking_period_msecs)
|
||||||
|
:
|
||||||
|
this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
|
||||||
|
|
||||||
|
public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
|
||||||
|
ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
|
||||||
|
vs = new MathStream(vs, "x=id*123 y=x*x");
|
||||||
|
mySetRealSource(vs);
|
||||||
|
//vs.onChange += _on_str_update;
|
||||||
|
}
|
||||||
|
|
||||||
|
// private void _on_str_update(object dp_obj) {
|
||||||
|
// DataPacket dp = dp_obj as DataPacket;
|
||||||
|
// IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
|
||||||
|
// BlueTable state = new BlueTable(out_schema);
|
||||||
|
// foreach (IBlueRow r in dp.newRows) {
|
||||||
|
// MutableBlueRow nr = new MutableBlueRow(out_schema, r);
|
||||||
|
// DateTime time = r.get_datetime("time");
|
||||||
|
// var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
||||||
|
// nr.setValue("time_txt", time_txt);
|
||||||
|
// state.addRow(nr);
|
||||||
|
// }
|
||||||
|
// var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
|
||||||
|
// myFireEvent(dp_out);
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using mbts;
|
||||||
|
|
||||||
|
using MongoDB.Bson;
|
||||||
|
using MongoDB.Driver;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
using System.Reactive.Subjects;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
|
||||||
|
readonly string collectionName;
|
||||||
|
readonly string databaseName;
|
||||||
|
readonly string url;
|
||||||
|
readonly object syncRoot = new();
|
||||||
|
readonly CancellationTokenSource cts = new();
|
||||||
|
readonly Subject<DataPacket> subject = new();
|
||||||
|
|
||||||
|
bool hasStarted = false;
|
||||||
|
IBlueRowFactory<BsonDocument>? rowFactory;
|
||||||
|
MongoClient? clientBacking;
|
||||||
|
IMongoDatabase? databaseBacking;
|
||||||
|
IMongoCollection<BsonDocument>? collectionBacking = null;
|
||||||
|
BlueTableImmutable? table = null;
|
||||||
|
|
||||||
|
public MongoCollectionStream(string collectionName)
|
||||||
|
: this(
|
||||||
|
collectionName,
|
||||||
|
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
|
||||||
|
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public MongoCollectionStream(string collectionName, string databaseName)
|
||||||
|
: this(
|
||||||
|
collectionName,
|
||||||
|
databaseName,
|
||||||
|
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public MongoCollectionStream(string collectionName, string databaseName, string url)
|
||||||
|
: this(collectionName, databaseName, url, null) { }
|
||||||
|
|
||||||
|
internal MongoCollectionStream(
|
||||||
|
string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
|
||||||
|
this.collectionName = collectionName;
|
||||||
|
this.databaseName = databaseName;
|
||||||
|
this.url = url;
|
||||||
|
this.rowFactory = rowFactory;
|
||||||
|
mySetRealSource(new ObservableViewStream(this));
|
||||||
|
}
|
||||||
|
|
||||||
|
MongoClient client => clientBacking ??= new(url);
|
||||||
|
IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
|
||||||
|
IMongoCollection<BsonDocument>? collection {
|
||||||
|
get => Volatile.Read(ref collectionBacking);
|
||||||
|
set => Volatile.Write(ref collectionBacking, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IDisposable Subscribe(IObserver<DataPacket> observer) {
|
||||||
|
lock (syncRoot) {
|
||||||
|
if (hasStarted) {
|
||||||
|
if (table is not null) {
|
||||||
|
// The first update has already been fired. Seed observer w/ current state.
|
||||||
|
observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
|
||||||
|
}
|
||||||
|
return subject.Subscribe(observer);
|
||||||
|
}
|
||||||
|
hasStarted = true;
|
||||||
|
}
|
||||||
|
var subscription = subject.Subscribe(observer);
|
||||||
|
// Retrieve collection snapshot to bootstrap sequence
|
||||||
|
initTable().ignore();
|
||||||
|
// Watch changes
|
||||||
|
Task.Run(watchChanges);
|
||||||
|
return Disposable.Create(Dispose);
|
||||||
|
|
||||||
|
void Dispose() {
|
||||||
|
Log.Gen.info("Disposing MongoDbChangeObservable subscription");
|
||||||
|
subscription.Dispose();
|
||||||
|
cts.Cancel();
|
||||||
|
cts.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
|
||||||
|
MongoSchemaAdaptor.InferSchema(docs);
|
||||||
|
|
||||||
|
async Task watchChanges() {
|
||||||
|
Log.Gen.info(
|
||||||
|
$"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}");
|
||||||
|
while (collection is null) {
|
||||||
|
await Task.Delay(100);
|
||||||
|
}
|
||||||
|
Log.notnull(collection);
|
||||||
|
using var cursor = collection.Watch(new ChangeStreamOptions {
|
||||||
|
FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
while (await cursor.MoveNextAsync(cts.Token)) {
|
||||||
|
/// Ignore while waiting for init.
|
||||||
|
lock (syncRoot) {
|
||||||
|
/// Note: <see cref="rowFactory"/> and <see cref="table"/> are initialized together
|
||||||
|
/// in the <see cref="initTable"/> fn since both depend on the schema, which is
|
||||||
|
/// inferred from the first incoming BsonDocument(s).
|
||||||
|
if (table is null || rowFactory is null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var bd = table.ToDataPacketBuilder();
|
||||||
|
foreach (var ch in cursor.Current) {
|
||||||
|
var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
|
||||||
|
var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
|
||||||
|
Log.Gen.info(
|
||||||
|
$"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
|
||||||
|
$"changes:\n\t\t{updDescJson}");
|
||||||
|
switch (ch.OperationType) {
|
||||||
|
case ChangeStreamOperationType.Insert: {
|
||||||
|
bd.AddOrUpdate(Row());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case ChangeStreamOperationType.Update: {
|
||||||
|
bd.Remove(OldRow().Id());
|
||||||
|
bd.AddOrUpdate(Row());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case ChangeStreamOperationType.Replace: {
|
||||||
|
bd.Remove(OldRow().Id());
|
||||||
|
bd.AddOrUpdate(Row());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case ChangeStreamOperationType.Delete: {
|
||||||
|
bd.Remove(OldRow().Id());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
IBlueRow Row() =>
|
||||||
|
rowFactory.GetBlueRow(ch.FullDocument);
|
||||||
|
|
||||||
|
IBlueRow OldRow() =>
|
||||||
|
rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
|
||||||
|
}
|
||||||
|
var dp = bd.ToDataPacket();
|
||||||
|
if (!dp.HasChanges()) {
|
||||||
|
Log.Gen.warn(
|
||||||
|
$"MongoCollectionStream. Ignoring update because it didn't change any rows.");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
lock (syncRoot) {
|
||||||
|
table = (BlueTableImmutable)dp.newRows;
|
||||||
|
subject.OnNext(dp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (TaskCanceledException ex) {
|
||||||
|
Log.Gen.warn(
|
||||||
|
$"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
[MemberNotNull(nameof(collection))]
|
||||||
|
async Task initTable() {
|
||||||
|
collection = database.GetCollection<BsonDocument>(collectionName);
|
||||||
|
var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||||
|
var docList = await cursor.ToListAsync();
|
||||||
|
rowFactory ??= getRowFactory(docList);
|
||||||
|
var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
|
||||||
|
foreach (var doc in docList) {
|
||||||
|
var row = rowFactory.GetBlueRow(doc);
|
||||||
|
firstTable.AddOrUpdate(row);
|
||||||
|
}
|
||||||
|
var dp = firstTable.ToDataPacket();
|
||||||
|
lock (syncRoot) {
|
||||||
|
table = (BlueTableImmutable)dp.newRows;
|
||||||
|
subject.OnNext(dp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,128 +1,133 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using MongoDB.Driver;
|
|
||||||
using MongoDB.Bson;
|
|
||||||
using BCG;
|
using BCG;
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
using ZstdSharp.Unsafe;
|
|
||||||
using System.Runtime.CompilerServices;
|
using MongoDB.Bson;
|
||||||
using mbts.spark;
|
using MongoDB.Driver;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Threading;
|
||||||
|
|
||||||
|
namespace bts {
|
||||||
|
// [DataStreamUserRespresentation("mongodb_sql")]
|
||||||
|
// public class MongoDBSQLStream : ViewStream, IWorker {
|
||||||
|
// private ViewStreamDatabase _vsd;
|
||||||
|
// private readonly string _connection_string, _db_name, _collection_name;
|
||||||
|
// private MongoClient _client;
|
||||||
|
// // make this a sorted table
|
||||||
|
// private BlueTable _table;
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
// // Access the database
|
||||||
[DataStreamUserRespresentation("mongodb_sql")]
|
// private IMongoDatabase _database;
|
||||||
public class MongoDBSQLStream : ViewStream, IWorker {
|
// private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||||
private ViewStreamDatabase _vsd;
|
// // mongodb://192.168.1.80:27017
|
||||||
private readonly string _connection_string, _db_name, _collection_name;
|
|
||||||
private MongoClient _client;
|
|
||||||
private BlueTable _table;
|
|
||||||
|
|
||||||
// Access the database
|
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||||
private IMongoDatabase _database;
|
// : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
|
||||||
// mongodb://192.168.1.80:27017
|
|
||||||
|
|
||||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
// }
|
||||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||||
|
// bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||||
|
// $"db_name: {db_name}");
|
||||||
|
// _connection_string = connection_string;
|
||||||
|
// _db_name = db_name;
|
||||||
|
// _collection_name = collection_name;
|
||||||
|
// _vsd = vsd;
|
||||||
|
// _client = new MongoClient(_connection_string);
|
||||||
|
// _database = _client.GetDatabase(_db_name);
|
||||||
|
// }
|
||||||
|
|
||||||
}
|
// protected override void myOnFirstSubscriber() {
|
||||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
// }
|
||||||
$"db_name: {db_name}");
|
|
||||||
_connection_string = connection_string;
|
|
||||||
_db_name = db_name;
|
|
||||||
_collection_name = collection_name;
|
|
||||||
_vsd = vsd;
|
|
||||||
_client = new MongoClient(_connection_string);
|
|
||||||
_database = _client.GetDatabase(_db_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
// protected override void myOnLastSubscriber() {
|
||||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
// MasterWorker.getInstance(false).removeWorker(this);
|
||||||
}
|
// }
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
// public void doWork() {
|
||||||
MasterWorker.getInstance(false).removeWorker(this);
|
// _read_mogodb();
|
||||||
}
|
// }
|
||||||
|
|
||||||
public void doWork() {
|
// private void _read_mogodb() {
|
||||||
_read_mogodb();
|
// var timer = Stopwatch.StartNew();
|
||||||
}
|
// // Access the collection
|
||||||
|
// IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||||
|
|
||||||
private void _read_mogodb() {
|
// // Fetch all documents from the collection
|
||||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
// var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||||
timer.start();
|
// _table = _table == null ? null : new BlueTable(_table.schema);
|
||||||
// Access the collection
|
// // Iterate through the documents and print values
|
||||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
// foreach (var document in documents.ToEnumerable()) {
|
||||||
|
// bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||||
|
// if (_table == null) {
|
||||||
|
// _init_table(document);
|
||||||
|
// }
|
||||||
|
// _add_row(document);
|
||||||
|
// }
|
||||||
|
// timer.Stop();
|
||||||
|
// Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
||||||
|
// timer.Restart();
|
||||||
|
// var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||||
|
// timer.Stop();
|
||||||
|
// Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
||||||
|
// if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||||
|
// return;
|
||||||
|
// myFireEvent(dp);
|
||||||
|
// }
|
||||||
|
|
||||||
// Fetch all documents from the collection
|
// private void _init_table(BsonDocument doc_row) {
|
||||||
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
// bts.Log.assert(_table == null);
|
||||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
// List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
// Iterate through the documents and print values
|
// cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||||
foreach (var document in documents.ToEnumerable()) {
|
// foreach (var n in doc_row.Names) {
|
||||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
// cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||||
if (_table == null) {
|
// }
|
||||||
_init_table(document);
|
// IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||||
}
|
// _table = new BlueTable(schema);
|
||||||
_add_row(document);
|
// }
|
||||||
}
|
|
||||||
timer.stop();
|
|
||||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
|
||||||
timer.start();
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
|
||||||
timer.stop();
|
|
||||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
|
||||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
|
||||||
return;
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _init_table(BsonDocument doc_row) {
|
// private void _add_row(BsonDocument doc) {
|
||||||
bts.Log.assert(_table == null);
|
// var key_val = doc["_id"];
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
// var key = key_val.ToString();
|
||||||
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
// int id;
|
||||||
foreach (var n in doc_row.Names) {
|
// if (!_id_map.TryGetValue(key, out id)) {
|
||||||
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
// id = _id_map.Count;
|
||||||
}
|
// _id_map.Add(key, id);
|
||||||
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
// }
|
||||||
_table = new BlueTable(schema);
|
// MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||||
}
|
// nr.setValue("id", id);
|
||||||
|
// foreach (var n in doc.Names) {
|
||||||
private void _add_row(BsonDocument doc) {
|
// var val = doc[n].ToString();
|
||||||
var key_val = doc["_id"];
|
// nr.setValue(n, val);
|
||||||
var key = key_val.ToString();
|
// }
|
||||||
int id;
|
// _table.addRow(nr);
|
||||||
if (!_id_map.TryGetValue(key, out id)) {
|
// }
|
||||||
id = _id_map.Count;
|
// }
|
||||||
_id_map.Add(key, id);
|
|
||||||
}
|
|
||||||
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
|
||||||
nr.setValue("id", id);
|
|
||||||
foreach (var n in doc.Names) {
|
|
||||||
var val = doc[n].ToString();
|
|
||||||
nr.setValue(n, val);
|
|
||||||
}
|
|
||||||
_table.addRow(nr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("mongodb")]
|
[DataStreamUserRespresentation("mongodb")]
|
||||||
public class MongoDBStream : ViewStream, IWorker {
|
public class MongoDBStream : ViewStream, IWorker {
|
||||||
private ViewStreamDatabase _vsd;
|
private ViewStreamDatabase _vsd;
|
||||||
private readonly string _connection_string, _db_name, _collection_name;
|
private readonly string _connection_string, _db_name, _collection_name;
|
||||||
|
private readonly TimeSpan _poll_period;
|
||||||
private MongoClient _client;
|
private MongoClient _client;
|
||||||
private BlueTable _table;
|
private BlueTable _table;
|
||||||
|
private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
|
||||||
|
private BlockingQueue _work_queue = new BlockingQueue();
|
||||||
|
|
||||||
// Access the database
|
// Access the database
|
||||||
private IMongoDatabase _database;
|
private IMongoDatabase _database;
|
||||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||||
// mongodb://192.168.1.80:27017
|
// mongodb://192.168.1.80:27017
|
||||||
|
|
||||||
public MongoDBStream(string connection_string, string db_name, string collection_name)
|
public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
|
||||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
: this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
|
||||||
|
|
||||||
}
|
}
|
||||||
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
|
||||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||||
$"db_name: {db_name}");
|
$"db_name: {db_name}");
|
||||||
_connection_string = connection_string;
|
_connection_string = connection_string;
|
||||||
@@ -130,24 +135,34 @@ namespace BCG {
|
|||||||
_collection_name = collection_name;
|
_collection_name = collection_name;
|
||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
_client = new MongoClient(_connection_string);
|
_client = new MongoClient(_connection_string);
|
||||||
|
_poll_period = poll_period;
|
||||||
_database = _client.GetDatabase(_db_name);
|
_database = _client.GetDatabase(_db_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
_vsd.addWorker(this);
|
||||||
|
//MasterWorker.getInstance(false).addWorker(this, _poll_period);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
MasterWorker.getInstance(false).removeWorker(this);
|
_vsd.removeWorker(this);
|
||||||
|
//MasterWorker.getInstance(false).removeWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private DateTime _last_poll_time = DateTime.MinValue;
|
||||||
public void doWork() {
|
public void doWork() {
|
||||||
_read_mogodb();
|
var now = DateTime.Now;
|
||||||
|
if (now - _last_poll_time > _poll_period) {
|
||||||
|
ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
|
||||||
|
_last_poll_time = DateTime.Now;
|
||||||
|
}
|
||||||
|
var dp = _table_queue.dequeue(0);
|
||||||
|
if (dp != null)
|
||||||
|
myFireEvent(dp);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _read_mogodb() {
|
private void _read_mogodb() {
|
||||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
var timer = Stopwatch.StartNew();
|
||||||
timer.start();
|
|
||||||
// Access the collection
|
// Access the collection
|
||||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||||
|
|
||||||
@@ -156,21 +171,22 @@ namespace BCG {
|
|||||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||||
// Iterate through the documents and print values
|
// Iterate through the documents and print values
|
||||||
foreach (var document in documents.ToEnumerable()) {
|
foreach (var document in documents.ToEnumerable()) {
|
||||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
//bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||||
if (_table == null) {
|
if (_table == null) {
|
||||||
_init_table(document);
|
_init_table(document);
|
||||||
}
|
}
|
||||||
_add_doc(document);
|
_add_doc(document);
|
||||||
}
|
}
|
||||||
timer.stop();
|
timer.Stop();
|
||||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
|
||||||
timer.start();
|
timer.Restart();
|
||||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||||
timer.stop();
|
timer.Stop();
|
||||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
|
||||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||||
return;
|
return;
|
||||||
myFireEvent(dp);
|
_table_queue.enqueue(dp);
|
||||||
|
//myFireEvent(dp);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _init_table(BsonDocument doc_row) {
|
private void _init_table(BsonDocument doc_row) {
|
||||||
@@ -209,4 +225,46 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
// [DataStreamUserRespresentation("mongodb_flat")]
|
||||||
|
// public class MongoDBFlatStream : ViewStream {
|
||||||
|
// public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
||||||
|
// :
|
||||||
|
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
||||||
|
|
||||||
|
// public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
||||||
|
// ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||||
|
// rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
||||||
|
// rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
||||||
|
// //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
||||||
|
// mySetRealSource(rs);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// [DataStreamUserRespresentation("newsreplay")]
|
||||||
|
// public class NewsReplayStream : ViewStream {
|
||||||
|
// public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
||||||
|
// int buffer_sz, string fire_period) :
|
||||||
|
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
||||||
|
|
||||||
|
// public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
||||||
|
// int buffer_sz, TimeSpan fire_period) {
|
||||||
|
// ViewStream r;
|
||||||
|
|
||||||
|
// r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||||
|
// r = new KeyValueToRelationalStream(r,
|
||||||
|
// "doc_id",
|
||||||
|
// "key",
|
||||||
|
// "value");
|
||||||
|
// r = new DateStandardizationStream(r,
|
||||||
|
// "created_at",
|
||||||
|
// "ISO8601");
|
||||||
|
// r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
||||||
|
|
||||||
|
// mySetRealSource(r);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using MongoDB.Bson;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public static class MongoDbExtensions {
|
||||||
|
public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
|
||||||
|
doc[elementName].AsBsonArray;
|
||||||
|
|
||||||
|
public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
|
||||||
|
doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
|
||||||
|
val.AsBsonArray : null;
|
||||||
|
|
||||||
|
public static string GetString(this BsonDocument doc, string elementName) =>
|
||||||
|
doc[elementName].AsString;
|
||||||
|
|
||||||
|
public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
|
||||||
|
doc.TryGetValue(elementName, out var val) && val.IsString ?
|
||||||
|
val.AsString : null;
|
||||||
|
|
||||||
|
public static int GetInt32(this BsonDocument doc, string elementName) =>
|
||||||
|
doc[elementName].AsInt32;
|
||||||
|
|
||||||
|
public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
|
||||||
|
doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
|
||||||
|
val.AsInt32 : null;
|
||||||
|
|
||||||
|
public static long GetInt64(this BsonDocument doc, string elementName) =>
|
||||||
|
doc[elementName].AsInt64;
|
||||||
|
|
||||||
|
public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
|
||||||
|
doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
|
||||||
|
val.AsInt64 : null;
|
||||||
|
|
||||||
|
public static double GetDouble(this BsonDocument doc, string elementName) =>
|
||||||
|
doc[elementName].AsDouble;
|
||||||
|
|
||||||
|
public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
|
||||||
|
doc.TryGetValue(elementName, out var val) && val.IsDouble ?
|
||||||
|
val.AsDouble : null;
|
||||||
|
|
||||||
|
public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
|
||||||
|
doc[elementName].ToUniversalTime();
|
||||||
|
|
||||||
|
public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
|
||||||
|
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
|
||||||
|
val.ToUniversalTime() : null;
|
||||||
|
|
||||||
|
public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
|
||||||
|
doc[elementName].ToUniversalTime();
|
||||||
|
|
||||||
|
public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
|
||||||
|
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
|
||||||
|
val.ToUniversalTime() : null;
|
||||||
|
|
||||||
|
public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
|
||||||
|
doc[elementName].AsObjectId;
|
||||||
|
|
||||||
|
public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
|
||||||
|
doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
|
||||||
|
val.AsObjectId : null;
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using MongoDB.Bson;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
public static class MongoSchemaAdaptor {
|
||||||
|
public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
|
||||||
|
var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
|
||||||
|
foreach (var doc in docs) {
|
||||||
|
foreach (var el in doc) {
|
||||||
|
if (schemaAdaptor.Contains(el.Name)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
switch (el.Value.BsonType) {
|
||||||
|
case BsonType.Array:
|
||||||
|
schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
|
||||||
|
break;
|
||||||
|
case BsonType.String:
|
||||||
|
schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
|
||||||
|
break;
|
||||||
|
case BsonType.Int32:
|
||||||
|
schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
|
||||||
|
break;
|
||||||
|
case BsonType.Int64:
|
||||||
|
schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
|
||||||
|
break;
|
||||||
|
case BsonType.Double:
|
||||||
|
schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
|
||||||
|
break;
|
||||||
|
case BsonType.DateTime:
|
||||||
|
schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
|
||||||
|
break;
|
||||||
|
case BsonType.Timestamp:
|
||||||
|
schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
|
||||||
|
break;
|
||||||
|
case BsonType.ObjectId:
|
||||||
|
schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
|
||||||
|
break;
|
||||||
|
case BsonType.Null:
|
||||||
|
Log.Gen.warn(
|
||||||
|
$"Ignoring column '{el.Name}' because at least one element is null");
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Unsupported BsonType '{el.Value.BsonType}'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return schemaAdaptor;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
namespace bts {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using ProtoBuf;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class ProjectData {
|
||||||
|
// schema from the base stream
|
||||||
|
public readonly IBlueTableSchema in_schema;
|
||||||
|
// this is the projected schema
|
||||||
|
public readonly IBlueTableSchema out_schema;
|
||||||
|
// projected cols to the input cols, for cols who's names are changed
|
||||||
|
public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
|
||||||
|
public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
|
||||||
|
IDictionary<string, string> mapped_to_orig_col_map_) {
|
||||||
|
in_schema = in_schema_;
|
||||||
|
out_schema = out_schema_;
|
||||||
|
mapped_to_orig_col_map = mapped_to_orig_col_map_;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class NewProjectRow : IBlueRow {
|
||||||
|
#region IBlueRow members
|
||||||
|
private ProjectData _project_data;
|
||||||
|
private IBlueRow _row;
|
||||||
|
public NewProjectRow(ProjectData project_data, IBlueRow row) {
|
||||||
|
_project_data = project_data;
|
||||||
|
_row = row;
|
||||||
|
}
|
||||||
|
public IBlueTableSchema schema { get { return _project_data.out_schema; } }
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region ICollection members
|
||||||
|
public int Count { get { return schema.columnNames.Length; } }
|
||||||
|
|
||||||
|
public bool IsSynchronized { get { return false; } }
|
||||||
|
|
||||||
|
public object SyncRoot { get { return this; } }
|
||||||
|
|
||||||
|
public void CopyTo(Array array, int index) {
|
||||||
|
int i = index;
|
||||||
|
foreach (object o in this as IEnumerable)
|
||||||
|
array.SetValue(o, i++);
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region IDictionary members
|
||||||
|
public bool IsFixedSize { get { return true; } }
|
||||||
|
|
||||||
|
public bool IsReadOnly { get { return true; } }
|
||||||
|
|
||||||
|
public object this[object key] {
|
||||||
|
get {
|
||||||
|
if (key is string)
|
||||||
|
return this[(string)key];
|
||||||
|
return this[(int)key];
|
||||||
|
}
|
||||||
|
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
}
|
||||||
|
|
||||||
|
public object this[int index] {
|
||||||
|
get {
|
||||||
|
return this[schema.columnInfo[index].colName];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public object this[string col_name] {
|
||||||
|
get {
|
||||||
|
string mapped_to_col = null;
|
||||||
|
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
|
||||||
|
return _row[mapped_to_col];
|
||||||
|
}
|
||||||
|
return _row[col_name];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ICollection Keys { get { return schema.columnNames; } }
|
||||||
|
|
||||||
|
public ICollection Values {
|
||||||
|
get {
|
||||||
|
object[] retVal = new object[schema.columnNames.Length];
|
||||||
|
CopyTo(retVal, 0);
|
||||||
|
return retVal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
|
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
|
public bool Contains(object key) { return schema[(string)key] != null; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
||||||
|
/// iterate through the values, typecast this into an IEnumerable
|
||||||
|
/// </summary>
|
||||||
|
/// <returns></returns>
|
||||||
|
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||||
|
|
||||||
|
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
||||||
|
|
||||||
|
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public class NewProjectStream : ViewStream, IStreamMapper {
|
||||||
|
|
||||||
|
private ViewStream _stream;
|
||||||
|
private string _project_expression;
|
||||||
|
private ProjectData _project_data = null;
|
||||||
|
//public ProjectData
|
||||||
|
public NewProjectStream(ViewStream vs, string project_expression) {
|
||||||
|
_stream = vs;
|
||||||
|
_project_expression = project_expression;
|
||||||
|
_stream = new MapStream(vs, this);
|
||||||
|
mySetRealSource(_stream);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
|
||||||
|
_project_data = _create_project_data(input_schema, _project_expression);
|
||||||
|
return _project_data.out_schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow map(IBlueRow row) {
|
||||||
|
return new NewProjectRow(_project_data, row);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
|
||||||
|
string project_expression) {
|
||||||
|
project_expression = project_expression.Trim();
|
||||||
|
string[] project_toks = project_expression.Split(' ');
|
||||||
|
HashSet<string> removed_cols = new HashSet<string>();
|
||||||
|
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
|
||||||
|
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
|
||||||
|
foreach (string p_tok in project_toks) {
|
||||||
|
if (p_tok.StartsWith("!")) {
|
||||||
|
string removed_col = p_tok.Substring(1);
|
||||||
|
removed_cols.Add(removed_col);
|
||||||
|
} else if (p_tok.Contains(":")) {
|
||||||
|
string[] map_toks = p_tok.Split(":");
|
||||||
|
string orig_col = map_toks[0];
|
||||||
|
string mapped_col = map_toks[1];
|
||||||
|
mapped_to_orig_map.Add(mapped_col, orig_col);
|
||||||
|
orig_to_mapped.Add(orig_col, mapped_col);
|
||||||
|
} else {
|
||||||
|
if (input_schema.columnNames.Contains(p_tok)) {
|
||||||
|
string orig_col = p_tok;
|
||||||
|
string mapped_col = orig_col;
|
||||||
|
orig_to_mapped.Add(orig_col, mapped_col);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// make the new schema
|
||||||
|
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
||||||
|
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
||||||
|
if (ci.colName == "id") {
|
||||||
|
// if there's a hidden col, other columns will show by default, but if cols are being
|
||||||
|
// projected epxlicitely, than others wil be hidden
|
||||||
|
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||||
|
new_cols.Add(nc);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (removed_cols.Contains(ci.colName)) {
|
||||||
|
continue;
|
||||||
|
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
||||||
|
string mapped_name = orig_to_mapped[ci.colName];
|
||||||
|
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
||||||
|
new_cols.Add(nc);
|
||||||
|
} else {
|
||||||
|
if (ci.colName == "id" || removed_cols.Count > 0) {
|
||||||
|
// if there's a hidden col, other columns will show by default, but if cols are being
|
||||||
|
// projected epxlicitely, than others wil be hidden
|
||||||
|
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||||
|
new_cols.Add(nc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
||||||
|
return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,20 +1,17 @@
|
|||||||
namespace BCG {
|
namespace BCG {
|
||||||
using mbts.spark;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("oldproject")]
|
||||||
public class ProjectStream : ViewStream
|
public class ProjectStream : ViewStream
|
||||||
{
|
{
|
||||||
private ProjectHelper myHelper;
|
private ProjectHelper myHelper;
|
||||||
private string myColExp, myTableName;
|
private string myColExp, myTableName;
|
||||||
private ViewStream myVs;
|
private ViewStream myVs;
|
||||||
private IDictionary<string, string> myDic;
|
private IDictionary<string, string> myDic;
|
||||||
private HighPerformanceTimer _timer = new();
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
||||||
@@ -70,10 +67,8 @@ namespace BCG {
|
|||||||
DataPacket dp = newValue as DataPacket;
|
DataPacket dp = newValue as DataPacket;
|
||||||
if (myHelper != null)
|
if (myHelper != null)
|
||||||
{
|
{
|
||||||
_timer.start();
|
|
||||||
IBlueRowCollection dels, adds;
|
IBlueRowCollection dels, adds;
|
||||||
BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
|
BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
|
||||||
_timer.stop();
|
|
||||||
if (dels.Count + adds.Count > 0)
|
if (dels.Count + adds.Count > 0)
|
||||||
myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
|
myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
|
||||||
new ProjectedTable (adds, myHelper),
|
new ProjectedTable (adds, myHelper),
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ namespace bts {
|
|||||||
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
|
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
|
||||||
myFireEvent(dp_out);
|
myFireEvent(dp_out);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -69,18 +69,14 @@ public class SortRow : IComparable, IBlueRow {
|
|||||||
/// The key can either be the string corrsponding to the column name or
|
/// The key can either be the string corrsponding to the column name or
|
||||||
/// the order of the column
|
/// the order of the column
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public object this[object key] {
|
public object? this[object key] {
|
||||||
get {
|
get => key is string str ? this[str] : this[(int)key];
|
||||||
if (key is string)
|
set => throw new NotSupportedException("The IBlueRow is readonly");
|
||||||
return this[(string)key];
|
|
||||||
return this[(int)key];
|
|
||||||
}
|
|
||||||
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public object this[int index] => row[index];
|
public object? this[int index] => row[index];
|
||||||
|
|
||||||
public object this[string colName] => row[colName];
|
public object? this[string colName] => row[colName];
|
||||||
|
|
||||||
public ICollection Keys => row.Keys;
|
public ICollection Keys => row.Keys;
|
||||||
|
|
||||||
|
|||||||
@@ -22,9 +22,12 @@ namespace BCG
|
|||||||
/// <param name="rows"></param>
|
/// <param name="rows"></param>
|
||||||
public void loadCollection (IBlueRowCollection rows)
|
public void loadCollection (IBlueRowCollection rows)
|
||||||
{
|
{
|
||||||
myFireEvent (new DataPacket (rows, new BlueTable(rows),
|
myFireEvent(new DataPacket(
|
||||||
myLastValue != null ?
|
rows,
|
||||||
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
|
new BlueTable(rows),
|
||||||
|
myLastValue is DataPacket dp
|
||||||
|
? new BlueTable(dp.newRows)
|
||||||
|
: new BlueTable(rows.schema)));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ namespace BCG
|
|||||||
public Guid subscription;
|
public Guid subscription;
|
||||||
}
|
}
|
||||||
private ThreadSafeQueue<Update> _queued_updates;
|
private ThreadSafeQueue<Update> _queued_updates;
|
||||||
TargetDoWorkRelay _do_work_relay;
|
TargetDoWorkRelay? _do_work_relay;
|
||||||
Guid _current_source_subscription;
|
Guid _current_source_subscription;
|
||||||
Guid _current_target_subscription;
|
Guid _current_target_subscription;
|
||||||
int _source_thread;
|
int _source_thread;
|
||||||
|
|||||||
@@ -3,24 +3,23 @@ using System.Collections.Generic;
|
|||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
|
/// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum DateConversion
|
public enum DateConversion {
|
||||||
{
|
|
||||||
ConvertDoubleToDateTime = 0,
|
ConvertDoubleToDateTime = 0,
|
||||||
ChopTime,
|
ChopTime,
|
||||||
UTCToCurrent,
|
UTCToCurrent,
|
||||||
ConvertStringToDateTime,
|
ConvertStringToDateTime,
|
||||||
ConvertDateTimeToMilliseconds,
|
ConvertDateTimeToMilliseconds,
|
||||||
SecondsOfDay
|
SecondsOfDay,
|
||||||
|
ISO8601,
|
||||||
|
EpochToDatetime
|
||||||
}
|
}
|
||||||
|
|
||||||
[DataStreamUserRespresentation("dateStandardize")]
|
[DataStreamUserRespresentation("dateStandardize")]
|
||||||
public class DateStandardizationStream : ViewStream, IStreamMapper
|
public class DateStandardizationStream : ViewStream, IStreamMapper {
|
||||||
{
|
|
||||||
private IBlueTableSchema mySchema;
|
private IBlueTableSchema mySchema;
|
||||||
private DateConversion myConversion;
|
private DateConversion myConversion;
|
||||||
private string myConversionColName;
|
private string myConversionColName;
|
||||||
@@ -31,12 +30,13 @@ namespace BCG
|
|||||||
{DateConversion.UTCToCurrent , typeof(DateTime)},
|
{DateConversion.UTCToCurrent , typeof(DateTime)},
|
||||||
{DateConversion.ConvertStringToDateTime , typeof(DateTime)},
|
{DateConversion.ConvertStringToDateTime , typeof(DateTime)},
|
||||||
{DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
|
{DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
|
||||||
{DateConversion.SecondsOfDay , typeof(Double)}
|
{DateConversion.SecondsOfDay , typeof(Double)},
|
||||||
|
{DateConversion.ISO8601 , typeof(DateTime)},
|
||||||
|
{DateConversion.EpochToDatetime , typeof(DateTime)}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
private static DateConversion _parse(string s)
|
private static DateConversion _parse(string s) {
|
||||||
{
|
|
||||||
DateConversion rv = DateConversion.ChopTime;
|
DateConversion rv = DateConversion.ChopTime;
|
||||||
Enum.TryParse<DateConversion>(s, false, out rv);
|
Enum.TryParse<DateConversion>(s, false, out rv);
|
||||||
return rv;
|
return rv;
|
||||||
@@ -45,11 +45,9 @@ namespace BCG
|
|||||||
/// for DSQA
|
/// for DSQA
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DateStandardizationStream(ViewStream vs, string colName, string conversion)
|
public DateStandardizationStream(ViewStream vs, string colName, string conversion)
|
||||||
: this(vs, colName, _parse(conversion))
|
: this(vs, colName, _parse(conversion)) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion)
|
public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion) {
|
||||||
{
|
|
||||||
myConversionColName = colName;
|
myConversionColName = colName;
|
||||||
myConversion = conversion;
|
myConversion = conversion;
|
||||||
mySetRealSource(new MapStream(vs, this));
|
mySetRealSource(new MapStream(vs, this));
|
||||||
@@ -57,24 +55,18 @@ namespace BCG
|
|||||||
|
|
||||||
#region IStreamMapper Members
|
#region IStreamMapper Members
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||||
{
|
if (mySchema == null) {
|
||||||
if (mySchema == null)
|
|
||||||
{
|
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
foreach (ColumnInfo c in inputSchema.columnInfo)
|
foreach (ColumnInfo c in inputSchema.columnInfo) {
|
||||||
{
|
if (c.colName == myConversionColName) {
|
||||||
if (c.colName == myConversionColName)
|
|
||||||
{
|
|
||||||
Type convType;
|
Type convType;
|
||||||
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType))
|
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) {
|
||||||
{
|
|
||||||
//BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
|
//BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
|
||||||
convType = typeof(DateTime);
|
convType = typeof(DateTime);
|
||||||
}
|
}
|
||||||
cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
|
cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
|
||||||
}
|
} else
|
||||||
else
|
|
||||||
cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
|
cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
|
||||||
}
|
}
|
||||||
mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
|
mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
|
||||||
@@ -86,56 +78,59 @@ namespace BCG
|
|||||||
|
|
||||||
#region IDynamicStreamMapper Members
|
#region IDynamicStreamMapper Members
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row)
|
public IBlueRow map(IBlueRow row) {
|
||||||
{
|
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
|
||||||
return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
|
rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
|
||||||
|
return rv;
|
||||||
|
//return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
|
||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
public static object mapDateTime(object objIn, DateConversion conv)
|
public static object mapDateTime(object obj_in, DateConversion conv) {
|
||||||
{
|
if (obj_in == DBNull.Value || obj_in == null) {
|
||||||
if (objIn == DBNull.Value || objIn == null)
|
|
||||||
{
|
|
||||||
return DBNull.Value;
|
return DBNull.Value;
|
||||||
}
|
} else {
|
||||||
else
|
switch (conv) {
|
||||||
{
|
case DateConversion.ChopTime: {
|
||||||
if (conv == DateConversion.ChopTime)
|
DateTime dtIn = (DateTime)obj_in;
|
||||||
{
|
return dtIn.Date;
|
||||||
DateTime dtIn = (DateTime)objIn;
|
}
|
||||||
return dtIn.Date;
|
case DateConversion.ConvertDoubleToDateTime: {
|
||||||
}
|
double dtIn = (double)obj_in;
|
||||||
else if (conv == DateConversion.ConvertDoubleToDateTime)
|
return DateTime.FromOADate(dtIn);
|
||||||
{
|
}
|
||||||
double dtIn = (double)objIn;
|
case DateConversion.UTCToCurrent: {
|
||||||
return DateTime.FromOADate(dtIn);
|
DateTime dtIn = (DateTime)obj_in;
|
||||||
}
|
return dtIn.ToLocalTime();
|
||||||
else if (conv == DateConversion.UTCToCurrent)
|
}
|
||||||
{
|
case DateConversion.ConvertStringToDateTime: {
|
||||||
DateTime dtIn = (DateTime)objIn;
|
return DateTime.Parse((string)obj_in);
|
||||||
return dtIn.ToLocalTime();
|
}
|
||||||
}
|
case DateConversion.ConvertDateTimeToMilliseconds: {
|
||||||
else if (conv == DateConversion.ConvertStringToDateTime)
|
DateTime dtIn = (DateTime)obj_in;
|
||||||
{
|
return dtIn.TimeOfDay.TotalMilliseconds;
|
||||||
return DateTime.Parse((string)objIn);
|
}
|
||||||
}
|
case DateConversion.SecondsOfDay: {
|
||||||
else if (conv == DateConversion.ConvertDateTimeToMilliseconds)
|
DateTime dtIn = (DateTime)obj_in;
|
||||||
{
|
DateTime start_of_day = dtIn.Date;
|
||||||
DateTime dtIn = (DateTime)objIn;
|
TimeSpan day_period = dtIn - start_of_day;
|
||||||
return dtIn.TimeOfDay.TotalMilliseconds;
|
return day_period.TotalSeconds;
|
||||||
}
|
}
|
||||||
else if (conv == DateConversion.SecondsOfDay)
|
case DateConversion.ISO8601: {
|
||||||
{
|
string iso8601Date = obj_in.ToString();
|
||||||
DateTime dtIn = (DateTime)objIn;
|
return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind);
|
||||||
DateTime start_of_day = dtIn.Date;
|
}
|
||||||
TimeSpan day_period = dtIn - start_of_day;
|
case DateConversion.EpochToDatetime: {
|
||||||
return day_period.TotalSeconds;
|
double d = Convert.ToDouble(obj_in);
|
||||||
}
|
long unix_time_seconds = (long)d;
|
||||||
else
|
DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||||
{
|
return epoch.AddSeconds(unix_time_seconds);
|
||||||
bts.Log.Gen.info("incorrect type");
|
}
|
||||||
return null;
|
default: {
|
||||||
|
bts.Log.Gen.info("incorrect type");
|
||||||
|
return DBNull.Value;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,29 +1,71 @@
|
|||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace BCG
|
using BCG.Utilities.General;
|
||||||
{
|
|
||||||
public class MSecondStream : TimeUnitStream
|
|
||||||
{
|
|
||||||
public MSecondStream (int intervalInms, int size)
|
|
||||||
: base (DateTime.Now, new TimeSpan (0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance)
|
|
||||||
{}
|
|
||||||
|
|
||||||
protected override DateTime myRoundTime (DateTime dt) {return dt;}
|
using bts;
|
||||||
|
|
||||||
|
namespace BCG {
|
||||||
|
[DataStreamUserRespresentation("milliseconds")]
|
||||||
|
public class MillisecondStream : ViewStream, IWorker {
|
||||||
|
private DateTime _last_fire_time = DateTime.MinValue;
|
||||||
|
private readonly TimeSpan _period;
|
||||||
|
private readonly int _size;
|
||||||
|
private readonly ViewStreamDatabase _vsd;
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
private bts.CircularBufferTable _table;
|
||||||
|
|
||||||
|
public MillisecondStream(int interval_msecs, int size)
|
||||||
|
: this(ViewStreamDatabase.instance, interval_msecs, size) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
|
||||||
|
_vsd = vsd;
|
||||||
|
_period = TimeSpan.FromMilliseconds(interval_msecs);
|
||||||
|
_size = size;
|
||||||
|
_schema = BlueDataUtility.form_schema("Milliseconds",
|
||||||
|
Tuple.Create("id", TypeCode.Int32),
|
||||||
|
Tuple.Create("time", TypeCode.DateTime),
|
||||||
|
Tuple.Create("display_time", TypeCode.String));
|
||||||
|
_table = new CircularBufferTable(_schema, _size);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_vsd.addWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_vsd.removeWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int _id = 0;
|
||||||
|
public void doWork() {
|
||||||
|
if (DateTime.Now - _last_fire_time >= _period) {
|
||||||
|
_last_fire_time = DateTime.Now;
|
||||||
|
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
|
||||||
|
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
|
||||||
|
_table.add_row(nr);
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public class MSecondStream : TimeUnitStream {
|
||||||
|
public MSecondStream(int intervalInms, int size)
|
||||||
|
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
|
protected override DateTime myRoundTime(DateTime dt) { return dt; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class DisplayableMillisecondStream : ViewStream, IStreamMapper
|
public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
|
||||||
{
|
private IBlueTableSchema mySchema = new BlueTableSchema("ms",
|
||||||
private IBlueTableSchema mySchema = new BlueTableSchema ("ms",
|
new ColumnInfo("id", 0, typeof(int)),
|
||||||
new ColumnInfo ("id", 0, typeof(int)),
|
new ColumnInfo("time", 1, typeof(string)));
|
||||||
new ColumnInfo ("time", 1, typeof(string)));
|
public DisplayableMillisecondStream(int intervalInMs, int sz) {
|
||||||
public DisplayableMillisecondStream (int intervalInMs, int sz)
|
mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
|
||||||
{
|
|
||||||
mySetRealSource (new MapStream (new MSecondStream (intervalInMs, sz), this));
|
|
||||||
}
|
}
|
||||||
#region IStreamMapper Members
|
#region IStreamMapper Members
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||||
{
|
|
||||||
return mySchema;
|
return mySchema;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,15 +73,15 @@ namespace BCG
|
|||||||
|
|
||||||
#region IDynamicStreamMapper Members
|
#region IDynamicStreamMapper Members
|
||||||
|
|
||||||
public IBlueRow map (IBlueRow row)
|
public IBlueRow map(IBlueRow row) {
|
||||||
{
|
DateTime time = row.get_datetime("time");
|
||||||
return new BlueRow (mySchema, row["id"], myToString ((DateTime)row["time"]));
|
var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
||||||
|
return new BlueRow(mySchema, row["id"], time_txt);
|
||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
private string myToString (DateTime dt)
|
private string myToString(DateTime dt) {
|
||||||
{
|
|
||||||
string sep = ":";
|
string sep = ":";
|
||||||
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
|
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,38 +8,36 @@ namespace BCG {
|
|||||||
private ViewStream mySrc;
|
private ViewStream mySrc;
|
||||||
private IBlueTableSchema mySchema = null;
|
private IBlueTableSchema mySchema = null;
|
||||||
|
|
||||||
public AlphabetizeColumnsStream (ViewStream src) {
|
public AlphabetizeColumnsStream(ViewStream src) {
|
||||||
mySrc = src;
|
mySrc = src;
|
||||||
}
|
}
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
|
||||||
mySrc.onChange += new ChangeDelegate(mySrc_onChange);
|
mySrc.onChange += new ChangeDelegate(mySrc_onChange);
|
||||||
}
|
}
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
|
||||||
mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
|
|
||||||
}
|
}
|
||||||
private void mySrc_onChange (object newValue) {
|
private void mySrc_onChange(object newValue) {
|
||||||
DataPacket dp = (DataPacket)newValue;
|
DataPacket dp = (DataPacket)newValue;
|
||||||
if (mySchema == null) {
|
if (mySchema == null) {
|
||||||
SortedList alphabetical = new SortedList ();
|
SortedList alphabetical = new SortedList();
|
||||||
foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo)
|
foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo)
|
||||||
alphabetical.Add (cinfo.colName,cinfo);
|
alphabetical.Add(cinfo.colName, cinfo);
|
||||||
ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count];
|
ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count];
|
||||||
alphabetical.Values.CopyTo (cinfos,0);
|
alphabetical.Values.CopyTo(cinfos, 0);
|
||||||
for (int i = 0; i < cinfos.Length; ++i)
|
for (int i = 0; i < cinfos.Length; ++i)
|
||||||
cinfos[i] = new ColumnInfo (cinfos[i].colName,i,cinfos[i].colType,cinfos[i].colValue);
|
cinfos[i] = new ColumnInfo(cinfos[i].colName, i, cinfos[i].colType);
|
||||||
mySchema = new BlueTableSchema (cinfos,dp.schema.tableName);
|
mySchema = new BlueTableSchema(cinfos, dp.schema.tableName);
|
||||||
}
|
}
|
||||||
BlueTable
|
BlueTable
|
||||||
deleted = new BlueTable (mySchema),
|
deleted = new BlueTable(mySchema),
|
||||||
added = new BlueTable (mySchema),
|
added = new BlueTable(mySchema),
|
||||||
newrows = new BlueTable (mySchema);
|
newrows = new BlueTable(mySchema);
|
||||||
deleted.addRows (dp.deletedRows);
|
deleted.addRows(dp.deletedRows);
|
||||||
added.addRows (dp.addedRows);
|
added.addRows(dp.addedRows);
|
||||||
newrows.addRows (dp.newRows);
|
newrows.addRows(dp.newRows);
|
||||||
|
|
||||||
myFireEvent (new DataPacket (newrows,added,deleted));
|
myFireEvent(new DataPacket(newrows, added, deleted));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,8 +1,6 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Text;
|
|
||||||
using mbts;
|
using mbts;
|
||||||
using mbts.spark;
|
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -183,7 +181,6 @@ namespace BCG
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public class MutableAppendStream : ViewStream
|
public class MutableAppendStream : ViewStream
|
||||||
{
|
{
|
||||||
HighPerformanceTimer _timer = new HighPerformanceTimer();
|
|
||||||
private bool mySubscriptionActive = false;
|
private bool mySubscriptionActive = false;
|
||||||
private MutableAppendTable myTable;
|
private MutableAppendTable myTable;
|
||||||
private int myMovingIndex;
|
private int myMovingIndex;
|
||||||
@@ -272,7 +269,6 @@ namespace BCG
|
|||||||
private void myHandleStreamChanges(object newValue)
|
private void myHandleStreamChanges(object newValue)
|
||||||
{
|
{
|
||||||
DataPacket dp = newValue as DataPacket;
|
DataPacket dp = newValue as DataPacket;
|
||||||
_timer.start();
|
|
||||||
myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
|
myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
|
||||||
BlueTable adds = new BlueTable(myTable.schema),
|
BlueTable adds = new BlueTable(myTable.schema),
|
||||||
dels = new BlueTable(myTable.schema);
|
dels = new BlueTable(myTable.schema);
|
||||||
@@ -287,7 +283,6 @@ namespace BCG
|
|||||||
IBlueRow nr = myTable.addRow(r, index);
|
IBlueRow nr = myTable.addRow(r, index);
|
||||||
adds.addRow(nr);
|
adds.addRow(nr);
|
||||||
}
|
}
|
||||||
_timer.stop();
|
|
||||||
myFireEvent(new DataPacket(myTable, adds, dels));
|
myFireEvent(new DataPacket(myTable, adds, dels));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,71 +1,61 @@
|
|||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Publishes the rows in the form of a circular buffer, ie if the buffer size is n
|
/// Publishes the rows in the form of a circular buffer, ie if the buffer size is n
|
||||||
/// then the n'th row being puclished results in the deletion of the 0th row
|
/// then the n'th row being puclished results in the deletion of the 0th row
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class CircularPublisher
|
public class CircularPublisher {
|
||||||
{
|
|
||||||
private CircularRowCollection myBuffer;
|
private CircularRowCollection myBuffer;
|
||||||
private PubTableBroker myBroker;
|
private PubTableBroker myBroker;
|
||||||
private int myBufferSize;
|
private int myBufferSize;
|
||||||
public CircularPublisher (int bufferSize)
|
public CircularPublisher(int bufferSize) {
|
||||||
{
|
|
||||||
myBufferSize = bufferSize;
|
myBufferSize = bufferSize;
|
||||||
}
|
|
||||||
|
|
||||||
public CircularPublisher (int bufferSize, IBlueRowCollection rows)
|
|
||||||
: this (bufferSize)
|
|
||||||
{
|
|
||||||
publishRows (rows);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void publishRow (IBlueRow nr)
|
public CircularPublisher(int bufferSize, IBlueRowCollection rows)
|
||||||
{
|
: this(bufferSize) {
|
||||||
if (myBuffer == null)
|
publishRows(rows);
|
||||||
{
|
|
||||||
myBuffer = new CircularRowCollection (
|
|
||||||
new BlueTableSchema (nr.schema.tableName, nr.schema.columnInfo),
|
|
||||||
myBufferSize);
|
|
||||||
myBroker = new PubTableBroker ();
|
|
||||||
myBroker.createTable (myBuffer);
|
|
||||||
}
|
|
||||||
IBlueRow br = myBuffer.addRowWithReturn (nr);
|
|
||||||
myBroker.makeBatchChanges (myBuffer,
|
|
||||||
myReturnCollection (br),
|
|
||||||
myReturnCollection (nr));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void publishRows (IBlueRowCollection nrs)
|
public void publishRow(IBlueRow nr) {
|
||||||
{
|
if (myBuffer == null) {
|
||||||
if (myBuffer == null)
|
myBuffer = new CircularRowCollection(
|
||||||
{
|
new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo),
|
||||||
myBuffer = new CircularRowCollection (
|
|
||||||
new BlueTableSchema (BlueDataUtility.extractRow (nrs, 0).schema.tableName,
|
|
||||||
BlueDataUtility.extractRow (nrs, 0).schema.columnInfo),
|
|
||||||
myBufferSize);
|
myBufferSize);
|
||||||
myBroker = new PubTableBroker ();
|
myBroker = new PubTableBroker();
|
||||||
myBroker.createTable (myBuffer);
|
myBroker.create_table(myBuffer);
|
||||||
}
|
}
|
||||||
BlueTable deletes = new BlueTable (myBuffer.schema);
|
IBlueRow br = myBuffer.addRowWithReturn(nr);
|
||||||
foreach (IBlueRow nr in nrs)
|
myBroker.make_batch_changes(myBuffer,
|
||||||
{
|
myReturnCollection(br),
|
||||||
IBlueRow dr = myBuffer.addRowWithReturn (nr);
|
myReturnCollection(nr));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void publishRows(IBlueRowCollection nrs) {
|
||||||
|
if (myBuffer == null) {
|
||||||
|
myBuffer = new CircularRowCollection(
|
||||||
|
new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName,
|
||||||
|
BlueDataUtility.extractRow(nrs, 0).schema.columnInfo),
|
||||||
|
myBufferSize);
|
||||||
|
myBroker = new PubTableBroker();
|
||||||
|
myBroker.create_table(myBuffer);
|
||||||
|
}
|
||||||
|
BlueTable deletes = new BlueTable(myBuffer.schema);
|
||||||
|
foreach (IBlueRow nr in nrs) {
|
||||||
|
IBlueRow dr = myBuffer.addRowWithReturn(nr);
|
||||||
if (dr != null)
|
if (dr != null)
|
||||||
deletes.addRow (dr);
|
deletes.addRow(dr);
|
||||||
}
|
}
|
||||||
myBroker.makeBatchChanges (myBuffer, deletes, nrs);
|
myBroker.make_batch_changes(myBuffer, deletes, nrs);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRowCollection myReturnCollection (IBlueRow r)
|
private IBlueRowCollection myReturnCollection(IBlueRow r) {
|
||||||
{
|
|
||||||
if (r == null)
|
if (r == null)
|
||||||
return new BlueTable (myBuffer.schema);
|
return new BlueTable(myBuffer.schema);
|
||||||
BlueTable retVal = new BlueTable (r.schema);
|
BlueTable retVal = new BlueTable(r.schema);
|
||||||
retVal.addRow (r);
|
retVal.addRow(r);
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,184 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq.Expressions;
|
|
||||||
using System.Xml.Serialization;
|
|
||||||
|
|
||||||
using BCG.Utilities;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
/// <summary>
|
|
||||||
/// This class will be used by the clients to publish tabular data to the pubsub
|
|
||||||
/// </summary>
|
|
||||||
public class PubTableBroker {
|
|
||||||
protected PublisherCore _publisher_core;
|
|
||||||
protected IDictionary<string, IPubTablePublisher> _publisher_map = new Dictionary<string, IPubTablePublisher>();
|
|
||||||
protected Dictionary<string, IBlueTableSchema> _schema_map = new Dictionary<string, IBlueTableSchema>();
|
|
||||||
|
|
||||||
protected bool myWaitingForFirstOp = false;
|
|
||||||
|
|
||||||
public PubTableBroker()
|
|
||||||
: this(new PublisherCore()) { }
|
|
||||||
|
|
||||||
public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; }
|
|
||||||
|
|
||||||
public PubTableBroker(IBlueRowCollection[] tablesToPublish)
|
|
||||||
: this(new PublisherCore(), tablesToPublish) { }
|
|
||||||
|
|
||||||
public PubTableBroker(PublisherCore publisher, IBlueRowCollection[] tablesToPublish) {
|
|
||||||
_publisher_core = publisher;
|
|
||||||
|
|
||||||
_publisher_core.beginTransaction();
|
|
||||||
foreach (IBlueRowCollection table in tablesToPublish)
|
|
||||||
createTable(table);
|
|
||||||
_publisher_core.commitTransaction();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void beginTransaction() {
|
|
||||||
myWaitingForFirstOp = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void commitTransaction() {
|
|
||||||
bts.Log.Gen.info("PubTableBroker.commitTransaction");
|
|
||||||
if (myWaitingForFirstOp)
|
|
||||||
myWaitingForFirstOp = false;
|
|
||||||
else
|
|
||||||
_publisher_core.commitTransaction();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void rollbackTransaction() {
|
|
||||||
bts.Log.Gen.info("PubTableBroker.rollbackTransaction");
|
|
||||||
if (myWaitingForFirstOp)
|
|
||||||
myWaitingForFirstOp = false;
|
|
||||||
else
|
|
||||||
_publisher_core.abortTransaction();
|
|
||||||
}
|
|
||||||
|
|
||||||
public virtual void createTable(IBlueRowCollection newTable) {
|
|
||||||
string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
|
|
||||||
foreach (IBlueRow r in newTable) {
|
|
||||||
log_msg += $"id: {r["id"]}";
|
|
||||||
}
|
|
||||||
bts.Log.Gen.info(log_msg);
|
|
||||||
_check_schema_consistency(newTable.schema);
|
|
||||||
myCheckTransaction();
|
|
||||||
IPubTablePublisher table_publisher = null;
|
|
||||||
var name_key = newTable.schema.tableName.ToLower();
|
|
||||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
|
|
||||||
table_publisher = _create_pubtable_publisher(name_key);
|
|
||||||
_publisher_map.Add(name_key, table_publisher);
|
|
||||||
_schema_map.Add(name_key, newTable.schema);
|
|
||||||
}
|
|
||||||
table_publisher.createTable(newTable);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
|
|
||||||
/// will be thrown
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="r"></param>
|
|
||||||
public virtual void addRow(IBlueRow r) {
|
|
||||||
string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
|
|
||||||
bts.Log.Gen.info(log_msg);
|
|
||||||
_check_schema_consistency(r.schema);
|
|
||||||
myCheckTransaction();
|
|
||||||
IPubTablePublisher table_publisher = null;
|
|
||||||
var name_key = r.schema.tableName.ToLower();
|
|
||||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
|
||||||
throw new ApplicationException("Row can not be added, as the table that has not been created yet");
|
|
||||||
else
|
|
||||||
table_publisher.addRow(r);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
|
|
||||||
/// will be thrown
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="r"></param>
|
|
||||||
public virtual void deleteRow(IBlueRow r) {
|
|
||||||
string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
|
|
||||||
bts.Log.Gen.info(log_msg);
|
|
||||||
_check_schema_consistency(r.schema);
|
|
||||||
myCheckTransaction();
|
|
||||||
IPubTablePublisher table_publisher = null;
|
|
||||||
var name_key = r.schema.tableName.ToLower();
|
|
||||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
|
||||||
throw new ApplicationException("Row can not be removed, as the table has not been created yet");
|
|
||||||
else
|
|
||||||
table_publisher.deleteRow(r);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
|
|
||||||
/// will be thrown
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="r"></param>
|
|
||||||
public virtual void changeRow(IBlueRow r) {
|
|
||||||
string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
|
|
||||||
bts.Log.Gen.info(log_msg);
|
|
||||||
_check_schema_consistency(r.schema);
|
|
||||||
myCheckTransaction();
|
|
||||||
IPubTablePublisher table_publisher = null;
|
|
||||||
var name_key = r.schema.tableName.ToLower();
|
|
||||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
|
||||||
throw new ApplicationException("Row can not be changed, as the table has not been created yet");
|
|
||||||
else
|
|
||||||
table_publisher.changeRow(r);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
|
|
||||||
/// will be thrown
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="r"></param>
|
|
||||||
public virtual void makeBatchChanges(IBlueRowCollection newRows,
|
|
||||||
IBlueRowCollection deletedRows,
|
|
||||||
IBlueRowCollection addedRows) {
|
|
||||||
string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
|
|
||||||
foreach (IBlueRow r in deletedRows)
|
|
||||||
log_msg += $"id: {r["id"]} ";
|
|
||||||
log_msg += "addedRows: ";
|
|
||||||
foreach (IBlueRow r in addedRows)
|
|
||||||
log_msg += $"id: {r["id"]}";
|
|
||||||
bts.Log.Gen.info(log_msg);
|
|
||||||
_check_schema_consistency(newRows.schema);
|
|
||||||
_check_schema_consistency(deletedRows.schema);
|
|
||||||
_check_schema_consistency(addedRows.schema);
|
|
||||||
IPubTablePublisher table_publisher = null;
|
|
||||||
var name_key = newRows.schema.tableName.ToLower();
|
|
||||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
|
||||||
throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
|
|
||||||
else
|
|
||||||
table_publisher.batchRemoveAdd(deletedRows, addedRows);
|
|
||||||
}
|
|
||||||
|
|
||||||
public virtual void deleteTableRows(string tableName) {
|
|
||||||
myCheckTransaction();
|
|
||||||
(_publisher_map[tableName.ToLower()] as IPubTablePublisher).createTable(
|
|
||||||
new BlueTable(_schema_map[tableName.ToLower()] as IBlueTableSchema));
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
|
|
||||||
|
|
||||||
public void clearTable(string tableName) { _publisher_core.clearState(PubStreamPrefixer.prefix(tableName)); }
|
|
||||||
|
|
||||||
protected void myCheckTransaction() {
|
|
||||||
if (myWaitingForFirstOp)
|
|
||||||
_publisher_core.beginTransaction();
|
|
||||||
myWaitingForFirstOp = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
|
|
||||||
bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
|
|
||||||
return new PubTablePublisher(tag, _publisher_core);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _check_schema_consistency(IBlueTableSchema schema_arg) {
|
|
||||||
IBlueTableSchema schema;
|
|
||||||
if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
|
|
||||||
bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
namespace BCG {
|
||||||
|
// this for testing purposes mainly, its wraps a ViewPublisher around a stream,
|
||||||
|
// and hence epublishes it out to the PS
|
||||||
|
public class PublisherStream : ViewStream {
|
||||||
|
private readonly ViewStream _main_stream;
|
||||||
|
private readonly string _name;
|
||||||
|
private ViewPublisher _vs_publisher;
|
||||||
|
private readonly PubTableBroker _broker = new PubTableBroker();
|
||||||
|
|
||||||
|
public PublisherStream(ViewStream main_stream, string name) {
|
||||||
|
_main_stream = main_stream;
|
||||||
|
_name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
|
||||||
|
_main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
|
||||||
|
_vs_publisher.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _handle_update(object o) {
|
||||||
|
myFireEvent(o);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,48 +7,32 @@ using BCG.Utilities;
|
|||||||
namespace BCG {
|
namespace BCG {
|
||||||
public class ViewPublisher : IDisposable {
|
public class ViewPublisher : IDisposable {
|
||||||
protected ViewPublisher() { }
|
protected ViewPublisher() { }
|
||||||
|
private readonly PubTableBroker _broker;
|
||||||
|
protected readonly ViewStream _stream;
|
||||||
|
private int _n_updates = 0;
|
||||||
|
|
||||||
/// If an name IsNullOrEmpty then it uses the name
|
/// If an name IsNullOrEmpty then it uses the name
|
||||||
/// in the schema (be careful with that because this
|
/// in the schema (be careful with that because this
|
||||||
/// could lead to double publishes if the schema name
|
/// could lead to double publishes if the schema name
|
||||||
/// is not uniquely chosen)
|
/// is not uniquely chosen)
|
||||||
public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
|
public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) {
|
||||||
myName = name;
|
_stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
|
||||||
myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
|
_broker = broker;
|
||||||
myBroker = br;
|
_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
|
||||||
myFirstTime = true;
|
|
||||||
myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Dispose() {
|
public void Dispose() {
|
||||||
myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
|
||||||
if (!String.IsNullOrEmpty(myName))
|
|
||||||
myBroker.clearTable(myName);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myChangeHandler(object o) {
|
private void _handle_update(object o) {
|
||||||
DataPacket dp = (DataPacket)o;
|
DataPacket dp = o as DataPacket;
|
||||||
if (myFirstTime) {
|
if (_n_updates == 0) {
|
||||||
myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName;
|
_broker.create_table(dp.newRows);
|
||||||
myLastRowCount = dp.newRows.Count;
|
_n_updates++;
|
||||||
myBroker.createTable(dp.newRows);
|
|
||||||
myFirstTime = false;
|
|
||||||
} else {
|
} else {
|
||||||
if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) {
|
_broker.make_batch_changes(dp.newRows, dp.deletedRows, dp.addedRows);
|
||||||
myLastRowCount = dp.newRows.Count;
|
|
||||||
myBroker.makeBatchChanges(dp.newRows, dp.deletedRows, dp.addedRows);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public int getLastRowPublishedCount() { return myLastRowCount; }
|
|
||||||
|
|
||||||
public bool hasPublished() { return !myFirstTime; }
|
|
||||||
|
|
||||||
private int myLastRowCount = -1;
|
|
||||||
private string myName;
|
|
||||||
private PubTableBroker myBroker;
|
|
||||||
protected ViewStream myView;
|
|
||||||
private bool myFirstTime;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
using System;
|
|
||||||
using BCG.Utilities;
|
|
||||||
|
|
||||||
namespace BCG
|
|
||||||
{
|
|
||||||
public class ViewStreamPublisher
|
|
||||||
{
|
|
||||||
private IDynamicStreamMapper myMapper;
|
|
||||||
private PubTableBroker myBroker;
|
|
||||||
private IBlueRowCollection myOutputTable;
|
|
||||||
private IBlueRowCollection myInputTable;
|
|
||||||
|
|
||||||
public ViewStreamPublisher (PubTableBroker broker)
|
|
||||||
{
|
|
||||||
myBroker = broker;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper)
|
|
||||||
: this (broker)
|
|
||||||
{
|
|
||||||
myMapper = mapper;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper, ViewStream vs)
|
|
||||||
: this (broker, mapper)
|
|
||||||
{
|
|
||||||
vs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChanges);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void myHandleChanges (object newValue)
|
|
||||||
{
|
|
||||||
DataPacket dp = newValue as DataPacket;
|
|
||||||
if (myMapper == null)
|
|
||||||
{
|
|
||||||
if (myOutputTable == null)
|
|
||||||
{
|
|
||||||
myBroker.createTable (dp.newRows);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
myBroker.makeBatchChanges (dp.newRows, dp.deletedRows, dp.addedRows);
|
|
||||||
}
|
|
||||||
myOutputTable = dp.newRows;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (myOutputTable != null)
|
|
||||||
{
|
|
||||||
BlueTable dRows = new BlueTable (myOutputTable.schema),
|
|
||||||
aRows = new BlueTable (myOutputTable.schema);
|
|
||||||
foreach (IBlueRow dRow in dp.deletedRows)
|
|
||||||
{
|
|
||||||
(myOutputTable as BlueTable).deleteRow ((int)dRow["id"]);
|
|
||||||
dRows.addRow (myMapper.map (dRow));
|
|
||||||
}
|
|
||||||
foreach (IBlueRow aRow in dp.addedRows)
|
|
||||||
{
|
|
||||||
IBlueRow r = myMapper.map (aRow);
|
|
||||||
(myOutputTable as BlueTable).addRow (r);
|
|
||||||
aRows.addRow (r);
|
|
||||||
}
|
|
||||||
myBroker.makeBatchChanges (myOutputTable, dRows, aRows);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (dp.newRows.Count > 0)
|
|
||||||
{
|
|
||||||
myOutputTable = new BlueTable (
|
|
||||||
myMapper.map (BlueDataUtility.extractRow (dp.newRows, 0)).schema);
|
|
||||||
myHandleChanges (newValue);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addRows (IBlueRowCollection aRows)
|
|
||||||
{
|
|
||||||
if (myInputTable != null)
|
|
||||||
(myInputTable as BlueTable).addRows (aRows);
|
|
||||||
else
|
|
||||||
myInputTable = new BlueTable (aRows);
|
|
||||||
myHandleChanges (new DataPacket(myInputTable, aRows, new BlueTable (aRows.schema)));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void deleteRows (IBlueRowCollection dRows)
|
|
||||||
{
|
|
||||||
// BlueEventLog.logAssert (myInputTable != null);
|
|
||||||
(myInputTable as BlueTable).deleteRows (dRows);
|
|
||||||
myHandleChanges (new DataPacket (myInputTable, new BlueTable (myInputTable.schema), dRows));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -6,7 +6,6 @@ using System.Reflection;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Globalization;
|
|
||||||
|
|
||||||
//using BCG.Utilities;
|
//using BCG.Utilities;
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
@@ -14,7 +13,8 @@ using System.Linq.Expressions;
|
|||||||
//using mbts.spark;
|
//using mbts.spark;
|
||||||
using mbts;
|
using mbts;
|
||||||
using BCG.Extensions;
|
using BCG.Extensions;
|
||||||
using System.Runtime.CompilerServices;
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Nodes;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public static class BlueDataUtility {
|
public static class BlueDataUtility {
|
||||||
@@ -49,12 +49,6 @@ namespace BCG {
|
|||||||
rv.Add(r.get<T>(col));
|
rv.Add(r.get<T>(col));
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
public static IEnumerable<T> get_colvalues<T>(IBlueRowCollection rows, string col) {
|
|
||||||
List<T> rv = new List<T>();
|
|
||||||
foreach (IBlueRow r in rows)
|
|
||||||
rv.Add(r.get<T>(col));
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
|
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
|
||||||
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
|
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
|
||||||
@@ -74,12 +68,17 @@ namespace BCG {
|
|||||||
// The values are returned as string values
|
// The values are returned as string values
|
||||||
// doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
|
// doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
|
||||||
public static IBlueRowCollection keyvalue_to_relational<T>(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
|
public static IBlueRowCollection keyvalue_to_relational<T>(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
|
||||||
HashSet<string> unique_cols = new HashSet<string>();
|
HashSet<string> keys = new HashSet<string>();
|
||||||
foreach (string col in get_column_values<string>(kv_rows, key_col)) {
|
foreach (string col in get_column_values<string>(kv_rows, key_col)) {
|
||||||
unique_cols.Add(col);
|
keys.Add(col);
|
||||||
}
|
}
|
||||||
//unique_cols.Add(doc_col);
|
if (keys.Contains("id")) {
|
||||||
var schema_cols = unique_cols.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
|
keys.Remove("id");
|
||||||
|
}
|
||||||
|
//unique_cols.Add(doc_col)
|
||||||
|
// all the keys become cols
|
||||||
|
var schema_cols = keys.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
|
||||||
|
//schema of the final table needs doc and id
|
||||||
schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
|
schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
|
||||||
schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
|
schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
|
||||||
var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
|
var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
|
||||||
@@ -91,6 +90,8 @@ namespace BCG {
|
|||||||
nr.setValue(doc_col, kv.Key);
|
nr.setValue(doc_col, kv.Key);
|
||||||
foreach (var r in kv.Value) {
|
foreach (var r in kv.Value) {
|
||||||
var col = r.get<string>(key_col);
|
var col = r.get<string>(key_col);
|
||||||
|
if (!keys.Contains(col))
|
||||||
|
continue;
|
||||||
var val = r.get<string>(val_col);
|
var val = r.get<string>(val_col);
|
||||||
nr.setValue(col, val);
|
nr.setValue(col, val);
|
||||||
}
|
}
|
||||||
@@ -118,6 +119,15 @@ namespace BCG {
|
|||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
|
||||||
|
Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
|
||||||
|
foreach (IBlueRow r in rows) {
|
||||||
|
var id = r.get<int>("id");
|
||||||
|
rv.Add(id, r);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
|
public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
|
List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
|
||||||
foreach (Tuple<Type, string> c in cols)
|
foreach (Tuple<Type, string> c in cols)
|
||||||
@@ -179,6 +189,35 @@ namespace BCG {
|
|||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
|
#region json
|
||||||
|
|
||||||
|
public static List<JsonNode> json_to_json_node_list(string jsonString) {
|
||||||
|
var jsonNodeList = new List<JsonNode>();
|
||||||
|
bool noe = String.IsNullOrEmpty(jsonString);
|
||||||
|
if (noe)
|
||||||
|
return jsonNodeList;
|
||||||
|
try {
|
||||||
|
// Parse the JSON string
|
||||||
|
JsonArray jsonArray = JsonNode.Parse(jsonString) as JsonArray;
|
||||||
|
|
||||||
|
if (jsonArray == null) {
|
||||||
|
Console.WriteLine("The JSON string must represent a list.");
|
||||||
|
return jsonNodeList;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Iterate through each JsonNode in the JsonArray
|
||||||
|
foreach (JsonNode jsonNode in jsonArray) {
|
||||||
|
jsonNodeList.Add(jsonNode);
|
||||||
|
}
|
||||||
|
} catch (JsonException ex) {
|
||||||
|
Console.WriteLine($"Invalid JSON: {ex.Message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
return jsonNodeList;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width) {
|
public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width) {
|
||||||
var rv = get_printed_rows(table, get_col_width);
|
var rv = get_printed_rows(table, get_col_width);
|
||||||
Console.WriteLine(rv);
|
Console.WriteLine(rv);
|
||||||
@@ -208,37 +247,98 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
#region Schema types
|
||||||
|
// these are the only types allowed in schemas
|
||||||
private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
|
private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
|
||||||
{
|
{
|
||||||
{ TypeCode.Boolean, typeof(bool) },
|
{ TypeCode.Boolean, typeof(bool) }, //3
|
||||||
{ TypeCode.Byte, typeof(byte) },
|
{ TypeCode.Byte, typeof(byte) }, //6
|
||||||
{ TypeCode.Char, typeof(char) },
|
{ TypeCode.Char, typeof(char) }, //4
|
||||||
{ TypeCode.DateTime, typeof(DateTime) },
|
// TODO
|
||||||
{ TypeCode.DBNull, typeof(DBNull) },
|
{ TypeCode.DateTime, typeof(DateTime) },//16
|
||||||
{ TypeCode.Decimal, typeof(decimal) },
|
{ TypeCode.Decimal, typeof(decimal) }, //15
|
||||||
{ TypeCode.Double, typeof(double) },
|
{ TypeCode.Double, typeof(double) }, //14
|
||||||
{ TypeCode.Empty, null },
|
//{ TypeCode.Empty, null },
|
||||||
{ TypeCode.Int16, typeof(short) },
|
{ TypeCode.Int16, typeof(short) }, //7
|
||||||
{ TypeCode.Int32, typeof(int) },
|
{ TypeCode.Int32, typeof(int) }, //9
|
||||||
{ TypeCode.Int64, typeof(long) },
|
{ TypeCode.Int64, typeof(long) }, //11
|
||||||
{ TypeCode.Object, typeof(object) },
|
{ TypeCode.SByte, typeof(sbyte) }, //5
|
||||||
{ TypeCode.SByte, typeof(sbyte) },
|
{ TypeCode.Single, typeof(Single) }, //13
|
||||||
{ TypeCode.Single, typeof(Single) },
|
{ TypeCode.String, typeof(string) }, //18
|
||||||
{ TypeCode.String, typeof(string) },
|
{ TypeCode.UInt16, typeof(UInt16) }, //8
|
||||||
{ TypeCode.UInt16, typeof(UInt16) },
|
{ TypeCode.UInt32, typeof(UInt32) }, //10
|
||||||
{ TypeCode.UInt32, typeof(UInt32) },
|
{ TypeCode.UInt64, typeof(UInt64) } //12
|
||||||
{ TypeCode.UInt64, typeof(UInt64) }
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
public static IEnumerable<TypeCode> valid_column_typecodes() {
|
||||||
|
return _typecode_to_type.Keys;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IEnumerable<Type> valid_column_types() {
|
||||||
|
return _typecode_to_type.Values;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool is_valid_col_type(Type t) {
|
||||||
|
TypeCode type_code = get_typecode(t);
|
||||||
|
return type_code != TypeCode.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
public static TypeCode get_typecode(Type t) {
|
public static TypeCode get_typecode(Type t) {
|
||||||
foreach (var kv in _typecode_to_type) {
|
foreach (var kv in _typecode_to_type) {
|
||||||
if (kv.Value == t)
|
if (kv.Value == t)
|
||||||
return kv.Key;
|
return kv.Key;
|
||||||
}
|
}
|
||||||
bts.Log.assert_d(false, $"type: {t} not found");
|
//bts.Log.assert_d(false, $"type: {t} not found");
|
||||||
return TypeCode.Empty;
|
return TypeCode.Empty;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Type get_type(TypeCode type_code) {
|
||||||
|
Type rv = null;
|
||||||
|
if (_typecode_to_type.TryGetValue(type_code, out rv)) {
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static TypeCode type_code(this ColumnInfo ci) {
|
||||||
|
return get_typecode(ci.colType);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
public static Tuple<Type, object> guess_type(string value) {
|
||||||
|
if (string.IsNullOrEmpty(value)) {
|
||||||
|
return Tuple.Create(typeof(string), (object)value);
|
||||||
|
}
|
||||||
|
|
||||||
|
int val = 0;
|
||||||
|
// Try to parse as integer
|
||||||
|
if (int.TryParse(value, out val)) {
|
||||||
|
return Tuple.Create(typeof(int), (object)val);
|
||||||
|
}
|
||||||
|
|
||||||
|
double dval = 0.0;
|
||||||
|
// Try to parse as double
|
||||||
|
if (double.TryParse(value, out dval)) {
|
||||||
|
return Tuple.Create(typeof(double), (object)dval);
|
||||||
|
}
|
||||||
|
|
||||||
|
DateTime dt;
|
||||||
|
// Try to parse as DateTime
|
||||||
|
if (DateTime.TryParse(value, out dt)) {
|
||||||
|
return Tuple.Create(typeof(DateTime), (object)dt);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool bval;
|
||||||
|
// Try to parse as boolean
|
||||||
|
if (bool.TryParse(value, out bval)) {
|
||||||
|
return Tuple.Create(typeof(bool), (object)bval);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If none of the above, return string type
|
||||||
|
return Tuple.Create(typeof(string), (object)value);
|
||||||
|
}
|
||||||
|
|
||||||
public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
|
public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
foreach (Tuple<string, TypeCode> c in cols)
|
foreach (Tuple<string, TypeCode> c in cols)
|
||||||
@@ -248,106 +348,6 @@ namespace BCG {
|
|||||||
return new BlueTableSchema(table_name, cis.ToArray());
|
return new BlueTableSchema(table_name, cis.ToArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static TypeCode convert_column_type(IEnumerable<string> col_vals) {
|
|
||||||
// bool
|
|
||||||
bool success = true;
|
|
||||||
foreach (var val in col_vals) {
|
|
||||||
bool parsed_val;
|
|
||||||
success = success && Boolean.TryParse(val, out parsed_val);
|
|
||||||
if (!success)
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (success)
|
|
||||||
return TypeCode.Boolean;
|
|
||||||
// int
|
|
||||||
success = true;
|
|
||||||
foreach (var val in col_vals) {
|
|
||||||
int parsed_val;
|
|
||||||
success = success && Int32.TryParse(val, out parsed_val);
|
|
||||||
if (!success)
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (success)
|
|
||||||
return TypeCode.Int32;
|
|
||||||
// try double
|
|
||||||
success = true;
|
|
||||||
foreach (var val in col_vals) {
|
|
||||||
double parsed_val;
|
|
||||||
success = success && Double.TryParse(val, out parsed_val);
|
|
||||||
if (!success)
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (success)
|
|
||||||
return TypeCode.Double;
|
|
||||||
// try DateTime
|
|
||||||
success = true;
|
|
||||||
foreach (var val in col_vals) {
|
|
||||||
DateTime parsed_val;
|
|
||||||
success = success && DateTime.TryParse(val, out parsed_val);
|
|
||||||
if (!success)
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (success)
|
|
||||||
return TypeCode.DateTime;
|
|
||||||
bts.Log.assert(false);
|
|
||||||
return TypeCode.Empty;
|
|
||||||
}
|
|
||||||
|
|
||||||
// public static Type guess_type(string val) {
|
|
||||||
// //DateTime
|
|
||||||
// {
|
|
||||||
// DateTime dt;
|
|
||||||
// var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt);
|
|
||||||
// if (success) {
|
|
||||||
// return typeof(DateTime);
|
|
||||||
// }
|
|
||||||
// DateTimeOffset dtoff;
|
|
||||||
// success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz",
|
|
||||||
// null, System.Globalization.DateTimeStyles.None, out dtoff);
|
|
||||||
|
|
||||||
// if (success) {
|
|
||||||
// return typeof(DateTime);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// //int
|
|
||||||
// {
|
|
||||||
// int i;
|
|
||||||
// var success = Int32.TryParse(val, out i);
|
|
||||||
// if (success) {
|
|
||||||
// return typeof(Int32);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// //double
|
|
||||||
// {
|
|
||||||
// double d;
|
|
||||||
// var success = Double.TryParse(val, out d);
|
|
||||||
// if (success) {
|
|
||||||
// return typeof(Double);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// //bool
|
|
||||||
// {
|
|
||||||
// bool b;
|
|
||||||
// var success = Boolean.TryParse(val, out b);
|
|
||||||
// if (success) {
|
|
||||||
// return typeof(bool);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// //string
|
|
||||||
// return typeof(string);
|
|
||||||
// }
|
|
||||||
|
|
||||||
private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) };
|
|
||||||
public static Type guess_type(string val) {
|
|
||||||
foreach (var t in _parsable_types) {
|
|
||||||
var type_val = convert_to_type(t, val);
|
|
||||||
if (type_val != null)
|
|
||||||
return t;
|
|
||||||
}
|
|
||||||
return typeof(string);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
|
public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
|
||||||
// just use one row to guess
|
// just use one row to guess
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
@@ -357,7 +357,7 @@ namespace BCG {
|
|||||||
cis.Add(ci);
|
cis.Add(ci);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
Type type = guess_type(r[ci.colName] as string);
|
var (type, _) = guess_type(r[ci.colName] as string);
|
||||||
cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
|
cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
|
||||||
}
|
}
|
||||||
return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
|
return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
|
||||||
@@ -365,37 +365,8 @@ namespace BCG {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
static object convert_to_type(Type t, string value) {
|
|
||||||
// Attempt to convert the string to the specified type
|
|
||||||
try {
|
|
||||||
return Convert.ChangeType(value, t);
|
|
||||||
} catch (Exception) {
|
|
||||||
if (t == typeof(double)) {
|
|
||||||
// some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
|
|
||||||
string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
|
|
||||||
cleaned_val = cleaned_val.Replace('−', '-');
|
|
||||||
double rv_d;
|
|
||||||
if (Double.TryParse(cleaned_val, out rv_d))
|
|
||||||
return rv_d;
|
|
||||||
else
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
// Handle conversion failure gracefully
|
|
||||||
// throw new ArgumentException("Failed to convert the string to the specified type.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// converts a row where all columns are string types to a schema where the best types are guesed
|
// converts a row where all columns are string types to a schema where the best types are guesed
|
||||||
public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
|
public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
|
||||||
Func<IBlueRow, IBlueTableSchema, IBlueRow> get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => {
|
|
||||||
MutableBlueRow rv = new MutableBlueRow(typed_schema);
|
|
||||||
foreach (ColumnInfo ci in typed_schema.columnInfo) {
|
|
||||||
var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString());
|
|
||||||
rv.setValue(ci.colName, typed_val);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
};
|
|
||||||
IBlueTableSchema typed_schema = get_typed_schema(string_rows);
|
IBlueTableSchema typed_schema = get_typed_schema(string_rows);
|
||||||
var rv = new BlueTable(typed_schema);
|
var rv = new BlueTable(typed_schema);
|
||||||
foreach (IBlueRow r in string_rows) {
|
foreach (IBlueRow r in string_rows) {
|
||||||
@@ -403,10 +374,38 @@ namespace BCG {
|
|||||||
rv.addRow(typed_row);
|
rv.addRow(typed_row);
|
||||||
}
|
}
|
||||||
return rv;
|
return rv;
|
||||||
|
|
||||||
|
static IBlueRow get_typed_row(IBlueRow string_row, IBlueTableSchema typed_schema) {
|
||||||
|
var rv = new MutableBlueRow(typed_schema);
|
||||||
|
foreach (var ci in typed_schema.columnInfo) {
|
||||||
|
var typed_val = convert(ci.colType, string_row[ci.colName].ToString());
|
||||||
|
rv.setValue(ci.colName, typed_val);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
static object convert(Type t, string value) {
|
||||||
|
// Attempt to convert the string to the specified type
|
||||||
|
try {
|
||||||
|
return Convert.ChangeType(value, t);
|
||||||
|
} catch (Exception) {
|
||||||
|
if (t == typeof(double)) {
|
||||||
|
// some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
|
||||||
|
string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
|
||||||
|
cleaned_val = cleaned_val.Replace('−', '-');
|
||||||
|
double rv_d;
|
||||||
|
if (Double.TryParse(cleaned_val, out rv_d))
|
||||||
|
return rv_d;
|
||||||
|
else
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
// Handle conversion failure gracefully
|
||||||
|
// throw new ArgumentException("Failed to convert the string to the specified type.");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
|
/// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
|
||||||
/// be alphabetic or not
|
/// be alphabetic or not
|
||||||
@@ -541,12 +540,12 @@ namespace BCG {
|
|||||||
foreach (PropertyInfo pi in t.GetProperties())
|
foreach (PropertyInfo pi in t.GetProperties())
|
||||||
if (myIsActiveProperty(pi))
|
if (myIsActiveProperty(pi))
|
||||||
colInfos.Add(new ColumnInfo(pi.Name, i++,
|
colInfos.Add(new ColumnInfo(pi.Name, i++,
|
||||||
ColumnInfo.isAllowedType(pi.PropertyType) ?
|
is_valid_col_type(pi.PropertyType) ?
|
||||||
pi.PropertyType : typeof(String)));
|
pi.PropertyType : typeof(String)));
|
||||||
foreach (FieldInfo fi in t.GetFields())
|
foreach (FieldInfo fi in t.GetFields())
|
||||||
if (myIsActiveField(fi))
|
if (myIsActiveField(fi))
|
||||||
colInfos.Add(new ColumnInfo(fi.Name, i++,
|
colInfos.Add(new ColumnInfo(fi.Name, i++,
|
||||||
ColumnInfo.isAllowedType(fi.FieldType) ? fi.FieldType : typeof(String)));
|
is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String)));
|
||||||
|
|
||||||
if (createIdField)
|
if (createIdField)
|
||||||
colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
|
colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
|
||||||
@@ -571,12 +570,12 @@ namespace BCG {
|
|||||||
foreach (PropertyInfo pi in t.GetProperties())
|
foreach (PropertyInfo pi in t.GetProperties())
|
||||||
if (myIsActiveProperty(pi))
|
if (myIsActiveProperty(pi))
|
||||||
rowArray[schema[pi.Name].colNo] =
|
rowArray[schema[pi.Name].colNo] =
|
||||||
ColumnInfo.isAllowedType(pi.PropertyType) ?
|
is_valid_col_type(pi.PropertyType) ?
|
||||||
pi.GetValue(o, null) : pi.GetValue(o, null).ToString();
|
pi.GetValue(o, null) : pi.GetValue(o, null).ToString();
|
||||||
foreach (FieldInfo fi in t.GetFields())
|
foreach (FieldInfo fi in t.GetFields())
|
||||||
if (myIsActiveField(fi))
|
if (myIsActiveField(fi))
|
||||||
rowArray[schema[fi.Name].colNo] =
|
rowArray[schema[fi.Name].colNo] =
|
||||||
ColumnInfo.isAllowedType(fi.FieldType) ?
|
is_valid_col_type(fi.FieldType) ?
|
||||||
fi.GetValue(o) : fi.GetValue(o).ToString();
|
fi.GetValue(o) : fi.GetValue(o).ToString();
|
||||||
if (idVal != null)
|
if (idVal != null)
|
||||||
rowArray[schema["id"].colNo] = idVal;
|
rowArray[schema["id"].colNo] = idVal;
|
||||||
@@ -590,12 +589,12 @@ namespace BCG {
|
|||||||
foreach (PropertyInfo pi in t.GetProperties())
|
foreach (PropertyInfo pi in t.GetProperties())
|
||||||
if (myIsActiveProperty(pi))
|
if (myIsActiveProperty(pi))
|
||||||
rowArray[schema[pi.Name].colNo] =
|
rowArray[schema[pi.Name].colNo] =
|
||||||
ColumnInfo.isAllowedType(pi.PropertyType) ?
|
is_valid_col_type(pi.PropertyType) ?
|
||||||
pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null));
|
pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null));
|
||||||
foreach (FieldInfo fi in t.GetFields())
|
foreach (FieldInfo fi in t.GetFields())
|
||||||
if (myIsActiveField(fi))
|
if (myIsActiveField(fi))
|
||||||
rowArray[schema[fi.Name].colNo] =
|
rowArray[schema[fi.Name].colNo] =
|
||||||
ColumnInfo.isAllowedType(fi.FieldType) ?
|
is_valid_col_type(fi.FieldType) ?
|
||||||
fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o));
|
fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o));
|
||||||
if (idVal != null)
|
if (idVal != null)
|
||||||
rowArray[schema["id"].colNo] = idVal;
|
rowArray[schema["id"].colNo] = idVal;
|
||||||
@@ -784,15 +783,7 @@ namespace BCG {
|
|||||||
/// <param name="r1"></param>
|
/// <param name="r1"></param>
|
||||||
/// <param name="r2"></param>
|
/// <param name="r2"></param>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) {
|
|
||||||
return computeRowEquality(r1, r2, r1.schema.columnNames);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static bool computeRowEquality(IBlueRow r1,
|
|
||||||
IBlueRow r2,
|
|
||||||
params string[] colNames) {
|
|
||||||
return compareRows(r1, r2, colNames) == 0 ? true : false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computes a string equivelant for the row, using the columns provided
|
/// Computes a string equivelant for the row, using the columns provided
|
||||||
@@ -830,7 +821,7 @@ namespace BCG {
|
|||||||
/// <param name="r2"></param>
|
/// <param name="r2"></param>
|
||||||
/// <param name="colNames"></param>
|
/// <param name="colNames"></param>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public static int compareRows(IBlueRow r1,
|
public static int compare_rows(IBlueRow r1,
|
||||||
IBlueRow r2,
|
IBlueRow r2,
|
||||||
params string[] colNames) {
|
params string[] colNames) {
|
||||||
int retVal = 0;
|
int retVal = 0;
|
||||||
@@ -981,7 +972,7 @@ namespace BCG {
|
|||||||
public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) {
|
public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) {
|
||||||
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
|
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
|
||||||
foreach (IBlueRow r in rows)
|
foreach (IBlueRow r in rows)
|
||||||
retVal = compareRows(r, retVal, cols) > 0 ? r : retVal;
|
retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal;
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -994,7 +985,7 @@ namespace BCG {
|
|||||||
public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) {
|
public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) {
|
||||||
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
|
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
|
||||||
foreach (IBlueRow r in rows)
|
foreach (IBlueRow r in rows)
|
||||||
retVal = compareRows(r, retVal, cols) < 0 ? r : retVal;
|
retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal;
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1073,8 +1064,58 @@ namespace BCG {
|
|||||||
out aRows);
|
out aRows);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) {
|
||||||
|
IBlueRowCollection d_rows, a_rows;
|
||||||
|
compute_differences_fast(rows2, rows1, out d_rows, out a_rows);
|
||||||
|
return d_rows.Count == 0 && a_rows.Count == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) {
|
||||||
|
return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) {
|
||||||
|
if (s1.tableName != s1.tableName)
|
||||||
|
return false;
|
||||||
|
if (s1.columnInfo.Length != s2.columnInfo.Length)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < s1.columnInfo.Length; i++) {
|
||||||
|
ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i];
|
||||||
|
if (!compute_equality(ci1, ci2))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool compute_equality(IBlueRow r1, IBlueRow r2) {
|
||||||
|
return compute_row_equality(r1, r2, r1.schema.columnNames);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool compute_row_equality(IBlueRow r1,
|
||||||
|
IBlueRow r2,
|
||||||
|
params string[] col_names) {
|
||||||
|
return compare_rows(r1, r2, col_names) == 0 ? true : false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string to_string(ColumnInfo ci) {
|
||||||
|
return ci.colNo + ";" + ci.colName + ";" + ci.colType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string to_string(this IBlueRow r) {
|
||||||
|
string rv = String.Empty;
|
||||||
|
foreach (ColumnInfo c in r.schema.columnInfo) {
|
||||||
|
rv += $"{c.colName}: {r[c.colName]}\n";
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
|
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
|
||||||
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
||||||
|
if (old_rows == null || old_rows.Count == 0) {
|
||||||
|
out_d_rows = new BlueTable(new_rows.schema);
|
||||||
|
out_a_rows = new BlueTable(new_rows);
|
||||||
|
return;
|
||||||
|
}
|
||||||
int[] col_indices = new int[new_rows.schema.columnNames.Length];
|
int[] col_indices = new int[new_rows.schema.columnNames.Length];
|
||||||
for (int i = 0; i < col_indices.Length; i++)
|
for (int i = 0; i < col_indices.Length; i++)
|
||||||
col_indices[i] = i;
|
col_indices[i] = i;
|
||||||
@@ -1083,6 +1124,11 @@ namespace BCG {
|
|||||||
|
|
||||||
public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
|
public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
|
||||||
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
||||||
|
if (old_rows == null || old_rows.Count == 0) {
|
||||||
|
out_d_rows = new BlueTable(new_rows.schema);
|
||||||
|
out_a_rows = new BlueTable(new_rows);
|
||||||
|
return;
|
||||||
|
}
|
||||||
int[] col_indices = new int[] { new_rows.schema["id"].colNo };
|
int[] col_indices = new int[] { new_rows.schema["id"].colNo };
|
||||||
compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
|
compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
|
||||||
}
|
}
|
||||||
@@ -1126,7 +1172,7 @@ namespace BCG {
|
|||||||
foreach (IBlueRow nr in c1)
|
foreach (IBlueRow nr in c1)
|
||||||
if (!BlueDataUtility.rowPresent(c2, nr))
|
if (!BlueDataUtility.rowPresent(c2, nr))
|
||||||
(aRows as BlueTable).addRow(nr);
|
(aRows as BlueTable).addRow(nr);
|
||||||
else if (BlueDataUtility.compareRows(nr, c2[(int)
|
else if (BlueDataUtility.compare_rows(nr, c2[(int)
|
||||||
nr["id"]], columns) != 0) {
|
nr["id"]], columns) != 0) {
|
||||||
(aRows as BlueTable).addRow(nr);
|
(aRows as BlueTable).addRow(nr);
|
||||||
(dRows as BlueTable).addRow(c2[(int)nr["id"]]);
|
(dRows as BlueTable).addRow(c2[(int)nr["id"]]);
|
||||||
@@ -1206,7 +1252,7 @@ namespace BCG {
|
|||||||
truelyAdds = new BlueTable(adds);
|
truelyAdds = new BlueTable(adds);
|
||||||
foreach (IBlueRow r in dels) {
|
foreach (IBlueRow r in dels) {
|
||||||
if (adds.containsID((int)r["id"]) &&
|
if (adds.containsID((int)r["id"]) &&
|
||||||
BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) {
|
BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) {
|
||||||
(truelyDels as BlueTable).deleteRow((int)r["id"]);
|
(truelyDels as BlueTable).deleteRow((int)r["id"]);
|
||||||
(truelyAdds as BlueTable).deleteRow((int)r["id"]);
|
(truelyAdds as BlueTable).deleteRow((int)r["id"]);
|
||||||
}
|
}
|
||||||
@@ -1582,6 +1628,13 @@ namespace BCG {
|
|||||||
return sb.ToString();
|
return sb.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static string id_string(IBlueRowCollection table) {
|
||||||
|
string rv = String.Empty;
|
||||||
|
foreach (IBlueRow r in table)
|
||||||
|
rv += $"{r["id"]} ";
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
|
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
|
||||||
BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
|
BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
|
||||||
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
|
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
|
||||||
@@ -1730,6 +1783,18 @@ namespace BCG {
|
|||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
|
||||||
|
if (rows == null || rows.Count() == 0)
|
||||||
|
return null;
|
||||||
|
IBlueRow max_row = rows.ElementAt(0);
|
||||||
|
foreach (IBlueRow r in rows) {
|
||||||
|
int c = compare_rows(r, max_row, cols_to_compare);
|
||||||
|
if (c > 0)
|
||||||
|
max_row = r;
|
||||||
|
}
|
||||||
|
return max_row;
|
||||||
|
}
|
||||||
|
|
||||||
public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
|
public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
|
||||||
foreach (KeyValuePair<string, bool> p in sort_map) {
|
foreach (KeyValuePair<string, bool> p in sort_map) {
|
||||||
int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
|
int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
|
||||||
@@ -2164,7 +2229,10 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) {
|
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) =>
|
||||||
|
createRowsFromCSVFileWithoutTypes(fileAddress, out var _);
|
||||||
|
|
||||||
|
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress, out IBlueTableSchema schema) {
|
||||||
StreamReader sr = new StreamReader(fileAddress);
|
StreamReader sr = new StreamReader(fileAddress);
|
||||||
string cols = sr.ReadLine();
|
string cols = sr.ReadLine();
|
||||||
string[] columns = cols.Split(',');
|
string[] columns = cols.Split(',');
|
||||||
@@ -2172,11 +2240,11 @@ namespace BCG {
|
|||||||
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
||||||
for (int i = 1; i < ci.Length; i++)
|
for (int i = 1; i < ci.Length; i++)
|
||||||
ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
|
ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
|
||||||
IBlueTableSchema bts = new BlueTableSchema(ci, "csvTable");
|
schema = new BlueTableSchema(ci, "csvTable");
|
||||||
BlueTable result = new BlueTable(bts);
|
BlueTable result = new BlueTable(schema);
|
||||||
int id = 0;
|
int id = 0;
|
||||||
while (!sr.EndOfStream)
|
while (!sr.EndOfStream)
|
||||||
result.addRow(createRowFromCSVFileRow(sr.ReadLine(), bts, id++));
|
result.addRow(createRowFromCSVFileRow(sr.ReadLine(), schema, id++));
|
||||||
sr.Close();
|
sr.Close();
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -2210,33 +2278,5 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
public class RowComparer<T> : IComparer<IBlueRow> {
|
|
||||||
private readonly IEnumerable<Tuple<string, bool>> _sort_cols;
|
|
||||||
public RowComparer(IEnumerable<Tuple<string, bool>> sort_cols) {
|
|
||||||
_sort_cols = sort_cols;
|
|
||||||
}
|
|
||||||
public int Compare(IBlueRow? x, IBlueRow? y) {
|
|
||||||
Func<object, object, int> compare_cols = (o1, o2) => {
|
|
||||||
return (o1 as IComparable).CompareTo(o2);
|
|
||||||
};
|
|
||||||
foreach (var c in _sort_cols) {
|
|
||||||
int i = compare_cols(x[c.Item1], y[c.Item1]);
|
|
||||||
|
|
||||||
}
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
public static IList<IBlueRow> get_sorted_row(IBlueRowCollection rows, IEnumerable<Tuple<string, bool>> sort_cols) {
|
|
||||||
List<IBlueRow> row_list = new List<IBlueRow>();
|
|
||||||
foreach (IBlueRow r in rows) {
|
|
||||||
row_list.Add(r);
|
|
||||||
}
|
|
||||||
var row_comparer = new RowComparer<IBlueRow>(sort_cols);
|
|
||||||
ParallelSort.QuicksortSequential<IBlueRow>(row_list, row_comparer);
|
|
||||||
return row_list;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,45 +1,42 @@
|
|||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
/// <summary>
|
||||||
/// <summary>
|
/// Allowed types
|
||||||
/// Allowed types
|
/// </summary>
|
||||||
/// </summary>
|
public class BlueDataValidTypes {
|
||||||
public class BlueDataValidTypes
|
public static readonly Type BOOL = typeof(bool);
|
||||||
{
|
public static readonly Type BYTE = typeof(byte);
|
||||||
public static readonly Type BOOL = typeof(bool);
|
public static readonly Type CHAR = typeof(char);
|
||||||
public static readonly Type BYTE = typeof(byte);
|
public static readonly Type DECIMAL = typeof(decimal);
|
||||||
public static readonly Type CHAR = typeof(char);
|
public static readonly Type DOUBLE = typeof(double);
|
||||||
public static readonly Type DECIMAL = typeof(decimal);
|
public static readonly Type SHORT = typeof(short);
|
||||||
public static readonly Type DOUBLE = typeof(double);
|
public static readonly Type INT = typeof(int);
|
||||||
public static readonly Type SHORT = typeof(short);
|
public static readonly Type LONG = typeof(long);
|
||||||
public static readonly Type INT = typeof(int);
|
public static readonly Type SBYTE = typeof(sbyte);
|
||||||
public static readonly Type LONG = typeof(long);
|
//public static readonly Type FLOAT = typeof(float);
|
||||||
public static readonly Type SBYTE = typeof(sbyte);
|
public static readonly Type STRING = typeof(string);
|
||||||
//public static readonly Type FLOAT = typeof(float);
|
//public static readonly Type USHORT = typeof(ushort);
|
||||||
public static readonly Type STRING = typeof(string);
|
public static readonly Type UINT = typeof(uint);
|
||||||
//public static readonly Type USHORT = typeof(ushort);
|
public static readonly Type ULONG = typeof(ulong);
|
||||||
public static readonly Type UINT = typeof(uint);
|
public static readonly Type DATETIME = typeof(DateTime);
|
||||||
public static readonly Type ULONG = typeof(ulong);
|
public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
|
||||||
public static readonly Type DATETIME = typeof(DateTime);
|
public static readonly Type SINGLE = typeof(Single);
|
||||||
public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
|
public static readonly Type BYTEARRAY = typeof(byte[]);
|
||||||
public static readonly Type SINGLE = typeof(Single);
|
|
||||||
public static readonly Type BYTEARRAY = typeof(byte[]);
|
|
||||||
|
|
||||||
public static int getValueTypeSize (Type t)
|
public static int getValueTypeSize(Type t) {
|
||||||
{
|
if (t == BOOL || t == BYTE || t == SBYTE)
|
||||||
if (t == BOOL || t == BYTE || t == SBYTE)
|
return 1;
|
||||||
return 1;
|
else if (t == SHORT || t == CHAR)
|
||||||
else if (t == SHORT || t == CHAR)
|
return 2;
|
||||||
return 2;
|
else if (t == INT || t == UINT || t == SINGLE)
|
||||||
else if (t == INT || t == UINT || t == SINGLE)
|
return 4;
|
||||||
return 4;
|
else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
|
||||||
else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
|
return 8;
|
||||||
return 8;
|
else if (t == DECIMAL)
|
||||||
else if (t == DECIMAL)
|
return 16;
|
||||||
return 16;
|
else
|
||||||
else
|
throw new ApplicationException("Unsupported Data Type");
|
||||||
throw new ApplicationException("Unsupported Data Type");
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
#nullable enable
|
||||||
|
namespace Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class BlueRowCollectionEqualityComparer : IEqualityComparer<IBlueRowCollection?> {
|
||||||
|
readonly IEqualityComparer<IBlueRow> rowComparer = BlueRowEqualityComparer.Instance;
|
||||||
|
|
||||||
|
public static BlueRowCollectionEqualityComparer Instance { get; } = new();
|
||||||
|
|
||||||
|
public bool Equals(IBlueRowCollection? x, IBlueRowCollection? y) {
|
||||||
|
if (x is null) {
|
||||||
|
return y is null;
|
||||||
|
}
|
||||||
|
if (x.Count != y?.Count) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return x.schema.columnInfo.SequenceEqual(y.schema.columnInfo)
|
||||||
|
&& x.Cast<IBlueRow>().OrderBy(r => r.Id()).SequenceEqual(
|
||||||
|
y.Cast<IBlueRow>().OrderBy(r => r.Id()),
|
||||||
|
rowComparer);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int GetHashCode([DisallowNull] IBlueRowCollection rows) {
|
||||||
|
var hash = new HashCode();
|
||||||
|
hash.Add(rows.schema.columnInfo.Length);
|
||||||
|
hash.Add(rows.Count);
|
||||||
|
return hash.ToHashCode();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
namespace Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class BlueRowEqualityComparer : IEqualityComparer<IBlueRow> {
|
||||||
|
public static BlueRowEqualityComparer Instance { get; } = new();
|
||||||
|
|
||||||
|
public bool Equals(IBlueRow x, IBlueRow y) =>
|
||||||
|
x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) &&
|
||||||
|
x.Values.Cast<object>().SequenceEqual(y.Values.Cast<object>());
|
||||||
|
|
||||||
|
public int GetHashCode([DisallowNull] IBlueRow obj) {
|
||||||
|
var hash = new HashCode();
|
||||||
|
foreach (var v in obj.Values) {
|
||||||
|
hash.Add(v?.GetHashCode() ?? 0);
|
||||||
|
}
|
||||||
|
return hash.ToHashCode();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,116 +1,117 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections.Generic;
|
||||||
using System.Linq;
|
// using System.Linq;
|
||||||
using System.Text;
|
// using System.Text;
|
||||||
|
|
||||||
/*
|
//TODORemove
|
||||||
* This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns.
|
// /*
|
||||||
* if groupNo is zero, then the group is not stord in a lookup table at all.
|
// * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns.
|
||||||
*
|
// * if groupNo is zero, then the group is not stord in a lookup table at all.
|
||||||
*
|
// *
|
||||||
*
|
// *
|
||||||
*/
|
// *
|
||||||
namespace BCG
|
// */
|
||||||
{
|
// namespace BCG
|
||||||
public class BlueRowLookupGroup
|
// {
|
||||||
{
|
// public class BlueRowLookupGroup
|
||||||
BlueTableSchema myEntrySchema;
|
// {
|
||||||
public IBlueTableSchema entrySchema
|
// BlueTableSchema myEntrySchema;
|
||||||
{
|
// public IBlueTableSchema entrySchema
|
||||||
get { return myEntrySchema; }
|
// {
|
||||||
//cant' set this value, have to build it from the row's schema
|
// get { return myEntrySchema; }
|
||||||
|
// //cant' set this value, have to build it from the row's schema
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
//for faster access and such
|
// //for faster access and such
|
||||||
private int[] myGroupColNos;
|
// private int[] myGroupColNos;
|
||||||
public int[] groupColNos
|
// public int[] groupColNos
|
||||||
{
|
// {
|
||||||
get { return myGroupColNos; }
|
// get { return myGroupColNos; }
|
||||||
|
|
||||||
}
|
// }
|
||||||
private string[] myGroupColNames;
|
// private string[] myGroupColNames;
|
||||||
public string[] groupColNames
|
// public string[] groupColNames
|
||||||
{
|
// {
|
||||||
get { return myGroupColNames; }
|
// get { return myGroupColNames; }
|
||||||
}
|
// }
|
||||||
|
|
||||||
private short myGroupNumber;
|
// private short myGroupNumber;
|
||||||
public short groupNumber
|
// public short groupNumber
|
||||||
{
|
// {
|
||||||
get { return myGroupNumber; }
|
// get { return myGroupNumber; }
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
/* how many bits are the ids into the table?
|
|
||||||
* In c# everything is stored as integers, but when it goes across the wire
|
|
||||||
* the data can be represented as a byte, a short, or an int
|
|
||||||
*/
|
|
||||||
private short myTableSize;
|
|
||||||
public short tableSize
|
|
||||||
{
|
|
||||||
get { return myTableSize; }
|
|
||||||
|
|
||||||
}
|
// /* how many bits are the ids into the table?
|
||||||
|
// * In c# everything is stored as integers, but when it goes across the wire
|
||||||
|
// * the data can be represented as a byte, a short, or an int
|
||||||
|
// */
|
||||||
|
// private short myTableSize;
|
||||||
|
// public short tableSize
|
||||||
|
// {
|
||||||
|
// get { return myTableSize; }
|
||||||
|
|
||||||
//if the group number is 0, the group is not stored in tables for serialization
|
// }
|
||||||
public bool isStoredInTable
|
|
||||||
{
|
|
||||||
get { return myGroupNumber != 0; }
|
|
||||||
|
|
||||||
}
|
// //if the group number is 0, the group is not stored in tables for serialization
|
||||||
public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
|
// public bool isStoredInTable
|
||||||
{
|
// {
|
||||||
|
// get { return myGroupNumber != 0; }
|
||||||
|
|
||||||
myGroupNumber = num;
|
// }
|
||||||
myTableSize = sizeInBits;
|
// public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
|
||||||
myGroupColNames = cols ;
|
// {
|
||||||
}
|
|
||||||
|
|
||||||
/*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
|
// myGroupNumber = num;
|
||||||
each of these entries in the lookup table. This function should be called by the LookupTableSchema
|
// myTableSize = sizeInBits;
|
||||||
during its constructor.
|
// myGroupColNames = cols ;
|
||||||
|
// }
|
||||||
This function also caches the column numbers for this group, for quicker access*/
|
|
||||||
public void buildEntrySchema(LookupTableSchema theSchema)
|
|
||||||
{
|
|
||||||
ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
|
|
||||||
myGroupColNos = new int[myGroupColNames.Length];
|
|
||||||
//the first column in our table is always the ID
|
|
||||||
int idColumn = 0;
|
|
||||||
columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
|
|
||||||
//go through each of the columns
|
|
||||||
for (int col = 1; col < columns.Length; ++col)
|
|
||||||
{
|
|
||||||
ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
|
|
||||||
if (columnInInputTable.isConstant)
|
|
||||||
columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
|
|
||||||
else
|
|
||||||
columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
|
|
||||||
|
|
||||||
myGroupColNos[col - 1] = columnInInputTable.colNo;
|
// /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
|
||||||
}
|
// each of these entries in the lookup table. This function should be called by the LookupTableSchema
|
||||||
|
// during its constructor.
|
||||||
|
|
||||||
|
// This function also caches the column numbers for this group, for quicker access*/
|
||||||
|
// public void buildEntrySchema(LookupTableSchema theSchema)
|
||||||
|
// {
|
||||||
|
// ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
|
||||||
|
// myGroupColNos = new int[myGroupColNames.Length];
|
||||||
|
// //the first column in our table is always the ID
|
||||||
|
// int idColumn = 0;
|
||||||
|
// columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
|
||||||
|
// //go through each of the columns
|
||||||
|
// for (int col = 1; col < columns.Length; ++col)
|
||||||
|
// {
|
||||||
|
// ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
|
||||||
|
// if (columnInInputTable.isConstant)
|
||||||
|
// columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
|
||||||
|
// else
|
||||||
|
// columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
|
||||||
|
|
||||||
myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
|
// myGroupColNos[col - 1] = columnInInputTable.colNo;
|
||||||
|
// }
|
||||||
}
|
|
||||||
|
|
||||||
/*takes a row as input, and extracts the members of the group from the row */
|
|
||||||
public object[] extractGroupFromRow(IBlueRow input)
|
|
||||||
{
|
|
||||||
int numCols = myGroupColNames.Length;
|
|
||||||
object[] results = new object[numCols];
|
|
||||||
|
|
||||||
for (int col = 0; col < numCols; ++col)
|
|
||||||
results[col] = input[myGroupColNos[col]];
|
|
||||||
|
|
||||||
return results;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
// myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
|
||||||
}
|
|
||||||
|
// }
|
||||||
|
|
||||||
|
// /*takes a row as input, and extracts the members of the group from the row */
|
||||||
|
// public object[] extractGroupFromRow(IBlueRow input)
|
||||||
|
// {
|
||||||
|
// int numCols = myGroupColNames.Length;
|
||||||
|
// object[] results = new object[numCols];
|
||||||
|
|
||||||
|
// for (int col = 0; col < numCols; ++col)
|
||||||
|
// results[col] = input[myGroupColNos[col]];
|
||||||
|
|
||||||
|
// return results;
|
||||||
|
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,8 +1,15 @@
|
|||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.DataStream;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Data;
|
using System.Data;
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
|
||||||
//using BCG.Utilities;
|
//using BCG.Utilities;
|
||||||
|
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
@@ -10,7 +17,7 @@ namespace BCG {
|
|||||||
/// Anologue of the dotNet DataTable.
|
/// Anologue of the dotNet DataTable.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class BlueTable : IBlueRowCollection {
|
public class BlueTable : IBlueRowCollection, IDictionary<int, IBlueRow> {
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Hashtable indexed on "id" to speed up removes
|
/// Hashtable indexed on "id" to speed up removes
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -86,6 +93,12 @@ namespace BCG {
|
|||||||
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||||
return (IBlueRow)row;
|
return (IBlueRow)row;
|
||||||
}
|
}
|
||||||
|
set {
|
||||||
|
if ((int)value["id"] != id) {
|
||||||
|
throw new ArgumentException("id arg not equal to row's actual id", nameof(id));
|
||||||
|
}
|
||||||
|
changeRow(value);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -202,6 +215,8 @@ namespace BCG {
|
|||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
|
public IDictionary<int, IBlueRow> AsDictionary() => this;
|
||||||
|
|
||||||
#region ICollection members
|
#region ICollection members
|
||||||
public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
|
public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
|
||||||
{ return myIdIndex.Values.GetEnumerator(); }
|
{ return myIdIndex.Values.GetEnumerator(); }
|
||||||
@@ -214,5 +229,55 @@ namespace BCG {
|
|||||||
|
|
||||||
public virtual int Count { get { return myIdIndex.Count; } }
|
public virtual int Count { get { return myIdIndex.Count; } }
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
|
#region IDictionary<int, IBlueRow> members
|
||||||
|
public void Add(int key, IBlueRow value) {
|
||||||
|
throw new NotImplementedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool IDictionary<int, IBlueRow>.ContainsKey(int key) =>
|
||||||
|
containsID(key);
|
||||||
|
|
||||||
|
bool IDictionary<int, IBlueRow>.Remove(int key) =>
|
||||||
|
deleteRow(key);
|
||||||
|
|
||||||
|
bool IDictionary<int, IBlueRow>.TryGetValue(
|
||||||
|
int key, [MaybeNullWhen(false)] out IBlueRow value) =>
|
||||||
|
tryGetValue(key, out value);
|
||||||
|
|
||||||
|
void ICollection<KeyValuePair<int, IBlueRow>>.Add(KeyValuePair<int, IBlueRow> item) =>
|
||||||
|
addRow(item.Value);
|
||||||
|
|
||||||
|
void ICollection<KeyValuePair<int, IBlueRow>>.Clear() =>
|
||||||
|
clear();
|
||||||
|
|
||||||
|
bool ICollection<KeyValuePair<int, IBlueRow>>.Contains(KeyValuePair<int, IBlueRow> item) =>
|
||||||
|
tryGetValue(item.Key, out var r)
|
||||||
|
&& BlueRowEqualityComparer.Instance.Equals(r, item.Value);
|
||||||
|
|
||||||
|
void ICollection<KeyValuePair<int, IBlueRow>>.CopyTo(
|
||||||
|
KeyValuePair<int, IBlueRow>[] array, int arrayIndex) {
|
||||||
|
foreach (DictionaryEntry de in myIdIndex) {
|
||||||
|
array[arrayIndex++] = new((int)de.Key, (IBlueRow)de.Value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ICollection<KeyValuePair<int, IBlueRow>>.Remove(KeyValuePair<int, IBlueRow> item) =>
|
||||||
|
deleteRow(item.Key);
|
||||||
|
|
||||||
|
IEnumerator<KeyValuePair<int, IBlueRow>> IEnumerable<KeyValuePair<int, IBlueRow>>.GetEnumerator() {
|
||||||
|
foreach (DictionaryEntry de in myIdIndex) {
|
||||||
|
yield return new((int)de.Key, (IBlueRow)de.Value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ICollection<int> IDictionary<int, IBlueRow>.Keys =>
|
||||||
|
GenericCollectionWrapper.Create<int>(myIdIndex.Keys);
|
||||||
|
|
||||||
|
ICollection<IBlueRow> IDictionary<int, IBlueRow>.Values =>
|
||||||
|
GenericCollectionWrapper.Create<IBlueRow>(myIdIndex.Values);
|
||||||
|
|
||||||
|
bool ICollection<KeyValuePair<int, IBlueRow>>.IsReadOnly => false;
|
||||||
|
#endregion
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
#nullable enable
|
||||||
|
namespace Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.Immutable;
|
||||||
|
|
||||||
|
// I would have named this ImmutableBlueTable, but that was already taken.
|
||||||
|
public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
|
||||||
|
readonly ImmutableDictionary<int, IBlueRow> rows;
|
||||||
|
|
||||||
|
public BlueTableImmutable(IBlueTableSchema schema)
|
||||||
|
: this(schema, ImmutableDictionary<int, IBlueRow>.Empty) { }
|
||||||
|
|
||||||
|
public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, IBlueRow> rows) {
|
||||||
|
this.schema = schema;
|
||||||
|
this.rows = rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Count => rows.Count;
|
||||||
|
public bool IsSynchronized => false;
|
||||||
|
public IBlueTableSchema schema { get; }
|
||||||
|
public object SyncRoot => this;
|
||||||
|
|
||||||
|
public IBlueRow this[int id] => rows[id];
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
rows.ContainsKey(id);
|
||||||
|
|
||||||
|
public void CopyTo(Array array, int index) {
|
||||||
|
foreach (var r in rows.Values) {
|
||||||
|
array.SetValue(r, index++);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IEnumerator GetEnumerator() =>
|
||||||
|
rows.Values.GetEnumerator();
|
||||||
|
|
||||||
|
public DataPacketBuilder ToDataPacketBuilder() =>
|
||||||
|
new(this);
|
||||||
|
|
||||||
|
public class DataPacketBuilder {
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
readonly ImmutableDictionary<int, IBlueRow>.Builder table;
|
||||||
|
readonly ImmutableDictionary<int, IBlueRow>.Builder added =
|
||||||
|
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
|
||||||
|
readonly ImmutableDictionary<int, IBlueRow>.Builder removed =
|
||||||
|
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
|
||||||
|
|
||||||
|
public DataPacketBuilder(BlueTableImmutable table) {
|
||||||
|
schema = table.schema;
|
||||||
|
this.table = table.rows.ToBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void AddOrUpdate(IBlueRow row, IEqualityComparer<IBlueRow>? comparer = null) {
|
||||||
|
var id = row.Id();
|
||||||
|
if (table.TryGetValue(id, out var orig)) {
|
||||||
|
if (comparer?.Equals(row, orig) ?? false) {
|
||||||
|
// No op: This row was already in the table (according to 'comparer').
|
||||||
|
// Remove it from 'added' and 'removed'.
|
||||||
|
added.Remove(id);
|
||||||
|
removed.Remove(id);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Change op: A row with this id was already in the table,
|
||||||
|
// but its values have changed (according to 'comparer').
|
||||||
|
removed[id] = orig;
|
||||||
|
}
|
||||||
|
added[id] = row;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Remove(int id) {
|
||||||
|
added.Remove(id);
|
||||||
|
if (table.TryGetValue(id, out var orig)) {
|
||||||
|
removed[id] = orig;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataPacket ToDataPacket(bool reset = true) {
|
||||||
|
table.RemoveRange(removed.Keys);
|
||||||
|
table.AddRange(added);
|
||||||
|
var newRows = new BlueTableImmutable(schema, table.ToImmutable());
|
||||||
|
var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
|
||||||
|
var removedRows = new BlueTableImmutable(schema, removed.ToImmutable());
|
||||||
|
if (reset) {
|
||||||
|
added.Clear();
|
||||||
|
removed.Clear();
|
||||||
|
}
|
||||||
|
return new(newRows, addedRows, removedRows);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BlueTableImmutable ToImmutable() =>
|
||||||
|
new(schema, table.ToImmutable());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,8 +8,7 @@ using System.Linq;
|
|||||||
|
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
|
/// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
|
||||||
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
|
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
|
||||||
@@ -17,112 +16,105 @@ namespace BCG
|
|||||||
/// the respective data, sorted according to the ColumnOrder.
|
/// the respective data, sorted according to the ColumnOrder.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class BlueTableSchema : IBlueTableSchema
|
public class BlueTableSchema : IBlueTableSchema {
|
||||||
{
|
public class BlueTableSchemaSerializationHelper {
|
||||||
public class BlueTableSchemaSerializationHelper
|
// public static object parse(Type t, string data) {
|
||||||
{
|
// if (t == BlueDataValidTypes.BOOL)
|
||||||
public static object parse(Type t, string data)
|
// return Boolean.Parse(data);
|
||||||
{
|
// else if (t == BlueDataValidTypes.BYTE)
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
// return Byte.Parse(data);
|
||||||
return Boolean.Parse(data);
|
// else if (t == BlueDataValidTypes.CHAR)
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
// return char.Parse(data);
|
||||||
return Byte.Parse(data);
|
// else if (t == BlueDataValidTypes.DATETIME)
|
||||||
else if (t == BlueDataValidTypes.CHAR)
|
// return DateTime.Parse(data);
|
||||||
return char.Parse(data);
|
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||||
else if (t == BlueDataValidTypes.DATETIME)
|
// return Double.Parse(data);
|
||||||
return DateTime.Parse(data);
|
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||||
else if (t == BlueDataValidTypes.DOUBLE)
|
// return Decimal.Parse(data);
|
||||||
return Double.Parse(data);
|
// else if (t == BlueDataValidTypes.INT)
|
||||||
else if (t == BlueDataValidTypes.DECIMAL)
|
// return int.Parse(data);
|
||||||
return Decimal.Parse(data);
|
// else if (t == BlueDataValidTypes.LONG)
|
||||||
else if (t == BlueDataValidTypes.INT)
|
// return long.Parse(data);
|
||||||
return int.Parse(data);
|
// else if (t == BlueDataValidTypes.SBYTE)
|
||||||
else if (t == BlueDataValidTypes.LONG)
|
// return SByte.Parse(data);
|
||||||
return long.Parse(data);
|
// else if (t == BlueDataValidTypes.STRING)
|
||||||
else if (t == BlueDataValidTypes.SBYTE)
|
// return data;
|
||||||
return SByte.Parse(data);
|
// else if (t == BlueDataValidTypes.UINT)
|
||||||
else if (t == BlueDataValidTypes.STRING)
|
// return uint.Parse(data);
|
||||||
return data;
|
// else if (t == BlueDataValidTypes.ULONG)
|
||||||
else if (t == BlueDataValidTypes.UINT)
|
// return ulong.Parse(data);
|
||||||
return uint.Parse(data);
|
// else if (t == BlueDataValidTypes.SINGLE)
|
||||||
else if (t == BlueDataValidTypes.ULONG)
|
// return Single.Parse(data);
|
||||||
return ulong.Parse(data);
|
// else
|
||||||
else if (t == BlueDataValidTypes.SINGLE)
|
// throw new ApplicationException("Can not be parsed");
|
||||||
return Single.Parse(data);
|
// }
|
||||||
else
|
// public static void serialize(BinaryWriter bw, Type t, object o) {
|
||||||
throw new ApplicationException("Can not be parsed");
|
// if (t == BlueDataValidTypes.BOOL)
|
||||||
}
|
// bw.Write((bool)o);
|
||||||
public static void serialize(BinaryWriter bw, Type t, object o)
|
// else if (t == BlueDataValidTypes.BYTE)
|
||||||
{
|
// bw.Write((byte)o);
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
// else if (t == BlueDataValidTypes.CHAR)
|
||||||
bw.Write((bool)o);
|
// bw.Write((char)o);
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
// else if (t == BlueDataValidTypes.DATETIME)
|
||||||
bw.Write((byte)o);
|
// bw.Write(((DateTime)o).Ticks);
|
||||||
else if (t == BlueDataValidTypes.CHAR)
|
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||||
bw.Write((char)o);
|
// bw.Write((double)o);
|
||||||
else if (t == BlueDataValidTypes.DATETIME)
|
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||||
bw.Write(((DateTime)o).Ticks);
|
// bw.Write((decimal)o);
|
||||||
else if (t == BlueDataValidTypes.DOUBLE)
|
// else if (t == BlueDataValidTypes.INT)
|
||||||
bw.Write((double)o);
|
// bw.Write((int)o);
|
||||||
else if (t == BlueDataValidTypes.DECIMAL)
|
// else if (t == BlueDataValidTypes.LONG)
|
||||||
bw.Write((decimal)o);
|
// bw.Write((long)o);
|
||||||
else if (t == BlueDataValidTypes.INT)
|
// else if (t == BlueDataValidTypes.SBYTE)
|
||||||
bw.Write((int)o);
|
// bw.Write((sbyte)o);
|
||||||
else if (t == BlueDataValidTypes.LONG)
|
// else if (t == BlueDataValidTypes.STRING)
|
||||||
bw.Write((long)o);
|
// bw.Write(o as string);
|
||||||
else if (t == BlueDataValidTypes.SBYTE)
|
// else if (t == BlueDataValidTypes.SINGLE)
|
||||||
bw.Write((sbyte)o);
|
// bw.Write((Single)o);
|
||||||
else if (t == BlueDataValidTypes.STRING)
|
// else if (t == BlueDataValidTypes.BYTEARRAY) {
|
||||||
bw.Write(o as string);
|
// byte[] bs = o as byte[];
|
||||||
else if (t == BlueDataValidTypes.SINGLE)
|
// bw.Write(bs.Length);
|
||||||
bw.Write((Single)o);
|
// bw.Write(bs);
|
||||||
else if (t == BlueDataValidTypes.BYTEARRAY)
|
// } else if (t == BlueDataValidTypes.UINT)
|
||||||
{
|
// bw.Write((uint)o);
|
||||||
byte[] bs = o as byte[];
|
// else if (t == BlueDataValidTypes.ULONG)
|
||||||
bw.Write(bs.Length);
|
// bw.Write((ulong)o);
|
||||||
bw.Write(bs);
|
// else
|
||||||
}
|
// throw new ApplicationException("Unsupported type");
|
||||||
else if (t == BlueDataValidTypes.UINT)
|
// }
|
||||||
bw.Write((uint)o);
|
|
||||||
else if (t == BlueDataValidTypes.ULONG)
|
|
||||||
bw.Write((ulong)o);
|
|
||||||
else
|
|
||||||
throw new ApplicationException("Unsupported type");
|
|
||||||
}
|
|
||||||
|
|
||||||
public static object deserialize(BinaryReader br, Type t)
|
// public static object deserialize(BinaryReader br, Type t) {
|
||||||
{
|
// if (t == BlueDataValidTypes.BOOL)
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
// return br.ReadBoolean();
|
||||||
return br.ReadBoolean();
|
// else if (t == BlueDataValidTypes.BYTE)
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
// return br.ReadByte();
|
||||||
return br.ReadByte();
|
// else if (t == BlueDataValidTypes.CHAR)
|
||||||
else if (t == BlueDataValidTypes.CHAR)
|
// return br.ReadChar();
|
||||||
return br.ReadChar();
|
// else if (t == BlueDataValidTypes.DATETIME)
|
||||||
else if (t == BlueDataValidTypes.DATETIME)
|
// return new DateTime(br.ReadInt64());
|
||||||
return new DateTime(br.ReadInt64());
|
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||||
else if (t == BlueDataValidTypes.DOUBLE)
|
// return br.ReadDouble();
|
||||||
return br.ReadDouble();
|
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||||
else if (t == BlueDataValidTypes.DECIMAL)
|
// return br.ReadDecimal();
|
||||||
return br.ReadDecimal();
|
// else if (t == BlueDataValidTypes.INT)
|
||||||
else if (t == BlueDataValidTypes.INT)
|
// return br.ReadInt32();
|
||||||
return br.ReadInt32();
|
// else if (t == BlueDataValidTypes.LONG)
|
||||||
else if (t == BlueDataValidTypes.LONG)
|
// return br.ReadInt64();
|
||||||
return br.ReadInt64();
|
// else if (t == BlueDataValidTypes.SBYTE)
|
||||||
else if (t == BlueDataValidTypes.SBYTE)
|
// return br.ReadSByte();
|
||||||
return br.ReadSByte();
|
// else if (t == BlueDataValidTypes.STRING)
|
||||||
else if (t == BlueDataValidTypes.STRING)
|
// return br.ReadString();
|
||||||
return br.ReadString();
|
// else if (t == BlueDataValidTypes.BYTEARRAY)
|
||||||
else if (t == BlueDataValidTypes.BYTEARRAY)
|
// return br.ReadBytes(br.ReadInt32());
|
||||||
return br.ReadBytes(br.ReadInt32());
|
// else if (t == BlueDataValidTypes.UINT)
|
||||||
else if (t == BlueDataValidTypes.UINT)
|
// return br.ReadUInt32();
|
||||||
return br.ReadUInt32();
|
// else if (t == BlueDataValidTypes.ULONG)
|
||||||
else if (t == BlueDataValidTypes.ULONG)
|
// return br.ReadUInt64();
|
||||||
return br.ReadUInt64();
|
// else if (t == BlueDataValidTypes.SINGLE)
|
||||||
else if (t == BlueDataValidTypes.SINGLE)
|
// return br.ReadSingle();
|
||||||
return br.ReadSingle();
|
// else
|
||||||
else
|
// throw new ApplicationException("Unsupported type");
|
||||||
throw new ApplicationException("Unsupported type");
|
// }
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected Hashtable myIndexByColName = new Hashtable();
|
protected Hashtable myIndexByColName = new Hashtable();
|
||||||
@@ -132,29 +124,24 @@ namespace BCG
|
|||||||
|
|
||||||
public BlueTableSchema() { }
|
public BlueTableSchema() { }
|
||||||
|
|
||||||
public BlueTableSchema(DataTable table)
|
public BlueTableSchema(DataTable table) {
|
||||||
{
|
|
||||||
myIndexByColOrder = new ColumnInfo[table.Columns.Count];
|
myIndexByColOrder = new ColumnInfo[table.Columns.Count];
|
||||||
myTableName = table.TableName;
|
myTableName = table.TableName;
|
||||||
foreach (DataColumn col in table.Columns)
|
foreach (DataColumn col in table.Columns) {
|
||||||
{
|
|
||||||
ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
|
ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
|
||||||
myIndexByColName[ci.colName] = ci;
|
myIndexByColName[ci.colName] = ci;
|
||||||
myIndexByColOrder[ci.colNo] = ci;
|
myIndexByColOrder[ci.colNo] = ci;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public BlueTableSchema(ColumnInfo[] cis, string tableName)
|
public BlueTableSchema(ColumnInfo[] cis, string tableName) {
|
||||||
{
|
|
||||||
myConstruct(cis, tableName);
|
myConstruct(cis, tableName);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myConstruct(ColumnInfo[] cis, string tableName)
|
private void myConstruct(ColumnInfo[] cis, string tableName) {
|
||||||
{
|
|
||||||
myIndexByColOrder = new ColumnInfo[cis.Length];
|
myIndexByColOrder = new ColumnInfo[cis.Length];
|
||||||
myTableName = tableName;
|
myTableName = tableName;
|
||||||
foreach (ColumnInfo ci in cis)
|
foreach (ColumnInfo ci in cis) {
|
||||||
{
|
|
||||||
myIndexByColOrder[ci.colNo] = ci;
|
myIndexByColOrder[ci.colNo] = ci;
|
||||||
myIndexByColName[ci.colName] = ci;
|
myIndexByColName[ci.colName] = ci;
|
||||||
}
|
}
|
||||||
@@ -169,15 +156,13 @@ namespace BCG
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="tableName">the table name</param>
|
/// <param name="tableName">the table name</param>
|
||||||
/// <param name="fromType">the plain old data type whose fields will be mirrored in the schema</param>
|
/// <param name="fromType">the plain old data type whose fields will be mirrored in the schema</param>
|
||||||
public BlueTableSchema(string tableName, Type fromType)
|
public BlueTableSchema(string tableName, Type fromType) {
|
||||||
{
|
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
int i = 0;
|
int i = 0;
|
||||||
// we sort the field name so that we have a deterministic order given
|
// we sort the field name so that we have a deterministic order given
|
||||||
// the same type.
|
// the same type.
|
||||||
bool foundId = false;
|
bool foundId = false;
|
||||||
foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; }))
|
foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; })) {
|
||||||
{
|
|
||||||
if (fi.Name.ToLower().Equals("id"))
|
if (fi.Name.ToLower().Equals("id"))
|
||||||
foundId = true;
|
foundId = true;
|
||||||
cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType));
|
cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType));
|
||||||
@@ -188,32 +173,25 @@ namespace BCG
|
|||||||
myConstruct(cis.ToArray(), tableName);
|
myConstruct(cis.ToArray(), tableName);
|
||||||
}
|
}
|
||||||
|
|
||||||
public ColumnInfo this[string colName]
|
public ColumnInfo this[string colName] {
|
||||||
{
|
get {
|
||||||
get
|
|
||||||
{
|
|
||||||
object retVal = myIndexByColName[colName];
|
object retVal = myIndexByColName[colName];
|
||||||
return retVal != null ? retVal as ColumnInfo : null;
|
return retVal != null ? retVal as ColumnInfo : null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public ColumnInfo this[int colNo]
|
public ColumnInfo this[int colNo] {
|
||||||
{
|
|
||||||
get { return myIndexByColOrder[colNo]; }
|
get { return myIndexByColOrder[colNo]; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The returned Array is sorted according to the Column No
|
/// The returned Array is sorted according to the Column No
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string[] columnNames
|
public string[] columnNames {
|
||||||
{
|
get {
|
||||||
get
|
if (myTableColNames == null) {
|
||||||
{
|
|
||||||
if (myTableColNames == null)
|
|
||||||
{
|
|
||||||
myTableColNames = new string[myIndexByColName.Count];
|
myTableColNames = new string[myIndexByColName.Count];
|
||||||
foreach (DictionaryEntry de in myIndexByColName)
|
foreach (DictionaryEntry de in myIndexByColName) {
|
||||||
{
|
|
||||||
ColumnInfo temp = (ColumnInfo)de.Value;
|
ColumnInfo temp = (ColumnInfo)de.Value;
|
||||||
myTableColNames[temp.colNo] = temp.colName;
|
myTableColNames[temp.colNo] = temp.colName;
|
||||||
}
|
}
|
||||||
@@ -225,8 +203,7 @@ namespace BCG
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// The returned Array is sorted according to the Column No
|
/// The returned Array is sorted according to the Column No
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ColumnInfo[] columnInfo
|
public ColumnInfo[] columnInfo {
|
||||||
{
|
|
||||||
get { return myIndexByColOrder; }
|
get { return myIndexByColOrder; }
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -234,49 +211,30 @@ namespace BCG
|
|||||||
|
|
||||||
|
|
||||||
#region ISerializedType methods
|
#region ISerializedType methods
|
||||||
public void serialize(BinaryWriter bw)
|
public void serialize(BinaryWriter bw) {
|
||||||
{
|
|
||||||
bw.Write(columnNames.Length);
|
bw.Write(columnNames.Length);
|
||||||
bw.Write(tableName);
|
bw.Write(tableName);
|
||||||
foreach (ColumnInfo ci in columnInfo)
|
foreach (ColumnInfo ci in columnInfo) {
|
||||||
{
|
|
||||||
bw.Write(ci.colName);
|
bw.Write(ci.colName);
|
||||||
bw.Write(ci.colType.ToString());
|
bw.Write(ci.colType.ToString());
|
||||||
if (ci.isConstant)
|
|
||||||
{
|
|
||||||
bw.Write(true);
|
|
||||||
BlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
{
|
||||||
bw.Write(false);
|
bw.Write(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deserialze(BinaryReader br)
|
public void deserialze(BinaryReader br) {
|
||||||
{
|
|
||||||
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
|
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
|
||||||
string name = br.ReadString();
|
string name = br.ReadString();
|
||||||
for (int i = 0; i < cis.Length; i++)
|
for (int i = 0; i < cis.Length; i++) {
|
||||||
{
|
|
||||||
string colName = br.ReadString();
|
string colName = br.ReadString();
|
||||||
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
|
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
|
||||||
bool isConstant = br.ReadBoolean();
|
cis[i] = new ColumnInfo(colName, i, t);
|
||||||
if (isConstant)
|
|
||||||
{
|
|
||||||
cis[i] = new ColumnInfo(colName, i, t, BlueTableSchemaSerializationHelper.deserialize(br, t));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
cis[i] = new ColumnInfo(colName, i, t);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
myConstruct(cis, name);
|
myConstruct(cis, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
public int getSerializedSize()
|
public int getSerializedSize() {
|
||||||
{
|
|
||||||
throw new NotImplementedException("Feature for provided constant values to" +
|
throw new NotImplementedException("Feature for provided constant values to" +
|
||||||
"non-primitive types in schema, not provided as of now");
|
"non-primitive types in schema, not provided as of now");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
#nullable enable
|
||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class BlueTableSchemaAdaptor<T> : IMutableBlueRowFactory<T> {
|
||||||
|
readonly string name;
|
||||||
|
readonly List<ColumnAdaptor> columns = new();
|
||||||
|
BlueTableSchema? schema;
|
||||||
|
|
||||||
|
public BlueTableSchemaAdaptor(string name) =>
|
||||||
|
this.name = name;
|
||||||
|
|
||||||
|
public IBlueTableSchema Schema =>
|
||||||
|
schema ??= new BlueTableSchema(columns.WrapSelectorRO(c => c.Info).ToArray(), name);
|
||||||
|
|
||||||
|
public bool Contains(string name) =>
|
||||||
|
columns.Any(c => c.Info.colName == name);
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, int> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r), typeof(int));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, int?> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, long> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r), typeof(long));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, long?> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, double> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r), typeof(double));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, double?> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, DateTime> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r), typeof(DateTime));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, DateTime?> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, string?> getValue) =>
|
||||||
|
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string));
|
||||||
|
|
||||||
|
public void AddColumn(string name, Func<T, object?> getValue, Type dataType) {
|
||||||
|
if (!ColumnInfo.isAllowedType(dataType)) {
|
||||||
|
throw new ArgumentException(
|
||||||
|
$"'{dataType}' is not a supported column type", nameof(dataType));
|
||||||
|
}
|
||||||
|
// Invalidate cached schema (set it to null)
|
||||||
|
schema = null;
|
||||||
|
columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetBlueRow(T item) =>
|
||||||
|
GetMutableBlueRow(item);
|
||||||
|
|
||||||
|
public MutableBlueRow GetMutableBlueRow(T item) {
|
||||||
|
var mbr = new MutableBlueRow(Schema);
|
||||||
|
foreach (var col in columns) {
|
||||||
|
mbr.setValue(col.Info.colNo, col.GetValue(item));
|
||||||
|
}
|
||||||
|
return mbr;
|
||||||
|
}
|
||||||
|
|
||||||
|
class ColumnAdaptor {
|
||||||
|
readonly Func<T, object?> getValue;
|
||||||
|
|
||||||
|
public ColumnAdaptor(ColumnInfo info, Func<T, object?> getValue) {
|
||||||
|
Info = info;
|
||||||
|
this.getValue = getValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ColumnInfo Info { get; }
|
||||||
|
|
||||||
|
public object? GetValue(T row) =>
|
||||||
|
getValue(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Data;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
namespace bts {
|
||||||
|
public class CircularBufferTable : IBlueRowCollection {
|
||||||
|
private BlueTable _table;
|
||||||
|
private List<IBlueRow> _ordered_list = new List<IBlueRow>();
|
||||||
|
private int _buffer_sz;
|
||||||
|
public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) {
|
||||||
|
_table = new BlueTable(schema);
|
||||||
|
_buffer_sz = buffer_sz;
|
||||||
|
}
|
||||||
|
|
||||||
|
// returns the row that might need to be deleted to keep
|
||||||
|
// the circular buffer intact
|
||||||
|
public IBlueRow add_row(IBlueRow row) {
|
||||||
|
_ordered_list.Add(row);
|
||||||
|
_table.addRow(row);
|
||||||
|
if (_ordered_list.Count > _buffer_sz) {
|
||||||
|
IBlueRow rv = _ordered_list.ElementAt(0);
|
||||||
|
_ordered_list.RemoveAt(0);
|
||||||
|
_table.deleteRow(rv);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] {
|
||||||
|
get {
|
||||||
|
return _table[id];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public IBlueTableSchema schema {
|
||||||
|
get {
|
||||||
|
return _table.schema;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public bool containsID(int id) {
|
||||||
|
return _table.containsID(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); }
|
||||||
|
public int Count { get { return _table.Count; } }
|
||||||
|
public bool IsSynchronized { get { return false; } }
|
||||||
|
public object SyncRoot { get { return this; } }
|
||||||
|
public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,56 +1,32 @@
|
|||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
|
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class ColumnInfo : IComparable
|
public class ColumnInfo : IComparable {
|
||||||
{
|
|
||||||
public static bool isAllowedType (Type t)
|
|
||||||
{
|
|
||||||
return
|
|
||||||
t.Equals (typeof (System.Int32)) ||
|
|
||||||
t.Equals (typeof (System.Int64)) ||
|
|
||||||
t.Equals (typeof (System.Double)) ||
|
|
||||||
t.Equals (typeof (System.String)) ||
|
|
||||||
t.Equals (typeof (System.DateTime)) ? true :
|
|
||||||
false;
|
|
||||||
}
|
|
||||||
|
|
||||||
//the default column info does not cach values in a lookup table, therefore it is in group 0
|
//the default column info does not cach values in a lookup table, therefore it is in group 0
|
||||||
public ColumnInfo(string colName, int colNo, Type colType)
|
public ColumnInfo(string colName, int colNo, Type colType, object colValue = null) {
|
||||||
: this(colName, colNo, colType, 0)
|
|
||||||
{
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
|
|
||||||
{
|
|
||||||
myColName = colName;
|
myColName = colName;
|
||||||
myColNo = colNo;
|
myColNo = colNo;
|
||||||
myColType = colType;
|
myColType = colType;
|
||||||
myLookupTableGroup = colGroup;
|
myColValue = colValue;
|
||||||
myColValue = null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ColumnInfo(string col_name, int col_no, TypeCode type_code)
|
||||||
public ColumnInfo (string colName, int colNo, object colValue)
|
: this(col_name, col_no, BlueDataUtility.get_type(type_code)) {
|
||||||
: this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;}
|
|
||||||
|
|
||||||
public ColumnInfo (string colName, int colNo, Type colType, object colValue)
|
|
||||||
: this (colName, colNo, colType)
|
|
||||||
{
|
|
||||||
|
|
||||||
myColValue = colValue == null ? DBNull.Value : colValue;
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static bool isAllowedType(Type type) =>
|
||||||
|
type == typeof(int) ||
|
||||||
|
type == typeof(long) ||
|
||||||
|
type == typeof(double) ||
|
||||||
|
type == typeof(string) ||
|
||||||
|
type == typeof(DateTime);
|
||||||
|
|
||||||
public int CompareTo (object obj)
|
public int CompareTo(object obj) {
|
||||||
{
|
|
||||||
int otherColNo = ((ColumnInfo)(obj)).colNo;
|
int otherColNo = ((ColumnInfo)(obj)).colNo;
|
||||||
if (myColNo < otherColNo)
|
if (myColNo < otherColNo)
|
||||||
return -1;
|
return -1;
|
||||||
@@ -59,37 +35,30 @@ namespace BCG
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
//public bool isCalculated{get{return myExpression != null;}}
|
public override bool Equals(object obj) {
|
||||||
|
if (obj is ColumnInfo) {
|
||||||
|
ColumnInfo other = obj as ColumnInfo;
|
||||||
|
return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override string ToString() {
|
||||||
|
return colName + " " + colNo + " " + colType;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// for backwards compatibility to allow build
|
||||||
public bool isConstant {get{return myColValue != null;}}
|
public bool isConstant {get{return myColValue != null;}}
|
||||||
|
|
||||||
private int myColNo;
|
private int myColNo;
|
||||||
private string myColName;
|
private string myColName;
|
||||||
private Type myColType;
|
private Type myColType;
|
||||||
//private string myExpression;
|
|
||||||
private object myColValue;
|
private object myColValue;
|
||||||
|
|
||||||
//no table will ever have more than 65,536 columns
|
public int colNo { get { return myColNo; } }
|
||||||
//if it does, we are in serious trouble
|
public string colName { get { return myColName; } }
|
||||||
private short myLookupTableGroup;
|
public Type colType { get { return myColType; } }
|
||||||
|
|
||||||
public int colNo{get{return myColNo;}}
|
|
||||||
public string colName {get{return myColName;}}
|
|
||||||
public Type colType {get{return myColType;}}
|
|
||||||
|
|
||||||
|
|
||||||
//this value corresponds to how the data in the column is serialized.
|
|
||||||
//if the group number is 0, the values are not stored in a table
|
|
||||||
//if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
|
|
||||||
//are stored together in a lookup table
|
|
||||||
|
|
||||||
|
|
||||||
public short lookupTableGroup
|
|
||||||
{
|
|
||||||
get { return myLookupTableGroup; }
|
|
||||||
set { myLookupTableGroup = value; }
|
|
||||||
}
|
|
||||||
//public string expression {get{return myExpression;}}
|
|
||||||
public object colValue {get{return myColValue;}}
|
public object colValue {get{return myColValue;}}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
#nullable enable
|
||||||
|
namespace Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class DataPacketEqualityComparer : IEqualityComparer<DataPacket?> {
|
||||||
|
readonly IEqualityComparer<IBlueRowCollection> rowCollectionComparer;
|
||||||
|
|
||||||
|
public DataPacketEqualityComparer(IEqualityComparer<IBlueRowCollection>? rowCollectionComparer = null) =>
|
||||||
|
this.rowCollectionComparer = rowCollectionComparer ?? BlueRowCollectionEqualityComparer.Instance;
|
||||||
|
|
||||||
|
public static DataPacketEqualityComparer Instance { get; } = new();
|
||||||
|
|
||||||
|
public bool Equals(DataPacket? x, DataPacket? y) {
|
||||||
|
if (x is null) {
|
||||||
|
return y is null;
|
||||||
|
}
|
||||||
|
return y is not null
|
||||||
|
&& x.schema.columnInfo.SequenceEqual(y.schema.columnInfo)
|
||||||
|
&& rowCollectionComparer.Equals(x.addedRows, y.addedRows)
|
||||||
|
&& rowCollectionComparer.Equals(x.deletedRows, y.deletedRows)
|
||||||
|
&& rowCollectionComparer.Equals(x.newRows, y.newRows);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int GetHashCode([DisallowNull] DataPacket obj) {
|
||||||
|
var hash = new HashCode();
|
||||||
|
hash.Add(obj.schema.columnNames.Length);
|
||||||
|
hash.Add(obj.addedRows.Count);
|
||||||
|
hash.Add(obj.deletedRows.Count);
|
||||||
|
return hash.ToHashCode();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
#nullable enable
|
||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
public interface IBlueRowFactory<T> {
|
||||||
|
IBlueTableSchema Schema { get; }
|
||||||
|
IBlueRow GetBlueRow(T item);
|
||||||
|
}
|
||||||
@@ -1,252 +1,252 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections;
|
// using System.Collections;
|
||||||
//using BCG.Utilities;
|
// //using BCG.Utilities;
|
||||||
|
|
||||||
namespace BCG
|
// namespace BCG
|
||||||
{
|
// {
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// Provides a multiple indexing mechanism, required to increase the peformance of the streams
|
// /// Provides a multiple indexing mechanism, required to increase the peformance of the streams
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
public class BlueTableIndexException : Exception
|
// public class BlueTableIndexException : Exception
|
||||||
{
|
// {
|
||||||
public BlueTableIndexException(string msg) : base(msg) { }
|
// public BlueTableIndexException(string msg) : base(msg) { }
|
||||||
}
|
// }
|
||||||
|
|
||||||
public interface IBlueTableIndex
|
// public interface IBlueTableIndex
|
||||||
{
|
// {
|
||||||
void addRow(IBlueRow row);
|
// void addRow(IBlueRow row);
|
||||||
void deleteRowOnPrimaryKey(object pkValue);
|
// void deleteRowOnPrimaryKey(object pkValue);
|
||||||
void deleteRowOnPrimaryKey(IBlueRow row);
|
// void deleteRowOnPrimaryKey(IBlueRow row);
|
||||||
void deleteRowOnBothKeys(object pkValue, object skValue);
|
// void deleteRowOnBothKeys(object pkValue, object skValue);
|
||||||
void deleteRowOnBothKeys(IBlueRow row);
|
// void deleteRowOnBothKeys(IBlueRow row);
|
||||||
ArrayList searchRowOnPrimaryKey(object pkValue);
|
// ArrayList searchRowOnPrimaryKey(object pkValue);
|
||||||
ArrayList searchRowOnPrimaryKey(IBlueRow row);
|
// ArrayList searchRowOnPrimaryKey(IBlueRow row);
|
||||||
ArrayList searchRowOnBothKeys(object pkValue, object skValue);
|
// ArrayList searchRowOnBothKeys(object pkValue, object skValue);
|
||||||
ArrayList searchRowOnBothKeys(IBlueRow row);
|
// ArrayList searchRowOnBothKeys(IBlueRow row);
|
||||||
|
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
/*
|
// /*
|
||||||
public class BlueTableDoubleIndex : IBlueTableIndex
|
// public class BlueTableDoubleIndex : IBlueTableIndex
|
||||||
{
|
// {
|
||||||
private IBlueRowCollection myTable;
|
// private IBlueRowCollection myTable;
|
||||||
private string myPrimaryIndex;
|
// private string myPrimaryIndex;
|
||||||
private string mySecondaryIndex;
|
// private string mySecondaryIndex;
|
||||||
private Hashtable myHt = new Hashtable ();
|
// private Hashtable myHt = new Hashtable ();
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Builds an index with 2 columns of the table acting as primary index and secondary index
|
|
||||||
/// respectively
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="table">
|
|
||||||
/// table to index
|
|
||||||
/// </param>
|
|
||||||
/// <param name="primaryIndex">
|
|
||||||
/// primary key to index upon
|
|
||||||
/// </param>
|
|
||||||
/// <param name="secondaryIndex">
|
|
||||||
/// column serving as the secondary key
|
|
||||||
/// </param>
|
|
||||||
public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
|
|
||||||
{
|
|
||||||
myTable = table;
|
|
||||||
myPrimaryIndex = primaryIndex;
|
|
||||||
mySecondaryIndex = secondaryIndex;
|
|
||||||
myInit ();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void myInit()
|
|
||||||
{
|
|
||||||
foreach (IBlueRow row in myTable)
|
|
||||||
addRow (row);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// To add a row to the index
|
// /// Builds an index with 2 columns of the table acting as primary index and secondary index
|
||||||
/// </summary>
|
// /// respectively
|
||||||
/// <param name="row">
|
// /// </summary>
|
||||||
/// row to be added
|
// /// <param name="table">
|
||||||
/// </param>
|
// /// table to index
|
||||||
public void addRow (IBlueRow row)
|
// /// </param>
|
||||||
{
|
// /// <param name="primaryIndex">
|
||||||
if (myHt[row[myPrimaryIndex]] != null)
|
// /// primary key to index upon
|
||||||
((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
|
// /// </param>
|
||||||
else
|
// /// <param name="secondaryIndex">
|
||||||
{
|
// /// column serving as the secondary key
|
||||||
TimeManager.start ();
|
// /// </param>
|
||||||
Hashtable ht = new Hashtable();
|
// public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
|
||||||
TimeManager.stop ();
|
// {
|
||||||
ht[row[mySecondaryIndex]] = row;
|
// myTable = table;
|
||||||
myHt[row[myPrimaryIndex]] = ht;
|
// myPrimaryIndex = primaryIndex;
|
||||||
}
|
// mySecondaryIndex = secondaryIndex;
|
||||||
}
|
// myInit ();
|
||||||
|
// }
|
||||||
|
|
||||||
/// <summary>
|
// private void myInit()
|
||||||
/// If index or the primary key has already been deleted appropriate exceptions are thrown
|
// {
|
||||||
/// </summary>
|
// foreach (IBlueRow row in myTable)
|
||||||
/// <param name="row"></param>
|
// addRow (row);
|
||||||
public void deleteRow (IBlueRow row)
|
// }
|
||||||
{
|
|
||||||
if (myHt[row[myPrimaryIndex]] == null)
|
|
||||||
{
|
|
||||||
throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
|
|
||||||
}
|
|
||||||
((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
|
|
||||||
if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
|
|
||||||
myHt.Remove(row[myPrimaryIndex]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// Returns an array of DataRow that match the primary key being passed as an argument
|
// /// To add a row to the index
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
/// <param name="o">
|
// /// <param name="row">
|
||||||
/// Primary Key of the table
|
// /// row to be added
|
||||||
/// </param>
|
// /// </param>
|
||||||
/// <returns>
|
// public void addRow (IBlueRow row)
|
||||||
/// Array of rows matching the criteria
|
// {
|
||||||
/// </returns>
|
// if (myHt[row[myPrimaryIndex]] != null)
|
||||||
public IBlueRow[] findValueFromPrimaryKey (object o)
|
// ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
|
||||||
{
|
// else
|
||||||
object oht = myHt[o];
|
// {
|
||||||
if (oht == null)
|
// TimeManager.start ();
|
||||||
return new IBlueRow[0];
|
// Hashtable ht = new Hashtable();
|
||||||
Hashtable ht = (Hashtable)oht;
|
// TimeManager.stop ();
|
||||||
IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
|
// ht[row[mySecondaryIndex]] = row;
|
||||||
ht.Values.CopyTo(rows, 0);
|
// myHt[row[myPrimaryIndex]] = ht;
|
||||||
return rows;
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
// /// <summary>
|
||||||
|
// /// If index or the primary key has already been deleted appropriate exceptions are thrown
|
||||||
|
// /// </summary>
|
||||||
|
// /// <param name="row"></param>
|
||||||
|
// public void deleteRow (IBlueRow row)
|
||||||
|
// {
|
||||||
|
// if (myHt[row[myPrimaryIndex]] == null)
|
||||||
|
// {
|
||||||
|
// throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
|
||||||
|
// }
|
||||||
|
// ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
|
||||||
|
// if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
|
||||||
|
// myHt.Remove(row[myPrimaryIndex]);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// /// <summary>
|
||||||
|
// /// Returns an array of DataRow that match the primary key being passed as an argument
|
||||||
|
// /// </summary>
|
||||||
|
// /// <param name="o">
|
||||||
|
// /// Primary Key of the table
|
||||||
|
// /// </param>
|
||||||
|
// /// <returns>
|
||||||
|
// /// Array of rows matching the criteria
|
||||||
|
// /// </returns>
|
||||||
|
// public IBlueRow[] findValueFromPrimaryKey (object o)
|
||||||
|
// {
|
||||||
|
// object oht = myHt[o];
|
||||||
|
// if (oht == null)
|
||||||
|
// return new IBlueRow[0];
|
||||||
|
// Hashtable ht = (Hashtable)oht;
|
||||||
|
// IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
|
||||||
|
// ht.Values.CopyTo(rows, 0);
|
||||||
|
// return rows;
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// returns as array of the primary keys
|
// /// returns as array of the primary keys
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
/// <returns>
|
// /// <returns>
|
||||||
/// object[] of primary keys
|
// /// object[] of primary keys
|
||||||
/// </returns>
|
// /// </returns>
|
||||||
public Array getPrimaryKeys()
|
// public Array getPrimaryKeys()
|
||||||
{
|
// {
|
||||||
object[] retval = new object [myHt.Keys.Count];
|
// object[] retval = new object [myHt.Keys.Count];
|
||||||
myHt.Keys.CopyTo (retval, 0);
|
// myHt.Keys.CopyTo (retval, 0);
|
||||||
return retval;
|
// return retval;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
*/
|
// */
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// Provides a 1-Level indexing structure.
|
// /// Provides a 1-Level indexing structure.
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
public class BlueTableSingleIndex : IBlueTableIndex
|
// public class BlueTableSingleIndex : IBlueTableIndex
|
||||||
{
|
// {
|
||||||
string myPrimaryKey;
|
// string myPrimaryKey;
|
||||||
string mySecondaryKey;
|
// string mySecondaryKey;
|
||||||
Hashtable myHt = new Hashtable();
|
// Hashtable myHt = new Hashtable();
|
||||||
|
|
||||||
public BlueTableSingleIndex(string primaryKey, string secondaryKey)
|
// public BlueTableSingleIndex(string primaryKey, string secondaryKey)
|
||||||
{
|
// {
|
||||||
myPrimaryKey = primaryKey;
|
// myPrimaryKey = primaryKey;
|
||||||
mySecondaryKey = secondaryKey;
|
// mySecondaryKey = secondaryKey;
|
||||||
}
|
// }
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// The primary key is the indexing key, and the secondary key is used as the key for
|
// /// The primary key is the indexing key, and the secondary key is used as the key for
|
||||||
/// searching and hence deletion
|
// /// searching and hence deletion
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
/// <param name="table"></param>
|
// /// <param name="table"></param>
|
||||||
/// <param name="primaryKey"></param>
|
// /// <param name="primaryKey"></param>
|
||||||
/// <param name="secondaryKey"></param>
|
// /// <param name="secondaryKey"></param>
|
||||||
public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
|
// public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
|
||||||
: this(primaryKey, secondaryKey)
|
// : this(primaryKey, secondaryKey)
|
||||||
{
|
// {
|
||||||
foreach (IBlueRow row in table)
|
// foreach (IBlueRow row in table)
|
||||||
addRow(row);
|
// addRow(row);
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// Adds row to the indexing new structure
|
// /// Adds row to the indexing new structure
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
/// <param name="row"></param>
|
// /// <param name="row"></param>
|
||||||
public void addRow(IBlueRow row)
|
// public void addRow(IBlueRow row)
|
||||||
{
|
// {
|
||||||
if (myHt[row[myPrimaryKey]] == null)
|
// if (myHt[row[myPrimaryKey]] == null)
|
||||||
myHt[row[myPrimaryKey]] = new ArrayList();
|
// myHt[row[myPrimaryKey]] = new ArrayList();
|
||||||
((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
|
// ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
|
||||||
}
|
// }
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
|
// /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
/// <param name="row"></param>
|
// /// <param name="row"></param>
|
||||||
/// <returns></returns>
|
// /// <returns></returns>
|
||||||
public ArrayList searchRowOnPrimaryKey(IBlueRow row)
|
// public ArrayList searchRowOnPrimaryKey(IBlueRow row)
|
||||||
{
|
// {
|
||||||
return searchRowOnPrimaryKey(row[myPrimaryKey]);
|
// return searchRowOnPrimaryKey(row[myPrimaryKey]);
|
||||||
}
|
// }
|
||||||
|
|
||||||
public ArrayList searchRowOnPrimaryKey(object pkValue)
|
// public ArrayList searchRowOnPrimaryKey(object pkValue)
|
||||||
{
|
// {
|
||||||
if (myHt[pkValue] == null)
|
// if (myHt[pkValue] == null)
|
||||||
return new ArrayList();
|
// return new ArrayList();
|
||||||
return (ArrayList)myHt[pkValue];
|
// return (ArrayList)myHt[pkValue];
|
||||||
}
|
// }
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// Gets rows based on the primary AND the secondary key
|
// /// Gets rows based on the primary AND the secondary key
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
/// <param name="row"></param>
|
// /// <param name="row"></param>
|
||||||
/// <returns></returns>
|
// /// <returns></returns>
|
||||||
public ArrayList searchRowOnBothKeys(IBlueRow row)
|
// public ArrayList searchRowOnBothKeys(IBlueRow row)
|
||||||
{
|
// {
|
||||||
return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
|
// return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
|
||||||
}
|
// }
|
||||||
|
|
||||||
public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
|
// public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
|
||||||
{
|
// {
|
||||||
if (myHt[pkValue] == null)
|
// if (myHt[pkValue] == null)
|
||||||
return new ArrayList();
|
// return new ArrayList();
|
||||||
ArrayList matchingList = (ArrayList)myHt[pkValue];
|
// ArrayList matchingList = (ArrayList)myHt[pkValue];
|
||||||
ArrayList retVal = new ArrayList();
|
// ArrayList retVal = new ArrayList();
|
||||||
foreach (IBlueRow r in matchingList)
|
// foreach (IBlueRow r in matchingList)
|
||||||
if (r[mySecondaryKey].Equals(skValue))
|
// if (r[mySecondaryKey].Equals(skValue))
|
||||||
retVal.Add(r);
|
// retVal.Add(r);
|
||||||
return retVal;
|
// return retVal;
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// Deletes rows based on the primary key only
|
// /// Deletes rows based on the primary key only
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
/// <param name="row"></param>
|
// /// <param name="row"></param>
|
||||||
public void deleteRowOnPrimaryKey(IBlueRow row)
|
// public void deleteRowOnPrimaryKey(IBlueRow row)
|
||||||
{
|
// {
|
||||||
deleteRowOnPrimaryKey(row[myPrimaryKey]);
|
// deleteRowOnPrimaryKey(row[myPrimaryKey]);
|
||||||
}
|
// }
|
||||||
|
|
||||||
public void deleteRowOnPrimaryKey(object pkValue)
|
// public void deleteRowOnPrimaryKey(object pkValue)
|
||||||
{
|
// {
|
||||||
if (myHt[pkValue] == null)
|
// if (myHt[pkValue] == null)
|
||||||
throw new BlueTableIndexException("Key to be deleted was not found in the index");
|
// throw new BlueTableIndexException("Key to be deleted was not found in the index");
|
||||||
myHt.Remove(pkValue);
|
// myHt.Remove(pkValue);
|
||||||
}
|
// }
|
||||||
|
|
||||||
public void deleteRowOnBothKeys(IBlueRow row)
|
// public void deleteRowOnBothKeys(IBlueRow row)
|
||||||
{
|
// {
|
||||||
deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
|
// deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
|
||||||
}
|
// }
|
||||||
|
|
||||||
public void deleteRowOnBothKeys(object pkValue, object skValue)
|
// public void deleteRowOnBothKeys(object pkValue, object skValue)
|
||||||
{
|
// {
|
||||||
if (myHt[pkValue] == null)
|
// if (myHt[pkValue] == null)
|
||||||
throw new BlueTableIndexException("Key to be deleted was not found in the index");
|
// throw new BlueTableIndexException("Key to be deleted was not found in the index");
|
||||||
ArrayList matchingList = (ArrayList)myHt[pkValue];
|
// ArrayList matchingList = (ArrayList)myHt[pkValue];
|
||||||
ArrayList aux = new ArrayList();
|
// ArrayList aux = new ArrayList();
|
||||||
foreach (IBlueRow r in matchingList)
|
// foreach (IBlueRow r in matchingList)
|
||||||
if (r[mySecondaryKey].Equals(skValue))
|
// if (r[mySecondaryKey].Equals(skValue))
|
||||||
aux.Add(r);
|
// aux.Add(r);
|
||||||
foreach (IBlueRow r in aux)
|
// foreach (IBlueRow r in aux)
|
||||||
matchingList.Remove(r);
|
// matchingList.Remove(r);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
#nullable enable
|
||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
public interface IMutableBlueRowFactory<T> : IBlueRowFactory<T> {
|
||||||
|
MutableBlueRow GetMutableBlueRow(T item);
|
||||||
|
|
||||||
|
IBlueRow IBlueRowFactory<T>.GetBlueRow(T item) =>
|
||||||
|
GetMutableBlueRow(item);
|
||||||
|
}
|
||||||
@@ -1,324 +1,324 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Data;
|
// using System.Data;
|
||||||
using System.Collections;
|
// using System.Collections;
|
||||||
using System.IO;
|
// using System.IO;
|
||||||
using System.Reflection;
|
// using System.Reflection;
|
||||||
using System.Collections.Generic;
|
// using System.Collections.Generic;
|
||||||
using System.Linq;
|
// using System.Linq;
|
||||||
|
|
||||||
using BCG.Utilities.General;
|
// using BCG.Utilities.General;
|
||||||
|
|
||||||
namespace BCG
|
// namespace BCG
|
||||||
{
|
// {
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
|
// /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
|
||||||
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
|
// /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
|
||||||
/// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
|
// /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
|
||||||
/// the respective data, sorted according to the ColumnOrder.
|
// /// the respective data, sorted according to the ColumnOrder.
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
[Serializable]
|
// [Serializable]
|
||||||
public class LookupTableSchema : IBlueTableSchema
|
// public class LookupTableSchema : IBlueTableSchema
|
||||||
{
|
// {
|
||||||
public class LookupTableSchemaSerializationHelper
|
// public class LookupTableSchemaSerializationHelper
|
||||||
{
|
// {
|
||||||
public static object parse(Type t, string data)
|
// public static object parse(Type t, string data)
|
||||||
{
|
// {
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
// if (t == BlueDataValidTypes.BOOL)
|
||||||
return Boolean.Parse(data);
|
// return Boolean.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
// else if (t == BlueDataValidTypes.BYTE)
|
||||||
return Byte.Parse(data);
|
// return Byte.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.CHAR)
|
// else if (t == BlueDataValidTypes.CHAR)
|
||||||
return char.Parse(data);
|
// return char.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.DATETIME)
|
// else if (t == BlueDataValidTypes.DATETIME)
|
||||||
return DateTime.Parse(data);
|
// return DateTime.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.DOUBLE)
|
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||||
return Double.Parse(data);
|
// return Double.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.DECIMAL)
|
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||||
return Decimal.Parse(data);
|
// return Decimal.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.INT)
|
// else if (t == BlueDataValidTypes.INT)
|
||||||
return int.Parse(data);
|
// return int.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.LONG)
|
// else if (t == BlueDataValidTypes.LONG)
|
||||||
return long.Parse(data);
|
// return long.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.SBYTE)
|
// else if (t == BlueDataValidTypes.SBYTE)
|
||||||
return SByte.Parse(data);
|
// return SByte.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.STRING)
|
// else if (t == BlueDataValidTypes.STRING)
|
||||||
return data;
|
// return data;
|
||||||
else if (t == BlueDataValidTypes.UINT)
|
// else if (t == BlueDataValidTypes.UINT)
|
||||||
return uint.Parse(data);
|
// return uint.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.ULONG)
|
// else if (t == BlueDataValidTypes.ULONG)
|
||||||
return ulong.Parse(data);
|
// return ulong.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.SINGLE)
|
// else if (t == BlueDataValidTypes.SINGLE)
|
||||||
return Single.Parse(data);
|
// return Single.Parse(data);
|
||||||
else
|
// else
|
||||||
throw new ApplicationException("Can not be parsed");
|
// throw new ApplicationException("Can not be parsed");
|
||||||
}
|
// }
|
||||||
public static void serialize(BinaryWriter bw, Type t, object o)
|
// public static void serialize(BinaryWriter bw, Type t, object o)
|
||||||
{
|
// {
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
// if (t == BlueDataValidTypes.BOOL)
|
||||||
bw.Write((bool)o);
|
// bw.Write((bool)o);
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
// else if (t == BlueDataValidTypes.BYTE)
|
||||||
bw.Write((byte)o);
|
// bw.Write((byte)o);
|
||||||
else if (t == BlueDataValidTypes.CHAR)
|
// else if (t == BlueDataValidTypes.CHAR)
|
||||||
bw.Write((char)o);
|
// bw.Write((char)o);
|
||||||
else if (t == BlueDataValidTypes.DATETIME)
|
// else if (t == BlueDataValidTypes.DATETIME)
|
||||||
bw.Write(((DateTime)o).Ticks);
|
// bw.Write(((DateTime)o).Ticks);
|
||||||
else if (t == BlueDataValidTypes.DOUBLE)
|
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||||
bw.Write((double)o);
|
// bw.Write((double)o);
|
||||||
else if (t == BlueDataValidTypes.DECIMAL)
|
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||||
bw.Write((decimal)o);
|
// bw.Write((decimal)o);
|
||||||
else if (t == BlueDataValidTypes.INT)
|
// else if (t == BlueDataValidTypes.INT)
|
||||||
bw.Write((int)o);
|
// bw.Write((int)o);
|
||||||
else if (t == BlueDataValidTypes.LONG)
|
// else if (t == BlueDataValidTypes.LONG)
|
||||||
bw.Write((long)o);
|
// bw.Write((long)o);
|
||||||
else if (t == BlueDataValidTypes.SBYTE)
|
// else if (t == BlueDataValidTypes.SBYTE)
|
||||||
bw.Write((sbyte)o);
|
// bw.Write((sbyte)o);
|
||||||
else if (t == BlueDataValidTypes.STRING)
|
// else if (t == BlueDataValidTypes.STRING)
|
||||||
bw.Write(o as string);
|
// bw.Write(o as string);
|
||||||
else if (t == BlueDataValidTypes.SINGLE)
|
// else if (t == BlueDataValidTypes.SINGLE)
|
||||||
bw.Write((Single)o);
|
// bw.Write((Single)o);
|
||||||
else if (t == BlueDataValidTypes.BYTEARRAY)
|
// else if (t == BlueDataValidTypes.BYTEARRAY)
|
||||||
{
|
// {
|
||||||
byte[] bs = o as byte[];
|
// byte[] bs = o as byte[];
|
||||||
bw.Write(bs.Length);
|
// bw.Write(bs.Length);
|
||||||
bw.Write(bs);
|
// bw.Write(bs);
|
||||||
}
|
// }
|
||||||
else if (t == BlueDataValidTypes.UINT)
|
// else if (t == BlueDataValidTypes.UINT)
|
||||||
bw.Write((uint)o);
|
// bw.Write((uint)o);
|
||||||
else if (t == BlueDataValidTypes.ULONG)
|
// else if (t == BlueDataValidTypes.ULONG)
|
||||||
bw.Write((ulong)o);
|
// bw.Write((ulong)o);
|
||||||
else
|
// else
|
||||||
throw new ApplicationException("Unsupported type");
|
// throw new ApplicationException("Unsupported type");
|
||||||
}
|
// }
|
||||||
|
|
||||||
public static object deserialize(BinaryReader br, Type t)
|
// public static object deserialize(BinaryReader br, Type t)
|
||||||
{
|
// {
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
// if (t == BlueDataValidTypes.BOOL)
|
||||||
return br.ReadBoolean();
|
// return br.ReadBoolean();
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
// else if (t == BlueDataValidTypes.BYTE)
|
||||||
return br.ReadByte();
|
// return br.ReadByte();
|
||||||
else if (t == BlueDataValidTypes.CHAR)
|
// else if (t == BlueDataValidTypes.CHAR)
|
||||||
return br.ReadChar();
|
// return br.ReadChar();
|
||||||
else if (t == BlueDataValidTypes.DATETIME)
|
// else if (t == BlueDataValidTypes.DATETIME)
|
||||||
return new DateTime(br.ReadInt64());
|
// return new DateTime(br.ReadInt64());
|
||||||
else if (t == BlueDataValidTypes.DOUBLE)
|
// else if (t == BlueDataValidTypes.DOUBLE)
|
||||||
return br.ReadDouble();
|
// return br.ReadDouble();
|
||||||
else if (t == BlueDataValidTypes.DECIMAL)
|
// else if (t == BlueDataValidTypes.DECIMAL)
|
||||||
return br.ReadDecimal();
|
// return br.ReadDecimal();
|
||||||
else if (t == BlueDataValidTypes.INT)
|
// else if (t == BlueDataValidTypes.INT)
|
||||||
return br.ReadInt32();
|
// return br.ReadInt32();
|
||||||
else if (t == BlueDataValidTypes.LONG)
|
// else if (t == BlueDataValidTypes.LONG)
|
||||||
return br.ReadInt64();
|
// return br.ReadInt64();
|
||||||
else if (t == BlueDataValidTypes.SBYTE)
|
// else if (t == BlueDataValidTypes.SBYTE)
|
||||||
return br.ReadSByte();
|
// return br.ReadSByte();
|
||||||
else if (t == BlueDataValidTypes.STRING)
|
// else if (t == BlueDataValidTypes.STRING)
|
||||||
return br.ReadString();
|
// return br.ReadString();
|
||||||
else if (t == BlueDataValidTypes.BYTEARRAY)
|
// else if (t == BlueDataValidTypes.BYTEARRAY)
|
||||||
return br.ReadBytes(br.ReadInt32());
|
// return br.ReadBytes(br.ReadInt32());
|
||||||
else if (t == BlueDataValidTypes.UINT)
|
// else if (t == BlueDataValidTypes.UINT)
|
||||||
return br.ReadUInt32();
|
// return br.ReadUInt32();
|
||||||
else if (t == BlueDataValidTypes.ULONG)
|
// else if (t == BlueDataValidTypes.ULONG)
|
||||||
return br.ReadUInt64();
|
// return br.ReadUInt64();
|
||||||
else if (t == BlueDataValidTypes.SINGLE)
|
// else if (t == BlueDataValidTypes.SINGLE)
|
||||||
return br.ReadSingle();
|
// return br.ReadSingle();
|
||||||
else
|
// else
|
||||||
throw new ApplicationException("Unsupported type");
|
// throw new ApplicationException("Unsupported type");
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
protected Hashtable myIndexByColName = new Hashtable();
|
// protected Hashtable myIndexByColName = new Hashtable();
|
||||||
protected ColumnInfo[] myIndexByColOrder;
|
// protected ColumnInfo[] myIndexByColOrder;
|
||||||
private string[] myTableColNames;
|
// private string[] myTableColNames;
|
||||||
private BlueRowLookupGroup[] myLookupGroups;
|
// private BlueRowLookupGroup[] myLookupGroups;
|
||||||
protected string myTableName;
|
// protected string myTableName;
|
||||||
|
|
||||||
public LookupTableSchema() { }
|
// public LookupTableSchema() { }
|
||||||
|
|
||||||
|
|
||||||
public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
|
// public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
|
||||||
{
|
// {
|
||||||
myConstruct(tableName, cis, groups);
|
// myConstruct(tableName, cis, groups);
|
||||||
}
|
// }
|
||||||
|
|
||||||
private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
|
// private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
|
||||||
{
|
// {
|
||||||
myIndexByColOrder = new ColumnInfo[cis.Length];
|
// myIndexByColOrder = new ColumnInfo[cis.Length];
|
||||||
myTableName = tableName;
|
// myTableName = tableName;
|
||||||
foreach (ColumnInfo ci in cis)
|
// foreach (ColumnInfo ci in cis)
|
||||||
{
|
// {
|
||||||
myIndexByColOrder[ci.colNo] = ci;
|
// myIndexByColOrder[ci.colNo] = ci;
|
||||||
myIndexByColName[ci.colName] = ci;
|
// myIndexByColName[ci.colName] = ci;
|
||||||
}
|
// }
|
||||||
myLookupGroups = groups;
|
// myLookupGroups = groups;
|
||||||
|
|
||||||
foreach (BlueRowLookupGroup group in groups)
|
// foreach (BlueRowLookupGroup group in groups)
|
||||||
{
|
// {
|
||||||
group.buildEntrySchema(this);
|
// group.buildEntrySchema(this);
|
||||||
|
|
||||||
|
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public ColumnInfo this[string colName]
|
// public ColumnInfo this[string colName]
|
||||||
{
|
// {
|
||||||
get
|
// get
|
||||||
{
|
// {
|
||||||
object retVal = myIndexByColName[colName];
|
// object retVal = myIndexByColName[colName];
|
||||||
return retVal != null ? retVal as ColumnInfo : null;
|
// return retVal != null ? retVal as ColumnInfo : null;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
public BlueRowLookupGroup[] lookupGroups
|
// public BlueRowLookupGroup[] lookupGroups
|
||||||
{
|
// {
|
||||||
get { return myLookupGroups; }
|
// get { return myLookupGroups; }
|
||||||
|
|
||||||
}
|
// }
|
||||||
public ColumnInfo this[int colNo]
|
// public ColumnInfo this[int colNo]
|
||||||
{
|
// {
|
||||||
get { return myIndexByColOrder[colNo]; }
|
// get { return myIndexByColOrder[colNo]; }
|
||||||
}
|
// }
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// The returned Array is sorted according to the Column No
|
// /// The returned Array is sorted according to the Column No
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
public string[] columnNames
|
// public string[] columnNames
|
||||||
{
|
// {
|
||||||
get
|
// get
|
||||||
{
|
// {
|
||||||
if (myTableColNames == null)
|
// if (myTableColNames == null)
|
||||||
{
|
// {
|
||||||
myTableColNames = new string[myIndexByColName.Count];
|
// myTableColNames = new string[myIndexByColName.Count];
|
||||||
foreach (DictionaryEntry de in myIndexByColName)
|
// foreach (DictionaryEntry de in myIndexByColName)
|
||||||
{
|
// {
|
||||||
ColumnInfo temp = (ColumnInfo)de.Value;
|
// ColumnInfo temp = (ColumnInfo)de.Value;
|
||||||
myTableColNames[temp.colNo] = temp.colName;
|
// myTableColNames[temp.colNo] = temp.colName;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
return myTableColNames;
|
// return myTableColNames;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
/// <summary>
|
// /// <summary>
|
||||||
/// The returned Array is sorted according to the Column No
|
// /// The returned Array is sorted according to the Column No
|
||||||
/// </summary>
|
// /// </summary>
|
||||||
public ColumnInfo[] columnInfo
|
// public ColumnInfo[] columnInfo
|
||||||
{
|
// {
|
||||||
get { return myIndexByColOrder; }
|
// get { return myIndexByColOrder; }
|
||||||
}
|
// }
|
||||||
|
|
||||||
public string tableName { get { return myTableName; } }
|
// public string tableName { get { return myTableName; } }
|
||||||
|
|
||||||
|
|
||||||
#region ISerializedType methods
|
// #region ISerializedType methods
|
||||||
public void serialize(BinaryWriter bw)
|
// public void serialize(BinaryWriter bw)
|
||||||
{
|
// {
|
||||||
bw.Write(columnNames.Length);
|
// bw.Write(columnNames.Length);
|
||||||
bw.Write(tableName);
|
// bw.Write(tableName);
|
||||||
|
|
||||||
//first, write the column names
|
// //first, write the column names
|
||||||
foreach (ColumnInfo ci in columnInfo)
|
// foreach (ColumnInfo ci in columnInfo)
|
||||||
{
|
// {
|
||||||
bw.Write(ci.colName);
|
// bw.Write(ci.colName);
|
||||||
bw.Write(ci.colType.ToString());
|
// bw.Write(ci.colType.ToString());
|
||||||
if (ci.isConstant)
|
// if (ci.isConstant)
|
||||||
{
|
// {
|
||||||
bw.Write(true);
|
// bw.Write(true);
|
||||||
LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
|
// LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
|
||||||
}
|
// }
|
||||||
else
|
// else
|
||||||
{
|
// {
|
||||||
bw.Write(false);
|
// bw.Write(false);
|
||||||
bw.Write(ci.lookupTableGroup);
|
// bw.Write(ci.lookupTableGroup);
|
||||||
}
|
// }
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
//this is here for error checking
|
// //this is here for error checking
|
||||||
bw.Write(myLookupGroups.Length);
|
// bw.Write(myLookupGroups.Length);
|
||||||
//now, write the size of each lookup table group
|
// //now, write the size of each lookup table group
|
||||||
foreach (BlueRowLookupGroup group in myLookupGroups)
|
// foreach (BlueRowLookupGroup group in myLookupGroups)
|
||||||
{
|
// {
|
||||||
bw.Write(group.groupNumber);
|
// bw.Write(group.groupNumber);
|
||||||
bw.Write(group.tableSize);
|
// bw.Write(group.tableSize);
|
||||||
bw.Write(group.groupColNames.Length);
|
// bw.Write(group.groupColNames.Length);
|
||||||
foreach (string colName in group.groupColNames)
|
// foreach (string colName in group.groupColNames)
|
||||||
{
|
// {
|
||||||
//write the index of this column in the table
|
// //write the index of this column in the table
|
||||||
bw.Write(this[colName].colNo);
|
// bw.Write(this[colName].colNo);
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
public void deserialze(BinaryReader br)
|
// public void deserialze(BinaryReader br)
|
||||||
{
|
// {
|
||||||
int numCols = br.ReadInt32();
|
// int numCols = br.ReadInt32();
|
||||||
//these are the final output columns that will be created
|
// //these are the final output columns that will be created
|
||||||
ColumnInfo[] columnInfos = new ColumnInfo[numCols];
|
// ColumnInfo[] columnInfos = new ColumnInfo[numCols];
|
||||||
|
|
||||||
Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
|
// Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
|
||||||
|
|
||||||
|
|
||||||
string tableName = br.ReadString();
|
// string tableName = br.ReadString();
|
||||||
for (int colNo = 0; colNo < numCols; ++colNo)
|
// for (int colNo = 0; colNo < numCols; ++colNo)
|
||||||
{
|
// {
|
||||||
string colName = br.ReadString();
|
// string colName = br.ReadString();
|
||||||
Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
|
// Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
|
||||||
bool colIsConst = br.ReadBoolean();
|
// bool colIsConst = br.ReadBoolean();
|
||||||
if (colIsConst)
|
// if (colIsConst)
|
||||||
{
|
// {
|
||||||
columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
|
// columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
|
||||||
|
|
||||||
}
|
// }
|
||||||
else
|
// else
|
||||||
{
|
// {
|
||||||
short thisColGroup = br.ReadInt16();
|
// short thisColGroup = br.ReadInt16();
|
||||||
columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
|
// columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
int numGroups = br.ReadInt32();
|
// int numGroups = br.ReadInt32();
|
||||||
BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
|
// BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
|
||||||
|
|
||||||
for (int group = 0; group < numGroups; ++group)
|
// for (int group = 0; group < numGroups; ++group)
|
||||||
{
|
// {
|
||||||
short groupNo = br.ReadInt16();
|
// short groupNo = br.ReadInt16();
|
||||||
short groupSizeInBits = br.ReadInt16();
|
// short groupSizeInBits = br.ReadInt16();
|
||||||
int numColsInGroup = br.ReadInt32();
|
// int numColsInGroup = br.ReadInt32();
|
||||||
int[] groupCols = new int[numColsInGroup];
|
// int[] groupCols = new int[numColsInGroup];
|
||||||
string[] groupColNames = new string[numColsInGroup];
|
// string[] groupColNames = new string[numColsInGroup];
|
||||||
for (int col = 0; col < numColsInGroup; ++col)
|
// for (int col = 0; col < numColsInGroup; ++col)
|
||||||
{
|
// {
|
||||||
groupCols[col] = br.ReadInt32();
|
// groupCols[col] = br.ReadInt32();
|
||||||
groupColNames[col] = columnInfos[groupCols[col]].colName;
|
// groupColNames[col] = columnInfos[groupCols[col]].colName;
|
||||||
}
|
// }
|
||||||
|
|
||||||
//create the group
|
// //create the group
|
||||||
theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
|
// theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
|
||||||
|
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
myConstruct(tableName, columnInfos, theGroups);
|
// myConstruct(tableName, columnInfos, theGroups);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
public int getSerializedSize()
|
// public int getSerializedSize()
|
||||||
{
|
// {
|
||||||
throw new NotImplementedException("Feature for provided constant values to" +
|
// throw new NotImplementedException("Feature for provided constant values to" +
|
||||||
"non-primitive types in schema, not provided as of now");
|
// "non-primitive types in schema, not provided as of now");
|
||||||
}
|
// }
|
||||||
|
|
||||||
#endregion
|
// #endregion
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -5,8 +5,7 @@ using System.IO;
|
|||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
//using BCG.Utilities;
|
//using BCG.Utilities;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The SerializedBlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
|
/// The SerializedBlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
|
||||||
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
|
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
|
||||||
@@ -17,12 +16,9 @@ namespace BCG
|
|||||||
/// memory streams, and uses this schema to unpack the values when they are needed.
|
/// memory streams, and uses this schema to unpack the values when they are needed.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class SerializedBlueTableSchema : IBlueTableSchema
|
public class SerializedBlueTableSchema : IBlueTableSchema {
|
||||||
{
|
public class SerializedBlueTableSchemaSerializationHelper {
|
||||||
public class SerializedBlueTableSchemaSerializationHelper
|
public static object parse(Type t, string data) {
|
||||||
{
|
|
||||||
public static object parse(Type t, string data)
|
|
||||||
{
|
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
if (t == BlueDataValidTypes.BOOL)
|
||||||
return Boolean.Parse(data);
|
return Boolean.Parse(data);
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
else if (t == BlueDataValidTypes.BYTE)
|
||||||
@@ -52,8 +48,7 @@ namespace BCG
|
|||||||
else
|
else
|
||||||
throw new ApplicationException("Can not be parsed");
|
throw new ApplicationException("Can not be parsed");
|
||||||
}
|
}
|
||||||
public static void serialize(BinaryWriter bw, Type t, object o)
|
public static void serialize(BinaryWriter bw, Type t, object o) {
|
||||||
{
|
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
if (t == BlueDataValidTypes.BOOL)
|
||||||
bw.Write((bool)o);
|
bw.Write((bool)o);
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
else if (t == BlueDataValidTypes.BYTE)
|
||||||
@@ -76,13 +71,11 @@ namespace BCG
|
|||||||
bw.Write(o as string);
|
bw.Write(o as string);
|
||||||
else if (t == BlueDataValidTypes.SINGLE)
|
else if (t == BlueDataValidTypes.SINGLE)
|
||||||
bw.Write((Single)o);
|
bw.Write((Single)o);
|
||||||
else if (t == BlueDataValidTypes.BYTEARRAY)
|
else if (t == BlueDataValidTypes.BYTEARRAY) {
|
||||||
{
|
|
||||||
byte[] bs = o as byte[];
|
byte[] bs = o as byte[];
|
||||||
bw.Write(bs.Length);
|
bw.Write(bs.Length);
|
||||||
bw.Write(bs);
|
bw.Write(bs);
|
||||||
}
|
} else if (t == BlueDataValidTypes.UINT)
|
||||||
else if (t == BlueDataValidTypes.UINT)
|
|
||||||
bw.Write((uint)o);
|
bw.Write((uint)o);
|
||||||
else if (t == BlueDataValidTypes.ULONG)
|
else if (t == BlueDataValidTypes.ULONG)
|
||||||
bw.Write((ulong)o);
|
bw.Write((ulong)o);
|
||||||
@@ -90,8 +83,7 @@ namespace BCG
|
|||||||
throw new ApplicationException("Unsupported type");
|
throw new ApplicationException("Unsupported type");
|
||||||
}
|
}
|
||||||
|
|
||||||
public static object deserialize(BinaryReader br, Type t)
|
public static object deserialize(BinaryReader br, Type t) {
|
||||||
{
|
|
||||||
if (t == BlueDataValidTypes.BOOL)
|
if (t == BlueDataValidTypes.BOOL)
|
||||||
return br.ReadBoolean();
|
return br.ReadBoolean();
|
||||||
else if (t == BlueDataValidTypes.BYTE)
|
else if (t == BlueDataValidTypes.BYTE)
|
||||||
@@ -132,45 +124,37 @@ namespace BCG
|
|||||||
public SerializedBlueTableSchema() { }
|
public SerializedBlueTableSchema() { }
|
||||||
|
|
||||||
//copies the schema from an existing schema, building the dispatch table
|
//copies the schema from an existing schema, building the dispatch table
|
||||||
public SerializedBlueTableSchema(IBlueTableSchema toCopy)
|
public SerializedBlueTableSchema(IBlueTableSchema toCopy) {
|
||||||
{
|
|
||||||
int numCols = toCopy.columnInfo.Length;
|
int numCols = toCopy.columnInfo.Length;
|
||||||
ColumnInfo[] theColumns = new ColumnInfo[toCopy.columnInfo.Length];
|
ColumnInfo[] theColumns = new ColumnInfo[toCopy.columnInfo.Length];
|
||||||
foreach (ColumnInfo toCopyCI in toCopy.columnInfo)
|
foreach (ColumnInfo toCopyCI in toCopy.columnInfo) {
|
||||||
{
|
|
||||||
theColumns[toCopyCI.colNo] = toCopyCI;
|
theColumns[toCopyCI.colNo] = toCopyCI;
|
||||||
}
|
}
|
||||||
myConstruct(theColumns, toCopy.tableName);
|
myConstruct(theColumns, toCopy.tableName);
|
||||||
}
|
}
|
||||||
|
|
||||||
public SerializedBlueTableSchema(DataTable table)
|
public SerializedBlueTableSchema(DataTable table) {
|
||||||
{
|
|
||||||
ColumnInfo[] theseCols = new ColumnInfo[table.Columns.Count];
|
ColumnInfo[] theseCols = new ColumnInfo[table.Columns.Count];
|
||||||
myTableName = table.TableName;
|
myTableName = table.TableName;
|
||||||
foreach (DataColumn col in table.Columns)
|
foreach (DataColumn col in table.Columns) {
|
||||||
{
|
|
||||||
theseCols[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
|
theseCols[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
|
||||||
}
|
}
|
||||||
myConstruct(theseCols, myTableName);
|
myConstruct(theseCols, myTableName);
|
||||||
}
|
}
|
||||||
|
|
||||||
public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName)
|
public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName) {
|
||||||
{
|
|
||||||
myConstruct(cis, tableName);
|
myConstruct(cis, tableName);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void serializeObject(object data, BinaryWriter bw, int colNo)
|
public void serializeObject(object data, BinaryWriter bw, int colNo) {
|
||||||
{
|
|
||||||
mySerializeDispatchTable[colNo](data, bw);
|
mySerializeDispatchTable[colNo](data, bw);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myConstruct(ColumnInfo[] cis, string tableName)
|
private void myConstruct(ColumnInfo[] cis, string tableName) {
|
||||||
{
|
|
||||||
myIndexByColOrder = new ColumnInfo[cis.Length];
|
myIndexByColOrder = new ColumnInfo[cis.Length];
|
||||||
myTableName = tableName;
|
myTableName = tableName;
|
||||||
foreach (ColumnInfo ci in cis)
|
foreach (ColumnInfo ci in cis) {
|
||||||
{
|
|
||||||
myIndexByColOrder[ci.colNo] = ci;
|
myIndexByColOrder[ci.colNo] = ci;
|
||||||
myIndexByColName[ci.colName] = ci;
|
myIndexByColName[ci.colName] = ci;
|
||||||
}
|
}
|
||||||
@@ -179,32 +163,25 @@ namespace BCG
|
|||||||
|
|
||||||
public SerializedBlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { }
|
public SerializedBlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { }
|
||||||
|
|
||||||
public ColumnInfo this[string colName]
|
public ColumnInfo this[string colName] {
|
||||||
{
|
get {
|
||||||
get
|
|
||||||
{
|
|
||||||
object retVal = myIndexByColName[colName];
|
object retVal = myIndexByColName[colName];
|
||||||
return retVal != null ? retVal as ColumnInfo : null;
|
return retVal != null ? retVal as ColumnInfo : null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public ColumnInfo this[int colNo]
|
public ColumnInfo this[int colNo] {
|
||||||
{
|
|
||||||
get { return myIndexByColOrder[colNo]; }
|
get { return myIndexByColOrder[colNo]; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The returned Array is sorted according to the Column No
|
/// The returned Array is sorted according to the Column No
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string[] columnNames
|
public string[] columnNames {
|
||||||
{
|
get {
|
||||||
get
|
if (myTableColNames == null) {
|
||||||
{
|
|
||||||
if (myTableColNames == null)
|
|
||||||
{
|
|
||||||
myTableColNames = new string[myIndexByColName.Count];
|
myTableColNames = new string[myIndexByColName.Count];
|
||||||
foreach (DictionaryEntry de in myIndexByColName)
|
foreach (DictionaryEntry de in myIndexByColName) {
|
||||||
{
|
|
||||||
ColumnInfo temp = (ColumnInfo)de.Value;
|
ColumnInfo temp = (ColumnInfo)de.Value;
|
||||||
myTableColNames[temp.colNo] = temp.colName;
|
myTableColNames[temp.colNo] = temp.colName;
|
||||||
}
|
}
|
||||||
@@ -216,64 +193,46 @@ namespace BCG
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// The returned Array is sorted according to the Column No
|
/// The returned Array is sorted according to the Column No
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ColumnInfo[] columnInfo
|
public ColumnInfo[] columnInfo {
|
||||||
{
|
|
||||||
get { return myIndexByColOrder; }
|
get { return myIndexByColOrder; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public string tableName { get { return myTableName; } }
|
public string tableName { get { return myTableName; } }
|
||||||
|
|
||||||
|
|
||||||
public int IdColumn
|
public int IdColumn {
|
||||||
{
|
|
||||||
get { return myIdCol; }
|
get { return myIdCol; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#region ISerializedType methods
|
#region ISerializedType methods
|
||||||
public void serialize(BinaryWriter bw)
|
public void serialize(BinaryWriter bw) {
|
||||||
{
|
|
||||||
bw.Write(columnNames.Length);
|
bw.Write(columnNames.Length);
|
||||||
bw.Write(tableName);
|
bw.Write(tableName);
|
||||||
foreach (ColumnInfo ci in columnInfo)
|
foreach (ColumnInfo ci in columnInfo) {
|
||||||
{
|
|
||||||
bw.Write(ci.colName);
|
bw.Write(ci.colName);
|
||||||
bw.Write(ci.colType.ToString());
|
bw.Write(ci.colType.ToString());
|
||||||
if (ci.isConstant)
|
bw.Write(false);
|
||||||
{
|
|
||||||
bw.Write(true);
|
|
||||||
SerializedBlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
bw.Write(false);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deserialze(BinaryReader br)
|
public void deserialze(BinaryReader br) {
|
||||||
{
|
|
||||||
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
|
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
|
||||||
string name = br.ReadString();
|
string name = br.ReadString();
|
||||||
for (int i = 0; i < cis.Length; i++)
|
for (int i = 0; i < cis.Length; i++) {
|
||||||
{
|
|
||||||
string colName = br.ReadString();
|
string colName = br.ReadString();
|
||||||
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
|
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
|
||||||
bool isConstant = br.ReadBoolean();
|
bool isConstant = br.ReadBoolean();
|
||||||
if (isConstant)
|
if (isConstant) {
|
||||||
{
|
cis[i] = new ColumnInfo(colName, i, t);
|
||||||
cis[i] = new ColumnInfo(colName, i, t, SerializedBlueTableSchemaSerializationHelper.deserialize(br, t));
|
} else {
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
cis[i] = new ColumnInfo(colName, i, t);
|
cis[i] = new ColumnInfo(colName, i, t);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
myConstruct(cis, name);
|
myConstruct(cis, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
public int getSerializedSize()
|
public int getSerializedSize() {
|
||||||
{
|
|
||||||
throw new NotImplementedException("Feature for provided constant values to" +
|
throw new NotImplementedException("Feature for provided constant values to" +
|
||||||
"non-primitive types in schema, not provided as of now");
|
"non-primitive types in schema, not provided as of now");
|
||||||
}
|
}
|
||||||
@@ -282,8 +241,7 @@ namespace BCG
|
|||||||
|
|
||||||
|
|
||||||
//builds a dispatch table used to pull values out of the memory streams in the rows.
|
//builds a dispatch table used to pull values out of the memory streams in the rows.
|
||||||
private void myBuildDispatchTable()
|
private void myBuildDispatchTable() {
|
||||||
{
|
|
||||||
int numCols = myIndexByColOrder.Length;
|
int numCols = myIndexByColOrder.Length;
|
||||||
mySerializeDispatchTable = new SerializeDelegate[numCols];
|
mySerializeDispatchTable = new SerializeDelegate[numCols];
|
||||||
myPullFromByteBufferDispatchTable = new PullFromByteBufferDelegate[numCols];
|
myPullFromByteBufferDispatchTable = new PullFromByteBufferDelegate[numCols];
|
||||||
@@ -292,18 +250,14 @@ namespace BCG
|
|||||||
|
|
||||||
myIdCol = idColumn.colNo;
|
myIdCol = idColumn.colNo;
|
||||||
|
|
||||||
for (int col = 0; col < numCols; ++col)
|
for (int col = 0; col < numCols; ++col) {
|
||||||
{
|
|
||||||
Type thisColType = myIndexByColOrder[col].colType;
|
Type thisColType = myIndexByColOrder[col].colType;
|
||||||
//we don't store serializers for the id column, or string columns.
|
//we don't store serializers for the id column, or string columns.
|
||||||
if (col == myIdCol || thisColType == typeof(string))
|
if (col == myIdCol || thisColType == typeof(string)) {
|
||||||
{
|
|
||||||
mySerializeDispatchTable[col] = null;
|
mySerializeDispatchTable[col] = null;
|
||||||
myColumnSizesTable[col] = 0;
|
myColumnSizesTable[col] = 0;
|
||||||
myPullFromByteBufferDispatchTable[col] = null;
|
myPullFromByteBufferDispatchTable[col] = null;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
mySerializeDispatchTable[col] = myGetDispatchMethodSerialize(thisColType);
|
mySerializeDispatchTable[col] = myGetDispatchMethodSerialize(thisColType);
|
||||||
myPullFromByteBufferDispatchTable[col] = myGetDispatchMethodPullFromBuffer(thisColType);
|
myPullFromByteBufferDispatchTable[col] = myGetDispatchMethodPullFromBuffer(thisColType);
|
||||||
myColumnSizesTable[col] = BlueDataValidTypes.getValueTypeSize(thisColType);
|
myColumnSizesTable[col] = BlueDataValidTypes.getValueTypeSize(thisColType);
|
||||||
@@ -315,16 +269,14 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public int getNullMapSize()
|
public int getNullMapSize() {
|
||||||
{
|
|
||||||
int numCols = myIndexByColOrder.Length;
|
int numCols = myIndexByColOrder.Length;
|
||||||
return Convert.ToInt32(Math.Ceiling((Convert.ToDouble(numCols) / 8.0)));
|
return Convert.ToInt32(Math.Ceiling((Convert.ToDouble(numCols) / 8.0)));
|
||||||
}
|
}
|
||||||
|
|
||||||
//this function reads the null array from inside the byte array.
|
//this function reads the null array from inside the byte array.
|
||||||
//it has to copy the bytes over.
|
//it has to copy the bytes over.
|
||||||
public BitArray getNullMap(byte[] theBytes)
|
public BitArray getNullMap(byte[] theBytes) {
|
||||||
{
|
|
||||||
int nullMapSize = getNullMapSize();
|
int nullMapSize = getNullMapSize();
|
||||||
//the null map starts after the id, so that we have instant access to the id.
|
//the null map starts after the id, so that we have instant access to the id.
|
||||||
int byteCursor = BlueDataValidTypes.getValueTypeSize(typeof(Int32));
|
int byteCursor = BlueDataValidTypes.getValueTypeSize(typeof(Int32));
|
||||||
@@ -336,8 +288,7 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
//gets the value either from the bytes stream but only if the value is not a string.
|
//gets the value either from the bytes stream but only if the value is not a string.
|
||||||
//doesn't work if you're getting the id.
|
//doesn't work if you're getting the id.
|
||||||
public unsafe object getValueNonString(byte[] theBytes, int colNo)
|
public unsafe object getValueNonString(byte[] theBytes, int colNo) {
|
||||||
{
|
|
||||||
if (colNo == myIdCol)
|
if (colNo == myIdCol)
|
||||||
throw new BlueDataException("Error: do not use getValueNonString to try to fetch the id of a row");
|
throw new BlueDataException("Error: do not use getValueNonString to try to fetch the id of a row");
|
||||||
BitArray nullMap = getNullMap(theBytes);
|
BitArray nullMap = getNullMap(theBytes);
|
||||||
@@ -345,8 +296,7 @@ namespace BCG
|
|||||||
return DBNull.Value;
|
return DBNull.Value;
|
||||||
//the rest of the row starts after the id and the null map
|
//the rest of the row starts after the id and the null map
|
||||||
int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
|
int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
|
||||||
for (int currentColumn = 0; currentColumn < colNo; ++currentColumn)
|
for (int currentColumn = 0; currentColumn < colNo; ++currentColumn) {
|
||||||
{
|
|
||||||
//pull the data from the binary reader, but only if this type is not null or a string
|
//pull the data from the binary reader, but only if this type is not null or a string
|
||||||
//if the type wer'e looking for is a string, it would have been returned already by the accesor
|
//if the type wer'e looking for is a string, it would have been returned already by the accesor
|
||||||
//in the bluerow.
|
//in the bluerow.
|
||||||
@@ -357,16 +307,14 @@ namespace BCG
|
|||||||
|
|
||||||
thisOffset += myColumnSizesTable[currentColumn];
|
thisOffset += myColumnSizesTable[currentColumn];
|
||||||
}
|
}
|
||||||
fixed (byte* thisBytePtr = &theBytes[thisOffset])
|
fixed (byte* thisBytePtr = &theBytes[thisOffset]) {
|
||||||
{
|
|
||||||
return myPullFromByteBufferDispatchTable[colNo](thisBytePtr);
|
return myPullFromByteBufferDispatchTable[colNo](thisBytePtr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray)
|
public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray) {
|
||||||
{
|
|
||||||
int numCols = myColumnSizesTable.Length;
|
int numCols = myColumnSizesTable.Length;
|
||||||
object[] theObjs = new Object[numCols];
|
object[] theObjs = new Object[numCols];
|
||||||
|
|
||||||
@@ -374,8 +322,7 @@ namespace BCG
|
|||||||
|
|
||||||
int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
|
int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
|
||||||
int thisStringNo = 0;
|
int thisStringNo = 0;
|
||||||
for (int currentColumn = 0; currentColumn < numCols; ++currentColumn)
|
for (int currentColumn = 0; currentColumn < numCols; ++currentColumn) {
|
||||||
{
|
|
||||||
//pull the data from the binary reader, but only if this type is not null or a string
|
//pull the data from the binary reader, but only if this type is not null or a string
|
||||||
//if the type wer'e looking for is a string, it would have been returned already by the accesor
|
//if the type wer'e looking for is a string, it would have been returned already by the accesor
|
||||||
//in the bluerow.
|
//in the bluerow.
|
||||||
@@ -384,10 +331,8 @@ namespace BCG
|
|||||||
theObjs[currentColumn] = DBNull.Value;
|
theObjs[currentColumn] = DBNull.Value;
|
||||||
else if (myIndexByColOrder[currentColumn].colType == typeof(string))
|
else if (myIndexByColOrder[currentColumn].colType == typeof(string))
|
||||||
theObjs[currentColumn] = stringArray[thisStringNo++];
|
theObjs[currentColumn] = stringArray[thisStringNo++];
|
||||||
else if (currentColumn != myIdCol)
|
else if (currentColumn != myIdCol) {
|
||||||
{
|
fixed (byte* thisGuy = &theBytes[thisOffset]) {
|
||||||
fixed (byte* thisGuy = &theBytes[thisOffset])
|
|
||||||
{
|
|
||||||
theObjs[currentColumn] = myPullFromByteBufferDispatchTable[currentColumn](thisGuy);
|
theObjs[currentColumn] = myPullFromByteBufferDispatchTable[currentColumn](thisGuy);
|
||||||
}
|
}
|
||||||
thisOffset += myColumnSizesTable[currentColumn];
|
thisOffset += myColumnSizesTable[currentColumn];
|
||||||
@@ -416,8 +361,7 @@ namespace BCG
|
|||||||
private void mySerializeDateTime(object data, BinaryWriter bw) { bw.Write(((DateTime)data).ToOADate()); }
|
private void mySerializeDateTime(object data, BinaryWriter bw) { bw.Write(((DateTime)data).ToOADate()); }
|
||||||
private void mySerializeSingle(object data, BinaryWriter bw) { bw.Write((Single)data); }
|
private void mySerializeSingle(object data, BinaryWriter bw) { bw.Write((Single)data); }
|
||||||
|
|
||||||
private SerializeDelegate myGetDispatchMethodSerialize(Type t)
|
private SerializeDelegate myGetDispatchMethodSerialize(Type t) {
|
||||||
{
|
|
||||||
if (t.Equals(BlueDataValidTypes.INT)) return new SerializeDelegate(mySerializeInt32);
|
if (t.Equals(BlueDataValidTypes.INT)) return new SerializeDelegate(mySerializeInt32);
|
||||||
else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new SerializeDelegate(mySerializeDouble);
|
else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new SerializeDelegate(mySerializeDouble);
|
||||||
else if (t.Equals(BlueDataValidTypes.CHAR)) return new SerializeDelegate(mySerializeChar);
|
else if (t.Equals(BlueDataValidTypes.CHAR)) return new SerializeDelegate(mySerializeChar);
|
||||||
@@ -448,16 +392,14 @@ namespace BCG
|
|||||||
private unsafe object myPullDecimal(byte* bytePtr) { return *((decimal*)bytePtr); }
|
private unsafe object myPullDecimal(byte* bytePtr) { return *((decimal*)bytePtr); }
|
||||||
private unsafe object myPullUInt(byte* bytePtr) { return *((UInt32*)bytePtr); }
|
private unsafe object myPullUInt(byte* bytePtr) { return *((UInt32*)bytePtr); }
|
||||||
private unsafe object myPullULong(byte* bytePtr) { return *((UInt64*)bytePtr); }
|
private unsafe object myPullULong(byte* bytePtr) { return *((UInt64*)bytePtr); }
|
||||||
private unsafe object myPullDateTime(byte* bytePtr)
|
private unsafe object myPullDateTime(byte* bytePtr) {
|
||||||
{
|
|
||||||
double thisDateTime = *((double*)bytePtr);
|
double thisDateTime = *((double*)bytePtr);
|
||||||
return DateTime.FromOADate(thisDateTime);
|
return DateTime.FromOADate(thisDateTime);
|
||||||
}
|
}
|
||||||
private unsafe object myPullSingle(byte* bytePtr) { return *((Single*)bytePtr); ; }
|
private unsafe object myPullSingle(byte* bytePtr) { return *((Single*)bytePtr); ; }
|
||||||
|
|
||||||
|
|
||||||
private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t)
|
private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t) {
|
||||||
{
|
|
||||||
if (t.Equals(BlueDataValidTypes.INT)) return new PullFromByteBufferDelegate(myPullInt32);
|
if (t.Equals(BlueDataValidTypes.INT)) return new PullFromByteBufferDelegate(myPullInt32);
|
||||||
else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new PullFromByteBufferDelegate(myPullDouble);
|
else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new PullFromByteBufferDelegate(myPullDouble);
|
||||||
else if (t.Equals(BlueDataValidTypes.CHAR)) return new PullFromByteBufferDelegate(myPullChar);
|
else if (t.Equals(BlueDataValidTypes.CHAR)) return new PullFromByteBufferDelegate(myPullChar);
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
namespace Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public static class BlueRowExtensions {
|
||||||
|
public static int Id(this IBlueRow row) =>
|
||||||
|
(int)(row["id"] ?? throw new ArgumentException(
|
||||||
|
"Row doesn't have an 'id' column", nameof(row)));
|
||||||
|
}
|
||||||
Loaded 100 of 528 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user