Compare commits
392
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6997f66505 | ||
|
|
d78442820d | ||
|
|
7f238a7b24 | ||
|
|
e113162925 | ||
|
|
1ca7f65a78 | ||
|
|
2148348e82 | ||
|
|
53ddaaf593 | ||
|
|
0430b87e35 | ||
|
|
4c93d20ce6 | ||
|
|
2840ac284d | ||
|
|
3e355d4afb | ||
|
|
922844e7ab | ||
|
|
1d3186e192 | ||
|
|
287e784d5e | ||
|
|
d29360f903 | ||
|
|
3681c1db4f | ||
|
|
bef4325666 | ||
|
|
32fd0f6f9b | ||
|
|
f1cb37277a | ||
|
|
4fa76a35e3 | ||
|
|
aded3a9e86 | ||
|
|
257b7c3f0f | ||
|
|
442f827af3 | ||
|
|
8e0e6a62bc | ||
|
|
f1ddd8ef6a | ||
|
|
66a1d19dd4 | ||
|
|
2ddf14d2b5 | ||
|
|
51aa380c3f | ||
|
|
b8d8dd5b49 | ||
|
|
d718a84f6c | ||
|
|
13552e29fe | ||
|
|
0ac95710b1 | ||
|
|
f23c7af88c | ||
|
|
345b76257b | ||
|
|
4996c27f01 | ||
|
|
9a9bcef348 | ||
|
|
8502b113fe | ||
|
|
979b2ccb6f | ||
|
|
6e12c5b895 | ||
|
|
2624a2c00d | ||
|
|
fb01953296 | ||
|
|
72a3358f44 | ||
|
|
2a505e8ce5 | ||
|
|
a4d4a938b0 | ||
|
|
c2922d5e02 | ||
|
|
8e109c6450 | ||
|
|
85b1909543 | ||
|
|
ba4e20552e | ||
|
|
b1f1987922 | ||
|
|
b13725bc09 | ||
|
|
85dbee2f00 | ||
|
|
14232f5644 | ||
|
|
89d49cd721 | ||
|
|
590bc1749a | ||
|
|
339767a51f | ||
|
|
47f2f16cd3 | ||
|
|
d1b28a86c8 | ||
|
|
50e7ad925e | ||
|
|
132201d2cf | ||
|
|
5543725ee2 | ||
|
|
9c535616df | ||
|
|
1dd2db41f2 | ||
|
|
a451c3e880 | ||
|
|
5485cc9c4f | ||
|
|
5e7e20df5a | ||
|
|
28f5181585 | ||
|
|
5e7a478bf5 | ||
|
|
0abc386d71 | ||
|
|
53b040342a | ||
|
|
e85f669534 | ||
|
|
935d414b7f | ||
|
|
89f4fe88f9 | ||
|
|
161e0a8c3b | ||
|
|
f5a6dc7a88 | ||
|
|
298a2f0617 | ||
|
|
c6fc1a0f89 | ||
|
|
6cd7baf903 | ||
|
|
7d7621d832 | ||
|
|
ef8e4c4998 | ||
|
|
ad2e631bbc | ||
|
|
dccc9f0099 | ||
|
|
bab5eb4c70 | ||
|
|
e0608c3791 | ||
|
|
e69c4782b5 | ||
|
|
7ab045b120 | ||
|
|
9bd4bc421d | ||
|
|
8b90debdd0 | ||
|
|
dc2e8b114a | ||
|
|
e064ab0433 | ||
|
|
42a9edbee3 | ||
|
|
92ea362d1e | ||
|
|
750228355a | ||
|
|
930c5e1187 | ||
|
|
b286f81067 | ||
|
|
36795bd851 | ||
|
|
857702fae7 | ||
|
|
2702d0d32e | ||
|
|
10e78b5e20 | ||
|
|
74027fc9cc | ||
|
|
af5af01b39 | ||
|
|
ca3fe805c7 | ||
|
|
40bf2d8b43 | ||
|
|
98fb021f3f | ||
|
|
b34a3421bb | ||
|
|
9dc61daefa | ||
|
|
32fe2533c7 | ||
|
|
9a49767b75 | ||
|
|
b020939c99 | ||
|
|
c660c52c3d | ||
|
|
37322fccb1 | ||
|
|
40ae9ff7a0 | ||
|
|
60bcbe67e4 | ||
|
|
b983f80bfa | ||
|
|
c58a2f90e0 | ||
|
|
2515030501 | ||
|
|
7146aca538 | ||
|
|
e0f01b6810 | ||
|
|
92e3094259 | ||
|
|
a9a5b563a0 | ||
|
|
84183d2741 | ||
|
|
acb44c836d | ||
|
|
cb5c7d2012 | ||
|
|
fefb848281 | ||
|
|
3af14cecd7 | ||
|
|
5df30e8857 | ||
|
|
d813a6a2c6 | ||
|
|
b3e28d5f7f | ||
|
|
5c11b83df5 | ||
|
|
5c5a0ca09a | ||
|
|
114873eb35 | ||
|
|
a9799b2a00 | ||
|
|
d5aeaa5e0f | ||
|
|
8a8c491fb1 | ||
|
|
450a6c62b6 | ||
|
|
a3887e6246 | ||
|
|
982ddd24d4 | ||
|
|
20e959fea3 | ||
|
|
a108823ea5 | ||
|
|
52c74cd800 | ||
|
|
a473cbc703 | ||
|
|
63c648a659 | ||
|
|
3ffe02254e | ||
|
|
f5de2b5355 | ||
|
|
979b51fb20 | ||
|
|
4efb0bfc42 | ||
|
|
f925be32a6 | ||
|
|
f3b5ff8970 | ||
|
|
fdea340492 | ||
|
|
5ce2ffa5d6 | ||
|
|
cdae65b098 | ||
|
|
a1d7607065 | ||
|
|
233d089c34 | ||
|
|
759f6dd59a | ||
|
|
80622ddaba | ||
|
|
434d4b6ad3 | ||
|
|
a90e8807ad | ||
|
|
6dc25bab01 | ||
|
|
00042d2471 | ||
|
|
de9d47cd00 | ||
|
|
461f690463 | ||
|
|
a425d3e6ba | ||
|
|
fe81b3107f | ||
|
|
d5d12c56c9 | ||
|
|
8979d5def8 | ||
|
|
922cf3170b | ||
|
|
5da807b5f0 | ||
|
|
c3507ab77b | ||
|
|
521346a5c5 | ||
|
|
a94d1f6f67 | ||
|
|
c69adca7ba | ||
|
|
212bd432af | ||
|
|
4361ac6519 | ||
|
|
93dc4c9c1f | ||
|
|
ebc39f203d | ||
|
|
ec88ffdbf3 | ||
|
|
26fecc73c6 | ||
|
|
05715eb489 | ||
|
|
70019c59e5 | ||
|
|
66f9e9aa4c | ||
|
|
6caa98a6b3 | ||
|
|
40930628fb | ||
|
|
d439e3e695 | ||
|
|
aae50cd5f8 | ||
|
|
587e340071 | ||
|
|
156ec8f3a8 | ||
|
|
7ca96f128b | ||
|
|
9aec630100 | ||
|
|
831cf840a7 | ||
|
|
8940325cfd | ||
|
|
aaf75acd89 | ||
|
|
13200de777 | ||
|
|
26f8a2a4ed | ||
|
|
5b77e4a36d | ||
|
|
13668374b7 | ||
|
|
405c6889c4 | ||
|
|
b2a15cc33b | ||
|
|
e31d1ad69b | ||
|
|
c88d1b26f0 | ||
|
|
b1446993b4 | ||
|
|
db0d415b5a | ||
|
|
3f1ad91beb | ||
|
|
639d17a625 | ||
|
|
072b8b2b02 | ||
|
|
658ffcad27 | ||
|
|
2b0f6e52b2 | ||
|
|
d721bba4db | ||
|
|
c387eb81c9 | ||
|
|
707ebcfbc4 | ||
|
|
1f4f3b032e | ||
|
|
f6e54254a9 | ||
|
|
b5adcb1a84 | ||
|
|
179b034bc7 | ||
|
|
2cf8a3963d | ||
|
|
49e4e05a3c | ||
|
|
ae9311a2a7 | ||
|
|
21c3d57324 | ||
|
|
9352f529de | ||
|
|
0b76ac4763 | ||
|
|
b887801cf5 | ||
|
|
c8ffa1489e | ||
|
|
f72ea96d20 | ||
|
|
7025ffbc4d | ||
|
|
bf989e1f70 | ||
|
|
933db6ea96 | ||
|
|
0e92e7f30c | ||
|
|
25c42e388f | ||
|
|
3187722159 | ||
|
|
352cb97818 | ||
|
|
0215b7990e | ||
|
|
1644b5b58b | ||
|
|
b2ea6820d7 | ||
|
|
5c9fc5ccc2 | ||
|
|
4bd6f149b9 | ||
|
|
293f7f3833 | ||
|
|
ad2f9a8624 | ||
|
|
99922725e1 | ||
|
|
60c3cc0015 | ||
|
|
775567968b | ||
|
|
e1555387ec | ||
|
|
d044f711a3 | ||
|
|
eba801845c | ||
|
|
04f85818c6 | ||
|
|
545dcdf9ff | ||
|
|
8c2d643e0c | ||
|
|
ae70f731b7 | ||
|
|
828b06d4e2 | ||
|
|
6fa8de176c | ||
|
|
fec918aefd | ||
|
|
aa57b3bb8c | ||
|
|
ab206ccf87 | ||
|
|
06ad7c51be | ||
|
|
60a52b3922 | ||
|
|
5426ba0aa7 | ||
|
|
b1fdee5a22 | ||
|
|
59461aadff | ||
|
|
2d3a9a77ae | ||
|
|
868b43ef8b | ||
|
|
fc7043f6e1 | ||
|
|
7bec823545 | ||
|
|
57c7868a24 | ||
|
|
0b0ff8f502 | ||
|
|
c9a8325713 | ||
|
|
d7e4c68c70 | ||
|
|
70f782bc79 | ||
|
|
7e142860a3 | ||
|
|
1956e3a804 | ||
|
|
65b50eda3a | ||
|
|
83b37c6661 | ||
|
|
6163887cd3 | ||
|
|
214641aa1e | ||
|
|
06bc7af232 | ||
|
|
36b6168dec | ||
|
|
f9aa9dd982 | ||
|
|
22f1ab44f9 | ||
|
|
e4c87c267d | ||
|
|
86899a6ff0 | ||
|
|
2324fd3ad2 | ||
|
|
288b7712db | ||
|
|
8850bf4903 | ||
|
|
11112b41d1 | ||
|
|
2a98889b67 | ||
|
|
8060518a28 | ||
|
|
68440d3df3 | ||
|
|
7d775afc4c | ||
|
|
50c7764ef7 | ||
|
|
fb8b2d28f3 | ||
|
|
bcc2c4a514 | ||
|
|
9352298be0 | ||
|
|
180bd11740 | ||
|
|
57ea841381 | ||
|
|
7d9d176fa2 | ||
|
|
a8f4ec07c8 | ||
|
|
58c5be268f | ||
|
|
bf4859a2a9 | ||
|
|
db1cf93994 | ||
|
|
ecb1c4fea0 | ||
|
|
a513933044 | ||
|
|
764d65b595 | ||
|
|
bc0d606252 | ||
|
|
ceb8a68902 | ||
|
|
0b3183fd30 | ||
|
|
b6dbe37818 | ||
|
|
c78b8bc8e1 | ||
|
|
314f3e3715 | ||
|
|
26a1d5fc03 | ||
|
|
37e5003cf9 | ||
|
|
07f3868ffc | ||
|
|
46f0b17138 | ||
|
|
33a8e06a3a | ||
|
|
74afdde8de | ||
|
|
0a92ad2eb1 | ||
|
|
56d207c3f1 | ||
|
|
9172e752e3 | ||
|
|
adc9f2acfc | ||
|
|
41d17e67d2 | ||
|
|
5cacff073b | ||
|
|
c8da92c250 | ||
|
|
c4e76076e3 | ||
|
|
6a715b9a68 | ||
|
|
1f28940c95 | ||
|
|
ebff40b88f | ||
|
|
a6d14a3ac3 | ||
|
|
af96bcb9e3 | ||
|
|
619702cb04 | ||
|
|
5baec19aff | ||
|
|
39ef30c1e3 | ||
|
|
c2f2d28914 | ||
|
|
e06bc3bdf5 | ||
|
|
e19c19c013 | ||
|
|
8f324bc507 | ||
|
|
12107c42ce | ||
|
|
91c6fb4e52 | ||
|
|
7044355547 | ||
|
|
583ae65fb2 | ||
|
|
552fec0edc | ||
|
|
fcf6927fbf | ||
|
|
23f6d827d5 | ||
|
|
f083d0abab | ||
|
|
a895e36922 | ||
|
|
59d7a72288 | ||
|
|
7731f2f4d1 | ||
|
|
41ae5897c0 | ||
|
|
c7949ccfba | ||
|
|
567af79c7f | ||
|
|
6bb4a7b25c | ||
|
|
bc7cef4c34 | ||
|
|
ab9dc361d6 | ||
|
|
39da0a4dde | ||
|
|
44e864d697 | ||
|
|
0a02da3aba | ||
|
|
836771a7f0 | ||
|
|
f386badcfb | ||
|
|
1d43a484db | ||
|
|
91042cb41e | ||
|
|
10c0e4247d | ||
|
|
8e34264e6d | ||
|
|
0a840d256b | ||
|
|
8a69c51cac | ||
|
|
0262fd40bc | ||
|
|
859dfe3544 | ||
|
|
9c624a2f79 | ||
|
|
dec1061b40 | ||
|
|
be4895faa6 | ||
|
|
3c9755f0dd | ||
|
|
61ed847d7d | ||
|
|
fc9094fea8 | ||
|
|
e36c0efc4b | ||
|
|
4158a14cff | ||
|
|
201a45cba0 | ||
|
|
ab31b9b253 | ||
|
|
780ddc79a7 | ||
|
|
d5a2adaed6 | ||
|
|
008ab3f068 | ||
|
|
7d926f588d | ||
|
|
707cd07876 | ||
|
|
f7ecb9090c | ||
|
|
a19e8a7315 | ||
|
|
cf189ecf81 | ||
|
|
4405b7adcb | ||
|
|
e51ee7b7b9 | ||
|
|
dfaa74f426 | ||
|
|
1d2ce59009 | ||
|
|
3602cc05bb | ||
|
|
ac88a81ac6 | ||
|
|
01fa8e0c63 | ||
|
|
322c06bfbe | ||
|
|
b68fceb362 | ||
|
|
59547e0f37 | ||
|
|
3ce100a9cf | ||
|
|
6513fb0cd5 | ||
|
|
fd0fb1f760 | ||
|
|
5db586c4d3 |
No files matched your search
@@ -87,6 +87,10 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
|
|||||||
# Pcap
|
# Pcap
|
||||||
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
||||||
|
|
||||||
|
# Databento (deps)
|
||||||
|
RUN apt-get update && apt-get -y install libzstd-dev
|
||||||
|
|
||||||
|
# Docker in Docker
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y \
|
||||||
docker.io \
|
docker.io \
|
||||||
docker-compose \
|
docker-compose \
|
||||||
|
|||||||
Vendored
+62
-5
@@ -16,11 +16,20 @@
|
|||||||
"program": "spark_api_server.exe",
|
"program": "spark_api_server.exe",
|
||||||
"args": [
|
"args": [
|
||||||
"run",
|
"run",
|
||||||
"--sparkd",
|
|
||||||
"http://127.0.0.1:8000/config"
|
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
},
|
},
|
||||||
|
"linux": {
|
||||||
|
"preLaunchTask": "Build Api Server (Debug)",
|
||||||
|
"program": "spark_api_server",
|
||||||
|
"args": [
|
||||||
|
"run",
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
},
|
||||||
|
"env": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||||
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -33,11 +42,19 @@
|
|||||||
"program": "spark_api_server.exe",
|
"program": "spark_api_server.exe",
|
||||||
"args": [
|
"args": [
|
||||||
"run",
|
"run",
|
||||||
"--sparkd",
|
|
||||||
"http://127.0.0.1:8000/config"
|
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
},
|
},
|
||||||
|
"linux": {
|
||||||
|
"program": "spark_api_server",
|
||||||
|
"args": [
|
||||||
|
"run",
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
},
|
||||||
|
"env": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||||
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -105,6 +122,18 @@
|
|||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "(gdb) Launch",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
|
"args": ["run"],
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
|
"environment": [],
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "DataStreamQueryAnalyzer",
|
"name": "DataStreamQueryAnalyzer",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
@@ -234,7 +263,35 @@
|
|||||||
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
||||||
},
|
},
|
||||||
"args": [
|
"args": [
|
||||||
"test-py-md-provider"
|
"run"
|
||||||
|
],
|
||||||
|
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
"setupCommands": [
|
||||||
|
{
|
||||||
|
"description": "Enable pretty-printing for gdb",
|
||||||
|
"text": "-enable-pretty-printing",
|
||||||
|
"ignoreFailures": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Set Disassembly Flavor to Intel",
|
||||||
|
"text": "-gdb-set disassembly-flavor intel",
|
||||||
|
"ignoreFailures": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Sparkd Web (Skip Build)",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"linux": {
|
||||||
|
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"web"
|
||||||
],
|
],
|
||||||
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
|
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
|
|||||||
+68
-74
@@ -1,5 +1,5 @@
|
|||||||
cmake_minimum_required(VERSION 3.12)
|
cmake_minimum_required(VERSION 3.12)
|
||||||
project(spark_root)
|
project(spark)
|
||||||
set(CMAKE_CXX_STANDARD 20)
|
set(CMAKE_CXX_STANDARD 20)
|
||||||
|
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
@@ -33,91 +33,85 @@ else()
|
|||||||
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
|
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
|
||||||
link_directories(${EXTERNALS_LIB})
|
|
||||||
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
||||||
if(DEFINED APP_DATA_DIR)
|
if(DEFINED APP_DATA_DIR)
|
||||||
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
||||||
endif()
|
endif()
|
||||||
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
||||||
if(WIN32)
|
|
||||||
# subdirs
|
|
||||||
add_subdirectory(src/lib_spark)
|
|
||||||
else()
|
|
||||||
if(BUILD_SPARKD)
|
|
||||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
|
||||||
# read by their code). We override some below.
|
|
||||||
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
|
||||||
|
|
||||||
# This allows us to override hardcoded exchange-urls with ones read from
|
if(BUILD_SPARKD)
|
||||||
# config (i.e., resolved at runtime rather than compile-time)
|
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||||
add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
|
# read by their code). We override some below.
|
||||||
|
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
||||||
|
|
||||||
# Enable ccapi services
|
# This allows us to override hardcoded exchange-urls with ones read from
|
||||||
add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
|
# config (i.e., resolved at runtime rather than compile-time)
|
||||||
add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
|
add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
|
||||||
# Enable ccapi exchanges
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
|
|
||||||
|
|
||||||
set(CCAPI_LOG_LEVEL
|
# Enable ccapi services
|
||||||
"Warn"
|
add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
|
||||||
CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
|
add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
|
||||||
string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
|
# Enable ccapi exchanges
|
||||||
add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
|
||||||
|
|
||||||
# ** Binance **
|
set(CCAPI_LOG_LEVEL
|
||||||
add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
|
"Warn"
|
||||||
add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
|
||||||
add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
|
||||||
|
add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
|
||||||
|
|
||||||
# ** Binance US **
|
# ** Binance **
|
||||||
add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
|
||||||
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
||||||
|
add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
||||||
|
|
||||||
# ** Binance USDS Futures **
|
# ** Binance US **
|
||||||
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
|
||||||
|
|
||||||
# ** Deribit **
|
# ** Binance USDS Futures **
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
|
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
||||||
add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
|
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
||||||
add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
|
|
||||||
add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
|
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
|
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
|
|
||||||
add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
|
|
||||||
add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
|
|
||||||
|
|
||||||
# ** Okx **
|
# ** Deribit **
|
||||||
add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
|
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
||||||
add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
|
add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
|
||||||
endif()
|
add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
|
||||||
|
add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
|
||||||
|
add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
|
||||||
|
add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
|
||||||
|
add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
|
||||||
|
add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
|
||||||
|
add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
|
||||||
|
|
||||||
# subdirs
|
# ** Okx **
|
||||||
if (BUILD_SPARKD)
|
add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
|
||||||
add_subdirectory(src/lib_system)
|
add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
|
||||||
add_subdirectory(src/lib_domain)
|
endif()
|
||||||
add_subdirectory(src/lib_python_md)
|
|
||||||
add_subdirectory(src/app_sparkd)
|
# subdirs
|
||||||
add_subdirectory(src/data_files)
|
add_subdirectory(src/lib_system)
|
||||||
endif()
|
add_subdirectory(src/lib_domain)
|
||||||
add_subdirectory(src/lib_spark)
|
add_subdirectory(src/lib_http)
|
||||||
if (BUILD_ADMIN_WEB_APP)
|
if (BUILD_SPARKD)
|
||||||
add_subdirectory(src/Spark.Admin.WebApp)
|
add_subdirectory(src/lib_python_md)
|
||||||
endif()
|
add_subdirectory(src/app_sparkd)
|
||||||
if(BUILD_DOCS)
|
add_subdirectory(src/data_files)
|
||||||
add_subdirectory(doc/user)
|
endif()
|
||||||
endif()
|
add_subdirectory(src/lib_spark)
|
||||||
if(BUILD_SDK)
|
if (BUILD_ADMIN_WEB_APP)
|
||||||
add_subdirectory(src/sdk)
|
add_subdirectory(src/Spark.Admin.WebApp)
|
||||||
endif()
|
endif()
|
||||||
|
if(BUILD_DOCS)
|
||||||
|
add_subdirectory(doc/user)
|
||||||
|
endif()
|
||||||
|
if(BUILD_SDK)
|
||||||
|
add_subdirectory(src/sdk)
|
||||||
endif()
|
endif()
|
||||||
+8
-8
@@ -28,7 +28,7 @@
|
|||||||
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
||||||
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
@@ -53,6 +53,7 @@
|
|||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
|
"CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
|
||||||
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
||||||
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
||||||
"BUILD_SDK": {
|
"BUILD_SDK": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
@@ -76,7 +77,7 @@
|
|||||||
"value": "ON"
|
"value": "ON"
|
||||||
},
|
},
|
||||||
"CCAPI_LOG_LEVEL": "Info",
|
"CCAPI_LOG_LEVEL": "Info",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -87,7 +88,7 @@
|
|||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_BUILD_TYPE": "Release",
|
"CMAKE_BUILD_TYPE": "Release",
|
||||||
"CCAPI_LOG_LEVEL": "Warn",
|
"CCAPI_LOG_LEVEL": "Warn",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -167,8 +168,8 @@
|
|||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
},
|
},
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -206,7 +207,7 @@
|
|||||||
"description": "Data Stream utils ported over from Edge",
|
"description": "Data Stream utils ported over from Edge",
|
||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -239,8 +240,7 @@
|
|||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"CONFIG_PROFILES": "default;data_stream",
|
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
|
|||||||
@@ -14,6 +14,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
|
||||||
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|x64 = Debug|x64
|
Debug|x64 = Debug|x64
|
||||||
@@ -70,6 +80,46 @@ Global
|
|||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
|
||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
||||||
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
macro(export_to_parent)
|
macro(export_to_parent)
|
||||||
foreach(var IN ITEMS ${ARGN})
|
foreach(var IN ITEMS ${ARGN})
|
||||||
set(${var} ${${var}} PARENT_SCOPE)
|
set(${var} ${${var}} PARENT_SCOPE)
|
||||||
|
|||||||
+103
-27
@@ -1,66 +1,142 @@
|
|||||||
include_guard(DIRECTORY)
|
include_guard(DIRECTORY)
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
|
include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
|
include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
|
||||||
|
|
||||||
function(init_library)
|
function(init_library)
|
||||||
set(_options SHARED STATIC)
|
set(_options SHARED STATIC)
|
||||||
set(_one_value_args SOURCE_DIR)
|
set(_one_value_args
|
||||||
|
NAME
|
||||||
|
SOURCE_DIR
|
||||||
|
INCLUDE_DIR
|
||||||
|
)
|
||||||
set(_multi_value_args)
|
set(_multi_value_args)
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
|
||||||
if (_ARGS_SHARED AND _ARGS_STATIC)
|
if (_ARGS_SHARED AND _ARGS_STATIC)
|
||||||
message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
|
message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
|
||||||
endif()
|
endif()
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
|
||||||
|
set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
|
||||||
|
|
||||||
set_ternary(_source_dir _ARGS_SOURCE_DIR _ARGS_SOURCE_DIR ${PROJECT_NAME})
|
# Cpp sources
|
||||||
# Glob cpp sources
|
|
||||||
file(
|
file(
|
||||||
GLOB_RECURSE ${PROJECT_NAME}_SOURCES
|
GLOB_RECURSE ${_name}_SOURCES
|
||||||
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
${_source_dir}/*.cpp
|
${_source_dir}/*.cpp
|
||||||
)
|
)
|
||||||
# Note: cpp files inside 'tests' dir(s) are considered tests.
|
# Note: cpp files inside 'tests' dir(s) are considered tests.
|
||||||
# Remove tests from main sources, and add them to test sources
|
# Remove tests from main sources, and add them to test sources
|
||||||
foreach(item ${${PROJECT_NAME}_SOURCES})
|
foreach(item ${${_name}_SOURCES})
|
||||||
if(${item} MATCHES "tests/*.cpp")
|
if(${item} MATCHES "tests/.*\.cpp$")
|
||||||
list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
|
list(APPEND ${_name}_TEST_SOURCES ${item})
|
||||||
list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
|
|
||||||
endif()
|
endif()
|
||||||
endforeach()
|
endforeach()
|
||||||
|
foreach(item ${${_name}_TEST_SOURCES})
|
||||||
|
list(REMOVE_ITEM ${_name}_SOURCES ${item})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
export_to_parent(
|
# Proto sources
|
||||||
${PROJECT_NAME}_SOURCES
|
file(
|
||||||
${PROJECT_NAME}_TEST_SOURCES
|
GLOB_RECURSE ${_name}_PROTO_SOURCES
|
||||||
|
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
${_source_dir}/protobufs/*.proto
|
||||||
|
)
|
||||||
|
update_deps_file("${${_name}_PROTO_SOURCES};")
|
||||||
|
protobuf_generate_cpp(
|
||||||
|
CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
|
||||||
|
CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
|
||||||
|
CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
|
||||||
|
PROTO_INCLUDE_DIRS
|
||||||
|
${EXTERNALS_INCLUDE}
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
PROTO_SOURCES
|
||||||
|
${${_name}_PROTO_SOURCES}
|
||||||
)
|
)
|
||||||
|
|
||||||
# Add main library
|
export_to_parent(
|
||||||
if (_ARGS_STATIC)
|
${_name}_SOURCES
|
||||||
add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES})
|
${_name}_TEST_SOURCES
|
||||||
elseif(_ARGS_SHARED)
|
${_name}_PROTO_SOURCES
|
||||||
add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
|
${_name}_CXX_PROTO_SOURCES
|
||||||
else()
|
${_name}_CXX_PROTO_HEADERS
|
||||||
add_library(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
|
)
|
||||||
endif()
|
|
||||||
|
|
||||||
|
# Add main lib
|
||||||
|
if (_ARGS_STATIC)
|
||||||
|
add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
|
||||||
|
elseif(_ARGS_SHARED)
|
||||||
|
add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
|
||||||
|
else()
|
||||||
|
add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
|
||||||
|
endif()
|
||||||
target_include_directories(
|
target_include_directories(
|
||||||
${PROJECT_NAME}
|
${_name}
|
||||||
|
SYSTEM
|
||||||
|
BEFORE
|
||||||
|
PUBLIC
|
||||||
|
$<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
|
||||||
|
$<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
|
||||||
|
)
|
||||||
|
target_include_directories(
|
||||||
|
${_name}
|
||||||
PUBLIC
|
PUBLIC
|
||||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
||||||
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
||||||
)
|
)
|
||||||
|
# Include protos (if any)
|
||||||
|
if (${_name}_PROTO_SOURCES)
|
||||||
|
target_include_directories(
|
||||||
|
${_name}
|
||||||
|
PUBLIC
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
|
||||||
|
)
|
||||||
|
endif()
|
||||||
target_link_directories(
|
target_link_directories(
|
||||||
${PROJECT_NAME}
|
${_name}
|
||||||
PUBLIC
|
PUBLIC
|
||||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
|
||||||
|
$<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
|
||||||
$<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
|
$<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
|
||||||
)
|
)
|
||||||
|
# Install main lib
|
||||||
|
foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
|
||||||
|
file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
|
||||||
|
install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
|
||||||
|
endforeach()
|
||||||
|
install_basic(
|
||||||
|
NAME ${_name}
|
||||||
|
INCLUDE_DIR ${_include_dir}
|
||||||
|
)
|
||||||
|
|
||||||
# Add tests executable (if there are any)
|
# Add tests executable (if there are any)
|
||||||
if (${PROJECT_NAME}_TEST_SOURCES)
|
if (${_name}_TEST_SOURCES)
|
||||||
add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
|
add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
|
||||||
target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
|
|
||||||
|
get_target_property(_lib_type ${_name} TYPE)
|
||||||
|
if (_lib_type STREQUAL "SHARED_LIBRARY")
|
||||||
|
# Set RPATH so the tests exe knows where to find the shared lib.
|
||||||
|
set_target_properties(
|
||||||
|
${_name}-tests
|
||||||
|
PROPERTIES
|
||||||
|
BUILD_RPATH "$ORIGIN"
|
||||||
|
INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Link main lib
|
||||||
|
target_link_directories(
|
||||||
|
${_name}-tests
|
||||||
|
PRIVATE
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
|
||||||
|
$<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
|
||||||
|
)
|
||||||
|
target_link_libraries(${_name}-tests PRIVATE ${_name})
|
||||||
|
|
||||||
# Catch2
|
# Catch2
|
||||||
find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
|
find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
|
||||||
target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
|
target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
|
||||||
endif()
|
endif()
|
||||||
endfunction()
|
endfunction()
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
function(init_shared_library)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
|
|
||||||
set(source_dir)
|
|
||||||
if (${ARGC} EQUAL 0)
|
|
||||||
set(source_dir ${PROJECT_NAME})
|
|
||||||
elseif(${ARGC} GREATER 1)
|
|
||||||
message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
|
|
||||||
else()
|
|
||||||
set(source_dir ${ARGV})
|
|
||||||
endif()
|
|
||||||
|
|
||||||
# Glob cpp sources
|
|
||||||
file(
|
|
||||||
GLOB_RECURSE ${PROJECT_NAME}_SOURCES
|
|
||||||
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
|
||||||
${source_dir}/*.cpp
|
|
||||||
)
|
|
||||||
# Note: cpp files inside 'tests' dir(s) are considered tests.
|
|
||||||
# Remove tests from main sources, and add them to test sources
|
|
||||||
foreach(item ${${PROJECT_NAME}_SOURCES})
|
|
||||||
if(${item} MATCHES "tests/*.cpp")
|
|
||||||
list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
|
|
||||||
list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
|
|
||||||
endif()
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
export_to_parent(
|
|
||||||
${PROJECT_NAME}_SOURCES
|
|
||||||
${PROJECT_NAME}_TEST_SOURCES
|
|
||||||
)
|
|
||||||
|
|
||||||
# Add main library
|
|
||||||
add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
|
|
||||||
target_include_directories(
|
|
||||||
${PROJECT_NAME}
|
|
||||||
PUBLIC
|
|
||||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
|
||||||
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
|
||||||
)
|
|
||||||
target_link_directories(
|
|
||||||
${PROJECT_NAME}
|
|
||||||
PUBLIC
|
|
||||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
|
|
||||||
$<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
|
|
||||||
)
|
|
||||||
|
|
||||||
# Add tests executable (if there are any)
|
|
||||||
if (${PROJECT_NAME}_TEST_SOURCES)
|
|
||||||
add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
|
|
||||||
target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
|
|
||||||
# Catch2
|
|
||||||
find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
|
|
||||||
target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
|
|
||||||
endif()
|
|
||||||
endfunction(init_shared_library)
|
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
||||||
|
|
||||||
|
function(install_basic)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args NAME INCLUDE_DIR)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
|
||||||
|
|
||||||
|
file(
|
||||||
|
GLOB_RECURSE ${_name}_HEADERS
|
||||||
|
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
${_include_dir}/*.h)
|
||||||
|
|
||||||
|
target_sources(${_name}
|
||||||
|
PUBLIC
|
||||||
|
FILE_SET HEADERS FILES ${${_name}_HEADERS})
|
||||||
|
|
||||||
|
install(
|
||||||
|
TARGETS ${_name}
|
||||||
|
EXPORT ${_name}Targets
|
||||||
|
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||||
|
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||||
|
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||||
|
FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
|
||||||
|
)
|
||||||
|
|
||||||
|
install(
|
||||||
|
EXPORT ${_name}Targets
|
||||||
|
FILE ${_name}Targets.cmake
|
||||||
|
NAMESPACE ${_name}::
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
|
||||||
|
|
||||||
|
pack_basic()
|
||||||
|
endfunction()
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
|
|
||||||
include(GNUInstallDirs)
|
|
||||||
|
|
||||||
function(install_basic_config)
|
|
||||||
file(
|
|
||||||
GLOB_RECURSE ${PROJECT_NAME}_HEADERS
|
|
||||||
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
|
||||||
*.h)
|
|
||||||
|
|
||||||
target_sources(${PROJECT_NAME}
|
|
||||||
PUBLIC
|
|
||||||
FILE_SET HEADERS FILES ${${PROJECT_NAME}_HEADERS})
|
|
||||||
|
|
||||||
install(
|
|
||||||
TARGETS ${PROJECT_NAME}
|
|
||||||
EXPORT ${PROJECT_NAME}Targets
|
|
||||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
|
||||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
|
||||||
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
|
||||||
FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
|
|
||||||
)
|
|
||||||
|
|
||||||
install(
|
|
||||||
EXPORT ${PROJECT_NAME}Targets
|
|
||||||
FILE ${PROJECT_NAME}Targets.cmake
|
|
||||||
NAMESPACE ${PROJECT_NAME}::
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
|
|
||||||
|
|
||||||
pack_basic()
|
|
||||||
endfunction()
|
|
||||||
+14
-6
@@ -1,6 +1,14 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
include(CMakePackageConfigHelpers)
|
include(CMakePackageConfigHelpers)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
||||||
|
|
||||||
function(pack_basic)
|
function(pack_basic)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args NAME)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
|
||||||
find_package(Git QUIET)
|
find_package(Git QUIET)
|
||||||
|
|
||||||
if(GIT_FOUND)
|
if(GIT_FOUND)
|
||||||
@@ -17,20 +25,20 @@ function(pack_basic)
|
|||||||
endif()
|
endif()
|
||||||
|
|
||||||
write_basic_package_version_file(
|
write_basic_package_version_file(
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
|
"${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
|
||||||
VERSION ${PROJECT_VERSION}
|
VERSION ${PROJECT_VERSION}
|
||||||
COMPATIBILITY AnyNewerVersion
|
COMPATIBILITY AnyNewerVersion
|
||||||
)
|
)
|
||||||
|
|
||||||
configure_package_config_file(
|
configure_package_config_file(
|
||||||
"${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
|
"${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
|
"${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
|
||||||
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
|
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
||||||
)
|
)
|
||||||
|
|
||||||
install(
|
install(
|
||||||
FILES
|
FILES
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
|
"${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
|
"${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
|
||||||
endfunction()
|
endfunction()
|
||||||
@@ -2,8 +2,16 @@ include_guard(GLOBAL)
|
|||||||
|
|
||||||
function(protobuf_generate_cpp)
|
function(protobuf_generate_cpp)
|
||||||
set(_options)
|
set(_options)
|
||||||
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR PROTOC)
|
set(_one_value_args
|
||||||
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
CXX_SOURCES
|
||||||
|
CXX_HEADERS
|
||||||
|
CXX_OUT_DIR
|
||||||
|
PROTOC
|
||||||
|
)
|
||||||
|
set(_multi_value_args
|
||||||
|
PROTO_INCLUDE_DIRS
|
||||||
|
PROTO_SOURCES
|
||||||
|
)
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
|
||||||
if (NOT DEFINED _ARGS_PROTOC)
|
if (NOT DEFINED _ARGS_PROTOC)
|
||||||
@@ -26,6 +34,7 @@ function(protobuf_generate_cpp)
|
|||||||
endif()
|
endif()
|
||||||
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
|
||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
||||||
|
message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT
|
OUTPUT
|
||||||
"${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
|
||||||
@@ -33,7 +42,7 @@ function(protobuf_generate_cpp)
|
|||||||
COMMAND
|
COMMAND
|
||||||
${_ARGS_PROTOC}
|
${_ARGS_PROTOC}
|
||||||
ARGS
|
ARGS
|
||||||
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
|
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
|
||||||
DEPENDS ${_abs_fil}
|
DEPENDS ${_abs_fil}
|
||||||
COMMENT "Running C++ protocol buffer compiler on ${_fil}"
|
COMMENT "Running C++ protocol buffer compiler on ${_fil}"
|
||||||
VERBATIM
|
VERBATIM
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
function(protobuf_generate_cpp_config)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args
|
||||||
|
PROTO_SOURCES_HEADER
|
||||||
|
PROTO_DEFAULTS_HEADER
|
||||||
|
)
|
||||||
|
set(_multi_value_args
|
||||||
|
PROTO_SOURCES
|
||||||
|
PROTO_DEFAULTS
|
||||||
|
)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
|
||||||
|
# Configure the 'config_proto_files' header. It provides an array of strings populated with the
|
||||||
|
# files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
|
||||||
|
# message with fields for each message defined in said files. A global instance of this 'config'
|
||||||
|
# proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
|
||||||
|
# (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
|
||||||
|
# associated with type `T` in this global instance.
|
||||||
|
message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
|
||||||
|
message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
|
||||||
|
message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
|
||||||
|
if (_ARGS_PROTO_SOURCES_HEADER)
|
||||||
|
set(PROTO_SOURCES)
|
||||||
|
set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
|
||||||
|
foreach(_proto_path ${_ARGS_PROTO_SOURCES})
|
||||||
|
list(APPEND PROTO_SOURCES ${_proto_path})
|
||||||
|
string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
|
||||||
|
endforeach()
|
||||||
|
list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
|
||||||
|
message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
|
||||||
|
configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Generate static defaults. Ideally, this would be a single string, but we use an array because
|
||||||
|
# the compiler complains if the string is longer than 65k.
|
||||||
|
set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
|
||||||
|
|
||||||
|
foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
|
||||||
|
file(READ ${_cfg_file_path} _cfg_file_content)
|
||||||
|
string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
|
||||||
|
string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
|
||||||
|
|
||||||
|
foreach(_line ${_cfg_lines})
|
||||||
|
string(APPEND _cfg_defaults_str "\n \"${_line}\",")
|
||||||
|
endforeach(_line)
|
||||||
|
endforeach(_cfg_file_path)
|
||||||
|
|
||||||
|
string(APPEND _cfg_defaults_str "};")
|
||||||
|
set(_old_cfg_defaults_str "")
|
||||||
|
|
||||||
|
if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
|
||||||
|
file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
|
||||||
|
file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
|
macro(set_first_defined RESULT)
|
||||||
|
foreach(_arg ${ARGN})
|
||||||
|
if (DEFINED ${_arg})
|
||||||
|
set(${RESULT} ${${_arg}})
|
||||||
|
break()
|
||||||
|
endif()
|
||||||
|
endforeach()
|
||||||
|
endmacro()
|
||||||
+6
-8
@@ -20,16 +20,14 @@ endfunction()
|
|||||||
|
|
||||||
function(sym_link_file)
|
function(sym_link_file)
|
||||||
set(_options)
|
set(_options)
|
||||||
set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
|
set(_one_value_args SOURCE DEST INSTALL_DEST)
|
||||||
set(_multi_value_args)
|
set(_multi_value_args)
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
add_custom_command(
|
file(
|
||||||
OUTPUT ${_ARGS_DEST}
|
CREATE_LINK
|
||||||
COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
|
${_ARGS_SOURCE}
|
||||||
)
|
${_ARGS_DEST}
|
||||||
add_custom_target(
|
SYMBOLIC
|
||||||
${_ARGS_TARGET_NAME} ALL
|
|
||||||
DEPENDS ${_ARGS_DEST}
|
|
||||||
)
|
)
|
||||||
if (DEFINED _ARGS_INSTALL_DEST)
|
if (DEFINED _ARGS_INSTALL_DEST)
|
||||||
install(
|
install(
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
|
||||||
|
namespace bts::@PROJECT_NAME@::protobufs {
|
||||||
|
|
||||||
|
inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
|
||||||
|
@PROTO_SOURCES_CXX_ARRAY_ITEMS@
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace bts::@PROJECT_NAME@::protobufs
|
||||||
+20
-16
@@ -1,15 +1,24 @@
|
|||||||
services:
|
services:
|
||||||
sparkd:
|
sparkd:
|
||||||
build:
|
build:
|
||||||
dockerfile: src/server/sparkd/Dockerfile
|
dockerfile: src/app_sparkd/Dockerfile
|
||||||
|
args:
|
||||||
|
- SOURCE_DIR=install/sparkd-release-gcc
|
||||||
|
- CONF_FILE=src/app_sparkd/spark.conf
|
||||||
command: [ "./sparkd", "run" ]
|
command: [ "./sparkd", "run" ]
|
||||||
ports: [ "8000:8000", "8001:8001", "8002:8002" ]
|
ports:
|
||||||
|
- 8000:8000
|
||||||
|
- 8001:8001
|
||||||
|
- 8002:8002
|
||||||
|
|
||||||
pubsub:
|
pubsub:
|
||||||
environment:
|
environment:
|
||||||
- DOTNET_ENVIRONMENT=Staging
|
- DOTNET_ENVIRONMENT=Staging
|
||||||
build:
|
build:
|
||||||
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
|
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
|
||||||
|
args:
|
||||||
|
- SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||||
|
- CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
|
||||||
ports:
|
ports:
|
||||||
- 22019:22019
|
- 22019:22019
|
||||||
command: [ "./PubSubServer" ]
|
command: [ "./PubSubServer" ]
|
||||||
@@ -17,20 +26,13 @@ services:
|
|||||||
md-publisher:
|
md-publisher:
|
||||||
build:
|
build:
|
||||||
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
|
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
|
||||||
|
args:
|
||||||
|
- SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||||
|
- CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
|
||||||
depends_on:
|
depends_on:
|
||||||
- pubsub
|
- pubsub
|
||||||
- sparkd
|
- sparkd
|
||||||
command:
|
command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
|
||||||
[
|
|
||||||
"dockerize",
|
|
||||||
"-wait",
|
|
||||||
"tcp://pubsub:22019",
|
|
||||||
"-wait",
|
|
||||||
"tcp://sparkd:8000",
|
|
||||||
"-timeout",
|
|
||||||
"10s",
|
|
||||||
"./DataStreamPublisher"
|
|
||||||
]
|
|
||||||
environment:
|
environment:
|
||||||
- DOTNET_ENVIRONMENT=MarketData
|
- DOTNET_ENVIRONMENT=MarketData
|
||||||
|
|
||||||
@@ -40,13 +42,15 @@ services:
|
|||||||
- 8443:8443
|
- 8443:8443
|
||||||
build:
|
build:
|
||||||
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
|
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
|
||||||
|
args:
|
||||||
|
- SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||||
|
- WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
|
||||||
|
- CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
|
||||||
command: [ "./Bts.DataStreamWebApp.Server" ]
|
command: [ "./Bts.DataStreamWebApp.Server" ]
|
||||||
environment:
|
environment:
|
||||||
- ASPNETCORE_ENVIRONMENT=Staging
|
- ASPNETCORE_ENVIRONMENT=Staging
|
||||||
|
|
||||||
api-server:
|
api-server:
|
||||||
profiles:
|
|
||||||
- api-server
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- sparkd
|
- sparkd
|
||||||
ports:
|
ports:
|
||||||
@@ -54,7 +58,7 @@ services:
|
|||||||
- 9443:9443
|
- 9443:9443
|
||||||
build:
|
build:
|
||||||
dockerfile: src/client/spark_api_server/Dockerfile
|
dockerfile: src/client/spark_api_server/Dockerfile
|
||||||
command: [ "./spark_api_server", "run" ]
|
command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
|
||||||
environment:
|
environment:
|
||||||
- ASPNETCORE_ENVIRONMENT=Staging
|
- ASPNETCORE_ENVIRONMENT=Staging
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ RUN locale-gen en_US.UTF-8
|
|||||||
ARG docker_user="bts"
|
ARG docker_user="bts"
|
||||||
ARG docker_uid="1000"
|
ARG docker_uid="1000"
|
||||||
ARG docker_pass="bts"
|
ARG docker_pass="bts"
|
||||||
# ARG entrypoint
|
|
||||||
|
|
||||||
RUN \
|
RUN \
|
||||||
groupadd -g "$docker_uid" "$docker_user" \
|
groupadd -g "$docker_uid" "$docker_user" \
|
||||||
|
|||||||
+1
-1
Submodule externals updated: cae5fd20da...4f5e385556.
@@ -1 +1 @@
|
|||||||
VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
|
VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
|
||||||
@@ -1 +1 @@
|
|||||||
VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
|
VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
module.exports = {
|
|
||||||
root: true,
|
|
||||||
env: { browser: true, es2020: true },
|
|
||||||
extends: [
|
|
||||||
'eslint:recommended',
|
|
||||||
'plugin:react/recommended',
|
|
||||||
'plugin:react/jsx-runtime',
|
|
||||||
'plugin:react-hooks/recommended',
|
|
||||||
],
|
|
||||||
ignorePatterns: ['dist', '.eslintrc.cjs'],
|
|
||||||
parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
|
|
||||||
settings: { react: { version: '18.3.1' } },
|
|
||||||
plugins: ['react-refresh'],
|
|
||||||
rules: {
|
|
||||||
"import/order": [
|
|
||||||
"error",
|
|
||||||
{
|
|
||||||
"groups": [
|
|
||||||
"builtin",
|
|
||||||
"external",
|
|
||||||
"internal",
|
|
||||||
["parent", "sibling", "index"]
|
|
||||||
],
|
|
||||||
"newlines-between": "always"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
'react-refresh/only-export-components': [
|
|
||||||
'warn',
|
|
||||||
{ allowConstantExport: true },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import importPlugin from "eslint-plugin-import";
|
||||||
|
import reactPkg from "eslint-plugin-react";
|
||||||
|
const { configs: reactConfigs } = reactPkg;
|
||||||
|
import js from "@eslint/js";
|
||||||
|
import defineConfig from "./vite.config.js";
|
||||||
|
|
||||||
|
export default [
|
||||||
|
js.configs.recommended,
|
||||||
|
importPlugin.flatConfigs.recommended,
|
||||||
|
reactConfigs.flat.recommended,
|
||||||
|
reactConfigs.flat["jsx-runtime"],
|
||||||
|
{
|
||||||
|
files: ["**/*.{js,jsx}"],
|
||||||
|
languageOptions: {
|
||||||
|
ecmaVersion: "latest",
|
||||||
|
sourceType: "module",
|
||||||
|
globals: {
|
||||||
|
console: "readonly"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
settings: {
|
||||||
|
"import/resolver": {
|
||||||
|
vite: {
|
||||||
|
viteConfig: defineConfig({ mode: "development" })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
rules: {
|
||||||
|
semi: "error",
|
||||||
|
"prefer-const": "error",
|
||||||
|
"no-unused-vars": "warn",
|
||||||
|
"import/no-dynamic-require": "warn",
|
||||||
|
"import/no-nodejs-modules": "warn",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
];
|
||||||
+1303
-1955
File diff suppressed because it is too large.
Load diff
@@ -10,35 +10,37 @@
|
|||||||
"preview": "vite preview"
|
"preview": "vite preview"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@angular/material": "^18.0.4",
|
"@angular/material": "^18.2.8",
|
||||||
"@emotion/react": "^11.11.4",
|
"@emotion/react": "^11.13.3",
|
||||||
"@emotion/styled": "^11.11.5",
|
"@emotion/styled": "^11.13.0",
|
||||||
"@fontsource/noto-sans": "^5.0.22",
|
"@fontsource/noto-sans": "^5.1.0",
|
||||||
"@material/data-table": "^15.0.0-canary.7f224ddd4.0",
|
"@material/data-table": "^15.0.0-canary.423edc3dc.0",
|
||||||
"@mui/icons-material": "^5.15.20",
|
"@mui/icons-material": "^6.1.4",
|
||||||
"@mui/material": "^5.15.20",
|
"@mui/material": "^6.1.4",
|
||||||
"@mui/x-data-grid": "^7.7.1",
|
"@mui/x-data-grid": "^7.20.0",
|
||||||
"bulma": "^1.0.1",
|
"bulma": "^1.0.2",
|
||||||
"classnames": "^2.5.1",
|
"classnames": "^2.5.1",
|
||||||
"react": "^18.3.1",
|
"react": "^18.3.1",
|
||||||
"react-data-grid": "^7.0.0-beta.44",
|
"react-data-grid": "^7.0.0-beta.47",
|
||||||
"react-data-table-component": "^7.6.2",
|
"react-data-table-component": "^7.6.2",
|
||||||
"react-dom": "^18.3.1",
|
"react-dom": "^18.3.1",
|
||||||
"react-json-view-lite": "^1.4.0",
|
"react-json-view-lite": "^1.5.0",
|
||||||
"react-pro-sidebar": "^1.1.0",
|
"react-pro-sidebar": "^1.1.0",
|
||||||
"react-use-websocket": "^4.8.1",
|
"react-use-websocket": "^4.9.0",
|
||||||
"rpc-websockets": "^7.11.2"
|
"rpc-websockets": "^9.0.4"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/react": "^18.3.3",
|
"@types/react": "^18.3.11",
|
||||||
"@types/react-dom": "^18.3.0",
|
"@types/react-dom": "^18.3.1",
|
||||||
"@vitejs/plugin-react": "^4.3.1",
|
"@vitejs/plugin-react": "^4.3.2",
|
||||||
"eslint": "^8.57.0",
|
"eslint": "^9.12.0",
|
||||||
"eslint-plugin-import": "^2.29.1",
|
"@eslint/js": "^9.12.0",
|
||||||
"eslint-plugin-react": "^7.34.2",
|
"eslint-plugin-import": "^2.31.0",
|
||||||
"eslint-plugin-react-hooks": "^4.6.2",
|
"eslint-plugin-react": "^7.37.1",
|
||||||
"eslint-plugin-react-refresh": "^0.4.7",
|
"eslint-plugin-react-hooks": "^5.0.0",
|
||||||
"react-mui-sidebar": "^1.3.5",
|
"eslint-plugin-react-refresh": "^0.4.12",
|
||||||
"vite": "^5.2.13"
|
"eslint-import-resolver-vite": "^2.0.1",
|
||||||
|
"react-mui-sidebar": "^1.3.8",
|
||||||
|
"vite": "^5.4.9"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { fileURLToPath, URL } from 'node:url';
|
import { fileURLToPath, URL } from 'node:url';
|
||||||
import { defineConfig, loadEnv } from 'vite';
|
import { defineConfig } from 'vite';
|
||||||
import plugin from '@vitejs/plugin-react';
|
import plugin from '@vitejs/plugin-react';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|||||||
@@ -1,7 +1,13 @@
|
|||||||
FROM bluetradingsystems/linux_dotnet7
|
FROM bluetradingsystems/linux_dotnet7
|
||||||
|
|
||||||
|
ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
|
||||||
|
ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
|
||||||
|
ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
|
||||||
|
ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
|
||||||
|
|
||||||
RUN dotnet dev-certs https --trust
|
RUN dotnet dev-certs https --trust
|
||||||
COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
|
COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
|
||||||
COPY --chown=bts:bts "build/Bts.DataStreamWebApp.Client/production/dist" "bin/wwwroot"
|
COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
|
||||||
COPY --chown=bts:bts "src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
|
COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
|
||||||
|
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
|
||||||
WORKDIR "/home/bts/bin"
|
WORKDIR "/home/bts/bin"
|
||||||
@@ -1,14 +1,4 @@
|
|||||||
{
|
{
|
||||||
"Kestrel": {
|
|
||||||
"Endpoints": {
|
|
||||||
"Http": {
|
|
||||||
"Url": "http://localhost:8080"
|
|
||||||
},
|
|
||||||
"Https": {
|
|
||||||
"Url": "https://localhost:8443"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"PubSub": {
|
"PubSub": {
|
||||||
"Routing": {
|
"Routing": {
|
||||||
"PublisherRoutes": [
|
"PublisherRoutes": [
|
||||||
|
|||||||
@@ -30,5 +30,15 @@
|
|||||||
"benzinga_news"
|
"benzinga_news"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
|
"Spark": {
|
||||||
|
"ConfigServer": "http://sparkd:8000/config"
|
||||||
|
},
|
||||||
|
"SparkApi": {
|
||||||
|
"IsEnabled": true,
|
||||||
|
"Credentials": {
|
||||||
|
"User": "bts",
|
||||||
|
"Password": ""
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
FROM bluetradingsystems/linux_dotnet7
|
|
||||||
|
|
||||||
RUN dotnet dev-certs https --trust
|
|
||||||
COPY --chown=bts:bts "bin" "bin"
|
|
||||||
COPY --chown=bts:bts "webapp.spark.conf" ".local/share/DataStream/conf/spark.conf"
|
|
||||||
WORKDIR "/home/bts/bin"
|
|
||||||
@@ -18,8 +18,11 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
|
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
|
||||||
|
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
|
||||||
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
||||||
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
||||||
|
<PackageReference Include="Parquet.Net" Version="5.0.1" />
|
||||||
|
<PackageReference Include="System.Data.SQLite" Version="1.0.118" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
@@ -11,6 +11,9 @@ public static class DataStreamExtensions {
|
|||||||
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
||||||
source as ViewStream ?? new ObservableViewStream(source);
|
source as ViewStream ?? new ObservableViewStream(source);
|
||||||
|
|
||||||
|
public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source) where T : IBlueRow =>
|
||||||
|
source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)));
|
||||||
|
|
||||||
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
||||||
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using System;
|
||||||
|
using BCG;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using Bts.DataStreamExpressions;
|
||||||
|
|
||||||
|
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
|
||||||
|
// So for example, if an expresison is passed
|
||||||
|
[DataStreamUserRespresentation("dsl")]
|
||||||
|
public class DSLStream : ViewStream {
|
||||||
|
private readonly string _dsl;
|
||||||
|
public DSLStream(string dsl) {
|
||||||
|
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
|
||||||
|
_dsl = dsl;
|
||||||
|
ViewStream rs = build_stream(dsl);
|
||||||
|
rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
|
||||||
|
static ViewStream? build_stream(string stream_expression) {
|
||||||
|
try {
|
||||||
|
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||||
|
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||||
|
return rv;
|
||||||
|
|
||||||
|
} catch (Exception ex) {
|
||||||
|
Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,12 +25,12 @@ namespace BCG {
|
|||||||
byte[] rem_req_bytes = rem_req.serialize();
|
byte[] rem_req_bytes = rem_req.serialize();
|
||||||
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
|
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
|
||||||
_source_str = new TableStream(_name);
|
_source_str = new TableStream(_name);
|
||||||
_source_str.onChange += new ChangeDelegate(_handle_data);
|
_source_str.onChange += _handle_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
//maybe there should be a way to turn remote requests off ?
|
//maybe there should be a way to turn remote requests off ?
|
||||||
_source_str.onChange -= new ChangeDelegate(_handle_data);
|
_source_str.onChange -= _handle_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
static string get_expression(params string[] lines) {
|
static string get_expression(params string[] lines) {
|
||||||
|
|||||||
@@ -21,13 +21,11 @@ namespace BCG
|
|||||||
: this (new TableStream (tableName))
|
: this (new TableStream (tableName))
|
||||||
{}
|
{}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber () {
|
protected override void myOnFirstSubscriber () =>
|
||||||
myVS.onChange += new ChangeDelegate(myHandleChanges);
|
myVS.onChange += myHandleChanges;
|
||||||
}
|
|
||||||
protected override void myOnLastSubscriber () {
|
|
||||||
myVS.onChange -= new ChangeDelegate(myHandleChanges);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber () =>
|
||||||
|
myVS.onChange -= myHandleChanges;
|
||||||
|
|
||||||
private void myHandleChanges(object newValue)
|
private void myHandleChanges(object newValue)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
|
myToCrush.onChange += myToCrush_onChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
|
|||||||
@@ -15,11 +15,11 @@ namespace BCG
|
|||||||
|
|
||||||
//set up an anonymous delegate for each of the streams
|
//set up an anonymous delegate for each of the streams
|
||||||
//we do this so that we can unsubscribe
|
//we do this so that we can unsubscribe
|
||||||
myDelegates = new ChangeDelegate[myToCrush.Length];
|
myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
|
||||||
for (int i = 0; i < myToCrush.Length; ++i)
|
for (int i = 0; i < myToCrush.Length; ++i)
|
||||||
{
|
{
|
||||||
int tempI = i;
|
int tempI = i;
|
||||||
myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
|
myDelegates[tempI] = new((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -27,7 +27,7 @@ namespace BCG
|
|||||||
private ViewStream[] myToCrush;
|
private ViewStream[] myToCrush;
|
||||||
private ITableCrusher myCrusher;
|
private ITableCrusher myCrusher;
|
||||||
|
|
||||||
private ChangeDelegate[] myDelegates;
|
private DataStreamChangeDelegate[] myDelegates;
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myDataSource.onChange += new ChangeDelegate(myDataSource_onChange);
|
myDataSource.onChange += myDataSource_onChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
|
|||||||
@@ -1,49 +0,0 @@
|
|||||||
// using System;
|
|
||||||
// using System.Collections.Generic;
|
|
||||||
// using System.Text;
|
|
||||||
// using System.IO;
|
|
||||||
|
|
||||||
// using System.Runtime.Serialization.Formatters.Binary;
|
|
||||||
|
|
||||||
// namespace BCG
|
|
||||||
// {
|
|
||||||
// public class DumpLoaderStream : ViewStream, ISideEffectStream
|
|
||||||
// {
|
|
||||||
// //wtf
|
|
||||||
// //public class Exception : System.Exception
|
|
||||||
// //{public Exception(string msg) : base(msg) { }}
|
|
||||||
|
|
||||||
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
|
|
||||||
// {
|
|
||||||
// mySetRealSource(
|
|
||||||
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
|
||||||
// ViewStreamDatabase.instance));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public DumpLoaderStream(string fileName)
|
|
||||||
// {
|
|
||||||
// if (fileName == null)
|
|
||||||
// throw new LoaderStream.Exception("File name cannot be null");
|
|
||||||
|
|
||||||
// for (int i = 0; i < 5; i++)
|
|
||||||
// try
|
|
||||||
// {
|
|
||||||
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
|
||||||
// {
|
|
||||||
// IClientDeserializer deser = new ClientUniversalDeserializer();
|
|
||||||
// byte[] the_ops;
|
|
||||||
// Array the_rows;
|
|
||||||
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
|
|
||||||
|
|
||||||
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
|
|
||||||
// }
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// catch (IOException)
|
|
||||||
// {
|
|
||||||
// System.Threading.Thread.Sleep(50);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
|
// This is the other side of the SaverStream, its "loads" the stream and fires once
|
||||||
[DataStreamUserRespresentation("loader")]
|
[DataStreamUserRespresentation("loader")]
|
||||||
public class LoaderStream : ViewStream {
|
public class LoaderStream : ViewStream {
|
||||||
|
|
||||||
|
|||||||
@@ -5,10 +5,11 @@ using System.Text;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using bts;
|
using BCG;
|
||||||
|
|
||||||
namespace BCG {
|
namespace bts {
|
||||||
// Writes all the udpates to a stream in a file.
|
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
|
||||||
|
// which just writes one update, as a create)
|
||||||
// File naming:
|
// File naming:
|
||||||
// token_epoch_yyyyMMdd_HHmmss
|
// token_epoch_yyyyMMdd_HHmmss
|
||||||
// File location: directory(passed)\token\
|
// File location: directory(passed)\token\
|
||||||
@@ -94,10 +95,10 @@ namespace BCG {
|
|||||||
void _on_change(object new_value) {
|
void _on_change(object new_value) {
|
||||||
DataPacket dp = new_value as DataPacket;
|
DataPacket dp = new_value as DataPacket;
|
||||||
byte[] serialized_update = null;
|
byte[] serialized_update = null;
|
||||||
bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
|
bool first_write = _serializer == null || _n_updates_in_current_file == 0;
|
||||||
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
||||||
if (_write_create) {
|
if (first_write) {
|
||||||
// new files always starts with create
|
// first file always starts with create
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
||||||
} else
|
} else
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
||||||
@@ -105,8 +106,8 @@ namespace BCG {
|
|||||||
long utc_epoch = epoch_now();
|
long utc_epoch = epoch_now();
|
||||||
|
|
||||||
//time, length of data, data
|
//time, length of data, data
|
||||||
bw.Write(utc_epoch);
|
//bw.Write(utc_epoch);
|
||||||
bw.Write(serialized_update.Length);
|
//bw.Write(serialized_update.Length);
|
||||||
bw.Write(serialized_update);
|
bw.Write(serialized_update);
|
||||||
_out_fs.Flush();
|
_out_fs.Flush();
|
||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
|
|||||||
@@ -1,190 +0,0 @@
|
|||||||
// using System;
|
|
||||||
// using System.Collections.Generic;
|
|
||||||
// using System.Linq;
|
|
||||||
// using System.Text;
|
|
||||||
// using System.IO;
|
|
||||||
// using System.Runtime.Serialization.Formatters.Binary;
|
|
||||||
// using BCG.Utilities.General;
|
|
||||||
// using bts.collections;
|
|
||||||
// using System.Reflection.Metadata;
|
|
||||||
|
|
||||||
|
|
||||||
// namespace BCG {
|
|
||||||
|
|
||||||
// public class PlaybackStream : ViewStream, IWorker {
|
|
||||||
// private readonly PlaybackType _type;
|
|
||||||
// private BlueTable _current_table;
|
|
||||||
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
|
||||||
// // these are the packets already downloaded from the current file
|
|
||||||
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
|
|
||||||
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
|
|
||||||
// private IEnumerable<string> _file_names;
|
|
||||||
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
|
|
||||||
// private DateTime _playback_t0, _playback_t1;
|
|
||||||
// private bool _all_files_read = false;
|
|
||||||
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
|
|
||||||
|
|
||||||
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
|
|
||||||
// _type = PlaybackType.Streaming;
|
|
||||||
// _playback_t0 = PersistorStream.parse_datetime(datetime_start);
|
|
||||||
// _playback_t1 = PersistorStream.parse_datetime(datetime_end);
|
|
||||||
// string token_directory_name = $"{directory_name}\\{token}";
|
|
||||||
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
|
|
||||||
// List<string> files_in_range = new List<string>();
|
|
||||||
// if (Directory.Exists(token_directory_name)) {
|
|
||||||
// _file_names =
|
|
||||||
// Directory.GetFiles(token_directory_name).Where(f => {
|
|
||||||
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
|
|
||||||
// return file_time > _playback_t0 && file_time < _playback_t1;
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
|
||||||
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
|
|
||||||
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
|
|
||||||
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
|
|
||||||
// if (snapshot_packet != null) {
|
|
||||||
// foreach (PlaybackPacket p in _snapshot_data.packets) {
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
|
||||||
// byte op = ops[i];
|
|
||||||
// if (op == StreamOpCode.create) {
|
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
// if (p.idx == snapshot_packet.idx) {
|
|
||||||
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
|
||||||
// private void _load_packets_from_current_file() {
|
|
||||||
// int curr_idx = _current_fs.Item1;
|
|
||||||
// FileStream curr_fs = _current_fs.Item2;
|
|
||||||
// _packets = new List<PlaybackPacket>();
|
|
||||||
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private bool current_file_all_replayed {
|
|
||||||
// get {
|
|
||||||
// bool rv = true;
|
|
||||||
// _packets.ForEach(p => rv = rv && p.replayed);
|
|
||||||
// return rv;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
|
||||||
// var fs = _current_fs.Item2;
|
|
||||||
// if (fs != null) {
|
|
||||||
// fs.Close();
|
|
||||||
// }
|
|
||||||
// ViewStreamDatabase.instance.removeWorker(this);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// // public event PlaybackStreamProgress onProgress;
|
|
||||||
// // public event Action onFileExhausted;
|
|
||||||
// private PlaybackPacket _locate_packet_to_replay() {
|
|
||||||
// foreach (var p in _packets) {
|
|
||||||
// // returns the first non replayed packet
|
|
||||||
// if (!p.replayed)
|
|
||||||
// return p;
|
|
||||||
// }
|
|
||||||
// //else
|
|
||||||
// {
|
|
||||||
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
|
|
||||||
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
|
|
||||||
// if (onProgress != null)
|
|
||||||
// onProgress(myFileReader.Position, myFileReader.Length);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// #region IWorker Members
|
|
||||||
|
|
||||||
// public void doWork() {
|
|
||||||
// if (_all_files_read)
|
|
||||||
// return;
|
|
||||||
// if (current_file_all_replayed) {
|
|
||||||
// _load_next_file();
|
|
||||||
// }
|
|
||||||
// PlaybackPacket curr_packet = _locate_packet_to_replay();
|
|
||||||
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// // what a stupid interface!
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
|
||||||
// _fire_packet(ops, row_array);
|
|
||||||
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
|
||||||
// } else {
|
|
||||||
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
|
|
||||||
// // this is the time we need to fire (replay) this packet
|
|
||||||
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
|
|
||||||
// long now_epoch = PersistorStream.epoch_now();
|
|
||||||
// // fire when we are past the
|
|
||||||
// if (now_epoch >= time_to_fire) {
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
|
||||||
// _fire_packet(ops, row_array);
|
|
||||||
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _fire_packet(byte[] ops, Array row_array) {
|
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
|
||||||
// byte op = ops[i];
|
|
||||||
// if (op == StreamOpCode.create) {
|
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// //_current_table.addRows(full_table);
|
|
||||||
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
|
||||||
// (myLastValue as DataPacket).newRows));
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _load_next_file() {
|
|
||||||
// var new_idx = _current_fs.Item1 + 1;
|
|
||||||
// if (new_idx >= _file_names.Count()) {
|
|
||||||
// _all_files_read = true;
|
|
||||||
// bts.Log.Gen.info("All files read");
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
// var prev_fs = _current_fs.Item2;
|
|
||||||
// if (prev_fs != null)
|
|
||||||
// prev_fs.Close();
|
|
||||||
// var new_filename = _file_names.ElementAt(new_idx);
|
|
||||||
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
|
|
||||||
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
|
|
||||||
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
|
|
||||||
// _load_packets_from_current_file();
|
|
||||||
// }
|
|
||||||
|
|
||||||
// #endregion
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
// using System;
|
|
||||||
// using System.Collections.Generic;
|
|
||||||
// using System.Linq;
|
|
||||||
// using System.Text;
|
|
||||||
// using System.IO;
|
|
||||||
|
|
||||||
// namespace BCG
|
|
||||||
// {
|
|
||||||
// public class RecorderStream : ViewStream
|
|
||||||
// {
|
|
||||||
// private ViewStream mySource;
|
|
||||||
// private DateTime myStartTime;
|
|
||||||
// private FileStream myOutFile;
|
|
||||||
// private string myFileName;
|
|
||||||
// private IBlueDataSerializer mySerializer;
|
|
||||||
|
|
||||||
// public RecorderStream(ViewStream src, string recordFile)
|
|
||||||
// {
|
|
||||||
// mySource = src;
|
|
||||||
// myFileName = recordFile;
|
|
||||||
|
|
||||||
// myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber()
|
|
||||||
// {
|
|
||||||
// myStartTime = DateTime.Now;
|
|
||||||
// mySource.onChange += new ChangeDelegate(mySource_onChange);
|
|
||||||
// myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber()
|
|
||||||
// {
|
|
||||||
// mySource.onChange -= mySource_onChange;
|
|
||||||
// myOutFile.Close();
|
|
||||||
// }
|
|
||||||
|
|
||||||
// void mySource_onChange(object newValue)
|
|
||||||
// {
|
|
||||||
// DataPacket dp = (DataPacket)newValue;
|
|
||||||
|
|
||||||
// BlueTable toWrite = new BlueTable(dp.schema);
|
|
||||||
|
|
||||||
// if (mySerializer == null)
|
|
||||||
// {
|
|
||||||
// mySerializer = new UniversalSerializer(dp.schema);
|
|
||||||
// }
|
|
||||||
// foreach (IBlueRow r in dp.newRows)
|
|
||||||
// toWrite.addRow(r);
|
|
||||||
|
|
||||||
// byte[] result;
|
|
||||||
|
|
||||||
// if (myLastValue == null)
|
|
||||||
// mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
|
|
||||||
// else
|
|
||||||
// mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
|
|
||||||
|
|
||||||
// BinaryWriter bw = new BinaryWriter(myOutFile);
|
|
||||||
// //write down what time this happened
|
|
||||||
// bw.Write((DateTime.Now - myStartTime).TotalSeconds);
|
|
||||||
// bw.Write(result.Length);
|
|
||||||
// bw.Write(result);
|
|
||||||
// myOutFile.Flush();
|
|
||||||
|
|
||||||
// myFireEvent(dp);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -5,7 +5,8 @@ using System.IO;
|
|||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
// this saves the total state in the filr provided in the cons
|
// this saves the updates as a Create on the same file, each update calls -save, which
|
||||||
|
// reqrites the whole data to the packet. The other side of this stream is the Loader
|
||||||
[DataStreamUserRespresentation("saver")]
|
[DataStreamUserRespresentation("saver")]
|
||||||
public class SaverStream : ViewStream {
|
public class SaverStream : ViewStream {
|
||||||
private ViewStream _stream;
|
private ViewStream _stream;
|
||||||
@@ -17,7 +18,8 @@ namespace bts {
|
|||||||
_filename = filename;
|
_filename = filename;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
|
protected override void myOnFirstSubscriber() =>
|
||||||
|
_stream.onChange += _handle_changes;
|
||||||
|
|
||||||
void _handle_changes(object dp_obj) {
|
void _handle_changes(object dp_obj) {
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
@@ -25,13 +27,13 @@ namespace bts {
|
|||||||
myFireEvent(dp_obj);
|
myFireEvent(dp_obj);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
|
protected override void myOnLastSubscriber() =>
|
||||||
|
_stream.onChange -= _handle_changes;
|
||||||
|
|
||||||
protected void _save(DataPacket dp) {
|
protected void _save(DataPacket dp) {
|
||||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||||
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||||
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
|
|||||||
@@ -37,12 +37,12 @@ namespace BCG.StreamWorld.Addition
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
|
myBaseStream.onChange += myHandleChanges;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
|
myBaseStream.onChange -= myHandleChanges;
|
||||||
myBaseTable.clear();
|
myBaseTable.clear();
|
||||||
myStateTable.clear();
|
myStateTable.clear();
|
||||||
}
|
}
|
||||||
@@ -228,13 +228,13 @@ namespace BCG
|
|||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region overridden methods
|
#region overridden methods
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() =>
|
||||||
{ myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
|
myParentStream.onChange += myHandleCallbacks;
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
myLastFiredRows = null;
|
myLastFiredRows = null;
|
||||||
myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
|
myParentStream.onChange -= myHandleCallbacks;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myFireEvent(object o)
|
protected override void myFireEvent(object o)
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class NullFilterStream : ViewStream {
|
||||||
|
public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
|
||||||
|
ViewStream rs = new FilterStream(parent_str,
|
||||||
|
r => {
|
||||||
|
bool is_dbnull = r.is_null(col_name);
|
||||||
|
if (is_dbnull) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (r.schema[col_name].type_code() == TypeCode.Double) {
|
||||||
|
return !Double.IsNaN(r.get<double>(col_name));
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,11 +36,11 @@ namespace BCG
|
|||||||
get{return myIsFrozen;}
|
get{return myIsFrozen;}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber ()
|
protected override void myOnFirstSubscriber () =>
|
||||||
{myStreamToFreeze.onChange += new DataStream.ChangeDelegate (myCallBack);}
|
myStreamToFreeze.onChange += myCallBack;
|
||||||
|
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber () =>
|
||||||
{myStreamToFreeze.onChange -= new DataStream.ChangeDelegate (myCallBack);}
|
myStreamToFreeze.onChange -= myCallBack;
|
||||||
|
|
||||||
internal void myCallBack(object o)
|
internal void myCallBack(object o)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -96,15 +96,15 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
|
myLeftSource.onChange += myLeft_onChange;
|
||||||
this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
|
myRightSource.onChange += myRight_onChange;
|
||||||
this.myVsd.addWorker(this);
|
this.myVsd.addWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
|
myLeftSource.onChange -= myLeft_onChange;
|
||||||
this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
|
myRightSource.onChange -= myRight_onChange;
|
||||||
this.myVsd.removeWorker(this);
|
this.myVsd.removeWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,102 +1,102 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections.Generic;
|
||||||
using System.Linq;
|
// using System.Linq;
|
||||||
using System.Text;
|
// using System.Text;
|
||||||
|
|
||||||
namespace BCG
|
// namespace BCG
|
||||||
{
|
// {
|
||||||
|
|
||||||
public class AverageGroupsStream : ViewStream
|
// public class AverageGroupsStream : ViewStream
|
||||||
{
|
// {
|
||||||
public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
|
// public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
|
||||||
{
|
// {
|
||||||
|
|
||||||
myColToAverage = colToAverage;
|
// myColToAverage = colToAverage;
|
||||||
myInput = input;
|
// myInput = input;
|
||||||
myKeyColumn = keyColumn;
|
// myKeyColumn = keyColumn;
|
||||||
}
|
// }
|
||||||
|
|
||||||
private ViewStream myInput;
|
// private ViewStream myInput;
|
||||||
private string myColToAverage;
|
// private string myColToAverage;
|
||||||
private BlueTableSchema mySchema;
|
// private BlueTableSchema mySchema;
|
||||||
private string myKeyColumn;
|
// private string myKeyColumn;
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
// protected override void myOnFirstSubscriber()
|
||||||
{
|
// {
|
||||||
myInput.onChange += new ChangeDelegate(myInput_onChange);
|
// myInput.onChange += new ChangeDelegate(myInput_onChange);
|
||||||
}
|
// }
|
||||||
protected override void myOnLastSubscriber()
|
// protected override void myOnLastSubscriber()
|
||||||
{
|
// {
|
||||||
myInput.onChange -= myInput_onChange;
|
// myInput.onChange -= myInput_onChange;
|
||||||
}
|
// }
|
||||||
private void myInput_onChange(object newValue)
|
// private void myInput_onChange(object newValue)
|
||||||
{
|
// {
|
||||||
if (mySchema == null)
|
// if (mySchema == null)
|
||||||
myBuildSchema(((DataPacket)newValue).schema);
|
// myBuildSchema(((DataPacket)newValue).schema);
|
||||||
|
|
||||||
IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
|
// IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
|
||||||
|
|
||||||
if (myLastValue == null)
|
// if (myLastValue == null)
|
||||||
myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
|
// myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
|
||||||
else
|
// else
|
||||||
{
|
// {
|
||||||
IBlueRowCollection added, deleted;
|
// IBlueRowCollection added, deleted;
|
||||||
BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
|
// BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
|
||||||
myFireEvent(new DataPacket(newRows, added, deleted));
|
// myFireEvent(new DataPacket(newRows, added, deleted));
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
private void myBuildSchema(IBlueTableSchema schema)
|
// private void myBuildSchema(IBlueTableSchema schema)
|
||||||
{
|
// {
|
||||||
List<ColumnInfo> columns = new List<ColumnInfo>();
|
// List<ColumnInfo> columns = new List<ColumnInfo>();
|
||||||
|
|
||||||
foreach (ColumnInfo ci in schema.columnInfo)
|
// foreach (ColumnInfo ci in schema.columnInfo)
|
||||||
columns.Add(ci);
|
// columns.Add(ci);
|
||||||
|
|
||||||
columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
|
// columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
|
||||||
mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
|
// mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private IBlueRowCollection myEvaluate(DataPacket packet)
|
// private IBlueRowCollection myEvaluate(DataPacket packet)
|
||||||
{
|
// {
|
||||||
|
|
||||||
IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
|
// IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
|
||||||
IDictionary<string, int> colToCount = new Dictionary<string, int>();
|
// IDictionary<string, int> colToCount = new Dictionary<string, int>();
|
||||||
IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
|
// IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
|
||||||
|
|
||||||
foreach (IBlueRow r in packet.newRows)
|
// foreach (IBlueRow r in packet.newRows)
|
||||||
{
|
// {
|
||||||
double thisVal = Convert.ToDouble(r[myColToAverage]);
|
// double thisVal = Convert.ToDouble(r[myColToAverage]);
|
||||||
|
|
||||||
string keyVal = r[myKeyColumn].ToString();
|
// string keyVal = r[myKeyColumn].ToString();
|
||||||
if (!colToCount.ContainsKey(keyVal))
|
// if (!colToCount.ContainsKey(keyVal))
|
||||||
{
|
// {
|
||||||
colToRunningTotal[keyVal] = thisVal;
|
// colToRunningTotal[keyVal] = thisVal;
|
||||||
colToCount[keyVal] = 1;
|
// colToCount[keyVal] = 1;
|
||||||
}
|
// }
|
||||||
else
|
// else
|
||||||
{
|
// {
|
||||||
colToRunningTotal[keyVal] += thisVal;
|
// colToRunningTotal[keyVal] += thisVal;
|
||||||
colToCount[keyVal] += 1;
|
// colToCount[keyVal] += 1;
|
||||||
}
|
// }
|
||||||
colToLastRow[keyVal] = r;
|
// colToLastRow[keyVal] = r;
|
||||||
}
|
// }
|
||||||
|
|
||||||
BlueTable result = new BlueTable(mySchema);
|
// BlueTable result = new BlueTable(mySchema);
|
||||||
foreach (string key in colToCount.Keys)
|
// foreach (string key in colToCount.Keys)
|
||||||
{
|
// {
|
||||||
object[] values = new object[mySchema.columnInfo.Length];
|
// object[] values = new object[mySchema.columnInfo.Length];
|
||||||
|
|
||||||
colToLastRow[key].CopyTo(values, 0);
|
// colToLastRow[key].CopyTo(values, 0);
|
||||||
values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
|
// values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
|
||||||
result.addRow(new BlueRow(mySchema, values));
|
// result.addRow(new BlueRow(mySchema, values));
|
||||||
}
|
// }
|
||||||
|
|
||||||
return result;
|
// return result;
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("Average")]
|
||||||
|
public class AverageNonIncGroupsStream : ViewStream {
|
||||||
|
public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
|
||||||
|
NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
|
||||||
|
(g_rows) => _find_mean(inp_col_name, g_rows));
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
|
||||||
|
IBlueRow inp_row = g_rows.ElementAt(0);
|
||||||
|
IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
|
||||||
|
double sum = 0.0;
|
||||||
|
inp_vals.ToList().ForEach(d => sum += d);
|
||||||
|
string out_col_name = $"average_{inp_col_name}";
|
||||||
|
double average = sum / inp_vals.Count();
|
||||||
|
IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
|
||||||
|
MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
|
||||||
|
out_row.setValue(out_col_name, average);
|
||||||
|
List<IBlueRow> rv = new List<IBlueRow>();
|
||||||
|
rv.Add(out_row);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -175,7 +175,7 @@ namespace BCG {
|
|||||||
//gets resubscribed, we start the process of building data again, and treat the callback as the first
|
//gets resubscribed, we start the process of building data again, and treat the callback as the first
|
||||||
//callback.
|
//callback.
|
||||||
myInit();
|
myInit();
|
||||||
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
myStream.onChange -= myChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myInit() {
|
protected override void myInit() {
|
||||||
|
|||||||
@@ -10,6 +10,6 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public MaxStream(ViewStream stream, string outputCol)
|
public MaxStream(ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
: this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -82,12 +82,12 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
|
myStream.onChange += myChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber ()
|
||||||
{
|
{
|
||||||
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
myStream.onChange -= myChangeHandler;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+42
-14
@@ -1,13 +1,37 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
using bts;
|
using bts;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
public class NonIncGroupByStream : ViewStream {
|
||||||
public class NonIncGroupByStream : ViewStream
|
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
||||||
{
|
Func<string[]> _groupby_cols_func;
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
||||||
|
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
||||||
|
_groupby_cols_func = groupby_cols_func;
|
||||||
|
_evaluate_func = evaluate_func;
|
||||||
|
}
|
||||||
|
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||||
|
BlueTable rv = null;
|
||||||
|
foreach (var kv in grouped_rows) {
|
||||||
|
List<IBlueRow> g_rows = kv.Value;
|
||||||
|
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
||||||
|
if (out_rows != null && out_rows.Count > 0) {
|
||||||
|
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
|
||||||
|
}
|
||||||
|
rv.addRows(out_rows);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
public string[] get_group_by_cols() {
|
||||||
|
return _groupby_cols_func();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
private ViewStream _main_stream;
|
private ViewStream _main_stream;
|
||||||
private INonIncGroupByEvaluator _groupby_func;
|
private INonIncGroupByEvaluator _groupby_func;
|
||||||
private BlueRowHasher _row_hasher;
|
private BlueRowHasher _row_hasher;
|
||||||
@@ -15,24 +39,29 @@ namespace BCG
|
|||||||
private Stopwatch _timer = new();
|
private Stopwatch _timer = new();
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
|
|
||||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
||||||
{
|
|
||||||
_main_stream = main_stream;
|
_main_stream = main_stream;
|
||||||
_groupby_func = groupby_func;
|
_groupby_func = groupby_func;
|
||||||
_groupby_cols = _groupby_func.get_group_by_cols();
|
_groupby_cols = _groupby_func.get_group_by_cols();
|
||||||
_row_hasher = new BlueRowHasher(_groupby_cols.Length);
|
_row_hasher = new BlueRowHasher(_groupby_cols.Length);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
||||||
|
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
||||||
|
|
||||||
private void _main_stream_onchange(object nv)
|
private void _main_stream_onchange(object nv) {
|
||||||
{
|
|
||||||
DataPacket dp = nv as DataPacket;
|
DataPacket dp = nv as DataPacket;
|
||||||
///grouping together
|
///grouping together
|
||||||
_timer.Restart();
|
_timer.Restart();
|
||||||
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
||||||
|
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
|
||||||
|
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
|
||||||
_timer.Stop();
|
_timer.Stop();
|
||||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
if (groupElapsedMs >= 50) {
|
if (groupElapsedMs >= 50) {
|
||||||
@@ -48,14 +77,13 @@ namespace BCG
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (applyElapsedMs >= 50)
|
if (applyElapsedMs >= 50) {
|
||||||
{
|
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
Log.DataFlow.info(
|
Log.DataFlow.info(
|
||||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||||
}
|
}
|
||||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -31,15 +31,15 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
myLeftOperand.master = this;
|
myLeftOperand.master = this;
|
||||||
myRightOperand.master = this;
|
myRightOperand.master = this;
|
||||||
myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
|
myLeftStream.onChange += myLeftOperand.changeHandler;
|
||||||
myRightStream.onChange += new DataStream.ChangeDelegate (myRightOperand.changeHandler);
|
myRightStream.onChange += myRightOperand.changeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber ()
|
||||||
{
|
{
|
||||||
myInit ();
|
myInit ();
|
||||||
myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
|
myLeftStream.onChange -= myLeftOperand.changeHandler;
|
||||||
myRightStream.onChange -= new DataStream.ChangeDelegate (myRightOperand.changeHandler);
|
myRightStream.onChange -= myRightOperand.changeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
public JoinOperand getOtherOperand (JoinOperand jo)
|
public JoinOperand getOtherOperand (JoinOperand jo)
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ namespace BCG
|
|||||||
IDictionary map = resolver.getMap ();
|
IDictionary map = resolver.getMap ();
|
||||||
map["id"] = "id";
|
map["id"] = "id";
|
||||||
myPs = new ProjectStream (myIjs, map);
|
myPs = new ProjectStream (myIjs, map);
|
||||||
myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
|
myPs.onChange += myProjectChangeHandler;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -129,26 +129,26 @@ namespace BCG
|
|||||||
private void myProjectChangeHandler (object o)
|
private void myProjectChangeHandler (object o)
|
||||||
{
|
{
|
||||||
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
|
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
|
||||||
myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
|
myIjs.onChange -= myInternalJoinChangeHandler;
|
||||||
myFireEvent ((o as ICloneable).Clone());
|
myFireEvent ((o as ICloneable).Clone());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber ()
|
protected override void myOnFirstSubscriber ()
|
||||||
{
|
{
|
||||||
myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
|
myIjs.onChange += myInternalJoinChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber ()
|
||||||
{
|
{
|
||||||
// how does this relate to the -= in myProjectChangeHandler?
|
// how does this relate to the -= in myProjectChangeHandler?
|
||||||
myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
|
myIjs.onChange -= myInternalJoinChangeHandler;
|
||||||
|
|
||||||
// if there is a project after the ijs we need to
|
// if there is a project after the ijs we need to
|
||||||
// unsubscribe to it otherwise references will be kept and
|
// unsubscribe to it otherwise references will be kept and
|
||||||
// a memory leak will occur
|
// a memory leak will occur
|
||||||
if (myPs != null)
|
if (myPs != null)
|
||||||
myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
|
myPs.onChange -= myProjectChangeHandler;
|
||||||
myPs = null;
|
myPs = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+21
-32
@@ -5,11 +5,10 @@ using mbts;
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
public class NonIncJoinBookkeeper : IDisposable {
|
||||||
public class NonIncJoinBookkeeper : IDisposable
|
|
||||||
{
|
|
||||||
public Action on_new_data;
|
public Action on_new_data;
|
||||||
private readonly NonIncJoinSpec _join_spec;
|
private readonly NonIncJoinSpec _join_spec;
|
||||||
private readonly int _join_position;
|
private readonly int _join_position;
|
||||||
@@ -26,41 +25,38 @@ namespace BCG
|
|||||||
|
|
||||||
private int[] _join_col_indices = null,
|
private int[] _join_col_indices = null,
|
||||||
_data_col_indices = null;
|
_data_col_indices = null;
|
||||||
|
|
||||||
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
|
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
||||||
{
|
|
||||||
_join_spec = the_join_spec;
|
_join_spec = the_join_spec;
|
||||||
_join_position = join_position;
|
_join_position = join_position;
|
||||||
_start_index_in_output = start_index_in_output;
|
_start_index_in_output = start_index_in_output;
|
||||||
_row_hasher = row_hasher;
|
_row_hasher = row_hasher;
|
||||||
_is_left_most = _join_position == 0;
|
_is_left_most = _join_position == 0;
|
||||||
_is_tag_along = _join_spec.join_col_names.Length == 0;
|
_is_tag_along = _join_spec.join_col_names.Length == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void init() {
|
||||||
_join_spec.stream.onChange += _on_stream_cb;
|
_join_spec.stream.onChange += _on_stream_cb;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void _on_stream_cb(object dp_obj)
|
public void _on_stream_cb(object dp_obj) {
|
||||||
{
|
Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
DataPacket dp = (DataPacket)dp_obj;
|
DataPacket dp = (DataPacket)dp_obj;
|
||||||
if (_schema == null)
|
if (_schema == null) {
|
||||||
{
|
|
||||||
_schema = dp.schema;
|
_schema = dp.schema;
|
||||||
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
||||||
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (_is_left_most)
|
if (_is_left_most) {
|
||||||
{
|
|
||||||
_row_list.Clear();
|
_row_list.Clear();
|
||||||
foreach (IBlueRow r in dp.newRows)
|
foreach (IBlueRow r in dp.newRows)
|
||||||
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
|
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
if (_is_tag_along)
|
if (_is_tag_along)
|
||||||
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
|
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
|
||||||
else
|
else {
|
||||||
{
|
|
||||||
_row_map.Clear();
|
_row_map.Clear();
|
||||||
foreach (IBlueRow r in dp.newRows)
|
foreach (IBlueRow r in dp.newRows)
|
||||||
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
||||||
@@ -79,31 +75,25 @@ namespace BCG
|
|||||||
|
|
||||||
public bool is_ready { get { return _schema != null; } }
|
public bool is_ready { get { return _schema != null; } }
|
||||||
|
|
||||||
public void constribute_cols_to_schema(List<ColumnInfo> cis)
|
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
|
||||||
{
|
|
||||||
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
||||||
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
||||||
|
|
||||||
public void copy_master_row_to_output_row(int row_index, object[] output_row)
|
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
|
||||||
{
|
|
||||||
IBlueRow this_row = _row_list[row_index].second;
|
IBlueRow this_row = _row_list[row_index].second;
|
||||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
|
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
|
||||||
{
|
if (_is_tag_along) {
|
||||||
if (_is_tag_along)
|
|
||||||
{
|
|
||||||
if (_solitary_row != null)
|
if (_solitary_row != null)
|
||||||
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
else
|
else
|
||||||
copy_dbnull_to_output_row(output_row);
|
copy_dbnull_to_output_row(output_row);
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
IBlueRow this_row = null;
|
IBlueRow this_row = null;
|
||||||
if (_row_map.TryGetValue(row_hash, out this_row))
|
if (_row_map.TryGetValue(row_hash, out this_row))
|
||||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
@@ -112,8 +102,7 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void copy_dbnull_to_output_row(object[] output_row)
|
private void copy_dbnull_to_output_row(object[] output_row) {
|
||||||
{
|
|
||||||
for (int i = 0; i < _data_col_indices.Length; i++)
|
for (int i = 0; i < _data_col_indices.Length; i++)
|
||||||
output_row[i + _start_index_in_output] = DBNull.Value;
|
output_row[i + _start_index_in_output] = DBNull.Value;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,63 +3,62 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
using mbts;
|
using mbts;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
public class NonIncJoinSpec {
|
||||||
public class NonIncJoinSpec
|
|
||||||
{
|
|
||||||
public ViewStream stream;
|
public ViewStream stream;
|
||||||
|
// join col names are the names on which the streams are joined (for equality),
|
||||||
|
// data col names are the names in the original stream
|
||||||
|
// projected col naes are their new names, ie if its x:l_x, then x is in the
|
||||||
|
// data_col_names and l_x in projected cols
|
||||||
public string[] join_col_names, data_col_names, projected_col_names;
|
public string[] join_col_names, data_col_names, projected_col_names;
|
||||||
|
|
||||||
public NonIncJoinSpec() { }
|
public NonIncJoinSpec() { }
|
||||||
|
|
||||||
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
|
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
|
||||||
: this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
|
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
|
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
|
||||||
{
|
|
||||||
stream = the_stream;
|
stream = the_stream;
|
||||||
join_col_names = the_join_col_names;
|
join_col_names = the_join_col_names;
|
||||||
data_col_names = the_data_col_names;
|
data_col_names = the_data_col_names;
|
||||||
projected_col_names = the_projected_col_names;
|
projected_col_names = the_projected_col_names;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
|
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
|
||||||
{
|
|
||||||
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
||||||
foreach (Triplet<ViewStream, string, string> r in join_specs)
|
foreach (Triplet<ViewStream, string, string> r in join_specs) {
|
||||||
{
|
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
|
||||||
string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
|
|
||||||
string[] visible_cols = r.third.Split(' ');
|
string[] visible_cols = r.third.Split(' ');
|
||||||
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
||||||
foreach (string s in visible_cols)
|
foreach (string s in visible_cols) {
|
||||||
{
|
|
||||||
string[] toks = s.Split(':');
|
string[] toks = s.Split(':');
|
||||||
if (toks.Length == 1)
|
if (toks.Length == 1) {
|
||||||
{
|
|
||||||
data_col_names.Add(toks[0]);
|
data_col_names.Add(toks[0]);
|
||||||
mapped_col_names.Add(toks[0]);
|
mapped_col_names.Add(toks[0]);
|
||||||
}
|
} else if (toks.Length == 2) {
|
||||||
else if (toks.Length == 2)
|
|
||||||
{
|
|
||||||
data_col_names.Add(toks[0]);
|
data_col_names.Add(toks[0]);
|
||||||
mapped_col_names.Add(toks[1]);
|
mapped_col_names.Add(toks[1]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
|
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
||||||
}
|
}
|
||||||
return rv.ToArray ();
|
return rv.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
public override string ToString() {
|
||||||
|
Func<string[], string> arr_txt = (a) => {
|
||||||
|
string rv = "";
|
||||||
|
foreach (string s in a) {
|
||||||
|
rv += "{s} ";
|
||||||
|
}
|
||||||
|
rv = rv.Trim();
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
|
||||||
|
return rv;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//[DataStreamUserRespresentation("ninctest")]
|
|
||||||
//public class NonIncJoinTestStream : ViewStream
|
|
||||||
//{
|
|
||||||
// public NonIncJoinTestStream (string key_cols, string data_cols)
|
|
||||||
// {
|
|
||||||
// NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
}
|
}
|
||||||
@@ -6,11 +6,9 @@ using System;
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
[DataStreamUserRespresentation("fastJoin")]
|
[DataStreamUserRespresentation("fastJoin")]
|
||||||
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
|
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
|
||||||
{
|
|
||||||
private readonly NonIncJoinSpec[] _join_specs;
|
private readonly NonIncJoinSpec[] _join_specs;
|
||||||
private readonly INonIncJoinInlineMapper _output_mapper;
|
private readonly INonIncJoinInlineMapper _output_mapper;
|
||||||
private readonly BlueRowHasher _row_hasher;
|
private readonly BlueRowHasher _row_hasher;
|
||||||
@@ -28,24 +26,19 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
||||||
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
||||||
})
|
}) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
||||||
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
|
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
||||||
params Triplet<ViewStream, string, string>[] join_specs)
|
params Triplet<ViewStream, string, string>[] join_specs)
|
||||||
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
|
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
||||||
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
|
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
|
||||||
{
|
|
||||||
_join_specs = join_specs;
|
_join_specs = join_specs;
|
||||||
_output_mapper = output_mapper;
|
_output_mapper = output_mapper;
|
||||||
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
|
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
|
||||||
@@ -53,12 +46,11 @@ namespace BCG
|
|||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
|
||||||
int data_col_no = 1;
|
int data_col_no = 1;
|
||||||
for (int jp = 0; jp < _join_specs.Length; jp++)
|
for (int jp = 0; jp < _join_specs.Length; jp++) {
|
||||||
{
|
|
||||||
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
|
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
|
||||||
|
Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
|
||||||
data_col_no += _join_specs[jp].data_col_names.Length;
|
data_col_no += _join_specs[jp].data_col_names.Length;
|
||||||
s.on_new_data += _stream_callback;
|
s.on_new_data += _stream_callback;
|
||||||
_book_keeping_data.Add(s);
|
_book_keeping_data.Add(s);
|
||||||
@@ -66,10 +58,11 @@ namespace BCG
|
|||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
_output_mapper.onMapperChange += _on_mapper_change;
|
_output_mapper.onMapperChange += _on_mapper_change;
|
||||||
_vsd.addWorker(this);
|
_vsd.addWorker(this);
|
||||||
|
foreach (var bkd in _book_keeping_data)
|
||||||
|
bkd.init();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
d.Dispose();
|
d.Dispose();
|
||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
@@ -77,30 +70,26 @@ namespace BCG
|
|||||||
_vsd.removeWorker(this);
|
_vsd.removeWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _stream_callback()
|
private void _stream_callback() {
|
||||||
{
|
Log.Gen.info("NonIncJoinStream._stream_callback received callback");
|
||||||
_is_dirty = true;
|
_is_dirty = true;
|
||||||
doWork();
|
doWork();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _on_mapper_change()
|
private void _on_mapper_change() {
|
||||||
{
|
|
||||||
_is_dirty = true;
|
_is_dirty = true;
|
||||||
doWork();
|
doWork();
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool _is_ready()
|
private bool _is_ready() {
|
||||||
{
|
|
||||||
bool rv = true;
|
bool rv = true;
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
rv &= d.is_ready;
|
rv &= d.is_ready;
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _construct_schema()
|
private void _construct_schema() {
|
||||||
{
|
if (_output_schema == null) {
|
||||||
if (_output_schema == null)
|
|
||||||
{
|
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
d.constribute_cols_to_schema(cis);
|
d.constribute_cols_to_schema(cis);
|
||||||
@@ -109,27 +98,23 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
|
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
private void _form_and_fire()
|
private void _form_and_fire() {
|
||||||
{
|
Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
|
||||||
if (_is_ready())
|
if (_is_ready()) {
|
||||||
{
|
|
||||||
_construct_schema();
|
_construct_schema();
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
||||||
List<object[]> output_rows = new List<object[]>();
|
List<object[]> output_rows = new List<object[]>();
|
||||||
|
|
||||||
for (int i = 0; i < left_row_list.Count; i++)
|
for (int i = 0; i < left_row_list.Count; i++) {
|
||||||
{
|
|
||||||
object[] or = new object[_output_schema.columnInfo.Length];
|
object[] or = new object[_output_schema.columnInfo.Length];
|
||||||
or[0] = i;
|
or[0] = i;
|
||||||
output_rows.Add(or);
|
output_rows.Add(or);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int ji = 0; ji < _book_keeping_data.Count; ji++)
|
for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
|
||||||
{
|
|
||||||
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
|
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
|
||||||
for (int i = 0; i < left_row_list.Count; i++)
|
for (int i = 0; i < left_row_list.Count; i++) {
|
||||||
{
|
|
||||||
if (ji == 0)
|
if (ji == 0)
|
||||||
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
|
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
|
||||||
else
|
else
|
||||||
@@ -139,19 +124,20 @@ namespace BCG
|
|||||||
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
||||||
timer.Stop();
|
timer.Stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (timer.ElapsedMilliseconds >= 250) {
|
//if (timer.ElapsedMilliseconds >= 250) {
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
||||||
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
|
+ $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
|
||||||
}
|
Log.Gen.info(log_msg);
|
||||||
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
//}
|
||||||
|
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
|
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
|
||||||
{
|
if (_output_mapper != null) {
|
||||||
if (_output_mapper != null)
|
|
||||||
{
|
|
||||||
IBlueRowCollection output_rows;
|
IBlueRowCollection output_rows;
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
output_rows = _output_mapper.map_rows(unmapped_rows);
|
output_rows = _output_mapper.map_rows(unmapped_rows);
|
||||||
@@ -160,17 +146,14 @@ namespace BCG
|
|||||||
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
||||||
}
|
}
|
||||||
return output_rows;
|
return output_rows;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
return unmapped_rows;
|
return unmapped_rows;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void doWork()
|
public void doWork() {
|
||||||
{
|
Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
|
||||||
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
|
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
|
||||||
{
|
|
||||||
_is_dirty = false;
|
_is_dirty = false;
|
||||||
_last_fire_time = DateTime.Now;
|
_last_fire_time = DateTime.Now;
|
||||||
_form_and_fire();
|
_form_and_fire();
|
||||||
|
|||||||
@@ -181,8 +181,8 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myVLeft.onChange += new ChangeDelegate(myVLeft_onChange);
|
myVLeft.onChange += myVLeft_onChange;
|
||||||
myVRight.onChange += new ChangeDelegate(myVRight_onChange);
|
myVRight.onChange += myVRight_onChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
void myVRight_onChange(object newValue)
|
void myVRight_onChange(object newValue)
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
namespace bts {
|
||||||
|
using BCG;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Diagnostics;
|
||||||
|
|
||||||
|
using ZstdSharp.Unsafe;
|
||||||
|
|
||||||
|
// functon that joins the 2 rows and returns the outputput row. The function
|
||||||
|
// can return a null which means the 2 rows dont need to be joined
|
||||||
|
using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
|
||||||
|
public class ProximityJoinStream : ViewStream, IWorker {
|
||||||
|
private ViewStream _lvs, _rvs;
|
||||||
|
private JoinFunc _join_func;
|
||||||
|
private bool _dirty = false;
|
||||||
|
private Stopwatch _stopwatch = new Stopwatch();
|
||||||
|
private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
|
||||||
|
public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
|
||||||
|
_lvs = lvs;
|
||||||
|
_rvs = rvs;
|
||||||
|
_join_func = join_func;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_lvs.onChange += _handle_data;
|
||||||
|
_rvs.onChange += _handle_data;
|
||||||
|
_vsd.addWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_lvs.onChange -= _handle_data;
|
||||||
|
_rvs.onChange -= _handle_data;
|
||||||
|
_vsd.removeWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _handle_data(object dp_obj) {
|
||||||
|
_dirty = true;
|
||||||
|
doWork();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
Func<bool> all_data_revcd = () => {
|
||||||
|
return _lvs.rows != null && _rvs.rows != null;
|
||||||
|
};
|
||||||
|
_stopwatch.Start();
|
||||||
|
if (_dirty && all_data_revcd()) {
|
||||||
|
_dirty = false;
|
||||||
|
BlueTable rv = null;
|
||||||
|
foreach (IBlueRow lr in _lvs.rows) {
|
||||||
|
foreach (IBlueRow rr in _rvs.rows) {
|
||||||
|
IBlueRow resr = _join_func(lr, rr);
|
||||||
|
if (resr == null)
|
||||||
|
continue;
|
||||||
|
rv = rv ?? new BlueTable(resr.schema);
|
||||||
|
rv.addRow(resr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_stopwatch.Stop();
|
||||||
|
Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.CodeDom;
|
||||||
|
using System.Configuration;
|
||||||
|
using System.Reactive.Disposables;
|
||||||
|
|
||||||
|
public class TimeProximityJoinStream : ViewStream {
|
||||||
|
private int _id = 0;
|
||||||
|
|
||||||
|
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
|
||||||
|
string pre_join_period, string post_join_period) :
|
||||||
|
this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
|
||||||
|
TimeBasedFilter.parse_duration(post_join_period)) { }
|
||||||
|
|
||||||
|
|
||||||
|
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
|
||||||
|
TimeSpan pre_join_period, TimeSpan post_join_period) {
|
||||||
|
Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
|
||||||
|
var id = 0;
|
||||||
|
lock (this) {
|
||||||
|
id = _id++;
|
||||||
|
}
|
||||||
|
IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
|
||||||
|
DateTime ldt = lr.get_datetime(l_time_col);
|
||||||
|
DateTime rdt = rr.get_datetime(r_time_col);
|
||||||
|
TimeSpan diff = ldt - rdt;
|
||||||
|
if (ldt < rdt) {
|
||||||
|
if (diff.Duration() > pre_join_period) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (diff > post_join_period)
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return create_row(lr, rr);
|
||||||
|
};
|
||||||
|
|
||||||
|
ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
|
||||||
|
mySetRealSource(rstr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -327,14 +327,14 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges);
|
myLeftStream.onChange += myHandleLeftStreamChanges;
|
||||||
myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges);
|
myRightStream.onChange += myHandleRightStreamChanges;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges);
|
myLeftStream.onChange -= myHandleLeftStreamChanges;
|
||||||
myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges);
|
myRightStream.onChange -= myHandleRightStreamChanges;
|
||||||
myLeftKeyToLeftRows.Clear();
|
myLeftKeyToLeftRows.Clear();
|
||||||
myRightKeyToRightRows.Clear();
|
myRightKeyToRightRows.Clear();
|
||||||
myLeftIDToResultRows.Clear();
|
myLeftIDToResultRows.Clear();
|
||||||
|
|||||||
@@ -23,15 +23,15 @@ namespace BCG
|
|||||||
myRightOperand.master = this;
|
myRightOperand.master = this;
|
||||||
//the events are not hooked onto the operands delegate, but rather a level of indirection
|
//the events are not hooked onto the operands delegate, but rather a level of indirection
|
||||||
//added in the form on myLeftChangeHandler
|
//added in the form on myLeftChangeHandler
|
||||||
myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftChangeHandler);
|
myLeftStream.onChange += myLeftChangeHandler;
|
||||||
myRightStream.onChange += new DataStream.ChangeDelegate (myRightChangeHandler);
|
myRightStream.onChange += myRightChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber ()
|
||||||
{
|
{
|
||||||
myInit ();
|
myInit ();
|
||||||
myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftChangeHandler);
|
myLeftStream.onChange -= myLeftChangeHandler;
|
||||||
myRightStream.onChange -= new DataStream.ChangeDelegate (myRightChangeHandler);
|
myRightStream.onChange -= myRightChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myInit()
|
protected override void myInit()
|
||||||
|
|||||||
@@ -2,9 +2,13 @@ using System;
|
|||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using Amazon.Runtime.Internal.Util;
|
||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public class LiteralStream : ViewStream {
|
public class LiteralStream : ViewStream {
|
||||||
|
|
||||||
@@ -22,6 +26,34 @@ namespace BCG {
|
|||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
|
||||||
|
public LiteralStream(string table_name, string rows) {
|
||||||
|
Func<string, int, IBlueRow> get_row = (json_row, id) => {
|
||||||
|
Dictionary<string, object> map = new Dictionary<string, object>();
|
||||||
|
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
|
||||||
|
//Logger.assert(col_vals.Length == 2);
|
||||||
|
foreach (string t in toks) {
|
||||||
|
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
|
||||||
|
string col = col_vals[0];
|
||||||
|
string val = col_vals[1];
|
||||||
|
map.Add(col, val);
|
||||||
|
}
|
||||||
|
map["id"] = id;
|
||||||
|
return new BlueRow(map);
|
||||||
|
};
|
||||||
|
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
||||||
|
BlueTable table = null;
|
||||||
|
foreach (string json_string in row_arr) {
|
||||||
|
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
||||||
|
get_row(json_string, table == null ? 0 : table.Count);
|
||||||
|
table = table ?? new BlueTable(r.schema);
|
||||||
|
table.addRow(r);
|
||||||
|
}
|
||||||
|
ViewStream str = new BlueTableStream(table);
|
||||||
|
str = new SchemaGuesserStream(str);
|
||||||
|
mySetRealSource(str);
|
||||||
|
}
|
||||||
|
|
||||||
public LiteralStream(string tableName, string schema, string value) {
|
public LiteralStream(string tableName, string schema, string value) {
|
||||||
string[] columns = schema.Split(' ');
|
string[] columns = schema.Split(' ');
|
||||||
|
|
||||||
@@ -55,7 +87,7 @@ namespace BCG {
|
|||||||
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
||||||
}
|
}
|
||||||
|
|
||||||
public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
// public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
||||||
|
|
||||||
// public LiteralStream(string value) {
|
// public LiteralStream(string value) {
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
using BCG;
|
||||||
|
|
||||||
|
using MongoDB.Driver.Core.WireProtocol.Messages;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
// the first arg is all the rows, the second arg is the index of the row that needs mapping, and
|
||||||
|
// the func returns the mapped row (ie mapped version of the index row)
|
||||||
|
using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
|
||||||
|
|
||||||
|
namespace bts {
|
||||||
|
// In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
|
||||||
|
// can use the state fo all rows.
|
||||||
|
public class GlobalMapStream : ViewStream {
|
||||||
|
private ViewStream _base_stream;
|
||||||
|
private readonly string _sort_col_name, _group_by_col;
|
||||||
|
private GlobalMap _map_func;
|
||||||
|
public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
|
||||||
|
_base_stream = base_stream;
|
||||||
|
_sort_col_name = sort_col_name;
|
||||||
|
_group_by_col = group_by_col;
|
||||||
|
_map_func = map_func;
|
||||||
|
}
|
||||||
|
|
||||||
|
public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
|
||||||
|
this(base_stream, sort_col_name, null, map_func) {
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_base_stream.onChange += _handle_stream_update;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_base_stream.onChange -= _handle_stream_update;
|
||||||
|
}
|
||||||
|
|
||||||
|
void _handle_stream_update(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
|
||||||
|
BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
|
||||||
|
:
|
||||||
|
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
|
||||||
|
BlueTable out_table = null;
|
||||||
|
foreach (var kv in groups) {
|
||||||
|
List<IBlueRow> group_rows = kv.Value;
|
||||||
|
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
|
||||||
|
uint i = 0;
|
||||||
|
foreach (IBlueRow r in group_rows) {
|
||||||
|
IBlueRow nr = _map_func(group_rows, i++);
|
||||||
|
out_table = out_table ?? new BlueTable(nr.schema);
|
||||||
|
out_table.addRow(nr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
|
||||||
|
// rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
|
||||||
|
// uint i = 0;
|
||||||
|
|
||||||
|
// BlueTable out_table = null;
|
||||||
|
// foreach (IBlueRow r in rows) {
|
||||||
|
// IBlueRow nr = _map_func(rows, i++);
|
||||||
|
// out_table = out_table ?? new BlueTable(nr.schema);
|
||||||
|
// out_table.addRow(nr);
|
||||||
|
// }
|
||||||
|
DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -119,13 +119,13 @@ namespace BCG {
|
|||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
|
myStream.onChange += myChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
myInit();
|
myInit();
|
||||||
if (myStream != null)
|
if (myStream != null)
|
||||||
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
|
myStream.onChange -= myChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnTableClear() {
|
protected override void myOnTableClear() {
|
||||||
|
|||||||
@@ -83,10 +83,11 @@ namespace BCG {
|
|||||||
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
|
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
|
protected override void myOnFirstSubscriber() =>
|
||||||
|
myStream.onChange += myHandleData;
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
myStream.onChange -= new ChangeDelegate(myHandleData);
|
myStream.onChange -= myHandleData;
|
||||||
myRows = null;
|
myRows = null;
|
||||||
myBaseRows = null;
|
myBaseRows = null;
|
||||||
myMap.Clear();
|
myMap.Clear();
|
||||||
|
|||||||
@@ -16,10 +16,10 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
_stream.onChange += new ChangeDelegate(_onstream_change);
|
_stream.onChange += _onstream_change;
|
||||||
}
|
}
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
_stream.onChange -= _onstream_change;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _onstream_change(object o) {
|
private void _onstream_change(object o) {
|
||||||
@@ -45,4 +45,4 @@ namespace BCG {
|
|||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -17,10 +17,10 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
_stream.onChange += new ChangeDelegate(_onstream_change);
|
_stream.onChange += _onstream_change;
|
||||||
}
|
}
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
_stream.onChange -= _onstream_change;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -41,7 +41,6 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
_table.addRows(added_rows);
|
_table.addRows(added_rows);
|
||||||
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
|
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -137,10 +137,24 @@ namespace bts {
|
|||||||
HashSet<string> removed_cols = new HashSet<string>();
|
HashSet<string> removed_cols = new HashSet<string>();
|
||||||
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
|
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
|
||||||
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
|
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
|
||||||
|
// by default every col will show up, so if we have the projection as (a:a1)
|
||||||
|
// it will convert a to a1,
|
||||||
|
// examples: if a schema has cols a b c
|
||||||
|
// project(parent_str, "a:aa ") -> aa b c
|
||||||
|
// project(parent_str, "a:aa !c") -> aa b
|
||||||
|
// project(parent_str, "a:aa !") -> aa
|
||||||
|
// project(parent_str, "a:aa b = (aa*aa)!") -> aa b
|
||||||
|
bool hide_all = false;
|
||||||
foreach (string p_tok in project_toks) {
|
foreach (string p_tok in project_toks) {
|
||||||
if (p_tok.StartsWith("!")) {
|
if (p_tok.StartsWith("!")) {
|
||||||
string removed_col = p_tok.Substring(1);
|
if (p_tok.Length == 1) {
|
||||||
removed_cols.Add(removed_col);
|
// if it is a "blanket "!"=> hide all cols except the one that
|
||||||
|
// are stated explicitly
|
||||||
|
hide_all = true;
|
||||||
|
} else {
|
||||||
|
string removed_col = p_tok.Substring(1);
|
||||||
|
removed_cols.Add(removed_col);
|
||||||
|
}
|
||||||
} else if (p_tok.Contains(":")) {
|
} else if (p_tok.Contains(":")) {
|
||||||
string[] map_toks = p_tok.Split(":");
|
string[] map_toks = p_tok.Split(":");
|
||||||
string orig_col = map_toks[0];
|
string orig_col = map_toks[0];
|
||||||
@@ -155,12 +169,11 @@ namespace bts {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// make the new schema
|
// make the new schema
|
||||||
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
||||||
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
||||||
if (ci.colName == "id") {
|
if (ci.colName == "id") {
|
||||||
// if there's a hidden col, other columns will show by default, but if cols are being
|
|
||||||
// projected epxlicitely, than others wil be hidden
|
|
||||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||||
new_cols.Add(nc);
|
new_cols.Add(nc);
|
||||||
continue;
|
continue;
|
||||||
@@ -172,12 +185,13 @@ namespace bts {
|
|||||||
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
||||||
new_cols.Add(nc);
|
new_cols.Add(nc);
|
||||||
} else {
|
} else {
|
||||||
if (ci.colName == "id" || removed_cols.Count > 0) {
|
if (hide_all)
|
||||||
// if there's a hidden col, other columns will show by default, but if cols are being
|
continue;
|
||||||
// projected epxlicitely, than others wil be hidden
|
//if (ci.colName == "id" || removed_cols.Count > 0) {
|
||||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
// unless columns are explicitely hidden, all columns show up
|
||||||
new_cols.Add(nc);
|
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||||
}
|
new_cols.Add(nc);
|
||||||
|
//}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
||||||
|
|||||||
@@ -54,12 +54,12 @@ namespace BCG {
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
|
myVs.onChange += myHandleChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
|
myVs.onChange -= myHandleChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
internal void myHandleChange (object newValue)
|
internal void myHandleChange (object newValue)
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
using System;
|
||||||
|
using System.Data;
|
||||||
|
using System.Data.SQLite;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Collections;
|
||||||
|
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Data.Entity;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
namespace bts.SilverPine {
|
||||||
|
|
||||||
|
public class DataRowWrapperRow : IBlueRow {
|
||||||
|
private readonly DataRow _row;
|
||||||
|
private readonly IBlueTableSchema _schema;
|
||||||
|
private readonly int _id;
|
||||||
|
public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
|
||||||
|
_row = row;
|
||||||
|
_id = id;
|
||||||
|
_schema = schema;
|
||||||
|
}
|
||||||
|
public virtual IBlueTableSchema schema { get { return _schema; } }
|
||||||
|
|
||||||
|
public virtual object this[string col_name] {
|
||||||
|
get {
|
||||||
|
ColumnInfo ci = _schema[col_name];
|
||||||
|
return this[ci.colNo];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public virtual object this[int index] {
|
||||||
|
get {
|
||||||
|
if (_schema[index].colName == "id")
|
||||||
|
return _id;
|
||||||
|
return _row[index];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual int Count { get { return _row.ItemArray.Length; } }
|
||||||
|
|
||||||
|
public virtual bool IsSynchronized { get { return false; } }
|
||||||
|
|
||||||
|
public virtual object SyncRoot { get { return this; } }
|
||||||
|
|
||||||
|
public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
|
||||||
|
|
||||||
|
public virtual bool IsFixedSize { get { return true; } }
|
||||||
|
|
||||||
|
virtual public bool IsReadOnly { get { return true; } }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The key can either be the string corrsponding to the column name or
|
||||||
|
/// the order of the column
|
||||||
|
/// </summary>
|
||||||
|
public virtual object this[object key] {
|
||||||
|
get {
|
||||||
|
if (key is string)
|
||||||
|
return this[(string)key];
|
||||||
|
else if (key is int)
|
||||||
|
return this[(int)key];
|
||||||
|
throw new BlueDataException("Wrong type while indexing the row");
|
||||||
|
}
|
||||||
|
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual ICollection Keys { get { return _schema.columnNames; } }
|
||||||
|
|
||||||
|
public virtual ICollection Values { get { return _row.ItemArray; } }
|
||||||
|
|
||||||
|
public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
|
public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
|
public virtual bool Contains(object key) { return _schema[(string)key] != null; }
|
||||||
|
|
||||||
|
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||||
|
|
||||||
|
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
||||||
|
|
||||||
|
public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public class DataTableWrapperTable : IBlueRowCollection {
|
||||||
|
|
||||||
|
public class DataTableWrapperTableEnumerator : IEnumerator {
|
||||||
|
private IEnumerator _enumerator;
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
private int _id = 0;
|
||||||
|
public DataTableWrapperTableEnumerator(DataTable table) {
|
||||||
|
_enumerator = table.Rows.GetEnumerator();
|
||||||
|
_schema = BlueDataUtility.getSchema(table);
|
||||||
|
|
||||||
|
}
|
||||||
|
public object Current {
|
||||||
|
get {
|
||||||
|
DataRow r = _enumerator.Current as DataRow;
|
||||||
|
return new DataRowWrapperRow(_schema, r, _id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public bool MoveNext() {
|
||||||
|
_id++;
|
||||||
|
return _enumerator.MoveNext();
|
||||||
|
}
|
||||||
|
public void Reset() {
|
||||||
|
_id = 0;
|
||||||
|
_enumerator.Reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private readonly DataTable _table;
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
private IEnumerator _enumerator;
|
||||||
|
|
||||||
|
public DataTableWrapperTable(DataTable table) {
|
||||||
|
_table = table;
|
||||||
|
_schema = BlueDataUtility.getSchema(_table);
|
||||||
|
_enumerator = new DataTableWrapperTableEnumerator(table);
|
||||||
|
}
|
||||||
|
public virtual IBlueRow this[int id] {
|
||||||
|
get {
|
||||||
|
// its assumed the rows index == id
|
||||||
|
DataRow dr = _table.Rows[id];
|
||||||
|
return new DataRowWrapperRow(_schema, dr, id);
|
||||||
|
}
|
||||||
|
set {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual IBlueTableSchema schema { get { return _schema; } }
|
||||||
|
|
||||||
|
public virtual bool containsID(int id) { return id < _table.Rows.Count; }
|
||||||
|
|
||||||
|
// private List<IBlueRow> _rows() {
|
||||||
|
// List<IBlueRow> rv = new List<IBlueRow>();
|
||||||
|
// int id = 0;
|
||||||
|
// foreach (DataRow r in _table.Rows) {
|
||||||
|
// rv.Add(new DataRowWrapperRow(r, id++));
|
||||||
|
// }
|
||||||
|
// return rv;
|
||||||
|
// }
|
||||||
|
|
||||||
|
public virtual IEnumerator GetEnumerator() {
|
||||||
|
// var rows = _rows();
|
||||||
|
// return rows.GetEnumerator();
|
||||||
|
return _enumerator;
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual bool IsSynchronized { get { return false; } }
|
||||||
|
|
||||||
|
public virtual object SyncRoot { get { return this; } }
|
||||||
|
|
||||||
|
public virtual void CopyTo(Array array, int index) {
|
||||||
|
Log.assert(false);
|
||||||
|
//_rows().CopyTo(array as IBlueRow[], index);
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual int Count {
|
||||||
|
get { return _table.Rows.Count; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class SQLiteHelper {
|
||||||
|
public static DataTable ExecuteSelectQuery(string connection_string, string query) {
|
||||||
|
// Create a new DataTable to store the results
|
||||||
|
DataTable dataTable = new DataTable();
|
||||||
|
|
||||||
|
// Create and open a SQLite connection using the provided connection string
|
||||||
|
using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
|
||||||
|
try {
|
||||||
|
connection.Open();
|
||||||
|
|
||||||
|
// Create a SQLite command with the provided query
|
||||||
|
using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
|
||||||
|
// Use SQLiteDataAdapter to fill the DataTable
|
||||||
|
using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
|
||||||
|
adapter.Fill(dataTable);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
Console.WriteLine("An error occurred: " + ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the populated DataTable
|
||||||
|
return dataTable;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("SQLite")]
|
||||||
|
public class SQLiteStream : ViewStream {
|
||||||
|
private readonly string _connection_string, _query;
|
||||||
|
private Stopwatch _stopwatch = new Stopwatch();
|
||||||
|
public SQLiteStream(string connection_string, string query) {
|
||||||
|
_connection_string = connection_string;
|
||||||
|
_query = query;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void _read_from_sqlite_nd_fire() {
|
||||||
|
_stopwatch.Start();
|
||||||
|
DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
|
||||||
|
_stopwatch.Stop();
|
||||||
|
Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||||
|
_stopwatch.Start();
|
||||||
|
IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
|
||||||
|
//IBlueRowCollection rows = new DataTableWrapperTable(dt);
|
||||||
|
_stopwatch.Stop();
|
||||||
|
Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||||
|
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_read_from_sqlite_nd_fire();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
// this value will be cleared by the base class
|
||||||
|
// but in this particular stream this behaviour
|
||||||
|
// is not usefull since any new subscribers
|
||||||
|
// would force the query to be recreated
|
||||||
|
//myLastDP = myLastValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -3,12 +3,15 @@ using System.Collections.Generic;
|
|||||||
using System.Text;
|
using System.Text;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using Microsoft.Data.Analysis;
|
||||||
|
using Parquet;
|
||||||
|
using Parquet.Data;
|
||||||
|
|
||||||
namespace BCG
|
|
||||||
{
|
namespace BCG {
|
||||||
[DataStreamUserRespresentation("CSVSaver")]
|
[DataStreamUserRespresentation("CSVSaver")]
|
||||||
public class CSVSaverStream : ViewStream, ISideEffectStream
|
public class CSVSaverStream : ViewStream, ISideEffectStream {
|
||||||
{
|
|
||||||
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
|
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
|
||||||
new ColumnInfo("fileName", 1, typeof(string)),
|
new ColumnInfo("fileName", 1, typeof(string)),
|
||||||
new ColumnInfo("nWrites", 2, typeof(int)),
|
new ColumnInfo("nWrites", 2, typeof(int)),
|
||||||
@@ -20,11 +23,9 @@ namespace BCG
|
|||||||
private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
|
private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
|
||||||
|
|
||||||
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
|
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
|
||||||
: this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
|
: this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
|
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
|
||||||
{
|
|
||||||
myStream = vs;
|
myStream = vs;
|
||||||
myFullyQualifiedFileName = fullyQualifiedFileName;
|
myFullyQualifiedFileName = fullyQualifiedFileName;
|
||||||
myOverwrite = overwrite;
|
myOverwrite = overwrite;
|
||||||
@@ -32,22 +33,20 @@ namespace BCG
|
|||||||
myFireResultSnapshot = fireResultSnapshot;
|
myFireResultSnapshot = fireResultSnapshot;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleChanges); }
|
protected override void myOnFirstSubscriber() =>
|
||||||
|
myStream.onChange += myHandleChanges;
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
|
protected override void myOnLastSubscriber() =>
|
||||||
|
myStream.onChange -= myHandleChanges;
|
||||||
|
|
||||||
private void myHandleChanges(object newValue)
|
private void myHandleChanges(object newValue) {
|
||||||
{
|
|
||||||
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
|
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
|
||||||
if (totalState.Count > 0 && myNWrites < myNMaxWrites)
|
if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
|
||||||
{
|
|
||||||
myNWrites++;
|
myNWrites++;
|
||||||
int i = 0;
|
int i = 0;
|
||||||
StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
|
StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
|
||||||
foreach (IBlueRow r in totalState)
|
foreach (IBlueRow r in totalState) {
|
||||||
{
|
if (i++ == 0) {
|
||||||
if (i++ == 0)
|
|
||||||
{
|
|
||||||
string header = String.Empty;
|
string header = String.Empty;
|
||||||
foreach (string col in r.schema.columnNames)
|
foreach (string col in r.schema.columnNames)
|
||||||
if (col != "id")
|
if (col != "id")
|
||||||
@@ -59,20 +58,17 @@ namespace BCG
|
|||||||
writer.Flush();
|
writer.Flush();
|
||||||
}
|
}
|
||||||
writer.Close();
|
writer.Close();
|
||||||
if (myFireResultSnapshot)
|
if (myFireResultSnapshot) {
|
||||||
{
|
|
||||||
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
|
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
|
||||||
BlueTable nRows = new BlueTable(myResultSchema);
|
BlueTable nRows = new BlueTable(myResultSchema);
|
||||||
nRows.addRow(nr);
|
nRows.addRow(nr);
|
||||||
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
|
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
|
||||||
}
|
} else
|
||||||
else
|
|
||||||
myFireEvent(newValue);
|
myFireEvent(newValue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private string myGetNewFileName()
|
private string myGetNewFileName() {
|
||||||
{
|
|
||||||
string[] tags = myFullyQualifiedFileName.Split('.');
|
string[] tags = myFullyQualifiedFileName.Split('.');
|
||||||
if (myOverwrite)
|
if (myOverwrite)
|
||||||
return tags[0] + ".csv";
|
return tags[0] + ".csv";
|
||||||
@@ -80,14 +76,11 @@ namespace BCG
|
|||||||
return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
|
return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string myGetCSVRow(IBlueRow r)
|
private static string myGetCSVRow(IBlueRow r) {
|
||||||
{
|
|
||||||
string rv = String.Empty;
|
string rv = String.Empty;
|
||||||
int i = 0;
|
int i = 0;
|
||||||
foreach (string col in r.schema.columnNames)
|
foreach (string col in r.schema.columnNames) {
|
||||||
{
|
if (col != "id") {
|
||||||
if (col != "id")
|
|
||||||
{
|
|
||||||
string colText = String.Empty;
|
string colText = String.Empty;
|
||||||
if (r[col] != null && r[col] != DBNull.Value)
|
if (r[col] != null && r[col] != DBNull.Value)
|
||||||
colText = r[col].ToString();
|
colText = r[col].ToString();
|
||||||
@@ -99,8 +92,7 @@ namespace BCG
|
|||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
private string myFormateDateTime(DateTime dt)
|
private string myFormateDateTime(DateTime dt) {
|
||||||
{
|
|
||||||
return
|
return
|
||||||
string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
|
string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
|
||||||
dt.Year.ToString("0000"),
|
dt.Year.ToString("0000"),
|
||||||
@@ -114,52 +106,49 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
|
|
||||||
[DataStreamUserRespresentation("CSVReader")]
|
[DataStreamUserRespresentation("CSVReader")]
|
||||||
public class CSVReaderStream : ViewStream
|
public class CSVReaderStream : ViewStream {
|
||||||
{
|
private readonly string _file_address;
|
||||||
private IBlueRowCollection myRows;
|
Stopwatch _stopwatch = new Stopwatch();
|
||||||
|
public CSVReaderStream(string fileAddress) {
|
||||||
public CSVReaderStream(string fileAddress)
|
_file_address = fileAddress;
|
||||||
{
|
|
||||||
myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
_stopwatch.Start();
|
||||||
myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
|
DataFrame df = DataFrame.LoadCsv(_file_address);
|
||||||
|
_stopwatch.Stop();
|
||||||
|
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||||
|
_stopwatch.Start();
|
||||||
|
IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
|
||||||
|
_stopwatch.Stop();
|
||||||
|
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||||
|
DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
[DataStreamUserRespresentation ("CSVReaderStreamMapper")]
|
[DataStreamUserRespresentation("CSVReaderStreamMapper")]
|
||||||
public class CSVReaderStreamMapperStream : ViewStream
|
public class CSVReaderStreamMapperStream : ViewStream {
|
||||||
{
|
public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
|
||||||
public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
|
|
||||||
{
|
|
||||||
mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
|
mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public class CSVReaderStreamMapper : IStreamMapper
|
public class CSVReaderStreamMapper : IStreamMapper {
|
||||||
{
|
|
||||||
private IBlueTableSchema mySchema;
|
private IBlueTableSchema mySchema;
|
||||||
private readonly string[] myOutputSchemaStringArray;
|
private readonly string[] myOutputSchemaStringArray;
|
||||||
|
|
||||||
public CSVReaderStreamMapper(string outputSchema)
|
public CSVReaderStreamMapper(string outputSchema) {
|
||||||
{
|
|
||||||
myOutputSchemaStringArray = outputSchema.Split(' ');
|
myOutputSchemaStringArray = outputSchema.Split(' ');
|
||||||
}
|
}
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||||
{
|
if (mySchema == null) {
|
||||||
if (mySchema == null)
|
|
||||||
{
|
|
||||||
ColumnInfo[] ci = inputSchema.columnInfo;
|
ColumnInfo[] ci = inputSchema.columnInfo;
|
||||||
foreach (string col in myOutputSchemaStringArray)
|
foreach (string col in myOutputSchemaStringArray) {
|
||||||
{
|
|
||||||
string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
|
string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
|
||||||
for (int i = 0; i < ci.Length; i++)
|
for (int i = 0; i < ci.Length; i++) {
|
||||||
{
|
if (ci[i].colName.ToLower().Equals(colName)) {
|
||||||
if (ci[i].colName.ToLower().Equals(colName))
|
|
||||||
{
|
|
||||||
Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
|
Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
|
||||||
ci[i] = new ColumnInfo(ci[i].colName, i, t);
|
ci[i] = new ColumnInfo(ci[i].colName, i, t);
|
||||||
break;
|
break;
|
||||||
@@ -171,74 +160,52 @@ namespace BCG
|
|||||||
return mySchema;
|
return mySchema;
|
||||||
}
|
}
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row)
|
public IBlueRow map(IBlueRow row) {
|
||||||
{
|
|
||||||
object[] vals = new object[row.schema.columnInfo.Length];
|
object[] vals = new object[row.schema.columnInfo.Length];
|
||||||
row.Values.CopyTo(vals, 0);
|
row.Values.CopyTo(vals, 0);
|
||||||
for (int i = 0; i < mySchema.columnInfo.Length; i++)
|
for (int i = 0; i < mySchema.columnInfo.Length; i++) {
|
||||||
{
|
|
||||||
Type t = mySchema.columnInfo[i].colType;
|
Type t = mySchema.columnInfo[i].colType;
|
||||||
try
|
try {
|
||||||
{
|
|
||||||
vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
|
vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
|
||||||
}
|
} catch {
|
||||||
catch
|
if (t.Equals(typeof(Int16))) {
|
||||||
{
|
|
||||||
if (t.Equals(typeof(Int16)))
|
|
||||||
{
|
|
||||||
Int16 result;
|
Int16 result;
|
||||||
if (Int16.TryParse((string)vals[i], out result))
|
if (Int16.TryParse((string)vals[i], out result)) {
|
||||||
{
|
|
||||||
vals[i] = result;
|
vals[i] = result;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
double r2;
|
double r2;
|
||||||
if (Double.TryParse((string)vals[i], out r2))
|
if (Double.TryParse((string)vals[i], out r2)) {
|
||||||
{
|
try {
|
||||||
try
|
|
||||||
{
|
|
||||||
vals[i] = (Int16)r2;
|
vals[i] = (Int16)r2;
|
||||||
continue;
|
continue;
|
||||||
}
|
} catch { }
|
||||||
catch { }
|
|
||||||
}
|
}
|
||||||
}
|
} else if (t.Equals(typeof(Int32))) {
|
||||||
else if (t.Equals(typeof(Int32)))
|
|
||||||
{
|
|
||||||
Int32 result;
|
Int32 result;
|
||||||
if (Int32.TryParse((string)vals[i], out result))
|
if (Int32.TryParse((string)vals[i], out result)) {
|
||||||
{
|
|
||||||
vals[i] = result;
|
vals[i] = result;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
double r2;
|
double r2;
|
||||||
if (Double.TryParse((string)vals[i], out r2))
|
if (Double.TryParse((string)vals[i], out r2)) {
|
||||||
{
|
try {
|
||||||
try
|
|
||||||
{
|
|
||||||
vals[i] = (Int32)r2;
|
vals[i] = (Int32)r2;
|
||||||
continue;
|
continue;
|
||||||
}
|
} catch { }
|
||||||
catch { }
|
|
||||||
}
|
}
|
||||||
}
|
} else if (t.Equals(typeof(Int64))) {
|
||||||
else if (t.Equals(typeof(Int64)))
|
|
||||||
{
|
|
||||||
Int64 result;
|
Int64 result;
|
||||||
if (Int64.TryParse((string)vals[i], out result))
|
if (Int64.TryParse((string)vals[i], out result)) {
|
||||||
{
|
|
||||||
vals[i] = result;
|
vals[i] = result;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
double r2;
|
double r2;
|
||||||
if (Double.TryParse((string)vals[i], out r2))
|
if (Double.TryParse((string)vals[i], out r2)) {
|
||||||
{
|
try {
|
||||||
try
|
|
||||||
{
|
|
||||||
vals[i] = (Int64)r2;
|
vals[i] = (Int64)r2;
|
||||||
continue;
|
continue;
|
||||||
}
|
} catch { }
|
||||||
catch { }
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
vals[i] = DBNull.Value;
|
vals[i] = DBNull.Value;
|
||||||
@@ -247,8 +214,7 @@ namespace BCG
|
|||||||
return new BlueRow(mySchema, vals);
|
return new BlueRow(mySchema, vals);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Type myGetType(string type)
|
private Type myGetType(string type) {
|
||||||
{
|
|
||||||
type = type.ToLower();
|
type = type.ToLower();
|
||||||
if (type.Contains("int"))
|
if (type.Contains("int"))
|
||||||
return typeof(int);
|
return typeof(int);
|
||||||
|
|||||||
+33
-24
@@ -1,31 +1,41 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
|
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
// if a table needs to be fired, it can be wrapped around this
|
||||||
[DataStreamUserRespresentation (false)]
|
public class BlueTableStream : ViewStream {
|
||||||
public class BlueRowCollectionStaticStream : ViewStream
|
private IBlueRowCollection _rows;
|
||||||
{
|
public BlueTableStream(IBlueRowCollection rows) {
|
||||||
public BlueRowCollectionStaticStream () {}
|
_rows = rows;
|
||||||
|
|
||||||
public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
|
|
||||||
|
|
||||||
public void loadCollection (IBlueRowCollection rows)
|
|
||||||
{
|
|
||||||
myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
|
|
||||||
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void crush_with_single_row(IBlueRow r)
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation(false)]
|
||||||
|
public class BlueRowCollectionStaticStream : ViewStream {
|
||||||
|
public BlueRowCollectionStaticStream() { }
|
||||||
|
|
||||||
|
public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
|
||||||
|
|
||||||
|
public void loadCollection(IBlueRowCollection rows) {
|
||||||
|
myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
|
||||||
|
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void crush_with_single_row(IBlueRow r) {
|
||||||
BlueTable t = new BlueTable(r.schema);
|
BlueTable t = new BlueTable(r.schema);
|
||||||
t.addRow(r);
|
t.addRow(r);
|
||||||
loadCollection(t);
|
loadCollection(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void loadPacket (DataPacket dp) {myFireEvent (dp);}
|
public void loadPacket(DataPacket dp) { myFireEvent(dp); }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
|
/// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
|
||||||
@@ -35,17 +45,16 @@ namespace BCG
|
|||||||
/// <param name="streamType"></param>
|
/// <param name="streamType"></param>
|
||||||
/// <param name="args"></param>
|
/// <param name="args"></param>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
|
public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
|
||||||
{
|
ArrayList totArgs = new ArrayList(args);
|
||||||
ArrayList totArgs = new ArrayList (args);
|
BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
|
||||||
BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
|
totArgs.Insert(0, v1);
|
||||||
totArgs.Insert (0, v1);
|
ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
|
||||||
ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
|
v1.loadCollection(rows);
|
||||||
v1.loadCollection (rows);
|
|
||||||
return retVal.rows;
|
return retVal.rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -35,14 +35,14 @@ namespace BCG
|
|||||||
#region ViewStream
|
#region ViewStream
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myStream.onChange +=new ChangeDelegate(myStreamChange);
|
myStream.onChange += myStreamChange;
|
||||||
myVsd.addWorker(this);
|
myVsd.addWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
myVsd.removeWorker(this);
|
myVsd.removeWorker(this);
|
||||||
myStream.onChange -= new ChangeDelegate(myStreamChange);
|
myStream.onChange -= myStreamChange;
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myParentStream.onChange += new ChangeDelegate(myParentStreamChangeHandler);
|
myParentStream.onChange += myParentStreamChangeHandler;
|
||||||
myVSD.addWorker(this);
|
myVSD.addWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,7 +36,7 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
myParentStream.onChange -= new ChangeDelegate(myParentStreamChangeHandler);
|
myParentStream.onChange -= myParentStreamChangeHandler;
|
||||||
myVSD.removeWorker(this);
|
myVSD.removeWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -53,11 +53,11 @@ namespace BCG
|
|||||||
if (r.is_subscription)
|
if (r.is_subscription)
|
||||||
{
|
{
|
||||||
_current_source_subscription = r.subscription_id;
|
_current_source_subscription = r.subscription_id;
|
||||||
_source_stream.onChange += new ChangeDelegate(_source_stream_onChange);
|
_source_stream.onChange += _source_stream_onChange;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
_source_stream.onChange -= new ChangeDelegate(_source_stream_onChange);
|
_source_stream.onChange -= _source_stream_onChange;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class DateTimeExpanstionStream : ViewStream {
|
||||||
|
public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
|
||||||
|
Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
|
||||||
|
IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
|
||||||
|
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
|
||||||
|
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month");
|
||||||
|
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour");
|
||||||
|
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min");
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
ViewStream rs = new MapStream(parent_stream,
|
||||||
|
s => get_schema(s),
|
||||||
|
r => {
|
||||||
|
DateTime dt = r.get_datetime(inp_col);
|
||||||
|
IBlueTableSchema s = get_schema(r.schema);
|
||||||
|
MutableBlueRow rv = new MutableBlueRow(s, r);
|
||||||
|
rv.setValue($"{inp_col}_date", dt.Date);
|
||||||
|
rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
|
||||||
|
rv.setValue($"{inp_col}_month", dt.Date.Month.ToString());
|
||||||
|
rv.setValue($"{inp_col}_hour", dt.Hour);
|
||||||
|
rv.setValue($"{inp_col}_min", dt.Minute);
|
||||||
|
return rv;
|
||||||
|
});
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,7 +24,7 @@ namespace BCG
|
|||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myKeyToTimeToValue = new Dictionary<string, SortedList<DateTime, double>>();
|
myKeyToTimeToValue = new Dictionary<string, SortedList<DateTime, double>>();
|
||||||
myToAverage.onChange += new ChangeDelegate(myToAverage_onChange);
|
myToAverage.onChange += myToAverage_onChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.CodeDom;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
// Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
|
||||||
|
// so:
|
||||||
|
// f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
|
||||||
|
[DataStreamUserRespresentation("ColDiff")]
|
||||||
|
public class ColumnDifferenceStream : ViewStream {
|
||||||
|
private readonly ViewStream _base_stream;
|
||||||
|
private readonly FilterStream.OnFilter _f1, _f2;
|
||||||
|
private readonly string _c1, _c2;
|
||||||
|
|
||||||
|
public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
|
||||||
|
_base_stream = base_stream;
|
||||||
|
_f1 = f1;
|
||||||
|
_f2 = f2;
|
||||||
|
_c1 = col1;
|
||||||
|
_c2 = col2;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
|
||||||
|
FilterStream.OnFilter f1_del = r => {
|
||||||
|
BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
|
||||||
|
return rv.filter(r);
|
||||||
|
};
|
||||||
|
FilterStream.OnFilter f2_del = r => {
|
||||||
|
BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
|
||||||
|
return rv.filter(r);
|
||||||
|
};
|
||||||
|
ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
|
||||||
|
// for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2]
|
||||||
|
// we should generlize this for any math operation
|
||||||
|
public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
|
||||||
|
// use GlobalMapStream
|
||||||
|
string diff_col_perc_name = $"{diff_col_name}_percentage";
|
||||||
|
System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
|
||||||
|
int minuend_idx = Convert.ToInt32(idx);
|
||||||
|
var inp_schema = rows[0].schema;
|
||||||
|
var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
|
||||||
|
out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
|
||||||
|
// minuend - subtrahend
|
||||||
|
IBlueRow minuend_row = rows.ElementAt(minuend_idx);
|
||||||
|
MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
|
||||||
|
int subtrahend_index = minuend_idx + row_offset;
|
||||||
|
double diff_val = Double.NaN, diff_val_perc = Double.NaN;
|
||||||
|
if (subtrahend_index >= 0) {
|
||||||
|
try {
|
||||||
|
IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
|
||||||
|
diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
|
||||||
|
diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
Log.assert(false, ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rv.setValue(diff_col_name, diff_val);
|
||||||
|
rv.setValue(diff_col_perc_name, diff_val_perc);
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
|
||||||
|
if (optional_perc_diff_filter != 0) {
|
||||||
|
rs = new FilterStream(rs, r => {
|
||||||
|
double diff_val_perc = r.get<double>(diff_col_perc_name);
|
||||||
|
double abs_diff_val_perc = Math.Abs(diff_val_perc);
|
||||||
|
return abs_diff_val_perc >= optional_perc_diff_filter;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_base_stream.onChange += _handle_base_stream_update;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_base_stream.onChange -= _handle_base_stream_update;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _handle_base_stream_update(object dp_obj) {
|
||||||
|
// could have been done with filter and cartesian prod, and then math
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
List<IBlueRow> rows = dp.newRows.to_list();
|
||||||
|
var f1_rows = rows.Where(r => _f1(r));
|
||||||
|
Log.assert(f1_rows.Count() == 1);
|
||||||
|
var f2_rows = rows.Where(r => _f2(r));
|
||||||
|
Log.assert(f2_rows.Count() == 1);
|
||||||
|
IBlueRow f1_r = f1_rows.ElementAt(0);
|
||||||
|
IBlueRow f2_r = f2_rows.ElementAt(0);
|
||||||
|
double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
|
||||||
|
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
|
||||||
|
BlueTable out_table = new BlueTable(out_schema);
|
||||||
|
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
|
||||||
|
out_row.setValue("diff", diff);
|
||||||
|
out_table.addRow(out_row);
|
||||||
|
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
// Given a sort order, an optional group by col, it observes the values of a column
|
||||||
|
// within that group (or the whole table, if grouping not specified) and returns all rows
|
||||||
|
// where the value changes, ie row[current][col] != row[current-1][col]
|
||||||
|
// TODO: a more general way to express a filter where row[current][col] != row[previous][col],
|
||||||
|
// or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
|
||||||
|
[DataStreamUserRespresentation("ColEvol")]
|
||||||
|
public class ColumnEvolutionStream : ViewStream {
|
||||||
|
private readonly ViewStream _base_stream;
|
||||||
|
private readonly string _sort_col;
|
||||||
|
private readonly string _evol_col;
|
||||||
|
public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
|
||||||
|
_base_stream = base_stream;
|
||||||
|
_sort_col = sort_col;
|
||||||
|
_evol_col = evolution_col;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_base_stream.onChange += _handle_base_stream_update;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
base.myOnLastSubscriber();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _handle_base_stream_update(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
List<IBlueRow> rows = dp.newRows.to_list();
|
||||||
|
rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
|
||||||
|
BlueTable rv = new BlueTable(dp.schema);
|
||||||
|
IBlueRow prev_row = rows.ElementAt(0);
|
||||||
|
foreach (IBlueRow r in rows) {
|
||||||
|
var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
|
||||||
|
if (cr != 0) {
|
||||||
|
rv.addRow(r);
|
||||||
|
}
|
||||||
|
prev_row = r;
|
||||||
|
}
|
||||||
|
DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -12,10 +12,10 @@ namespace BCG {
|
|||||||
mySrc = src;
|
mySrc = src;
|
||||||
}
|
}
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
mySrc.onChange += new ChangeDelegate(mySrc_onChange);
|
mySrc.onChange += mySrc_onChange;
|
||||||
}
|
}
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
|
mySrc.onChange -= mySrc_onChange;
|
||||||
}
|
}
|
||||||
private void mySrc_onChange(object newValue) {
|
private void mySrc_onChange(object newValue) {
|
||||||
DataPacket dp = (DataPacket)newValue;
|
DataPacket dp = (DataPacket)newValue;
|
||||||
|
|||||||
@@ -28,14 +28,14 @@ namespace BCG
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
myLeftStream.onChange += new ChangeDelegate(myHandleLeftChanges);
|
myLeftStream.onChange += myHandleLeftChanges;
|
||||||
myRightstream.onChange += new ChangeDelegate(myHandleRightChanges);
|
myRightstream.onChange += myHandleRightChanges;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
myLeftStream.onChange -= new ChangeDelegate(myHandleLeftChanges);
|
myLeftStream.onChange -= myHandleLeftChanges;
|
||||||
myRightstream.onChange -= new ChangeDelegate(myHandleRightChanges);
|
myRightstream.onChange -= myHandleRightChanges;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleLeftChanges (object newValue)
|
private void myHandleLeftChanges (object newValue)
|
||||||
|
|||||||
@@ -222,7 +222,7 @@ namespace BCG
|
|||||||
pj = new ProjectStream(pj, "!");
|
pj = new ProjectStream(pj, "!");
|
||||||
myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
|
myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
|
||||||
if (mySubscriptionActive)
|
if (mySubscriptionActive)
|
||||||
pj.onChange += new ChangeDelegate(myHandleStreamChanges);
|
pj.onChange += myHandleStreamChanges;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void removeStream(ViewStream vs)
|
public void removeStream(ViewStream vs)
|
||||||
@@ -235,7 +235,7 @@ namespace BCG
|
|||||||
index = p.Key;
|
index = p.Key;
|
||||||
if (mySubscriptionActive)
|
if (mySubscriptionActive)
|
||||||
{
|
{
|
||||||
p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
|
p.Value.second.onChange -= myHandleStreamChanges;
|
||||||
if (myTable != null)
|
if (myTable != null)
|
||||||
{
|
{
|
||||||
IBlueRowCollection delRows = myTable.removeTable(p.Key);
|
IBlueRowCollection delRows = myTable.removeTable(p.Key);
|
||||||
@@ -254,14 +254,14 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
mySubscriptionActive = true;
|
mySubscriptionActive = true;
|
||||||
foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
|
foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
|
||||||
p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
|
p.Value.second.onChange += myHandleStreamChanges;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
{
|
{
|
||||||
mySubscriptionActive = false;
|
mySubscriptionActive = false;
|
||||||
foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
|
foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
|
||||||
p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
|
p.Value.second.onChange -= myHandleStreamChanges;
|
||||||
if (myTable != null)
|
if (myTable != null)
|
||||||
myTable.clear();
|
myTable.clear();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,11 +14,11 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
mySrc.onChange += new ChangeDelegate(mySrc_onChange);
|
mySrc.onChange += mySrc_onChange;
|
||||||
}
|
}
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber ()
|
||||||
{
|
{
|
||||||
mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
|
mySrc.onChange -= mySrc_onChange;
|
||||||
}
|
}
|
||||||
private void mySrc_onChange (object newValue) {
|
private void mySrc_onChange (object newValue) {
|
||||||
DataPacket dp = (DataPacket)newValue;
|
DataPacket dp = (DataPacket)newValue;
|
||||||
|
|||||||
@@ -21,15 +21,15 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
myLeftOp = new UnionOperand (true, this);
|
myLeftOp = new UnionOperand (true, this);
|
||||||
myRightOp = new UnionOperand (false, this);
|
myRightOp = new UnionOperand (false, this);
|
||||||
myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
|
myLeftVs.onChange += myLeftOp.myChangeHandler;
|
||||||
myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
|
myRightVs.onChange += myRightOp.myChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber ()
|
protected override void myOnLastSubscriber ()
|
||||||
{
|
{
|
||||||
myInit ();
|
myInit ();
|
||||||
myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
|
myLeftVs.onChange -= myLeftOp.myChangeHandler;
|
||||||
myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
|
myRightVs.onChange -= myRightOp.myChangeHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void startSession ()
|
public void startSession ()
|
||||||
@@ -116,4 +116,4 @@ namespace BCG
|
|||||||
receiveStream(dp, isLeft);
|
receiveStream(dp, isLeft);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -14,11 +14,11 @@ namespace BCG {
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
_vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
|
_vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
|
||||||
_main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
|
_main_stream.onChange += _handle_update;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
_main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
|
_main_stream.onChange -= _handle_update;
|
||||||
_vs_publisher.Dispose();
|
_vs_publisher.Dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -28,4 +28,4 @@ namespace BCG {
|
|||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -20,11 +20,11 @@ namespace BCG {
|
|||||||
public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
|
public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
|
||||||
_stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
|
_stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
|
||||||
_broker = broker;
|
_broker = broker;
|
||||||
_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
|
_stream.onChange += _handle_update;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Dispose() {
|
public void Dispose() {
|
||||||
_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
|
_stream.onChange -= _handle_update;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _handle_update(object o) {
|
private void _handle_update(object o) {
|
||||||
|
|||||||
@@ -15,10 +15,74 @@ using mbts;
|
|||||||
using BCG.Extensions;
|
using BCG.Extensions;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.Json.Nodes;
|
using System.Text.Json.Nodes;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
using BCG;
|
||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
|
||||||
|
namespace bts {
|
||||||
|
|
||||||
|
// public class CompositeRow : IBlueRow {
|
||||||
|
// private IBlueRow _lrow, _rrow;
|
||||||
|
// private BlueRow _out_row;
|
||||||
|
// public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
|
||||||
|
// _lrow = lrow;
|
||||||
|
// _rrow = rrow;
|
||||||
|
// IBlueTableSchema schema = _lrow.schema;
|
||||||
|
// List<object> vals = new List<object>();
|
||||||
|
// foreach (var o in _lrow) {
|
||||||
|
// vals.Add(o);
|
||||||
|
// }
|
||||||
|
// foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
|
||||||
|
// if (ci.colName == "id") {
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// vals.Add(_rrow[ci.colNo]);
|
||||||
|
// schema = schema.add_column(ci.type_code(), ci.colName);
|
||||||
|
// }
|
||||||
|
// _out_row = new BlueRow(schema, vals.ToArray());
|
||||||
|
// }
|
||||||
|
// public IBlueTableSchema schema {
|
||||||
|
// get { return _out_row.schema; }
|
||||||
|
// }
|
||||||
|
// public object? this[int index] {
|
||||||
|
// get { return _out_row[index]; }
|
||||||
|
// }
|
||||||
|
// public object? this[string colName] {
|
||||||
|
// get {
|
||||||
|
// return _out_row[colName];
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public static class BlueDataUtility {
|
public static class BlueDataUtility {
|
||||||
|
public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
|
||||||
|
IBlueTableSchema schema = lr.schema;
|
||||||
|
List<object> vals = new List<object>();
|
||||||
|
foreach (ColumnInfo ci in lr.schema.columnInfo) {
|
||||||
|
if (ci.colName == "id") {
|
||||||
|
vals.Add(id);
|
||||||
|
}
|
||||||
|
vals.Add(lr[ci.colName]);
|
||||||
|
}
|
||||||
|
foreach (ColumnInfo ci in rr.schema.columnInfo) {
|
||||||
|
if (ci.colName == "id") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
schema = schema.add_column(ci.type_code(), ci.colName);
|
||||||
|
vals.Add(rr[ci.colName]);
|
||||||
|
}
|
||||||
|
IBlueRow r = new BlueRow(schema, vals.ToArray());
|
||||||
|
return r;
|
||||||
|
}
|
||||||
#region Extensions
|
#region Extensions
|
||||||
|
public static bool is_null(this IBlueRow r, string col_name) {
|
||||||
|
object col_val = r[col_name];
|
||||||
|
return col_val == null || DBNull.Value.Equals(col_val);
|
||||||
|
}
|
||||||
|
|
||||||
public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
|
public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
|
||||||
return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
|
return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
|
||||||
}
|
}
|
||||||
@@ -49,6 +113,17 @@ namespace BCG {
|
|||||||
rv.Add(r.get<T>(col));
|
rv.Add(r.get<T>(col));
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
|
||||||
|
List<T> rv = new List<T>();
|
||||||
|
int i = 0;
|
||||||
|
int n_copied = 0;
|
||||||
|
foreach (IBlueRow r in rows)
|
||||||
|
if (i++ >= start_index && n_copied < count) {
|
||||||
|
rv.Add(r.get<T>(col));
|
||||||
|
n_copied++;
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
|
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
|
||||||
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
|
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
|
||||||
@@ -112,13 +187,17 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
|
public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
|
||||||
List<IBlueRow> rv = new List<IBlueRow>();
|
List<IBlueRow> rv = new List<IBlueRow>();
|
||||||
foreach (IBlueRow r in rows)
|
foreach (IBlueRow r in rows)
|
||||||
rv.Add(r);
|
rv.Add(r);
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
|
||||||
|
return rows.to_list();
|
||||||
|
}
|
||||||
|
|
||||||
public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
|
public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
|
||||||
Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
|
Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
|
||||||
foreach (IBlueRow r in rows) {
|
foreach (IBlueRow r in rows) {
|
||||||
@@ -135,6 +214,11 @@ namespace BCG {
|
|||||||
return new BlueTableSchema(input_schema.tableName, cis.ToArray());
|
return new BlueTableSchema(input_schema.tableName, cis.ToArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
|
||||||
|
Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
|
||||||
|
return add_columns(input_schema, tup);
|
||||||
|
}
|
||||||
|
|
||||||
public static IBlueRowCollection to_collection(this IBlueRow r) {
|
public static IBlueRowCollection to_collection(this IBlueRow r) {
|
||||||
if (r == null)
|
if (r == null)
|
||||||
return null;
|
return null;
|
||||||
@@ -191,6 +275,18 @@ namespace BCG {
|
|||||||
|
|
||||||
#region json
|
#region json
|
||||||
|
|
||||||
|
public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
|
||||||
|
Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
|
||||||
|
return dictionary;
|
||||||
|
}
|
||||||
|
public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
|
||||||
|
Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
|
||||||
|
if (!dictionary.ContainsKey("id")) {
|
||||||
|
dictionary.Add("id", id);
|
||||||
|
}
|
||||||
|
return new BlueRow(dictionary);
|
||||||
|
}
|
||||||
|
|
||||||
public static List<JsonNode> json_to_json_node_list(string jsonString) {
|
public static List<JsonNode> json_to_json_node_list(string jsonString) {
|
||||||
var jsonNodeList = new List<JsonNode>();
|
var jsonNodeList = new List<JsonNode>();
|
||||||
bool noe = String.IsNullOrEmpty(jsonString);
|
bool noe = String.IsNullOrEmpty(jsonString);
|
||||||
@@ -209,7 +305,7 @@ namespace BCG {
|
|||||||
foreach (JsonNode jsonNode in jsonArray) {
|
foreach (JsonNode jsonNode in jsonArray) {
|
||||||
jsonNodeList.Add(jsonNode);
|
jsonNodeList.Add(jsonNode);
|
||||||
}
|
}
|
||||||
} catch (JsonException ex) {
|
} catch (System.Text.Json.JsonException ex) {
|
||||||
Console.WriteLine($"Invalid JSON: {ex.Message}");
|
Console.WriteLine($"Invalid JSON: {ex.Message}");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -486,7 +582,16 @@ namespace BCG {
|
|||||||
ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
|
ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
|
||||||
foreach (DataColumn col in table.Columns)
|
foreach (DataColumn col in table.Columns)
|
||||||
cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
|
cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
|
||||||
return new BlueTableSchema(cis, table.TableName);
|
IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
|
||||||
|
rv = rv.add_id_col_if_missing();
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
|
||||||
|
IBlueTableSchema rv = schema;
|
||||||
|
if (!schema.columnNames.Contains("id")) {
|
||||||
|
rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -744,10 +849,16 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
|
public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
|
||||||
BlueTable retVal = new BlueTable(getSchema(dt));
|
IBlueTableSchema schema = getSchema(dt);
|
||||||
foreach (DataRow dr in dt.Rows)
|
BlueTable rv = new BlueTable(schema);
|
||||||
retVal.addRow(dr);
|
foreach (DataRow dr in dt.Rows) {
|
||||||
return retVal;
|
MutableBlueRow nr = new MutableBlueRow(schema);
|
||||||
|
foreach (DataColumn col in dr.Table.Columns)
|
||||||
|
nr.setValue(col.ColumnName, dr[col.ColumnName]);
|
||||||
|
nr.setValue("id", rv.Count);
|
||||||
|
rv.addRow(nr);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
|
public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
|
||||||
@@ -819,20 +930,20 @@ namespace BCG {
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="r1"></param>
|
/// <param name="r1"></param>
|
||||||
/// <param name="r2"></param>
|
/// <param name="r2"></param>
|
||||||
/// <param name="colNames"></param>
|
/// <param name="col_names"></param>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public static int compare_rows(IBlueRow r1,
|
public static int compare_rows(IBlueRow r1,
|
||||||
IBlueRow r2,
|
IBlueRow r2,
|
||||||
params string[] colNames) {
|
params string[] col_names) {
|
||||||
int retVal = 0;
|
int comp_res = 0;
|
||||||
foreach (string colName in colNames) {
|
foreach (string cn in col_names) {
|
||||||
object val1 = r1[colName];
|
object val1 = r1[cn];
|
||||||
object val2 = r2[colName];
|
object val2 = r2[cn];
|
||||||
retVal = _compare_row_cols(val1, val2);
|
comp_res = _compare_row_cols(val1, val2);
|
||||||
if (retVal != 0)
|
if (comp_res != 0)
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
return retVal;
|
return comp_res;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static int _compare_row_cols(object v1, object v2) {
|
private static int _compare_row_cols(object v1, object v2) {
|
||||||
|
|||||||
@@ -96,3 +96,107 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
|
|||||||
new(schema, table.ToImmutable());
|
new(schema, table.ToImmutable());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
|
||||||
|
readonly ImmutableDictionary<int, T> rows;
|
||||||
|
|
||||||
|
public BlueTableImmutable(IBlueTableSchema schema)
|
||||||
|
: this(schema, ImmutableDictionary<int, T>.Empty) { }
|
||||||
|
|
||||||
|
public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, T> rows) {
|
||||||
|
this.schema = schema;
|
||||||
|
this.rows = rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Count => rows.Count;
|
||||||
|
public bool IsSynchronized => false;
|
||||||
|
public IBlueTableSchema schema { get; }
|
||||||
|
public object SyncRoot => this;
|
||||||
|
|
||||||
|
public T this[int id] => rows[id];
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
rows.ContainsKey(id);
|
||||||
|
|
||||||
|
public void CopyTo(Array array, int index) {
|
||||||
|
foreach (var r in rows.Values) {
|
||||||
|
array.SetValue(r, index++);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IEnumerator<T> GetEnumerator() =>
|
||||||
|
rows.Values.GetEnumerator();
|
||||||
|
|
||||||
|
IEnumerator IEnumerable.GetEnumerator() =>
|
||||||
|
GetEnumerator();
|
||||||
|
|
||||||
|
public DataPacketBuilder ToDataPacketBuilder() =>
|
||||||
|
new(this);
|
||||||
|
|
||||||
|
public class DataPacketBuilder {
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
readonly ImmutableDictionary<int, T>.Builder table;
|
||||||
|
readonly ImmutableDictionary<int, T>.Builder added =
|
||||||
|
ImmutableDictionary<int, T>.Empty.ToBuilder();
|
||||||
|
readonly ImmutableDictionary<int, T>.Builder removed =
|
||||||
|
ImmutableDictionary<int, T>.Empty.ToBuilder();
|
||||||
|
|
||||||
|
public DataPacketBuilder(BlueTableImmutable<T> table) {
|
||||||
|
schema = table.schema;
|
||||||
|
this.table = table.rows.ToBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
|
||||||
|
var id = row.Id();
|
||||||
|
if (table.TryGetValue(id, out var orig)) {
|
||||||
|
if (comparer?.Equals(row, orig) ?? false) {
|
||||||
|
// No op: This row was already in the table (according to 'comparer').
|
||||||
|
// Remove it from 'added' and 'removed'.
|
||||||
|
added.Remove(id);
|
||||||
|
removed.Remove(id);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Change op: A row with this id was already in the table,
|
||||||
|
// but its values have changed (according to 'comparer').
|
||||||
|
removed[id] = orig;
|
||||||
|
}
|
||||||
|
added[id] = row;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Remove(int id) {
|
||||||
|
added.Remove(id);
|
||||||
|
if (table.TryGetValue(id, out var orig)) {
|
||||||
|
removed[id] = orig;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataPacket ToDataPacket(bool reset = true) {
|
||||||
|
table.RemoveRange(removed.Keys);
|
||||||
|
table.AddRange(added);
|
||||||
|
var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
|
||||||
|
var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
|
||||||
|
var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
|
||||||
|
if (reset) {
|
||||||
|
added.Clear();
|
||||||
|
removed.Clear();
|
||||||
|
}
|
||||||
|
return new(newRows, addedRows, removedRows);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IDataPacket<T> ToDataPacketT(bool reset = true) {
|
||||||
|
table.RemoveRange(removed.Keys);
|
||||||
|
table.AddRange(added);
|
||||||
|
var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
|
||||||
|
var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
|
||||||
|
var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
|
||||||
|
if (reset) {
|
||||||
|
added.Clear();
|
||||||
|
removed.Clear();
|
||||||
|
}
|
||||||
|
return new DataPacket<T>(newRows, addedRows, removedRows);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BlueTableImmutable<T> ToImmutable() =>
|
||||||
|
new(schema, table.ToImmutable());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,7 +24,16 @@ namespace BCG
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
object? this[int index] {get;}
|
object? this[int index] {get;}
|
||||||
object? this[string colName] {get;}
|
object? this[string colName] {get;}
|
||||||
|
bool IDictionary.IsFixedSize => true;
|
||||||
|
bool IDictionary.IsReadOnly => true;
|
||||||
|
object ICollection.SyncRoot => this;
|
||||||
|
bool ICollection.IsSynchronized => false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// public interface IBlueRow<T> : IBlueRow where T : IBlueRow {
|
||||||
|
// T Value => (T)this;
|
||||||
|
// }
|
||||||
|
|
||||||
namespace Extensions
|
namespace Extensions
|
||||||
{
|
{
|
||||||
public static class IBlueRowExtensions
|
public static class IBlueRowExtensions
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
@@ -16,4 +17,10 @@ namespace BCG
|
|||||||
IBlueTableSchema schema {get;}
|
IBlueTableSchema schema {get;}
|
||||||
bool containsID (int id);
|
bool containsID (int id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
|
||||||
|
new T this[int id] { get; }
|
||||||
|
|
||||||
|
IBlueRow IBlueRowCollection.this[int id] => this[id];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -314,6 +314,18 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public class DataPacket<T> : IDataPacket<T> where T : IBlueRow {
|
||||||
|
public DataPacket(IBlueRowCollection<T> newRows, IBlueRowCollection<T> addedRows, IBlueRowCollection<T> deletedRows) {
|
||||||
|
this.newRows = newRows;
|
||||||
|
this.addedRows = addedRows;
|
||||||
|
this.deletedRows = deletedRows;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRowCollection<T> newRows { get; }
|
||||||
|
public IBlueRowCollection<T> addedRows { get; }
|
||||||
|
public IBlueRowCollection<T> deletedRows { get; }
|
||||||
|
}
|
||||||
|
|
||||||
public static class DataPacketExtensions {
|
public static class DataPacketExtensions {
|
||||||
public static bool HasChanges(this DataPacket dataPacket) =>
|
public static bool HasChanges(this DataPacket dataPacket) =>
|
||||||
dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;
|
dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;
|
||||||
|
|||||||
@@ -53,16 +53,15 @@ namespace BCG
|
|||||||
|
|
||||||
public event Action<Exception> ErrorChanged;
|
public event Action<Exception> ErrorChanged;
|
||||||
|
|
||||||
public delegate void ChangeDelegate(object newValue);
|
public event DataStreamChangeDelegate onChange
|
||||||
public event ChangeDelegate onChange
|
|
||||||
{
|
{
|
||||||
add
|
add
|
||||||
{
|
{
|
||||||
ChangeDelegate currValue = value;
|
DataStreamChangeDelegate currValue = value;
|
||||||
if (myTestMode)
|
if (myTestMode)
|
||||||
{
|
{
|
||||||
ChangeDelegateWrapper w = new ChangeDelegateWrapper (value);
|
ChangeDelegateWrapper w = new ChangeDelegateWrapper (value);
|
||||||
currValue = new ChangeDelegate (w.wrapperFunction);
|
currValue = new DataStreamChangeDelegate (w.wrapperFunction);
|
||||||
myDelMap.Add (value, currValue);
|
myDelMap.Add (value, currValue);
|
||||||
}
|
}
|
||||||
if (myRealSource != null)
|
if (myRealSource != null)
|
||||||
@@ -70,7 +69,7 @@ namespace BCG
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
myOnChangeDelegate =
|
myOnChangeDelegate =
|
||||||
(ChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);
|
(DataStreamChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);
|
||||||
|
|
||||||
|
|
||||||
if (myOnChangeDelegate.GetInvocationList().Length == 1) // first subscriber
|
if (myOnChangeDelegate.GetInvocationList().Length == 1) // first subscriber
|
||||||
@@ -95,11 +94,11 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
remove
|
remove
|
||||||
{
|
{
|
||||||
ChangeDelegate currValue = value;
|
var currValue = value;
|
||||||
// BlueTrace.WriteViewStreamUnsubscribe(this.ID);
|
// BlueTrace.WriteViewStreamUnsubscribe(this.ID);
|
||||||
if (myTestMode)
|
if (myTestMode)
|
||||||
{
|
{
|
||||||
currValue = myDelMap[value] as ChangeDelegate;
|
currValue = myDelMap[value] as DataStreamChangeDelegate;
|
||||||
myDelMap.Remove (value);
|
myDelMap.Remove (value);
|
||||||
}
|
}
|
||||||
if (myRealSource != null)
|
if (myRealSource != null)
|
||||||
@@ -107,7 +106,7 @@ namespace BCG
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
myOnChangeDelegate =
|
myOnChangeDelegate =
|
||||||
(ChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);
|
(DataStreamChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);
|
||||||
|
|
||||||
// this is need to avoid a callback in a reentrant add.
|
// this is need to avoid a callback in a reentrant add.
|
||||||
if (myPendingAdds.Count > 0)
|
if (myPendingAdds.Count > 0)
|
||||||
@@ -175,7 +174,7 @@ namespace BCG
|
|||||||
|
|
||||||
// this was made internal for testing purposes only.
|
// this was made internal for testing purposes only.
|
||||||
// don't mess with it.
|
// don't mess with it.
|
||||||
internal ChangeDelegate myOnChangeDelegate;
|
internal DataStreamChangeDelegate myOnChangeDelegate;
|
||||||
protected object myLastValue;
|
protected object myLastValue;
|
||||||
protected DataStream myRealSource;
|
protected DataStream myRealSource;
|
||||||
private Stack myPendingAdds;
|
private Stack myPendingAdds;
|
||||||
@@ -184,9 +183,9 @@ namespace BCG
|
|||||||
internal class ChangeDelegateWrapper
|
internal class ChangeDelegateWrapper
|
||||||
{
|
{
|
||||||
readonly Stopwatch timer = new();
|
readonly Stopwatch timer = new();
|
||||||
private DataStream.ChangeDelegate myDel;
|
private DataStreamChangeDelegate myDel;
|
||||||
|
|
||||||
internal ChangeDelegateWrapper (DataStream.ChangeDelegate del) {myDel = del;}
|
internal ChangeDelegateWrapper (DataStreamChangeDelegate del) {myDel = del;}
|
||||||
|
|
||||||
internal void wrapperFunction (object o)
|
internal void wrapperFunction (object o)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
namespace BCG;
|
||||||
|
|
||||||
|
public delegate void DataStreamChangeDelegate(object newValue);
|
||||||
|
public delegate void DataStreamChangeDelegate<T>(IDataPacket<T> dp) where T : IBlueRow;
|
||||||
@@ -9,6 +9,9 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
|
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
|
||||||
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||||
|
<PackageReference Include="OpenAI" Version="2.0.0" />
|
||||||
|
<PackageReference Include="SimMetrics.Net" Version="1.0.5" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
@@ -17,8 +20,8 @@
|
|||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
|
<Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
|
||||||
<OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
|
<OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
|
||||||
</Proto>
|
</Proto>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
|
|||||||
@@ -8,11 +8,12 @@ namespace BCG {
|
|||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
public class PrimitiveManager {
|
public class PrimitiveManager {
|
||||||
private IDictionary myTypeLookupTable = new Hashtable();
|
|
||||||
private static PrimitiveManager mySelf;
|
private static PrimitiveManager mySelf;
|
||||||
private ArrayList myAssemblyNames = new ArrayList();
|
readonly List<string> myAssemblyNames = new();
|
||||||
|
readonly Dictionary<string, Type> myTypeLookupTable = new();
|
||||||
|
|
||||||
private PrimitiveManager() {
|
private PrimitiveManager() {
|
||||||
myLoadAllTypes();
|
myLoadAllTypes();
|
||||||
@@ -95,15 +96,14 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Returns the Type name corresponding to the primitive name
|
/// Returns the Type corresponding to the primitive name
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="friendlyName"></param>
|
/// <param name="friendlyName"></param>
|
||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public string find(string friendlyName) {
|
public Type find(string friendlyName) =>
|
||||||
if (myTypeLookupTable[friendlyName] == null)
|
myTypeLookupTable.TryGetValue(friendlyName, out var t)
|
||||||
throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
|
? t
|
||||||
return (myTypeLookupTable[friendlyName] as Type).Name;
|
: throw new ArgumentException($"'{friendlyName}' is not the friendly name of any of the loaded types", nameof(friendlyName));
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Constructs and returns a ViewStream of type corresponding to the friendly name
|
/// Constructs and returns a ViewStream of type corresponding to the friendly name
|
||||||
@@ -118,12 +118,8 @@ namespace BCG {
|
|||||||
constructorArgs = new object[args.Count];
|
constructorArgs = new object[args.Count];
|
||||||
args.CopyTo(constructorArgs, 0);
|
args.CopyTo(constructorArgs, 0);
|
||||||
}
|
}
|
||||||
if (myTypeLookupTable[friendlyName] == null)
|
var type = find(friendlyName);
|
||||||
throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
|
return Activator.CreateInstance(type, constructorArgs) as ViewStream;
|
||||||
return
|
|
||||||
constructorArgs == null ?
|
|
||||||
Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream :
|
|
||||||
Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -12,4 +12,16 @@ namespace BCG
|
|||||||
//this is an ICollection of IDictionary
|
//this is an ICollection of IDictionary
|
||||||
void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
|
void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public interface IDataPacket<T> : IDataPacket where T : IBlueRow {
|
||||||
|
IBlueRowCollection<T> newRows { get; }
|
||||||
|
IBlueRowCollection<T> addedRows { get; }
|
||||||
|
IBlueRowCollection<T> deletedRows { get; }
|
||||||
|
|
||||||
|
void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
|
||||||
|
new_rows = newRows;
|
||||||
|
added_rows = addedRows;
|
||||||
|
deleted_rows = deletedRows;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Loaded 100 of 951 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user