Compare commits
840
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9fbf84eeb7 | ||
|
|
0bea6e223a | ||
|
|
005207e061 | ||
|
|
809a66323d | ||
|
|
5c4ee1a262 | ||
|
|
504f441d2e | ||
|
|
fd51025b00 | ||
|
|
f4afd92db2 | ||
|
|
b1fe5b52f6 | ||
|
|
164ca306f5 | ||
|
|
a6d23ef4ef | ||
|
|
c0f4090d70 | ||
|
|
63c94e09e7 | ||
|
|
fc94ec4d46 | ||
|
|
e2fdb569c6 | ||
|
|
804c27eb49 | ||
|
|
6448b4cf90 | ||
|
|
e01a825aff | ||
|
|
00649e0c14 | ||
|
|
ae1e9ecb29 | ||
|
|
ff65acba05 | ||
|
|
dc08945719 | ||
|
|
a739ca9a75 | ||
|
|
ed7b187876 | ||
|
|
53e1e92a6a | ||
|
|
eee7af494a | ||
|
|
91a7c35edf | ||
|
|
3eaef1a1b6 | ||
|
|
4a70b4910d | ||
|
|
7697f73ada | ||
|
|
029875e0a0 | ||
|
|
9dc7611c0a | ||
|
|
782cdd7430 | ||
|
|
4184278d94 | ||
|
|
0b3fa813f1 | ||
|
|
e60f2159e1 | ||
|
|
610861be20 | ||
|
|
1787460c35 | ||
|
|
588b86710f | ||
|
|
3022a25a10 | ||
|
|
cf30649dd2 | ||
|
|
afb8444815 | ||
|
|
fc59247958 | ||
|
|
14c417eeec | ||
|
|
62a6e4340f | ||
|
|
cce85e45b7 | ||
|
|
ff584ee19a | ||
|
|
f146859677 | ||
|
|
cf55e4bc1d | ||
|
|
a0b1078572 | ||
|
|
afd229b7c7 | ||
|
|
bfc7615c9a | ||
|
|
0023574905 | ||
|
|
7e79a21cd4 | ||
|
|
6b0030559b | ||
|
|
77caef2640 | ||
|
|
edb8a9d516 | ||
|
|
153631191a | ||
|
|
026a8af893 | ||
|
|
3739532947 | ||
|
|
0cb3cadc11 | ||
|
|
36db69dc67 | ||
|
|
c7c755a964 | ||
|
|
0e460353a7 | ||
|
|
5198236b45 | ||
|
|
767fa25d61 | ||
|
|
0858eaac23 | ||
|
|
593601551d | ||
|
|
036df46b1c | ||
|
|
84697a6d65 | ||
|
|
bb550d3f88 | ||
|
|
85bd2da414 | ||
|
|
cc55fe1acf | ||
|
|
bb64760c62 | ||
|
|
8b9525f503 | ||
|
|
eccfd1ee30 | ||
|
|
7b933026bb | ||
|
|
ffe859703a | ||
|
|
4756e243fa | ||
|
|
0133159296 | ||
|
|
38afd3055f | ||
|
|
4f184c2566 | ||
|
|
f608497094 | ||
|
|
9035b95612 | ||
|
|
7483665b5a | ||
|
|
5f141a8030 | ||
|
|
83f1b5b567 | ||
|
|
06ad3a1d9e | ||
|
|
01de13bb43 | ||
|
|
155a396af5 | ||
|
|
399f9e61e2 | ||
|
|
1f67456cc5 | ||
|
|
9ff7821b59 | ||
|
|
c598f831d0 | ||
|
|
3ca4cedf54 | ||
|
|
d919ec5474 | ||
|
|
98c2d738b2 | ||
|
|
cf3f7e6862 | ||
|
|
ae868ab74d | ||
|
|
bd22f4779d | ||
|
|
27cbd51b1c | ||
|
|
02626ce35c | ||
|
|
d219804caf | ||
|
|
1e0ec5bb46 | ||
|
|
c281bc49b5 | ||
|
|
b3566155a3 | ||
|
|
a4e39afbb9 | ||
|
|
efce01d28e | ||
|
|
d362022b89 | ||
|
|
e7d751fb68 | ||
|
|
50eefc65d4 | ||
|
|
8770de4a87 | ||
|
|
e799b6e24b | ||
|
|
69e338c1ca | ||
|
|
e8bb64a80f | ||
|
|
8e2ede596f | ||
|
|
9772d0eb77 | ||
|
|
687e2477c0 | ||
|
|
1bd0ba9950 | ||
|
|
ddabc744f2 | ||
|
|
74b88b003e | ||
|
|
aaf1be09c2 | ||
|
|
e3bd9a457c | ||
|
|
59f62b6ec9 | ||
|
|
b46e75f7d7 | ||
|
|
3f100feaca | ||
|
|
093da2a938 | ||
|
|
2f17c2b8c5 | ||
|
|
33bfdc517a | ||
|
|
d17be48e6d | ||
|
|
cbaa1df641 | ||
|
|
8c23665d65 | ||
|
|
c7db105533 | ||
|
|
3f9923239e | ||
|
|
9ae9451359 | ||
|
|
c33e5ddb02 | ||
|
|
54388d3b92 | ||
|
|
17ada3c57c | ||
|
|
046f412ec4 | ||
|
|
c94d11f525 | ||
|
|
2b69846381 | ||
|
|
bdd5e08b60 | ||
|
|
7c9330650d | ||
|
|
98fca5ad7f | ||
|
|
8a0fcce90d | ||
|
|
112527f9ed | ||
|
|
fc1e20538d | ||
|
|
83acbecf55 | ||
|
|
06c494291c | ||
|
|
fe74aed048 | ||
|
|
b94cdb561e | ||
|
|
06bd8791c0 | ||
|
|
b5850cdb02 | ||
|
|
545d76142f | ||
|
|
edc2cea60e | ||
|
|
e6466f8309 | ||
|
|
501be8d6a5 | ||
|
|
7dea9b3792 | ||
|
|
a3f49b1ce8 | ||
|
|
765237d701 | ||
|
|
97c52410a2 | ||
|
|
8b43d93e26 | ||
|
|
14baa7d62e | ||
|
|
a054f4cc63 | ||
|
|
5311128607 | ||
|
|
075acdc8eb | ||
|
|
e1fdf6306b | ||
|
|
3b9fb2759b | ||
|
|
3f89d3b7f7 | ||
|
|
a6dd0d842d | ||
|
|
d050d2d19a | ||
|
|
942d00a8b1 | ||
|
|
31cb25ac2e | ||
|
|
cbb43a22de | ||
|
|
9e7b4849f0 | ||
|
|
6e72890153 | ||
|
|
c238fce057 | ||
|
|
b8f5db4ee2 | ||
|
|
87c8b9862a | ||
|
|
fbc2d8633d | ||
|
|
d12d798fa5 | ||
|
|
e75be2da2f | ||
|
|
3ae3a7094e | ||
|
|
bafa2f71ba | ||
|
|
63bd29a49e | ||
|
|
5fdc9ca48b | ||
|
|
a19e7bc7b4 | ||
|
|
04b0a89644 | ||
|
|
363a43cb8b | ||
|
|
c9f9c5f0f6 | ||
|
|
ab84a05644 | ||
|
|
4681f22e71 | ||
|
|
a25e055e35 | ||
|
|
2386612b51 | ||
|
|
3606dac145 | ||
|
|
bf936c9d13 | ||
|
|
1aeeb7170e | ||
|
|
78ad8a3aed | ||
|
|
5ca8dbafaf | ||
|
|
b464b9466e | ||
|
|
7a5c676e99 | ||
|
|
1e8d699dd1 | ||
|
|
aadd2dd594 | ||
|
|
5163ca4cae | ||
|
|
b5d520d568 | ||
|
|
83e4465a3b | ||
|
|
54cfb343a8 | ||
|
|
4e9b360e23 | ||
|
|
16bc93b1a0 | ||
|
|
f56d4d9062 | ||
|
|
36cc186626 | ||
|
|
2e78af1756 | ||
|
|
70fae06a30 | ||
|
|
07b59559a2 | ||
|
|
cd1709563b | ||
|
|
94f22ddf40 | ||
|
|
7f6c08f956 | ||
|
|
1d737cdf82 | ||
|
|
b645192355 | ||
|
|
5ad7755b28 | ||
|
|
df4ca124c3 | ||
|
|
2c8b9ee288 | ||
|
|
f02f28e347 | ||
|
|
c8a89c1e8d | ||
|
|
7ae9e07a30 | ||
|
|
a130e47218 | ||
|
|
a377f6de7f | ||
|
|
855373c9e6 | ||
|
|
e1b535b633 | ||
|
|
3b9ac71460 | ||
|
|
9a5075daaf | ||
|
|
4e7fe811a8 | ||
|
|
0a6733180d | ||
|
|
f2169c6bd4 | ||
|
|
23ef478391 | ||
|
|
236a5a9362 | ||
|
|
cb560bbe33 | ||
|
|
248f2644ca | ||
|
|
b87d86f4d6 | ||
|
|
8515196dcb | ||
|
|
e5e042f016 | ||
|
|
da3593326d | ||
|
|
313d3a2377 | ||
|
|
22f0e126d0 | ||
|
|
c76b061e20 | ||
|
|
0107b65281 | ||
|
|
6b5496b799 | ||
|
|
42f9daf9cb | ||
|
|
bc5ddade7f | ||
|
|
d8d198f7f4 | ||
|
|
af9eb670c1 | ||
|
|
d91837c377 | ||
|
|
7725ae33ef | ||
|
|
9f56abaa38 | ||
|
|
95444d4a4c | ||
|
|
a7e1c18fe2 | ||
|
|
d11fd23cb0 | ||
|
|
8499ac0dfb | ||
|
|
f4afe348e6 | ||
|
|
096424b60b | ||
|
|
b4f5e66466 | ||
|
|
69a0bcc9cd | ||
|
|
d72ebdf421 | ||
|
|
8e93856767 | ||
|
|
19dae6cd34 | ||
|
|
24f42550dd | ||
|
|
c2150ae450 | ||
|
|
40dbfaa068 | ||
|
|
3756a8a8b7 | ||
|
|
46c8dcb18f | ||
|
|
024480f32e | ||
|
|
ba525372bb | ||
|
|
8aa0f1b674 | ||
|
|
fc14215d9b | ||
|
|
091ed5a327 | ||
|
|
30bb9d0837 | ||
|
|
9cb2c580d6 | ||
|
|
2e2150ac1a | ||
|
|
21d9435a67 | ||
|
|
e7e334fb5a | ||
|
|
61f04c4605 | ||
|
|
24fb2e1e39 | ||
|
|
ece141c448 | ||
|
|
617bac8e74 | ||
|
|
ab217401eb | ||
|
|
77e5c1481b | ||
|
|
a0f77f92a3 | ||
|
|
0acdcb150d | ||
|
|
ac2f3b9bc6 | ||
|
|
0e522f90e5 | ||
|
|
40ad1f1b48 | ||
|
|
1cb75d399f | ||
|
|
cddecc0543 | ||
|
|
484e36b330 | ||
|
|
8a9c22fa2a | ||
|
|
a6e8853cb9 | ||
|
|
dafa193283 | ||
|
|
23651970c9 | ||
|
|
043efab5a2 | ||
|
|
5476466d55 | ||
|
|
54cb5a2ab2 | ||
|
|
29bf7d8d7e | ||
|
|
855d2fd790 | ||
|
|
8a6a652882 | ||
|
|
3f288704bf | ||
|
|
98bc78463d | ||
|
|
6d271ad7cd | ||
|
|
2a17d656a5 | ||
|
|
10436f0f3b | ||
|
|
86b19f0ef0 | ||
|
|
502024dd10 | ||
|
|
623012b0b0 | ||
|
|
5859f0f2d1 | ||
|
|
eeffb735ae | ||
|
|
b602997f3a | ||
|
|
3530a77b38 | ||
|
|
afdf635b31 | ||
|
|
4d4aefc0a7 | ||
|
|
960a8af77a | ||
|
|
89828659a4 | ||
|
|
76f83fd7f5 | ||
|
|
d4e40948d2 | ||
|
|
912ab512d2 | ||
|
|
5d3eb92175 | ||
|
|
924386084b | ||
|
|
acab3ac392 | ||
|
|
d06a62f6ee | ||
|
|
14e3eee0bc | ||
|
|
7277bf52f5 | ||
|
|
b81dc7f5cf | ||
|
|
b1a4635345 | ||
|
|
7c7c689170 | ||
|
|
116506c8da | ||
|
|
3feda49ef1 | ||
|
|
10318be7bc | ||
|
|
f7d525a527 | ||
|
|
d2fe9f167a | ||
|
|
696d91a5af | ||
|
|
d575ba0ea8 | ||
|
|
058b131760 | ||
|
|
57f8f3ff58 | ||
|
|
775df2850d | ||
|
|
cb2b15f10b | ||
|
|
60362a4c95 | ||
|
|
c6d7a8f38c | ||
|
|
7931c62b46 | ||
|
|
58518451c8 | ||
|
|
e3bad78aef | ||
|
|
8e9ab8f2eb | ||
|
|
8512f414d6 | ||
|
|
513fc353cf | ||
|
|
4a04beafd2 | ||
|
|
79a3251f26 | ||
|
|
8785e0fecd | ||
|
|
04251c96b6 | ||
|
|
c2d400af1a | ||
|
|
133cc16628 | ||
|
|
16f7aede93 | ||
|
|
2b9488faff | ||
|
|
51b6d592d3 | ||
|
|
1eb4ceba0a | ||
|
|
dbf5cadb44 | ||
|
|
90f1bb0851 | ||
|
|
547d83e1b8 | ||
|
|
e888149c41 | ||
|
|
b944c3980f | ||
|
|
c2a8266106 | ||
|
|
4277ba61b9 | ||
|
|
35cfe9533b | ||
|
|
d49efef93d | ||
|
|
502ed13456 | ||
|
|
716c4dd674 | ||
|
|
4a2b189a07 | ||
|
|
a42edaf9fa | ||
|
|
cbc142ab15 | ||
|
|
3980f2d030 | ||
|
|
00f6d269c9 | ||
|
|
10b2d38de8 | ||
|
|
6d4d8f6ac1 | ||
|
|
5db89e1e8d | ||
|
|
379a46afa2 | ||
|
|
d9eff37184 | ||
|
|
3d055de139 | ||
|
|
648be7dd97 | ||
|
|
2cac9affe4 | ||
|
|
30ff3ba707 | ||
|
|
9d3afd68aa | ||
|
|
8c3de1789b | ||
|
|
b251e1d47e | ||
|
|
6ccd48a8ba | ||
|
|
41d84f7f73 | ||
|
|
795062864d | ||
|
|
1ff72e9444 | ||
|
|
e13b5066a3 | ||
|
|
5ba7901258 | ||
|
|
bc6eb59110 | ||
|
|
5ae5a5dcbf | ||
|
|
7966bcce77 | ||
|
|
ff86b10895 | ||
|
|
ea3c0717d8 | ||
|
|
d0abca3fd9 | ||
|
|
7895fe7357 | ||
|
|
6f5f4f5cc2 | ||
|
|
066a0ba693 | ||
|
|
c2b67b4813 | ||
|
|
ecad456a2d | ||
|
|
2cb3cfdcdd | ||
|
|
0ebc8e5b4b | ||
|
|
2072105c0f | ||
|
|
d663c86aa0 | ||
|
|
4a2dfa46a1 | ||
|
|
6e35db6a24 | ||
|
|
b45dc3bc2d | ||
|
|
a1912907b8 | ||
|
|
b1b3dcdeb8 | ||
|
|
03dc8ef767 | ||
|
|
975bf56c2a | ||
|
|
3f58c2ce84 | ||
|
|
96f28acf91 | ||
|
|
c2c2658e21 | ||
|
|
d6b8e359cd | ||
|
|
84a0850d95 | ||
|
|
4effdc4160 | ||
|
|
b9fe39f6e3 | ||
|
|
b555850015 | ||
|
|
23794fa30d | ||
|
|
90b5a309a3 | ||
|
|
07beda3fc6 | ||
|
|
e3c1ea70f2 | ||
|
|
93b93969f5 | ||
|
|
0a10111846 | ||
|
|
eee00cc407 | ||
|
|
0eb58eaba8 | ||
|
|
12f6733d1e | ||
|
|
93a0984ed1 | ||
|
|
b7e9c393a0 | ||
|
|
aa602b3fd6 | ||
|
|
b4b8cabdae | ||
|
|
cfe06869b3 | ||
|
|
b177ed1cc0 | ||
|
|
ff17a64933 | ||
|
|
81a1fe11f6 | ||
|
|
ad5bf88fcf | ||
|
|
e15f155ac9 | ||
|
|
6d6822cd65 | ||
|
|
ace0c70516 | ||
|
|
e9e5059bc2 | ||
|
|
cde72f51a0 | ||
|
|
e1fd49ae32 | ||
|
|
274f3f98e4 | ||
|
|
a4b7d84318 | ||
|
|
be17205094 | ||
|
|
6ef9bee8e4 | ||
|
|
2c8b3563f1 | ||
|
|
71f7e64ccb | ||
|
|
c31b44ecfb | ||
|
|
19efb77e07 | ||
|
|
76de640421 | ||
|
|
fb030742d3 | ||
|
|
73608f1b45 | ||
|
|
58aebe7002 | ||
|
|
299cb0b29e | ||
|
|
06663f711e | ||
|
|
51479f4885 | ||
|
|
5b520ba241 | ||
|
|
f0651a5341 | ||
|
|
897e0990a2 | ||
|
|
30e1ae5c3c | ||
|
|
efe83f81ec | ||
|
|
5fb21ada97 | ||
|
|
58edd07f6b | ||
|
|
9d81354c3a | ||
|
|
0f37be89a4 | ||
|
|
f6586fe636 | ||
|
|
7827ccc0e7 | ||
|
|
af6a2acc0a | ||
|
|
84abf59a55 | ||
|
|
2fc1d30bf1 | ||
|
|
4dcd5d63a1 | ||
|
|
63e5697b20 | ||
|
|
8e5cb100a7 | ||
|
|
938bee1817 | ||
|
|
d48af15480 | ||
|
|
e3b2e98cd9 | ||
|
|
ee53fb854c | ||
|
|
d36e2d61f6 | ||
|
|
bfff3ea5fe | ||
|
|
059c03d289 | ||
|
|
8c561f7886 | ||
|
|
288567d1eb | ||
|
|
ecfc5405fe | ||
|
|
8db34a2a11 | ||
|
|
91094c4d84 | ||
|
|
dae8c85d73 | ||
|
|
abdd98082f | ||
|
|
2f6aa785ab | ||
|
|
7acf1f7043 | ||
|
|
1c3c7a078a | ||
|
|
93ace52bfa | ||
|
|
b5d97c0b1b | ||
|
|
284aa53ad4 | ||
|
|
8a9952f52d | ||
|
|
748aafaab3 | ||
|
|
3ba30c70de | ||
|
|
e72f465999 | ||
|
|
f3d93f7749 | ||
|
|
8d142b8a74 | ||
|
|
8210b9f35f | ||
|
|
12b20b05ce | ||
|
|
72efbccdbd | ||
|
|
33643e9eab | ||
|
|
0f6dc6f40b | ||
|
|
8c778f5e41 | ||
|
|
f308654200 | ||
|
|
bdf5b980fc | ||
|
|
914ac597cd | ||
|
|
4aa6a77680 | ||
|
|
6b59dd84ee | ||
|
|
4e2c5f2d17 | ||
|
|
9a1b94a1b0 | ||
|
|
13b3c55771 | ||
|
|
d0a6fcdb9c | ||
|
|
1eb13028c1 | ||
|
|
424e5ff16b | ||
|
|
bed389885b | ||
|
|
85cf25a57a | ||
|
|
61471d4675 | ||
|
|
d83435d49d | ||
|
|
582fc10dce | ||
|
|
5969985d24 | ||
|
|
fabe7d665c | ||
|
|
7ce75bb8e7 | ||
|
|
b8e1c3cd60 | ||
|
|
4909944694 | ||
|
|
7258af09b4 | ||
|
|
7f2ce6c1a9 | ||
|
|
4835725e1c | ||
|
|
6a9e8de475 | ||
|
|
4639951acc | ||
|
|
d8de211aac | ||
|
|
87394bc60b | ||
|
|
beba30f65c | ||
|
|
2d58424d8f | ||
|
|
d1bbfa3791 | ||
|
|
aab17f460f | ||
|
|
461ef53322 | ||
|
|
2486bc90b2 | ||
|
|
1c5cbaf60b | ||
|
|
b8de447cfb | ||
|
|
8bfc2d40f9 | ||
|
|
c09fd71c94 | ||
|
|
f3b037553e | ||
|
|
d386d93166 | ||
|
|
5217742c71 | ||
|
|
c59ec95460 | ||
|
|
70f01583ec | ||
|
|
66d2f9eb63 | ||
|
|
4625bed8b8 | ||
|
|
f18fb332b5 | ||
|
|
8746a6f5ca | ||
|
|
702d47abb5 | ||
|
|
bf2a54717c | ||
|
|
9db779fb25 | ||
|
|
f55ddec708 | ||
|
|
6e942dd2d4 | ||
|
|
142310a17a | ||
|
|
7f99a89a95 | ||
|
|
bfce663b4b | ||
|
|
33c743643d | ||
|
|
858e0a389b | ||
|
|
9c3bb85674 | ||
|
|
f4930e8cd4 | ||
|
|
f21e7f3602 | ||
|
|
b56c35aca4 | ||
|
|
e6ac83196e | ||
|
|
7b6f74fc4a | ||
|
|
f754953fac | ||
|
|
2dd1143d9f | ||
|
|
e1ee3275d8 | ||
|
|
2549e406f3 | ||
|
|
d5024f1858 | ||
|
|
3623ac96db | ||
|
|
19834625f9 | ||
|
|
c8fc649b6b | ||
|
|
6b1fc1c413 | ||
|
|
6f086532ac | ||
|
|
dc66282225 | ||
|
|
e2c74c0bb3 | ||
|
|
e89d50eec1 | ||
|
|
8bebbc278d | ||
|
|
2e6d12c9c1 | ||
|
|
102a5e23ea | ||
|
|
582dc791db | ||
|
|
afb35d8c25 | ||
|
|
3a32ed5fe6 | ||
|
|
4e01697327 | ||
|
|
22032c1742 | ||
|
|
0a9bce317c | ||
|
|
40e44db597 | ||
|
|
4a447e44e1 | ||
|
|
b1f4654b3c | ||
|
|
9d350334c0 | ||
|
|
ac597f505f | ||
|
|
147d9ceb1f | ||
|
|
377804dfb9 | ||
|
|
0d26401961 | ||
|
|
889e226151 | ||
|
|
986d53f4e0 | ||
|
|
d13ba2968a | ||
|
|
1a192c8117 | ||
|
|
5e36d4801c | ||
|
|
2cf6411ca7 | ||
|
|
e99df129d5 | ||
|
|
d07c01bd17 | ||
|
|
d69673c524 | ||
|
|
4ee595cdaf | ||
|
|
6fcaf07df0 | ||
|
|
492d48d886 | ||
|
|
b6a56e768b | ||
|
|
3e2b5976a8 | ||
|
|
edd3f6cbd6 | ||
|
|
2e9347341b | ||
|
|
2061e38349 | ||
|
|
650dd57b6d | ||
|
|
82747f7dc0 | ||
|
|
04b26904a3 | ||
|
|
f123e4c72c | ||
|
|
b887cbebab | ||
|
|
14e975bc7e | ||
|
|
064f557057 | ||
|
|
4d106b5e6c | ||
|
|
9fbeaf0d68 | ||
|
|
4018ac8b78 | ||
|
|
81c9503300 | ||
|
|
477210926d | ||
|
|
0aa48a79a9 | ||
|
|
9e71c7af02 | ||
|
|
4647751409 | ||
|
|
a3087b874b | ||
|
|
5b2f5c5c8f | ||
|
|
479227090e | ||
|
|
2dfcf320f5 | ||
|
|
b4c7213cb2 | ||
|
|
c7dfd69b90 | ||
|
|
4fdc866485 | ||
|
|
f5d5ca9a1c | ||
|
|
cfafcf77b9 | ||
|
|
8ebe783139 | ||
|
|
1851a96a58 | ||
|
|
0b334ef870 | ||
|
|
ee811a729a | ||
|
|
cb15be276d | ||
|
|
7d5af6729b | ||
|
|
e5679cf082 | ||
|
|
98f89b9ad4 | ||
|
|
ba4f8f67e3 | ||
|
|
9c78380490 | ||
|
|
f6166f0674 | ||
|
|
6334a902e7 | ||
|
|
082ca62fff | ||
|
|
9dbd8dd7c6 | ||
|
|
8f7a4d0df6 | ||
|
|
f372dd56b3 | ||
|
|
86eba20d61 | ||
|
|
35a985280d | ||
|
|
f5e1b3319c | ||
|
|
65c3948e4f | ||
|
|
002bdff9a3 | ||
|
|
bb6e5dd548 | ||
|
|
7c1aed3339 | ||
|
|
22ae51fb99 | ||
|
|
a53d347f81 | ||
|
|
838dbcd886 | ||
|
|
5ed2309d22 | ||
|
|
4d41704dd2 | ||
|
|
bd1000e518 | ||
|
|
dd4aa3c2ed | ||
|
|
fee9d3f597 | ||
|
|
416f9aef31 | ||
|
|
48f5e4c201 | ||
|
|
4438b4ca8d | ||
|
|
73356e0d61 | ||
|
|
4a285fa581 | ||
|
|
cfa82c2177 | ||
|
|
0410a0431a | ||
|
|
02843f14ff | ||
|
|
f1f1d4259c | ||
|
|
71fc43af68 | ||
|
|
350144ffd5 | ||
|
|
da227a5550 | ||
|
|
0c92cf3a87 | ||
|
|
1188af391e | ||
|
|
0bb85cf915 | ||
|
|
d77f5996fe | ||
|
|
940c41a274 | ||
|
|
e671960dc3 | ||
|
|
8fa34b6b39 | ||
|
|
f9d5429b5b | ||
|
|
2c342482eb | ||
|
|
3aefac26c2 | ||
|
|
1554594cdd | ||
|
|
727a0f38b5 | ||
|
|
7a0e8cdb26 | ||
|
|
a4e97b04d7 | ||
|
|
463980a6ef | ||
|
|
9b25857042 | ||
|
|
ef19e7d76b | ||
|
|
471faec2cb | ||
|
|
ce96d42db8 | ||
|
|
187894ca84 | ||
|
|
63492c4575 | ||
|
|
e150c611f6 | ||
|
|
bb837d6c26 | ||
|
|
965e541ac8 | ||
|
|
1b232aa610 | ||
|
|
25439b094a | ||
|
|
f927ef2c5e | ||
|
|
d42389b75a | ||
|
|
671cbcf03a | ||
|
|
e589e23f41 | ||
|
|
11fadc0a3e | ||
|
|
b8c2d93866 | ||
|
|
67a23dd196 | ||
|
|
ff22c4eb5f | ||
|
|
5addff4c50 | ||
|
|
142e1bd2da | ||
|
|
d05d91bbe7 | ||
|
|
3a90ca9cb5 | ||
|
|
85a46727be | ||
|
|
624fd2c976 | ||
|
|
16d599d2d6 | ||
|
|
c00e4fcb6b | ||
|
|
03e19afe1d | ||
|
|
058e11eadc | ||
|
|
6ff52e7107 | ||
|
|
ee4c28ed41 | ||
|
|
50d26f0060 | ||
|
|
36378e6074 | ||
|
|
8cde8ee6e1 | ||
|
|
4f8fad444e | ||
|
|
832c645a30 | ||
|
|
691343d9b6 | ||
|
|
461cbad338 | ||
|
|
fecbcbb6b6 | ||
|
|
aab8e83543 | ||
|
|
420b46741e | ||
|
|
e8f65f95bd | ||
|
|
0e893b9b5d | ||
|
|
0593ae0320 | ||
|
|
960149b9c5 | ||
|
|
b5f5f2f3d5 | ||
|
|
bd8c97244a | ||
|
|
cb187e6997 | ||
|
|
4004ed2267 | ||
|
|
a00347ac3f | ||
|
|
c4d08840bd | ||
|
|
84c0684b63 | ||
|
|
bca8df6d99 | ||
|
|
98f8e58f3f | ||
|
|
76f18f3443 | ||
|
|
b67d748ba1 | ||
|
|
0113721355 | ||
|
|
715f7f7c70 | ||
|
|
89955597c1 | ||
|
|
96d9aae048 | ||
|
|
f28d32946f | ||
|
|
924efb2c3f | ||
|
|
50abb1d784 | ||
|
|
b5ef39ee4b | ||
|
|
5b61b4fe69 | ||
|
|
2d8523ae67 | ||
|
|
f0eef447d3 | ||
|
|
8604d5672c | ||
|
|
30822fb1f1 | ||
|
|
826bbf76a9 | ||
|
|
eb8c97a1b7 | ||
|
|
5517c58724 | ||
|
|
e9bcdd0c65 | ||
|
|
83b37c6661 | ||
|
|
6163887cd3 | ||
|
|
214641aa1e | ||
|
|
06bc7af232 | ||
|
|
36b6168dec | ||
|
|
f9aa9dd982 | ||
|
|
22f1ab44f9 | ||
|
|
e4c87c267d | ||
|
|
86899a6ff0 | ||
|
|
2324fd3ad2 | ||
|
|
288b7712db | ||
|
|
8850bf4903 | ||
|
|
11112b41d1 | ||
|
|
2a98889b67 | ||
|
|
8060518a28 | ||
|
|
68440d3df3 | ||
|
|
7d775afc4c | ||
|
|
50c7764ef7 | ||
|
|
fb8b2d28f3 | ||
|
|
bcc2c4a514 | ||
|
|
9352298be0 | ||
|
|
180bd11740 | ||
|
|
57ea841381 | ||
|
|
7d9d176fa2 | ||
|
|
a8f4ec07c8 | ||
|
|
58c5be268f | ||
|
|
bf4859a2a9 | ||
|
|
db1cf93994 | ||
|
|
ecb1c4fea0 | ||
|
|
a513933044 | ||
|
|
764d65b595 | ||
|
|
bc0d606252 | ||
|
|
ceb8a68902 | ||
|
|
0b3183fd30 | ||
|
|
b6dbe37818 | ||
|
|
c78b8bc8e1 | ||
|
|
314f3e3715 | ||
|
|
26a1d5fc03 | ||
|
|
37e5003cf9 | ||
|
|
07f3868ffc | ||
|
|
46f0b17138 | ||
|
|
33a8e06a3a | ||
|
|
322c06bfbe | ||
|
|
b68fceb362 | ||
|
|
5db586c4d3 | ||
|
|
31df1a3827 | ||
|
|
7e219aec7c | ||
|
|
37f8b98df9 | ||
|
|
f06beff22f | ||
|
|
6aa39a716a | ||
|
|
5088b5ebe0 | ||
|
|
010dd3a26f | ||
|
|
1a18755176 | ||
|
|
233e2576af | ||
|
|
543c655bca | ||
|
|
050e21e22b | ||
|
|
672ad6d237 | ||
|
|
0a8f6e866f | ||
|
|
b177ffef04 | ||
|
|
393f304be4 | ||
|
|
151816bf5e | ||
|
|
87b6a2044c |
No files matched your search
@@ -9,7 +9,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"fable": {
|
"fable": {
|
||||||
"version": "4.0.6",
|
"version": "4.24.0",
|
||||||
"commands": [
|
"commands": [
|
||||||
"fable"
|
"fable"
|
||||||
]
|
]
|
||||||
@@ -27,4 +27,4 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -65,7 +65,7 @@ RUN \
|
|||||||
&& apt-get install -y cmake
|
&& apt-get install -y cmake
|
||||||
|
|
||||||
# Dotnet
|
# Dotnet
|
||||||
RUN apt-get update && apt-get install -y dotnet7
|
RUN apt-get update && apt-get install -y dotnet8
|
||||||
|
|
||||||
# Node
|
# Node
|
||||||
ARG node_major=18
|
ARG node_major=18
|
||||||
@@ -90,6 +90,12 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
|
|||||||
# Databento (deps)
|
# Databento (deps)
|
||||||
RUN apt-get update && apt-get -y install libzstd-dev
|
RUN apt-get update && apt-get -y install libzstd-dev
|
||||||
|
|
||||||
|
# Docker in Docker
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
docker.io \
|
||||||
|
docker-compose \
|
||||||
|
docker-compose-v2
|
||||||
|
|
||||||
# Set up default executables (update-alternatives)
|
# Set up default executables (update-alternatives)
|
||||||
RUN \
|
RUN \
|
||||||
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
||||||
|
|||||||
@@ -1,15 +1,10 @@
|
|||||||
{
|
{
|
||||||
"dockerFile": "Dockerfile",
|
"dockerFile": "Dockerfile",
|
||||||
// Creates local dir to mount to HOME in docker
|
|
||||||
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
|
|
||||||
"hostRequirements": {
|
"hostRequirements": {
|
||||||
"memory": "4gb",
|
"memory": "8gb",
|
||||||
"storage": "6gb"
|
"storage": "10gb"
|
||||||
},
|
},
|
||||||
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
|
|
||||||
"containerUser": "bts",
|
"containerUser": "bts",
|
||||||
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
|
|
||||||
"workspaceFolder": "/home/bts/spark",
|
|
||||||
"updateRemoteUserUID": true,
|
"updateRemoteUserUID": true,
|
||||||
"shutdownAction": "none",
|
"shutdownAction": "none",
|
||||||
// Note: If running multiple containers on the same host, each must use a
|
// Note: If running multiple containers on the same host, each must use a
|
||||||
@@ -17,31 +12,20 @@
|
|||||||
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
||||||
"runArgs": [
|
"runArgs": [
|
||||||
"--shm-size",
|
"--shm-size",
|
||||||
"2gb",
|
"8gb"
|
||||||
"-p",
|
|
||||||
"8000:8000",
|
|
||||||
"-p",
|
|
||||||
"8001:8001",
|
|
||||||
"-p",
|
|
||||||
"8002:8002",
|
|
||||||
"-p",
|
|
||||||
"8003:8003"
|
|
||||||
],
|
],
|
||||||
"mounts": [
|
"mounts": [
|
||||||
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
|
"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
|
||||||
],
|
],
|
||||||
"customizations": {
|
"customizations": {
|
||||||
"vscode": {
|
"vscode": {
|
||||||
"settings": {
|
|
||||||
"clangd.arguments": [
|
|
||||||
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"extensions": [
|
"extensions": [
|
||||||
"eamodio.gitlens",
|
"eamodio.gitlens",
|
||||||
"llvm-vs-code-extensions.vscode-clangd",
|
"llvm-vs-code-extensions.vscode-clangd",
|
||||||
|
"ms-python.python",
|
||||||
"ms-vscode.cmake-tools",
|
"ms-vscode.cmake-tools",
|
||||||
"ms-vscode.cpptools"
|
"ms-vscode.cpptools",
|
||||||
|
"xaver.clang-format"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# This file defines the global default configuration for docker deployments.
|
||||||
|
# It is tracked in source control. For temporary modifications, please create
|
||||||
|
# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
|
||||||
|
# If a variable is defined in both files, the value in .env.local will be used.
|
||||||
|
#
|
||||||
|
# About the deployment procedure:
|
||||||
|
# We use Docker Compose to build images and launch containers for various services
|
||||||
|
# that make up our software suite:
|
||||||
|
# - sparkd
|
||||||
|
# - TODO: pubsub
|
||||||
|
# - TODO: md-publisher
|
||||||
|
# - TODO: webapp (DSQA on the web)
|
||||||
|
# - TODO: api-server (websocket based)
|
||||||
|
# From inside a devcontainer, you can launch all services in separate containers
|
||||||
|
# on your machine by running:
|
||||||
|
#
|
||||||
|
# tools/deploy.sh
|
||||||
|
#
|
||||||
|
# This script builds release versions for each service, packaged (generally) in
|
||||||
|
# self-contained directories. Next, it runs a few "docker compose" commands to
|
||||||
|
# build docker images loaded with the self-contained binaries from the previous
|
||||||
|
# step, and start containers with those images to run each service. To view the
|
||||||
|
# commands that deploy.sh runs internally (without actually running them), run:
|
||||||
|
#
|
||||||
|
# tools/deploy.sh --dry-run
|
||||||
|
#
|
||||||
|
# For additional information/help, run:
|
||||||
|
#
|
||||||
|
# tools/deploy.sh --help
|
||||||
|
#
|
||||||
|
|
||||||
|
### Sparkd ###
|
||||||
|
# The internal image for sparkd to use. For more info, view:
|
||||||
|
# src/server/sparkd/Dockerfile
|
||||||
|
SPARKD__IMAGE=sparkd
|
||||||
|
# The config dir for this service to use
|
||||||
|
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
|
||||||
|
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
|
||||||
|
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
|
||||||
|
# SPARKD__DATA_DIR=/home/bts/sparkdata/data
|
||||||
|
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
|
||||||
|
SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
|
||||||
|
SPARKD__CONFIG_PORT=8000
|
||||||
|
SPARKD__QUICKBUS_PORT=8001
|
||||||
|
SPARKD__SIM_FIX_PORT=8002
|
||||||
|
SPARKD__SHM_SIZE=4gb
|
||||||
|
SPARKD__PYTHON_REQUIREMENTS=requirements.txt
|
||||||
|
|
||||||
|
# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
|
||||||
@@ -11,6 +11,7 @@ cmake-build-debug/
|
|||||||
**.vscode/*
|
**.vscode/*
|
||||||
!.vscode/launch.json
|
!.vscode/launch.json
|
||||||
!.vscode/tasks.json
|
!.vscode/tasks.json
|
||||||
|
!.vscode/*.env
|
||||||
.vs
|
.vs
|
||||||
obj/
|
obj/
|
||||||
*.csproj.user
|
*.csproj.user
|
||||||
@@ -59,3 +60,5 @@ trace_data/
|
|||||||
.devcontainer/
|
.devcontainer/
|
||||||
*.Generated.cs
|
*.Generated.cs
|
||||||
*.lst
|
*.lst
|
||||||
|
.env.local
|
||||||
|
.secrets
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[submodule "externals"]
|
[submodule "externals"]
|
||||||
path = externals
|
path = externals
|
||||||
url = git@bitbucket.org:bluetradesys/externals.git
|
url = https://bitbucket.org/bluetradesys/externals.git
|
||||||
[submodule "vcpkg"]
|
[submodule "vcpkg"]
|
||||||
path = vcpkg
|
path = vcpkg
|
||||||
url = https://github.com/microsoft/vcpkg.git
|
url = https://github.com/microsoft/vcpkg.git
|
||||||
Vendored
+164
-12
@@ -1,6 +1,19 @@
|
|||||||
{
|
{
|
||||||
"version": "0.2.0",
|
"version": "0.2.0",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": "Python Debugger: Current File with Arguments",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "${file}",
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"args": "${command:pickArgs}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": ".NET Core Attach",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "attach"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "DataStreamPublisher",
|
"name": "DataStreamPublisher",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
@@ -17,15 +30,59 @@
|
|||||||
"--config",
|
"--config",
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "(gdb) Launch",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
|
"args": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
|
"environment": [],
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "spreader tests2",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"[spreader2]"
|
||||||
|
],
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
|
"environment": [],
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "spreader tests",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"[spreader]"
|
||||||
|
],
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
|
"environment": [],
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "DataStreamQueryAnalyzer",
|
"name": "DataStreamQueryAnalyzer",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"preLaunchTask": "Build DataStream (Debug)",
|
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
||||||
"program": "",
|
"program": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
"program": "DataStreamQueryAnalyzer.exe",
|
||||||
@@ -33,7 +90,7 @@
|
|||||||
"--config",
|
"--config",
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
"env": {
|
"env": {
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
||||||
},
|
},
|
||||||
@@ -57,7 +114,7 @@
|
|||||||
"--config",
|
"--config",
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -69,7 +126,19 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
"preLaunchTask": "Build Spark (Debug)",
|
||||||
"program": "Spark.exe",
|
"program": "Spark.exe",
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Spark (Skip Build)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"program": "Spark.exe",
|
||||||
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -86,27 +155,100 @@
|
|||||||
"--sparkd",
|
"--sparkd",
|
||||||
"${input:sparkConfigServer}"
|
"${input:sparkConfigServer}"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Spark Test",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"preLaunchTask": "Build Spark (Debug)",
|
||||||
|
"program": "Spark.Console.exe",
|
||||||
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkTest}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Spark Test (Skip Build)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"program": "Spark.Console.exe",
|
||||||
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkTest}"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Sparkd",
|
"name": "Sparkd",
|
||||||
"type": "coreclr",
|
"type": "cppdbg",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"preLaunchTask": "Build Sparkd (Debug | Clang)",
|
|
||||||
"program": "",
|
"program": "",
|
||||||
"linux": {
|
"linux": {
|
||||||
"program": "sparkd",
|
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
||||||
},
|
},
|
||||||
"args": [
|
"args": [
|
||||||
"run"
|
"run"
|
||||||
],
|
],
|
||||||
"cwd": "build/sparkd-debug-clang/server",
|
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"externalConsole": false,
|
||||||
}
|
"MIMode": "gdb",
|
||||||
|
"setupCommands": [
|
||||||
|
{
|
||||||
|
"description": "Enable pretty-printing for gdb",
|
||||||
|
"text": "-enable-pretty-printing",
|
||||||
|
"ignoreFailures": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Set Disassembly Flavor to Intel",
|
||||||
|
"text": "-gdb-set disassembly-flavor intel",
|
||||||
|
"ignoreFailures": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Sparkd (Skip Build)",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"linux": {
|
||||||
|
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
"setupCommands": [
|
||||||
|
{
|
||||||
|
"description": "Enable pretty-printing for gdb",
|
||||||
|
"text": "-enable-pretty-printing",
|
||||||
|
"ignoreFailures": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Set Disassembly Flavor to Intel",
|
||||||
|
"text": "-gdb-set disassembly-flavor intel",
|
||||||
|
"ignoreFailures": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
],
|
],
|
||||||
"inputs": [
|
"inputs": [
|
||||||
{
|
{
|
||||||
@@ -137,5 +279,15 @@
|
|||||||
"description": "Config Server URL",
|
"description": "Config Server URL",
|
||||||
"default": "http://localhost:8000/config",
|
"default": "http://localhost:8000/config",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkTest",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Test to run",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkdTest",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Test to run",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
Vendored
+1
@@ -0,0 +1 @@
|
|||||||
|
PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
|
||||||
Vendored
+118
-16
@@ -363,7 +363,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -383,7 +383,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -403,7 +403,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -427,7 +427,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -450,7 +450,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -473,7 +473,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -496,7 +496,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -519,7 +519,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -541,7 +541,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -564,7 +564,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -586,7 +586,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -601,7 +601,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -617,7 +617,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -633,7 +633,22 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Spark (Skip Build)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "Spark.exe",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -658,7 +673,7 @@
|
|||||||
"${input:sparkConfigQuery}",
|
"${input:sparkConfigQuery}",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -684,6 +699,24 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Sparkd (Skip Build)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"label": "Run Sparkd Config Query",
|
"label": "Run Sparkd Config Query",
|
||||||
"dependsOn": "Build Sparkd",
|
"dependsOn": "Build Sparkd",
|
||||||
@@ -705,6 +738,65 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"label": "Test Spark (Debug)",
|
||||||
|
"dependsOn": "Build Spark (Debug)",
|
||||||
|
"type": "process",
|
||||||
|
"icon": {
|
||||||
|
"id": "coverage",
|
||||||
|
"color": "terminal.ansiYellow",
|
||||||
|
},
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "Spark.Console.exe",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkTest}"
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Test Sparkd",
|
||||||
|
"dependsOn": "Build Sparkd",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkdTest}",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "coverage",
|
||||||
|
"color": "terminal.ansiYellow",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Test Sparkd (Skip Build)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkdTest}",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "coverage",
|
||||||
|
"color": "terminal.ansiYellow",
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
@@ -823,6 +915,16 @@
|
|||||||
"id": "sparkdConfigQuery",
|
"id": "sparkdConfigQuery",
|
||||||
"type": "promptString",
|
"type": "promptString",
|
||||||
"description": "Config Type(s)"
|
"description": "Config Type(s)"
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkTest",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Test to run",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkdTest",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Test to run",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+24
-45
@@ -1,7 +1,10 @@
|
|||||||
cmake_minimum_required(VERSION 3.12)
|
cmake_minimum_required(VERSION 3.12)
|
||||||
project(spark)
|
project(spark)
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
|
||||||
set(CMAKE_CXX_STANDARD 20)
|
set(CMAKE_CXX_STANDARD 20)
|
||||||
|
|
||||||
|
get_version_from_git()
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
set(platform "windows")
|
set(platform "windows")
|
||||||
else()
|
else()
|
||||||
@@ -19,9 +22,9 @@ set(
|
|||||||
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
|
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
|
||||||
# api server, etc.).
|
# api server, etc.).
|
||||||
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
|
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
|
||||||
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
|
set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
|
||||||
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
|
set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
|
||||||
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
||||||
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
|
||||||
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
|
||||||
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
|
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
|
||||||
@@ -33,8 +36,8 @@ else()
|
|||||||
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
|
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
||||||
link_directories(${SPARK_LIB})
|
link_directories(${EXTERNALS_LIB})
|
||||||
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
||||||
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
||||||
@@ -44,14 +47,8 @@ endif()
|
|||||||
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
# subdirs
|
# subdirs
|
||||||
add_subdirectory(src/lib)
|
add_subdirectory(src/lib_spark)
|
||||||
else()
|
else()
|
||||||
# make sparkd look for libspark.so in its directory
|
|
||||||
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
|
|
||||||
set(CMAKE_INSTALL_RPATH ".")
|
|
||||||
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
|
||||||
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
|
||||||
|
|
||||||
if(BUILD_SPARKD)
|
if(BUILD_SPARKD)
|
||||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||||
# read by their code). We override some below.
|
# read by their code). We override some below.
|
||||||
@@ -89,9 +86,15 @@ else()
|
|||||||
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
||||||
|
|
||||||
# ** Binance USDS Futures **
|
# ** Binance USDS Futures **
|
||||||
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
|
||||||
|
add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
|
||||||
|
|
||||||
|
# ** Binance Coin Futures **
|
||||||
|
add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
||||||
|
add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
||||||
|
add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
||||||
|
|
||||||
# ** Deribit **
|
# ** Deribit **
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
||||||
@@ -110,11 +113,15 @@ else()
|
|||||||
endif()
|
endif()
|
||||||
|
|
||||||
# subdirs
|
# subdirs
|
||||||
|
add_subdirectory(src/lib_spark)
|
||||||
if (BUILD_SPARKD)
|
if (BUILD_SPARKD)
|
||||||
add_subdirectory(src/server/sparkd)
|
add_subdirectory(src/lib_python_interop)
|
||||||
|
add_subdirectory(src/lib_onixs_fix)
|
||||||
|
add_subdirectory(src/lib_binance)
|
||||||
|
add_subdirectory(src/lib_okx_execution)
|
||||||
|
add_subdirectory(src/app_sparkd)
|
||||||
add_subdirectory(src/data_files)
|
add_subdirectory(src/data_files)
|
||||||
endif()
|
endif()
|
||||||
add_subdirectory(src/lib)
|
|
||||||
if (BUILD_ADMIN_WEB_APP)
|
if (BUILD_ADMIN_WEB_APP)
|
||||||
add_subdirectory(src/Spark.Admin.WebApp)
|
add_subdirectory(src/Spark.Admin.WebApp)
|
||||||
endif()
|
endif()
|
||||||
@@ -124,32 +131,4 @@ else()
|
|||||||
if(BUILD_SDK)
|
if(BUILD_SDK)
|
||||||
add_subdirectory(src/sdk)
|
add_subdirectory(src/sdk)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(BUILD_SPARKD)
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT ${CMAKE_BINARY_DIR}/server
|
|
||||||
COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
|
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
|
||||||
COMMAND
|
|
||||||
ln ARGS -sf
|
|
||||||
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
|
||||||
${CMAKE_BINARY_DIR}/server
|
|
||||||
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
|
|
||||||
COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
|
|
||||||
COMMENT "Creating server runtime layout"
|
|
||||||
DEPENDS sparkd data_files_dir)
|
|
||||||
|
|
||||||
add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
|
|
||||||
|
|
||||||
install(
|
|
||||||
DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
|
|
||||||
DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
|
|
||||||
install(
|
|
||||||
FILES
|
|
||||||
"${SPARK_LIB}/libACE-5.7.5.so"
|
|
||||||
"${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
|
|
||||||
DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
|
|
||||||
endif()
|
|
||||||
endif()
|
endif()
|
||||||
+22
-22
@@ -26,13 +26,13 @@
|
|||||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||||
"CMAKE_CXX_STANDARD": "20",
|
"CMAKE_CXX_STANDARD": "20",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
|
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
|
||||||
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
||||||
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
|
||||||
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -57,12 +57,18 @@
|
|||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
|
||||||
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "OFF"
|
||||||
},
|
},
|
||||||
"CMAKE_CXX_STANDARD": "20",
|
"CMAKE_CXX_STANDARD": "20",
|
||||||
"CMAKE_INSTALL_RPATH": ".",
|
"CMAKE_INSTALL_RPATH": ".",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
|
"CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
|
||||||
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
||||||
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
|
"DATABENTO_USE_EXTERNAL_JSON": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
||||||
"BUILD_SDK": {
|
"BUILD_SDK": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
@@ -87,7 +93,7 @@
|
|||||||
"value": "ON"
|
"value": "ON"
|
||||||
},
|
},
|
||||||
"CCAPI_LOG_LEVEL": "Info",
|
"CCAPI_LOG_LEVEL": "Info",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -98,7 +104,7 @@
|
|||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CMAKE_BUILD_TYPE": "Release",
|
"CMAKE_BUILD_TYPE": "Release",
|
||||||
"CCAPI_LOG_LEVEL": "Warn",
|
"CCAPI_LOG_LEVEL": "Warn",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -180,8 +186,12 @@
|
|||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CMAKE_EXPORT_COMPILE_COMMANDS": {
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"type": "BOOL",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
||||||
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -221,7 +231,7 @@
|
|||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"SHARED_MEMORY_PREFIX": "data_stream"
|
"SHARED_MEMORY_PREFIX": "data_stream"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -256,8 +266,7 @@
|
|||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
@@ -302,16 +311,7 @@
|
|||||||
"lhs": "${hostSystemName}",
|
"lhs": "${hostSystemName}",
|
||||||
"rhs": "Linux"
|
"rhs": "Linux"
|
||||||
},
|
},
|
||||||
"hidden": true,
|
"hidden": true
|
||||||
"targets": [
|
|
||||||
"data_files_dir",
|
|
||||||
"admin_web_app_bundle",
|
|
||||||
"install",
|
|
||||||
"libspark",
|
|
||||||
"rapidjson_project",
|
|
||||||
"server_layout",
|
|
||||||
"sparkd"
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -14,6 +14,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
|
||||||
|
EndProject
|
||||||
|
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
|
||||||
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|x64 = Debug|x64
|
Debug|x64 = Debug|x64
|
||||||
@@ -70,6 +98,102 @@ Global
|
|||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
|
||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
@@ -80,4 +204,19 @@ Global
|
|||||||
GlobalSection(Performance) = preSolution
|
GlobalSection(Performance) = preSolution
|
||||||
HasPerformanceSessions = true
|
HasPerformanceSessions = true
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
|
GlobalSection(NestedProjects) = preSolution
|
||||||
|
{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
|
||||||
|
{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
|
EndGlobalSection
|
||||||
EndGlobal
|
EndGlobal
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
|
macro(export_to_parent)
|
||||||
|
foreach(var IN ITEMS ${ARGN})
|
||||||
|
set(${var} ${${var}} PARENT_SCOPE)
|
||||||
|
endforeach()
|
||||||
|
endmacro()
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
|
function(get_version_from_git)
|
||||||
|
find_package(Git QUIET)
|
||||||
|
if(NOT Git_FOUND)
|
||||||
|
message(WARNING "Git not found")
|
||||||
|
return()
|
||||||
|
endif()
|
||||||
|
|
||||||
|
execute_process(
|
||||||
|
COMMAND ${GIT_EXECUTABLE} describe --tags --always
|
||||||
|
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
||||||
|
OUTPUT_VARIABLE GIT_TAG
|
||||||
|
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||||
|
RESULT_VARIABLE GIT_RESULT
|
||||||
|
)
|
||||||
|
|
||||||
|
if(NOT GIT_RESULT EQUAL 0)
|
||||||
|
message(WARNING "Failed to get git tag")
|
||||||
|
return()
|
||||||
|
endif()
|
||||||
|
|
||||||
|
execute_process(
|
||||||
|
COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
|
||||||
|
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
||||||
|
OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
|
||||||
|
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||||
|
)
|
||||||
|
set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
|
||||||
|
set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
|
||||||
|
|
||||||
|
string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
|
||||||
|
if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
|
||||||
|
set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
|
||||||
|
set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||||
|
set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
|
||||||
|
set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
|
||||||
|
set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
|
||||||
|
set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
|
||||||
|
|
||||||
|
set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}")
|
||||||
|
set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
|
||||||
|
set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
|
||||||
|
set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
|
||||||
|
else()
|
||||||
|
message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
|
||||||
|
|
||||||
|
function(init_library)
|
||||||
|
set(_options SHARED STATIC)
|
||||||
|
set(_one_value_args
|
||||||
|
NAME
|
||||||
|
SOURCE_DIR
|
||||||
|
INCLUDE_DIR
|
||||||
|
)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
if (_ARGS_SHARED AND _ARGS_STATIC)
|
||||||
|
message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
|
||||||
|
endif()
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
|
||||||
|
set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
|
||||||
|
|
||||||
|
# Cpp sources
|
||||||
|
file(
|
||||||
|
GLOB_RECURSE ${_name}_SOURCES
|
||||||
|
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
${_source_dir}/*.cpp
|
||||||
|
)
|
||||||
|
# Note: cpp files inside 'tests' dir(s) are considered tests.
|
||||||
|
# Remove tests from main sources, and add them to test sources
|
||||||
|
foreach(item ${${_name}_SOURCES})
|
||||||
|
if(${item} MATCHES "tests/.*\.cpp$")
|
||||||
|
list(APPEND ${_name}_TEST_SOURCES ${item})
|
||||||
|
endif()
|
||||||
|
endforeach()
|
||||||
|
foreach(item ${${_name}_TEST_SOURCES})
|
||||||
|
list(REMOVE_ITEM ${_name}_SOURCES ${item})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
# Proto sources
|
||||||
|
file(
|
||||||
|
GLOB_RECURSE ${_name}_PROTO_SOURCES
|
||||||
|
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
${_source_dir}/protobufs/*.proto
|
||||||
|
)
|
||||||
|
update_deps_file("${${_name}_PROTO_SOURCES};")
|
||||||
|
protobuf_generate_cpp(
|
||||||
|
CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
|
||||||
|
CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
|
||||||
|
CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
|
||||||
|
PROTO_INCLUDE_DIRS
|
||||||
|
${EXTERNALS_INCLUDE}
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
PROTO_SOURCES
|
||||||
|
${${_name}_PROTO_SOURCES}
|
||||||
|
)
|
||||||
|
|
||||||
|
export_to_parent(
|
||||||
|
${_name}_SOURCES
|
||||||
|
${_name}_TEST_SOURCES
|
||||||
|
${_name}_PROTO_SOURCES
|
||||||
|
${_name}_CXX_PROTO_SOURCES
|
||||||
|
${_name}_CXX_PROTO_HEADERS
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add main lib
|
||||||
|
if (_ARGS_STATIC)
|
||||||
|
add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
|
||||||
|
elseif(_ARGS_SHARED)
|
||||||
|
add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
|
||||||
|
else()
|
||||||
|
add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
|
||||||
|
endif()
|
||||||
|
target_include_directories(
|
||||||
|
${_name}
|
||||||
|
SYSTEM
|
||||||
|
BEFORE
|
||||||
|
PUBLIC
|
||||||
|
$<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
|
||||||
|
$<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
|
||||||
|
)
|
||||||
|
target_include_directories(
|
||||||
|
${_name}
|
||||||
|
PUBLIC
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
||||||
|
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
||||||
|
)
|
||||||
|
# Include protos (if any)
|
||||||
|
if (${_name}_PROTO_SOURCES)
|
||||||
|
target_include_directories(
|
||||||
|
${_name}
|
||||||
|
PUBLIC
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
target_link_directories(
|
||||||
|
${_name}
|
||||||
|
PUBLIC
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
|
||||||
|
$<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
|
||||||
|
$<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
|
||||||
|
)
|
||||||
|
# Install main lib
|
||||||
|
foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
|
||||||
|
file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
|
||||||
|
install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
|
||||||
|
endforeach()
|
||||||
|
install_basic(
|
||||||
|
NAME ${_name}
|
||||||
|
INCLUDE_DIR ${_include_dir}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add tests executable (if there are any)
|
||||||
|
if (${_name}_TEST_SOURCES AND NOT WIN32)
|
||||||
|
add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
|
||||||
|
|
||||||
|
get_target_property(_lib_type ${_name} TYPE)
|
||||||
|
if (_lib_type STREQUAL "SHARED_LIBRARY")
|
||||||
|
# Set RPATH so the tests exe knows where to find the shared lib.
|
||||||
|
set_target_properties(
|
||||||
|
${_name}-tests
|
||||||
|
PROPERTIES
|
||||||
|
BUILD_RPATH "$ORIGIN"
|
||||||
|
INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Link main lib
|
||||||
|
target_link_directories(
|
||||||
|
${_name}-tests
|
||||||
|
PRIVATE
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
|
||||||
|
$<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
|
||||||
|
)
|
||||||
|
target_link_libraries(${_name}-tests PRIVATE ${_name})
|
||||||
|
|
||||||
|
# Catch2
|
||||||
|
find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
|
||||||
|
target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
||||||
|
|
||||||
|
function(init_python_module)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args
|
||||||
|
NAME
|
||||||
|
)
|
||||||
|
set(_multi_value_args
|
||||||
|
SOURCES
|
||||||
|
)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
|
||||||
|
|
||||||
|
set(module_proj ${_name}_module)
|
||||||
|
set(${module_proj}_SOURCES ${_sources})
|
||||||
|
pybind11_add_module(${module_proj} SHARED ${_sources})
|
||||||
|
target_link_libraries(
|
||||||
|
${module_proj}
|
||||||
|
PUBLIC
|
||||||
|
pybind11::embed
|
||||||
|
spark
|
||||||
|
${_name}
|
||||||
|
)
|
||||||
|
target_include_directories(
|
||||||
|
${module_proj}
|
||||||
|
PRIVATE
|
||||||
|
$<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
|
||||||
|
)
|
||||||
|
target_include_directories(
|
||||||
|
${module_proj}
|
||||||
|
PUBLIC
|
||||||
|
${Python3_INCLUDE_DIRS}
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
||||||
|
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
||||||
|
)
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
||||||
|
DEPENDS ${_name} ${module_proj}
|
||||||
|
COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
|
||||||
|
COMMENT "Generating py stubs"
|
||||||
|
)
|
||||||
|
add_custom_target(
|
||||||
|
"${_name}_py_stubs"
|
||||||
|
ALL
|
||||||
|
DEPENDS
|
||||||
|
${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
||||||
|
)
|
||||||
|
install(
|
||||||
|
TARGETS ${module_proj}
|
||||||
|
EXPORT ${module_proj}Targets
|
||||||
|
RUNTIME
|
||||||
|
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||||
|
LIBRARY
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||||
|
ARCHIVE
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||||
|
)
|
||||||
|
foreach(dir ${pybind11_INCLUDE_DIRS})
|
||||||
|
install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
|
||||||
|
endforeach()
|
||||||
|
install(
|
||||||
|
EXPORT ${module_proj}Targets
|
||||||
|
FILE ${module_proj}Targets.cmake
|
||||||
|
NAMESPACE ${_name}::
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
||||||
|
)
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
|
||||||
|
function(init_python_plugin)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args NAME PLUGIN_NAME)
|
||||||
|
set(_multi_value_args SOURCES)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
set(_plugin_name ${_ARGS_PLUGIN_NAME})
|
||||||
|
if (NOT DEFINED _ARGS_PLUGIN_NAME)
|
||||||
|
set(_plugin_name ${_name}_plugins)
|
||||||
|
endif()
|
||||||
|
set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
|
||||||
|
if (NOT DEFINED _ARGS_SOURCES)
|
||||||
|
file(
|
||||||
|
GLOB_RECURSE ${_plugin_name}_SOURCES
|
||||||
|
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
${_plugin_name}/*.py)
|
||||||
|
endif()
|
||||||
|
set(src_outs)
|
||||||
|
foreach(_src ${${_plugin_name}_SOURCES})
|
||||||
|
set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
|
||||||
|
set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT ${_src_out}
|
||||||
|
COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
|
||||||
|
DEPENDS ${_src_in}
|
||||||
|
COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
|
||||||
|
)
|
||||||
|
list(APPEND src_outs ${_src_out})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
|
||||||
|
|
||||||
|
add_library(${_plugin_name} INTERFACE)
|
||||||
|
add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
|
||||||
|
|
||||||
|
set_target_properties(
|
||||||
|
${_plugin_name}
|
||||||
|
PROPERTIES
|
||||||
|
INTERFACE_PYTHON_SCRIPTS_DIR
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
|
||||||
|
)
|
||||||
|
|
||||||
|
set_target_properties(
|
||||||
|
${_plugin_name}
|
||||||
|
PROPERTIES
|
||||||
|
INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
|
||||||
|
)
|
||||||
|
|
||||||
|
install(
|
||||||
|
TARGETS ${_plugin_name}
|
||||||
|
EXPORT ${_plugin_name}Targets
|
||||||
|
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||||
|
)
|
||||||
|
|
||||||
|
install(
|
||||||
|
EXPORT ${_plugin_name}Targets
|
||||||
|
FILE ${_plugin_name}Targets.cmake
|
||||||
|
NAMESPACE ${_name}::
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
||||||
|
)
|
||||||
|
install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
||||||
|
|
||||||
|
function(install_basic)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args NAME INCLUDE_DIR)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
|
||||||
|
|
||||||
|
file(
|
||||||
|
GLOB_RECURSE ${_name}_HEADERS
|
||||||
|
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
${_include_dir}/*.h)
|
||||||
|
|
||||||
|
target_sources(${_name}
|
||||||
|
PUBLIC
|
||||||
|
FILE_SET HEADERS FILES ${${_name}_HEADERS})
|
||||||
|
|
||||||
|
install(
|
||||||
|
TARGETS ${_name}
|
||||||
|
EXPORT ${_name}Targets
|
||||||
|
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||||
|
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||||
|
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||||
|
FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
|
||||||
|
)
|
||||||
|
|
||||||
|
install(
|
||||||
|
EXPORT ${_name}Targets
|
||||||
|
FILE ${_name}Targets.cmake
|
||||||
|
NAMESPACE ${_name}::
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
|
||||||
|
|
||||||
|
pack_basic()
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(CMakePackageConfigHelpers)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
||||||
|
|
||||||
|
function(pack_basic)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args NAME)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
|
||||||
|
find_package(Git QUIET)
|
||||||
|
|
||||||
|
if(GIT_FOUND)
|
||||||
|
execute_process(
|
||||||
|
COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
|
||||||
|
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
||||||
|
OUTPUT_VARIABLE PROJECT_VERSION
|
||||||
|
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
|
||||||
|
set(PROJECT_VERSION "0.0.0")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
write_basic_package_version_file(
|
||||||
|
"${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
|
||||||
|
VERSION ${PROJECT_VERSION}
|
||||||
|
COMPATIBILITY AnyNewerVersion
|
||||||
|
)
|
||||||
|
|
||||||
|
configure_package_config_file(
|
||||||
|
"${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
|
||||||
|
"${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
|
||||||
|
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
||||||
|
)
|
||||||
|
|
||||||
|
install(
|
||||||
|
FILES
|
||||||
|
"${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
|
||||||
|
"${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
|
||||||
|
endfunction()
|
||||||
+82
-63
@@ -1,12 +1,23 @@
|
|||||||
set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
|
set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
|
||||||
set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
|
set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
|
||||||
set(PROTOBUF_STATIC_LIB "libprotobuf")
|
set(PROTOBUF_STATIC_LIB "libprotobuf")
|
||||||
|
set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
|
||||||
|
set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
|
||||||
|
|
||||||
function(Protobuf_Generate_Cpp SRCS HDRS)
|
function(Protobuf_Generate_Cpp)
|
||||||
set(${SRCS})
|
set(_options)
|
||||||
set(${HDRS})
|
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
|
||||||
|
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set(${_ARGS_CXX_SOURCES})
|
||||||
|
set(${_ARGS_CXX_HEADERS})
|
||||||
|
|
||||||
foreach(FIL ${ARGN})
|
set(_proto_include_cli_args)
|
||||||
|
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
||||||
|
list(APPEND _proto_include_cli_args -I ${_dir})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
foreach(FIL ${_ARGS_PROTO_SOURCES})
|
||||||
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
|
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
|
||||||
get_filename_component(FIL_WE ${FIL} NAME_WE)
|
get_filename_component(FIL_WE ${FIL} NAME_WE)
|
||||||
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
|
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
|
||||||
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
|
|||||||
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
|
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
|
||||||
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
|
||||||
|
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
|
OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
COMMAND
|
COMMAND
|
||||||
${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
|
${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
|
||||||
${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
|
|
||||||
DEPENDS ${ABS_FIL}
|
DEPENDS ${ABS_FIL}
|
||||||
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
|
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
|
||||||
VERBATIM)
|
VERBATIM)
|
||||||
endforeach()
|
endforeach()
|
||||||
|
|
||||||
set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
|
set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
|
||||||
|
|
||||||
# remove some warnings from protobuf generated files
|
# remove some warnings from protobuf generated files
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
|
set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
|
||||||
"/wd4309 /wd4838")
|
"/wd4309 /wd4838")
|
||||||
else()
|
else()
|
||||||
set_source_files_properties(
|
set_source_files_properties(
|
||||||
${${SRCS}}
|
${${_ARGS_CXX_SOURCES}}
|
||||||
PROPERTIES
|
PROPERTIES
|
||||||
COMPILE_FLAGS
|
COMPILE_FLAGS
|
||||||
"-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
"-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
||||||
)
|
)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
set(${SRCS}
|
set(${_ARGS_CXX_SOURCES}
|
||||||
${${SRCS}}
|
${${_ARGS_CXX_SOURCES}}
|
||||||
PARENT_SCOPE)
|
PARENT_SCOPE)
|
||||||
set(${HDRS}
|
set(${_ARGS_CXX_HEADERS}
|
||||||
${${HDRS}}
|
${${_ARGS_CXX_HEADERS}}
|
||||||
PARENT_SCOPE)
|
PARENT_SCOPE)
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
function(Service_Compiler_Generate_Cpp SRCS HDRS)
|
function(Service_Compiler_Generate_Cpp)
|
||||||
set(${SRCS})
|
set(_options)
|
||||||
set(${HDRS})
|
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
|
||||||
|
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set(${_ARGS_CXX_SOURCES})
|
||||||
|
set(${_ARGS_CXX_HEADERS})
|
||||||
|
|
||||||
foreach(FIL ${ARGN})
|
set(_proto_include_cli_args)
|
||||||
|
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
||||||
|
list(APPEND _proto_include_cli_args -I ${_dir})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
foreach(FIL ${_ARGS_PROTO_SOURCES})
|
||||||
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
|
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
|
||||||
get_filename_component(FIL_WE ${FIL} NAME_WE)
|
get_filename_component(FIL_WE ${FIL} NAME_WE)
|
||||||
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
|
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
|
||||||
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
|
|||||||
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
|
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
|
||||||
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
|
||||||
|
|
||||||
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
|
||||||
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
|
||||||
|
|
||||||
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
|
||||||
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
|
||||||
|
|
||||||
set(TEMPLATES
|
set(TEMPLATES
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
|
||||||
)
|
)
|
||||||
|
|
||||||
|
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
|
set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
|
OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
||||||
COMMAND SET PYTHONPATH=${SPARK_PYTHON}
|
COMMAND SET PYTHONPATH=${SPARK_PYTHON}
|
||||||
COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
|
COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
|
||||||
COMMAND
|
COMMAND
|
||||||
${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
|
${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
|
||||||
--plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
|
--plugin=protoc-gen-service=${_plugin_cmd}
|
||||||
--service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
|
--service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
|
||||||
${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
|
${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
|
||||||
DEPENDS ${ABS_FIL} ${TEMPLATES}
|
DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
|
|
||||||
COMMENT "Running C++ service compiler on ${FIL}"
|
COMMENT "Running C++ service compiler on ${FIL}"
|
||||||
VERBATIM)
|
VERBATIM)
|
||||||
else()
|
else()
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
|
OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
||||||
COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
|
COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
|
||||||
ARGS
|
ARGS
|
||||||
${Protobuf_PROTOC_EXECUTABLE}
|
${Protobuf_PROTOC_EXECUTABLE}
|
||||||
--cpp_out=${CMAKE_BINARY_DIR}/src
|
--cpp_out=${_ARGS_CXX_OUT_DIR}
|
||||||
--plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
|
--plugin=protoc-gen-service=${_plugin_py}
|
||||||
--service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
|
--service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
|
||||||
-I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
|
${_proto_include_cli_args} ${ABS_FIL}
|
||||||
DEPENDS ${ABS_FIL} ${TEMPLATES}
|
DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
|
|
||||||
COMMENT "Running C++ service compiler on ${FIL}"
|
COMMENT "Running C++ service compiler on ${FIL}"
|
||||||
VERBATIM)
|
VERBATIM)
|
||||||
endif()
|
endif()
|
||||||
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
|
|||||||
|
|
||||||
# remove some warnings from protobuf generated files
|
# remove some warnings from protobuf generated files
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
|
set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
|
||||||
"/wd4309 /wd4838")
|
"/wd4309 /wd4838")
|
||||||
else()
|
else()
|
||||||
set_source_files_properties(
|
set_source_files_properties(
|
||||||
${${SRCS}}
|
${${_ARGS_CXX_SOURCES}}
|
||||||
PROPERTIES
|
PROPERTIES
|
||||||
COMPILE_FLAGS
|
COMPILE_FLAGS
|
||||||
"-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
"-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
||||||
)
|
)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
set(${SRCS}
|
set(${_ARGS_CXX_SOURCES}
|
||||||
${${SRCS}}
|
${${_ARGS_CXX_SOURCES}}
|
||||||
PARENT_SCOPE)
|
PARENT_SCOPE)
|
||||||
set(${HDRS}
|
set(${_ARGS_CXX_HEADERS}
|
||||||
${${HDRS}}
|
${${_ARGS_CXX_HEADERS}}
|
||||||
PARENT_SCOPE)
|
PARENT_SCOPE)
|
||||||
|
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
|
function(protobuf_generate_cpp)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args
|
||||||
|
CXX_SOURCES
|
||||||
|
CXX_HEADERS
|
||||||
|
CXX_OUT_DIR
|
||||||
|
PROTOC
|
||||||
|
)
|
||||||
|
set(_multi_value_args
|
||||||
|
PROTO_INCLUDE_DIRS
|
||||||
|
PROTO_SOURCES
|
||||||
|
)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
|
||||||
|
if (NOT DEFINED _ARGS_PROTOC)
|
||||||
|
if (WIN32)
|
||||||
|
set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
|
||||||
|
else()
|
||||||
|
set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
|
||||||
|
endif()
|
||||||
|
endif()
|
||||||
|
set(${_ARGS_CXX_SOURCES})
|
||||||
|
set(${_ARGS_CXX_HEADERS})
|
||||||
|
|
||||||
|
set(_proto_include_cli_args)
|
||||||
|
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
||||||
|
list(APPEND _proto_include_cli_args -I ${_dir})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
foreach(_fil ${_ARGS_PROTO_SOURCES})
|
||||||
|
get_filename_component(_abs_fil ${_fil} ABSOLUTE)
|
||||||
|
get_filename_component(_fil_we ${_fil} NAME_WE)
|
||||||
|
get_filename_component(_fil_dir ${_fil} DIRECTORY)
|
||||||
|
if(_fil_dir)
|
||||||
|
set(_fil_we "${_fil_dir}/${_fil_we}")
|
||||||
|
endif()
|
||||||
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
|
||||||
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
||||||
|
message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
||||||
|
# TODO: Pass proto sources by relative path (so this fn can be called from
|
||||||
|
# arbitrary subdirs).
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
|
||||||
|
COMMAND
|
||||||
|
${_ARGS_PROTOC}
|
||||||
|
ARGS
|
||||||
|
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
|
||||||
|
DEPENDS ${_abs_fil}
|
||||||
|
COMMENT "Running C++ protocol buffer compiler on ${_fil}"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
set_source_files_properties(
|
||||||
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
${${_ARGS_CXX_HEADERS}}
|
||||||
|
PROPERTIES
|
||||||
|
GENERATED TRUE
|
||||||
|
)
|
||||||
|
|
||||||
|
# remove some warnings from protobuf generated files
|
||||||
|
if(WIN32)
|
||||||
|
set_source_files_properties(
|
||||||
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
PROPERTIES
|
||||||
|
COMPILE_FLAGS "/wd4309 /wd4838"
|
||||||
|
)
|
||||||
|
else()
|
||||||
|
set_source_files_properties(
|
||||||
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
PROPERTIES
|
||||||
|
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
||||||
|
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
||||||
|
endfunction()
|
||||||
|
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
|
function(protobuf_generate_cpp_config)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args
|
||||||
|
PROTO_SOURCES_HEADER
|
||||||
|
PROTO_DEFAULTS_HEADER
|
||||||
|
)
|
||||||
|
set(_multi_value_args
|
||||||
|
PROTO_SOURCES
|
||||||
|
PROTO_DEFAULTS
|
||||||
|
)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
|
||||||
|
# Configure the 'config_proto_files' header. It provides an array of strings populated with the
|
||||||
|
# files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
|
||||||
|
# message with fields for each message defined in said files. A global instance of this 'config'
|
||||||
|
# proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
|
||||||
|
# (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
|
||||||
|
# associated with type `T` in this global instance.
|
||||||
|
message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
|
||||||
|
message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
|
||||||
|
message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
|
||||||
|
if (_ARGS_PROTO_SOURCES_HEADER)
|
||||||
|
set(PROTO_SOURCES)
|
||||||
|
set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
|
||||||
|
foreach(_proto_path ${_ARGS_PROTO_SOURCES})
|
||||||
|
list(APPEND PROTO_SOURCES ${_proto_path})
|
||||||
|
string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
|
||||||
|
endforeach()
|
||||||
|
list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
|
||||||
|
message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
|
||||||
|
configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Generate static defaults. Ideally, this would be a single string, but we use an array because
|
||||||
|
# the compiler complains if the string is longer than 65k.
|
||||||
|
set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
|
||||||
|
|
||||||
|
foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
|
||||||
|
file(READ ${_cfg_file_path} _cfg_file_content)
|
||||||
|
string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
|
||||||
|
string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
|
||||||
|
|
||||||
|
foreach(_line ${_cfg_lines})
|
||||||
|
string(APPEND _cfg_defaults_str "\n \"${_line}\",")
|
||||||
|
endforeach(_line)
|
||||||
|
endforeach(_cfg_file_path)
|
||||||
|
|
||||||
|
string(APPEND _cfg_defaults_str "};")
|
||||||
|
set(_old_cfg_defaults_str "")
|
||||||
|
|
||||||
|
if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
|
||||||
|
file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
|
||||||
|
file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
|
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
|
||||||
|
function(protobuf_generate_fs)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
|
||||||
|
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
|
||||||
|
if (NOT DEFINED _ARGS_TARGET_NAME)
|
||||||
|
set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
|
||||||
|
endif()
|
||||||
|
if (NOT DEFINED _ARGS_PROTOC)
|
||||||
|
set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
set(_proto_include_cli_args)
|
||||||
|
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
||||||
|
list(APPEND _proto_include_cli_args -I ${_dir})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
set(_outputs)
|
||||||
|
foreach(_proto_source ${_ARGS_PROTO_SOURCES})
|
||||||
|
# get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
|
||||||
|
get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
|
||||||
|
# get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
|
||||||
|
list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
|
||||||
|
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
|
||||||
|
set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT ${_outputs}
|
||||||
|
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
COMMAND ${_venv_exec}
|
||||||
|
${_ARGS_PROTOC}
|
||||||
|
--plugin=protoc-gen-service=${_plugin_py}
|
||||||
|
--service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
|
||||||
|
${_proto_include_cli_args}
|
||||||
|
${_ARGS_PROTO_SOURCES}
|
||||||
|
DEPENDS
|
||||||
|
${_venv_exec}
|
||||||
|
${_plugin_py}
|
||||||
|
${_template_fs}
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
|
||||||
|
${_ARGS_PROTO_SOURCES}
|
||||||
|
)
|
||||||
|
add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
|
||||||
|
function(protobuf_generate_js)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args TARGET_NAME OUTPUT)
|
||||||
|
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
|
||||||
|
if (NOT DEFINED _ARGS_TARGET_NAME)
|
||||||
|
set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
set(_proto_include_cli_args)
|
||||||
|
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
||||||
|
list(APPEND _proto_include_cli_args -p ${_dir})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT ${_ARGS_OUTPUT}
|
||||||
|
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
COMMAND npx
|
||||||
|
--no-install pbjs
|
||||||
|
-t json-module
|
||||||
|
-w commonjs
|
||||||
|
-o ${_ARGS_OUTPUT}
|
||||||
|
${_proto_include_cli_args}
|
||||||
|
${_ARGS_PROTO_SOURCES}
|
||||||
|
DEPENDS
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
|
||||||
|
${_ARGS_PROTO_SOURCES}
|
||||||
|
)
|
||||||
|
add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
|
macro(set_first_defined RESULT)
|
||||||
|
foreach(_arg ${ARGN})
|
||||||
|
if (DEFINED ${_arg})
|
||||||
|
set(${RESULT} ${${_arg}})
|
||||||
|
break()
|
||||||
|
endif()
|
||||||
|
endforeach()
|
||||||
|
endmacro()
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
|
||||||
|
|
||||||
|
function(sym_link_dir)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT ${_ARGS_DEST}
|
||||||
|
COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
|
||||||
|
)
|
||||||
|
set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
|
||||||
|
add_custom_target(
|
||||||
|
${_ARGS_TARGET_NAME} ALL
|
||||||
|
DEPENDS ${_ARGS_DEST}
|
||||||
|
COMMENT "${_comment}"
|
||||||
|
)
|
||||||
|
endfunction()
|
||||||
|
|
||||||
|
function(sym_link_file)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args SOURCE DEST INSTALL_DEST)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
file(
|
||||||
|
CREATE_LINK
|
||||||
|
${_ARGS_SOURCE}
|
||||||
|
${_ARGS_DEST}
|
||||||
|
SYMBOLIC
|
||||||
|
)
|
||||||
|
if (DEFINED _ARGS_INSTALL_DEST)
|
||||||
|
install(
|
||||||
|
FILES
|
||||||
|
${_ARGS_SOURCE}
|
||||||
|
DESTINATION ${_ARGS_INSTALL_DEST}
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
|
|
||||||
|
function(sym_link_local_lib LIB_NAME)
|
||||||
|
set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
|
||||||
|
file(
|
||||||
|
CREATE_LINK
|
||||||
|
"${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
|
||||||
|
"${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
|
||||||
|
SYMBOLIC
|
||||||
|
)
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
|
||||||
|
|
||||||
|
function(sym_link_python_module)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
|
||||||
|
"${_multi_value_args}" ${ARGN})
|
||||||
|
if(NOT DEFINED _ARGS_LIB_NAME)
|
||||||
|
message(
|
||||||
|
FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
|
||||||
|
endif()
|
||||||
|
set(_lib_name ${_ARGS_LIB_NAME})
|
||||||
|
set(_module_name ${_ARGS_MODULE_NAME})
|
||||||
|
if(NOT DEFINED _ARGS_MODULE_NAME)
|
||||||
|
set(_module_name ${_lib_name}_module)
|
||||||
|
endif()
|
||||||
|
set(_plugin_name ${_ARGS_PLUGIN_NAME})
|
||||||
|
set(_deps ${_lib_name} ${_module_name})
|
||||||
|
if(DEFINED _ARGS_PLUGIN_NAME)
|
||||||
|
list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
|
||||||
|
endif()
|
||||||
|
add_dependencies(${PROJECT_NAME} ${_deps})
|
||||||
|
sym_link_local_lib(${_lib_name})
|
||||||
|
set(_module_suffix cpython-310-x86_64-linux-gnu.so)
|
||||||
|
sym_link_file(
|
||||||
|
SOURCE
|
||||||
|
${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
|
||||||
|
DEST
|
||||||
|
${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
|
||||||
|
INSTALL_DEST
|
||||||
|
${CMAKE_INSTALL_BINDIR}/)
|
||||||
|
if(DEFINED _ARGS_PLUGIN_NAME)
|
||||||
|
get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
|
||||||
|
sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
|
||||||
|
${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
|
macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
|
||||||
|
if(${CONDITION})
|
||||||
|
set(${RESULT} ${TRUE_VALUE})
|
||||||
|
else()
|
||||||
|
set(${RESULT} ${FALSE_VALUE})
|
||||||
|
endif()
|
||||||
|
endmacro()
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
|
||||||
|
namespace bts::@PROJECT_NAME@::protobufs {
|
||||||
|
|
||||||
|
inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
|
||||||
|
@PROTO_SOURCES_CXX_ARRAY_ITEMS@
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace bts::@PROJECT_NAME@::protobufs
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
#ifndef BTS_SPARK_VERSION_H
|
||||||
|
#define BTS_SPARK_VERSION_H
|
||||||
|
|
||||||
|
#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
|
||||||
|
#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
|
||||||
|
#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
|
||||||
|
#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
|
||||||
|
#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
|
||||||
|
#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
cmake_minimum_required(VERSION 3.5)
|
|
||||||
|
|
||||||
# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
|
|
||||||
# setting it explicitly prevents a compile-warning)
|
|
||||||
cmake_policy(SET CMP0076 NEW)
|
|
||||||
|
|
||||||
# add protos to the library to facilitate source_group
|
|
||||||
target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
|
|
||||||
|
|
||||||
function(assign_source_group)
|
|
||||||
foreach(_source IN ITEMS ${ARGN})
|
|
||||||
if(IS_ABSOLUTE "${_source}")
|
|
||||||
file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
|
|
||||||
else()
|
|
||||||
set(_source_rel "${_source}")
|
|
||||||
endif()
|
|
||||||
get_filename_component(_source_path "${_source_rel}" PATH)
|
|
||||||
string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
|
|
||||||
source_group("${_source_path_msvc}" FILES "${_source}")
|
|
||||||
set(SOURCE_FILES "${_source}")
|
|
||||||
endforeach()
|
|
||||||
endfunction(assign_source_group)
|
|
||||||
|
|
||||||
assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
|
|
||||||
+114
@@ -0,0 +1,114 @@
|
|||||||
|
services:
|
||||||
|
sparkd:
|
||||||
|
volumes:
|
||||||
|
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
||||||
|
build:
|
||||||
|
dockerfile: src/app_sparkd/Dockerfile
|
||||||
|
args:
|
||||||
|
- IMAGE=${SPARKD__IMAGE}
|
||||||
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||||
|
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||||
|
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||||
|
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||||
|
command: >
|
||||||
|
sh -c "
|
||||||
|
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
|
||||||
|
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
||||||
|
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||||
|
fi
|
||||||
|
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
||||||
|
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
||||||
|
set -a
|
||||||
|
. /home/bts/sparkdata/conf/.env.local
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
if [ -d /home/bts/sparkdata/conf/plugins ]; then
|
||||||
|
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
|
||||||
|
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
|
||||||
|
fi
|
||||||
|
./sparkd run"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
- ${SPARKD__CONFIG_DIR}/.env
|
||||||
|
ports:
|
||||||
|
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
||||||
|
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
||||||
|
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: "${SPARKD__MEMORY:-4gb}"
|
||||||
|
|
||||||
|
sparkd-symbols:
|
||||||
|
volumes:
|
||||||
|
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
||||||
|
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
|
||||||
|
build:
|
||||||
|
dockerfile: src/app_sparkd/Dockerfile
|
||||||
|
args:
|
||||||
|
- IMAGE=${SPARKD__IMAGE}
|
||||||
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||||
|
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||||
|
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||||
|
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||||
|
command: >
|
||||||
|
sh -c "
|
||||||
|
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
|
||||||
|
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
||||||
|
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||||
|
fi
|
||||||
|
ls -alh /home/bts/sparkdata/conf/
|
||||||
|
cat /home/bts/sparkdata/conf/.env.local
|
||||||
|
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
||||||
|
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
||||||
|
set -a
|
||||||
|
. /home/bts/sparkdata/conf/.env.local
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
if [ -d /home/bts/sparkdata/conf/plugins ]; then
|
||||||
|
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
|
||||||
|
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
|
||||||
|
fi
|
||||||
|
./sparkd run"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
- ${SPARKD__CONFIG_DIR}/.env
|
||||||
|
ports:
|
||||||
|
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
||||||
|
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
||||||
|
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: any
|
||||||
|
delay: 3s
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: "${SPARKD__MEMORY:-4gb}"
|
||||||
|
|
||||||
|
sparkd-tail:
|
||||||
|
image: bluetradingsystems/linux_dotnet7
|
||||||
|
volumes:
|
||||||
|
- type: volume
|
||||||
|
read_only: true
|
||||||
|
source: sparkd_data
|
||||||
|
target: /home/bts/sparkdata/data/logging.sparkd
|
||||||
|
volume:
|
||||||
|
subpath: logging.sparkd
|
||||||
|
command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
|
||||||
|
|
||||||
|
sparkd-ping-test:
|
||||||
|
depends_on:
|
||||||
|
- sparkd
|
||||||
|
profiles:
|
||||||
|
- ping-test
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile_inline: |
|
||||||
|
FROM bluetradingsystems/linux_dotnet7
|
||||||
|
USER "root"
|
||||||
|
RUN apt-get update && apt-get -y install iputils-ping
|
||||||
|
command: [ "ping", "sparkd" ]
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
sparkd_data:
|
||||||
|
name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
|
||||||
@@ -124,8 +124,8 @@ service OrderService {
|
|||||||
```
|
```
|
||||||
### C++ Example
|
### C++ Example
|
||||||
```cpp
|
```cpp
|
||||||
#include "lib/api/public/ServiceContext.h"
|
#include "spark/api/public/ServiceContext.h"
|
||||||
#include "lib/api/services/OrderService.client.pb.h"
|
#include "spark/api/services/OrderService.client.pb.h"
|
||||||
using namespace bts::api;
|
using namespace bts::api;
|
||||||
ServiceContext ctx;
|
ServiceContext ctx;
|
||||||
auto orders = ctx.locate<orderman::OrderService>();
|
auto orders = ctx.locate<orderman::OrderService>();
|
||||||
|
|||||||
@@ -29,9 +29,9 @@ namespace sample {
|
|||||||
```
|
```
|
||||||
|
|
||||||
```cpp
|
```cpp
|
||||||
#include "lib/api/public/ServiceContext.h"
|
#include "spark/api/public/ServiceContext.h"
|
||||||
#include "spark/lib/domain_model/Security.h"
|
#include "spark/domain_model/Security.h"
|
||||||
#include "lib/api/services/MarketDataService.client.pb.h"
|
#include "spark/api/services/MarketDataService.client.pb.h"
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
using namespace bts::api;
|
using namespace bts::api;
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
FROM ubuntu:22.04
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y pkg-config
|
||||||
|
RUN apt-get update && apt-get install -y locales
|
||||||
|
RUN apt-get update && apt-get install -y ca-certificates
|
||||||
|
RUN apt-get update && apt-get install -y libssh2-1-dev
|
||||||
|
RUN apt-get update && apt-get install -y software-properties-common
|
||||||
|
RUN apt-get update && apt-get install -y sudo
|
||||||
|
|
||||||
|
# GCC
|
||||||
|
ARG gcc_major=13
|
||||||
|
RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
|
||||||
|
&& apt-get update \
|
||||||
|
&& apt-get install -y g++-$gcc_major
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y dotnet7
|
||||||
|
|
||||||
|
RUN locale-gen en_US.UTF-8
|
||||||
|
|
||||||
|
ARG docker_user="bts"
|
||||||
|
ARG docker_uid="1000"
|
||||||
|
ARG docker_pass="bts"
|
||||||
|
|
||||||
|
RUN \
|
||||||
|
groupadd -g "$docker_uid" "$docker_user" \
|
||||||
|
&& useradd \
|
||||||
|
-m \
|
||||||
|
-u "$docker_uid" \
|
||||||
|
-g "$docker_uid" \
|
||||||
|
-G sudo \
|
||||||
|
--password "$(openssl passwd -1 "$docker_pass")" \
|
||||||
|
"$docker_user"
|
||||||
|
|
||||||
|
USER bts
|
||||||
|
WORKDIR /home/bts
|
||||||
-1
Submodule docker/linux deleted from 56c61231f3.
Submodule docker/windows deleted from de7d40c09d.
+1
-1
Submodule externals updated: ff5f02bbde...4f6734703c.
@@ -1,22 +1,42 @@
|
|||||||
astroid==2.7.3
|
astroid==2.7.3
|
||||||
autopep8==1.5.7
|
autopep8==1.5.7
|
||||||
|
blinker==1.9.0
|
||||||
|
boto3==1.37.9
|
||||||
|
botocore==1.37.9
|
||||||
certifi==2023.5.7
|
certifi==2023.5.7
|
||||||
charset-normalizer==3.1.0
|
charset-normalizer==3.1.0
|
||||||
|
click==8.1.8
|
||||||
colorama==0.4.4
|
colorama==0.4.4
|
||||||
|
Flask==3.1.0
|
||||||
idna==3.4
|
idna==3.4
|
||||||
isort==5.9.3
|
isort==5.9.3
|
||||||
|
itsdangerous==2.2.0
|
||||||
|
Jinja2==3.1.6
|
||||||
|
jmespath==1.0.1
|
||||||
lazy-object-proxy==1.6.0
|
lazy-object-proxy==1.6.0
|
||||||
|
MarkupSafe==3.0.2
|
||||||
mccabe==0.6.1
|
mccabe==0.6.1
|
||||||
|
numpy==2.2.6
|
||||||
platformdirs==2.3.0
|
platformdirs==2.3.0
|
||||||
protobuf==4.21.12
|
protobuf==4.21.12
|
||||||
pybars3==0.9.7
|
pybars3==0.9.7
|
||||||
|
pybind11==2.13.6
|
||||||
|
pybind11-stubgen==2.5.3
|
||||||
|
pybit==5.11.0
|
||||||
pycodestyle==2.7.0
|
pycodestyle==2.7.0
|
||||||
pycparser==2.20
|
pycparser==2.20
|
||||||
|
pycryptodome==3.23.0
|
||||||
pylint==2.10.2
|
pylint==2.10.2
|
||||||
PyMeta3==0.5.1
|
PyMeta3==0.5.1
|
||||||
|
python-dateutil==2.9.0.post0
|
||||||
|
python-dotenv==1.0.1
|
||||||
requests==2.31.0
|
requests==2.31.0
|
||||||
|
s3transfer==0.11.4
|
||||||
|
six==1.17.0
|
||||||
termcolor==1.1.0
|
termcolor==1.1.0
|
||||||
toml==0.10.2
|
toml==0.10.2
|
||||||
urllib3==2.0.3
|
urllib3==2.0.3
|
||||||
|
websocket-client==1.8.0
|
||||||
|
Werkzeug==3.1.3
|
||||||
wrapt==1.12.1
|
wrapt==1.12.1
|
||||||
yapf==0.31.0
|
yapf==0.31.0
|
||||||
@@ -18,6 +18,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\clie
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
|
||||||
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|x64 = Debug|x64
|
Debug|x64 = Debug|x64
|
||||||
@@ -90,6 +98,38 @@ Global
|
|||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,14 +1,14 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.0" />
|
<PackageReference Include="YaccLexTools" Version="1.2.3" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -24,8 +24,6 @@ namespace QUT.GplexBuffers
|
|||||||
public BufferException(string message) : base(message) { }
|
public BufferException(string message) : base(message) { }
|
||||||
public BufferException(string message, Exception innerException)
|
public BufferException(string message, Exception innerException)
|
||||||
: base(message, innerException) { }
|
: base(message, innerException) { }
|
||||||
protected BufferException(SerializationInfo info, StreamingContext context)
|
|
||||||
: base(info, context) { }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
internal abstract class ScanBuff
|
internal abstract class ScanBuff
|
||||||
@@ -60,12 +58,10 @@ namespace QUT.GplexBuffers
|
|||||||
return new BuildBuffer(source);
|
return new BuildBuffer(source);
|
||||||
}
|
}
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
||||||
{
|
{
|
||||||
return new BuildBuffer(source, fallbackCodePage);
|
return new BuildBuffer(source, fallbackCodePage);
|
||||||
}
|
}
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -353,17 +349,15 @@ namespace QUT.GplexBuffers
|
|||||||
{
|
{
|
||||||
FileStream fStrm = (stream as FileStream);
|
FileStream fStrm = (stream as FileStream);
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
if (fStrm != null) FileName = fStrm.Name;
|
||||||
NextBlk = BlockReaderFactory.Raw(stream);
|
NextBlk = BlockReaderFactory.Get(stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public BuildBuffer(Stream stream, int fallbackCodePage)
|
public BuildBuffer(Stream stream, int fallbackCodePage)
|
||||||
{
|
{
|
||||||
FileStream fStrm = (stream as FileStream);
|
FileStream fStrm = (stream as FileStream);
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
if (fStrm != null) FileName = fStrm.Name;
|
||||||
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Marks a conservative lower bound for the buffer,
|
/// Marks a conservative lower bound for the buffer,
|
||||||
@@ -434,43 +428,19 @@ namespace QUT.GplexBuffers
|
|||||||
//
|
//
|
||||||
internal static class BlockReaderFactory
|
internal static class BlockReaderFactory
|
||||||
{
|
{
|
||||||
public static BlockReader Raw(Stream stream)
|
|
||||||
{
|
|
||||||
return delegate(char[] block, int index, int number)
|
|
||||||
{
|
|
||||||
byte[] b = new byte[number];
|
|
||||||
int count = stream.Read(b, 0, number);
|
|
||||||
int i = 0;
|
|
||||||
int j = index;
|
|
||||||
for (; i < count; i++, j++)
|
|
||||||
block[j] = (char)b[i];
|
|
||||||
return count;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
|
||||||
public static BlockReader Get(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
{
|
||||||
Encoding encoding;
|
Encoding encoding;
|
||||||
int preamble = Preamble(stream);
|
int preamble = Preamble(stream);
|
||||||
|
|
||||||
if (preamble != 0) // There is a valid BOM here!
|
if (preamble != 0) // There is a valid BOM here!
|
||||||
encoding = Encoding.GetEncoding(preamble);
|
encoding = Encoding.GetEncoding(preamble);
|
||||||
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
|
else if (fallbackCodePage >= 0) // Anything but "guess"
|
||||||
return Raw(stream);
|
|
||||||
else if (fallbackCodePage != -2) // Anything but "guess"
|
|
||||||
encoding = Encoding.GetEncoding(fallbackCodePage);
|
encoding = Encoding.GetEncoding(fallbackCodePage);
|
||||||
else // This is the "guess" option
|
else // This is the "guess" option
|
||||||
{
|
encoding = GuessEncoding(stream);
|
||||||
int guess = new Guesser(stream).GuessCodePage();
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (guess == -1) // ==> this is a 7-bit file
|
|
||||||
encoding = Encoding.ASCII;
|
|
||||||
else if (guess == 65001)
|
|
||||||
encoding = Encoding.UTF8;
|
|
||||||
else // ==> use the machine default
|
|
||||||
encoding = Encoding.Default;
|
|
||||||
}
|
|
||||||
StreamReader reader = new StreamReader(stream, encoding);
|
StreamReader reader = new StreamReader(stream, encoding);
|
||||||
return reader.Read;
|
return reader.Read;
|
||||||
}
|
}
|
||||||
@@ -488,6 +458,14 @@ namespace QUT.GplexBuffers
|
|||||||
int b2 = stream.ReadByte();
|
int b2 = stream.ReadByte();
|
||||||
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
||||||
return 65001; // UTF8
|
return 65001; // UTF8
|
||||||
|
|
||||||
|
int b3 = stream.ReadByte();
|
||||||
|
// Check for UTF-32 BOM (Byte Order Mark)
|
||||||
|
if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
|
||||||
|
return 12001; // UTF-32BE
|
||||||
|
if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
|
||||||
|
return 12000; // UTF-32LE
|
||||||
|
|
||||||
//
|
//
|
||||||
// There is no unicode preamble, so we
|
// There is no unicode preamble, so we
|
||||||
// return denoter for the machine default.
|
// return denoter for the machine default.
|
||||||
@@ -495,7 +473,150 @@ namespace QUT.GplexBuffers
|
|||||||
stream.Seek(0, SeekOrigin.Begin);
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
#endif // !BYTEMODE
|
|
||||||
|
private static Encoding GuessEncoding(Stream stream)
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (IsUtf16(stream, out Encoding encoding))
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return encoding;
|
||||||
|
}
|
||||||
|
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (IsUtf8(stream))
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return Encoding.UTF8;
|
||||||
|
}
|
||||||
|
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (IsAscii(stream))
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return Encoding.ASCII;
|
||||||
|
}
|
||||||
|
|
||||||
|
#if !NET462_OR_GREATER
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (IsLatin1(stream))
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return Encoding.Latin1;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return Encoding.Default;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsUtf8(Stream stream)
|
||||||
|
{
|
||||||
|
int b;
|
||||||
|
while ((b = stream.ReadByte()) != -1)
|
||||||
|
{
|
||||||
|
if ((b & 0x80) == 0) // 0xxxxxxx
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
|
||||||
|
{
|
||||||
|
if (!IsValidUtf8ContinuationByte(stream)) return false;
|
||||||
|
}
|
||||||
|
else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
|
||||||
|
{
|
||||||
|
if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
|
||||||
|
}
|
||||||
|
else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
||||||
|
{
|
||||||
|
if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsValidUtf8ContinuationByte(Stream stream)
|
||||||
|
{
|
||||||
|
int b = stream.ReadByte();
|
||||||
|
return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsUtf16(Stream stream, out Encoding encoding)
|
||||||
|
{
|
||||||
|
encoding = Encoding.Default;
|
||||||
|
|
||||||
|
if (stream.Length >= 2)
|
||||||
|
{
|
||||||
|
// Read the first two bytes
|
||||||
|
int b0 = stream.ReadByte();
|
||||||
|
int b1 = stream.ReadByte();
|
||||||
|
|
||||||
|
// Check for UTF-16 BOM (Byte Order Mark)
|
||||||
|
if (b0 == 0xFF && b1 == 0xFE)
|
||||||
|
{
|
||||||
|
encoding = Encoding.Unicode;
|
||||||
|
return true; // UTF-16LE
|
||||||
|
}
|
||||||
|
if (b0 == 0xFE && b1 == 0xFF)
|
||||||
|
{
|
||||||
|
encoding = Encoding.BigEndianUnicode;
|
||||||
|
return true; // UTF-16BE
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset the stream position
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
|
||||||
|
// Check for common UTF-16 patterns
|
||||||
|
byte[] buffer = new byte[2];
|
||||||
|
while (stream.Read(buffer, 0, 2) == 2)
|
||||||
|
{
|
||||||
|
if (buffer[0] == 0x00 && buffer[1] != 0x00)
|
||||||
|
{
|
||||||
|
encoding = Encoding.BigEndianUnicode;
|
||||||
|
return true; // Likely UTF-16BE
|
||||||
|
}
|
||||||
|
if (buffer[0] != 0x00 && buffer[1] == 0x00)
|
||||||
|
{
|
||||||
|
encoding = Encoding.Unicode;
|
||||||
|
return true; // Likely UTF-16LE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsAscii(Stream stream)
|
||||||
|
{
|
||||||
|
int b;
|
||||||
|
while ((b = stream.ReadByte()) != -1)
|
||||||
|
{
|
||||||
|
// Latin1 includes byte values from 0x00 to 0xFF
|
||||||
|
if (b > 0x7F)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsLatin1(Stream stream)
|
||||||
|
{
|
||||||
|
int b;
|
||||||
|
while ((b = stream.ReadByte()) != -1)
|
||||||
|
{
|
||||||
|
// Latin1 includes byte values from 0x00 to 0xFF
|
||||||
|
if (b > 0xFF)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
#endregion Buffer classes
|
#endregion Buffer classes
|
||||||
@@ -540,180 +661,6 @@ namespace QUT.GplexBuffers
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#region guesser
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
// ==============================================================
|
|
||||||
// ============ Encoding Guesser =============
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class provides a simple finite state automaton that
|
|
||||||
/// scans the file looking for (1) valid UTF-8 byte patterns,
|
|
||||||
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
|
|
||||||
/// The method then guesses whether it is UTF-8 or maybe some
|
|
||||||
/// local machine default encoding. This works well for the
|
|
||||||
/// various Latin encodings.
|
|
||||||
/// </summary>
|
|
||||||
internal class Guesser
|
|
||||||
{
|
|
||||||
ScanBuff buffer;
|
|
||||||
|
|
||||||
public int GuessCodePage() { return Scan(); }
|
|
||||||
|
|
||||||
const int maxAccept = 10;
|
|
||||||
const int initial = 0;
|
|
||||||
const int eofNum = 0;
|
|
||||||
const int goStart = -1;
|
|
||||||
const int INITIAL = 0;
|
|
||||||
const int EndToken = 0;
|
|
||||||
|
|
||||||
#region user code
|
|
||||||
/*
|
|
||||||
* Reads the bytes of a file to determine if it is
|
|
||||||
* UTF-8 or a single-byte code page file.
|
|
||||||
*/
|
|
||||||
public long utfX;
|
|
||||||
public long uppr;
|
|
||||||
#endregion user code
|
|
||||||
|
|
||||||
int state;
|
|
||||||
int currentStart = startState[0];
|
|
||||||
int code;
|
|
||||||
|
|
||||||
#region ScannerTables
|
|
||||||
static int[] startState = new int[] { 11, 0 };
|
|
||||||
|
|
||||||
#region CharacterMap
|
|
||||||
static sbyte[] map = new sbyte[256] {
|
|
||||||
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
||||||
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
static sbyte[][] nextState = new sbyte[][] {
|
|
||||||
new sbyte[] {0, 0, 0, 0, 0, 0},
|
|
||||||
new sbyte[] {-1, -1, 10, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, -1, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 8, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 5, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 6, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 7, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, -1, 9, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, 1, 2, 3, 4, 2}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
|
|
||||||
// Reason for suppression: cannot have self-reference in array initializer.
|
|
||||||
static Guesser()
|
|
||||||
{
|
|
||||||
nextState[7] = nextState[2];
|
|
||||||
nextState[9] = nextState[2];
|
|
||||||
nextState[10] = nextState[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
int NextState()
|
|
||||||
{
|
|
||||||
if (code == ScanBuff.EndOfFile)
|
|
||||||
return eofNum;
|
|
||||||
else
|
|
||||||
return nextState[state][map[code]];
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
public Guesser(System.IO.Stream file) { SetSource(file); }
|
|
||||||
|
|
||||||
public void SetSource(System.IO.Stream source)
|
|
||||||
{
|
|
||||||
this.buffer = new BuildBuffer(source);
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
|
|
||||||
int Scan()
|
|
||||||
{
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
int next;
|
|
||||||
state = currentStart;
|
|
||||||
while ((next = NextState()) == goStart)
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
while ((next = NextState()) > eofNum)
|
|
||||||
{
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
if (state <= maxAccept)
|
|
||||||
{
|
|
||||||
#region ActionSwitch
|
|
||||||
#pragma warning disable 162
|
|
||||||
switch (state)
|
|
||||||
{
|
|
||||||
case eofNum:
|
|
||||||
switch (currentStart)
|
|
||||||
{
|
|
||||||
case 11:
|
|
||||||
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
|
|
||||||
else if (uppr * 10 > utfX) return 0; /* default code page */
|
|
||||||
else return 65001; /* UTF-8 encoding */
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return EndToken;
|
|
||||||
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
|
|
||||||
case 2: // Recognized '{Upper128}', Shortest string "\x80"
|
|
||||||
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
|
|
||||||
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
|
|
||||||
uppr++;
|
|
||||||
break;
|
|
||||||
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
|
|
||||||
uppr += 3;
|
|
||||||
break;
|
|
||||||
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
|
|
||||||
utfX += 3;
|
|
||||||
break;
|
|
||||||
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
|
|
||||||
utfX += 2;
|
|
||||||
break;
|
|
||||||
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
|
|
||||||
utfX++;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
#pragma warning restore 162
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // end class Guesser
|
|
||||||
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endregion
|
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
|
|
||||||
// End of code copied from embedded resource
|
// End of code copied from embedded resource
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("AddColumnsStream")]
|
||||||
|
public class AddColumnsStream: ViewStream {
|
||||||
|
public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
|
||||||
|
ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
using System;
|
||||||
|
|
||||||
|
namespace BCG {
|
||||||
|
[DataStreamUserRespresentation("CountStream")]
|
||||||
|
public class CountStream : ViewStream {
|
||||||
|
private DataPacket myLastFiredDataPacket;
|
||||||
|
private IBlueTableSchema mySchema = new BlueTableSchema("count",
|
||||||
|
new ColumnInfo("id", 0, typeof(int)),
|
||||||
|
new ColumnInfo("count", 1, typeof(int)));
|
||||||
|
private int myId = 0;
|
||||||
|
private ViewStream myVS;
|
||||||
|
|
||||||
|
public CountStream(ViewStream vs) {
|
||||||
|
myVS = vs;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CountStream(string tableName)
|
||||||
|
: this(new TableStream(tableName)) { }
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
myVS.onChange += new ChangeDelegate(myHandleChanges);
|
||||||
|
}
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
myVS.onChange -= new ChangeDelegate(myHandleChanges);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void myHandleChanges(object newValue) {
|
||||||
|
myFireEvent(
|
||||||
|
myLastFiredDataPacket =
|
||||||
|
myLastFiredDataPacket == null ?
|
||||||
|
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
||||||
|
new BlueTable(mySchema))
|
||||||
|
:
|
||||||
|
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
||||||
|
myLastFiredDataPacket.newRows)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private IBlueRowCollection myGetRows(object dp, int id) {
|
||||||
|
BlueTable retVal = new BlueTable(mySchema);
|
||||||
|
retVal.addRow
|
||||||
|
(new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
|
||||||
|
return retVal;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
[DataStreamUserRespresentation("DateTimeExpanderStream")]
|
||||||
|
public class DateTimeExpanstionStream : ViewStream {
|
||||||
|
public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
|
||||||
|
Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
|
||||||
|
IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
|
||||||
|
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
|
||||||
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
|
||||||
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
|
||||||
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
|
||||||
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
|
||||||
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
ViewStream rs = new MapStream(parent_stream,
|
||||||
|
s => get_schema(s),
|
||||||
|
r => {
|
||||||
|
DateTime dt = r.get_datetime(inp_col);
|
||||||
|
IBlueTableSchema s = get_schema(r.schema);
|
||||||
|
MutableBlueRow rv = new MutableBlueRow(s, r);
|
||||||
|
rv.setValue($"{inp_col}_date", dt.Date);
|
||||||
|
rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
|
||||||
|
rv.setValue($"{inp_col}_day_of_month", dt.Day);
|
||||||
|
rv.setValue($"{inp_col}_month", dt.Date.Month);
|
||||||
|
rv.setValue($"{inp_col}_hour", dt.Hour);
|
||||||
|
rv.setValue($"{inp_col}_minute", dt.Minute);
|
||||||
|
rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
|
||||||
|
return rv;
|
||||||
|
});
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
using System;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("FilterStream")]
|
||||||
|
public class FilterStream : ViewStream {
|
||||||
|
public class RegExpFilter : IStreamFilter {
|
||||||
|
private Regex[] myRegExps;
|
||||||
|
private string[] myCols;
|
||||||
|
|
||||||
|
public RegExpFilter(string col, string regExp)
|
||||||
|
: this(new string[] { col }, new string[] { regExp }) { }
|
||||||
|
|
||||||
|
public RegExpFilter(string[] cols, string[] regExps) {
|
||||||
|
myCols = cols;
|
||||||
|
myRegExps = new Regex[regExps.Length];
|
||||||
|
for (int i = 0; i < regExps.Length; i++)
|
||||||
|
myRegExps[i] = new Regex(regExps[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#region IStreamFilter Members
|
||||||
|
public bool filter(IBlueRow dr) {
|
||||||
|
bool rv = true;
|
||||||
|
for (int i = 0; i < myCols.Length; i++) {
|
||||||
|
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
|
||||||
|
if (!rv)
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
private BlueTable myLastFiredRows;
|
||||||
|
private IBlueRowCollection myLastReceivedRows;
|
||||||
|
private IStreamFilter myFilter;
|
||||||
|
private ViewStream myParentStream;
|
||||||
|
private string myFilterString;
|
||||||
|
private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
|
||||||
|
|
||||||
|
#region constructors
|
||||||
|
public delegate bool OnFilter(IBlueRow r);
|
||||||
|
private class OnFilterWrapper : IStreamFilter {
|
||||||
|
public OnFilterWrapper(OnFilter f) { this.f = f; }
|
||||||
|
|
||||||
|
public bool filter(IBlueRow dr) { return f(dr); }
|
||||||
|
|
||||||
|
private OnFilter f;
|
||||||
|
}
|
||||||
|
public FilterStream(ViewStream vs, OnFilter f)
|
||||||
|
: this(vs, new OnFilterWrapper(f)) { }
|
||||||
|
|
||||||
|
|
||||||
|
public FilterStream(ViewStream vs, IStreamFilter filter) {
|
||||||
|
{
|
||||||
|
myParentStream = vs;
|
||||||
|
myFilter = filter;
|
||||||
|
myFilterString = filter.ToString();
|
||||||
|
if (filter is IDynamicStreamFilter)
|
||||||
|
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public FilterStream(ViewStream vs, string filterExpression)
|
||||||
|
: this(vs, new BooleanExpressionFilter(filterExpression)) {
|
||||||
|
myFilterString = filterExpression;
|
||||||
|
}
|
||||||
|
|
||||||
|
public FilterStream(ViewStream vs, string col, string regExp)
|
||||||
|
: this(vs, new RegExpFilter(col, regExp)) {
|
||||||
|
myFilterString = col + " , " + regExp;
|
||||||
|
}
|
||||||
|
|
||||||
|
public FilterStream(ViewStream vs)
|
||||||
|
: this(vs, new BinaryFilter(true)) {
|
||||||
|
myFilterString = true.ToString();
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region overridden methods
|
||||||
|
protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
myLastFiredRows = null;
|
||||||
|
myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myFireEvent(object o) {
|
||||||
|
DataPacket dp = o as DataPacket;
|
||||||
|
if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
|
||||||
|
base.myFireEvent(o);
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
private void myHandleCallbacks(object newValue) {
|
||||||
|
DataPacket retVal;
|
||||||
|
DataPacket arrivedPacket = newValue as DataPacket;
|
||||||
|
myLastReceivedRows = arrivedPacket.newRows;
|
||||||
|
if (myLastFiredRows != null) {
|
||||||
|
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
|
||||||
|
new BlueTable(myLastFiredRows.schema));
|
||||||
|
foreach (IBlueRow r in arrivedPacket.deletedRows)
|
||||||
|
if (myLastFiredRows.containsID((int)r["id"])) {
|
||||||
|
myLastFiredRows.deleteRow(r);
|
||||||
|
(retVal.deletedRows as BlueTable).addRow(r);
|
||||||
|
}
|
||||||
|
foreach (IBlueRow r in arrivedPacket.addedRows)
|
||||||
|
if (myFilter.filter(r)) {
|
||||||
|
myLastFiredRows.addRow(r);
|
||||||
|
(retVal.addedRows as BlueTable).addRow(r);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
myLastFiredRows = new BlueTable(arrivedPacket.schema);
|
||||||
|
retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
|
||||||
|
foreach (IBlueRow r in arrivedPacket.newRows)
|
||||||
|
if (myFilter.filter(r))
|
||||||
|
myLastFiredRows.addRow(r);
|
||||||
|
}
|
||||||
|
myFireEvent(retVal);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IStreamFilter filter {
|
||||||
|
set {
|
||||||
|
{
|
||||||
|
myFilter = value;
|
||||||
|
if (numberOfSubscribers > 0) {
|
||||||
|
if (myLastReceivedRows != null) {
|
||||||
|
BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
|
||||||
|
foreach (IBlueRow r in myLastReceivedRows)
|
||||||
|
if (myFilter.filter(r))
|
||||||
|
nRows.addRow(r);
|
||||||
|
IBlueRowCollection dRows = new BlueTable(nRows.schema),
|
||||||
|
aRows = new BlueTable(nRows.schema);
|
||||||
|
BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
|
||||||
|
myLastFiredRows = nRows;
|
||||||
|
myFireEvent(new DataPacket(nRows, aRows, dRows));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
get {
|
||||||
|
return myFilter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//public string filterString { set { filter = new BooleanExpressionFilter(value); } }
|
||||||
|
|
||||||
|
public override string ToString() {
|
||||||
|
return GetType().ToString() + " " + myFilterString;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void myHandleFilterChange() {
|
||||||
|
filter = myFilter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
//using bts.SilverPine;
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
namespace bts.BlueOrb {
|
||||||
|
[DataStreamUserRespresentation("GroupByStream")]
|
||||||
|
public class GroupByStream : ViewStream {
|
||||||
|
|
||||||
|
public interface INonIncGroupByEvaluator {
|
||||||
|
// grouped_rows
|
||||||
|
IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
||||||
|
string[] get_group_by_cols();
|
||||||
|
}
|
||||||
|
|
||||||
|
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
||||||
|
Func<string[]> _groupby_cols_func;
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
||||||
|
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
||||||
|
_groupby_cols_func = groupby_cols_func;
|
||||||
|
_evaluate_func = evaluate_func;
|
||||||
|
}
|
||||||
|
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||||
|
BlueTable rv = null;
|
||||||
|
foreach (var kv in grouped_rows) {
|
||||||
|
List<IBlueRow> g_rows = kv.Value;
|
||||||
|
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
||||||
|
if (out_rows != null && out_rows.Count > 0) {
|
||||||
|
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
|
||||||
|
}
|
||||||
|
rv.addRows(out_rows);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
public string[] get_group_by_cols() {
|
||||||
|
return _groupby_cols_func();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
private ViewStream _main_stream;
|
||||||
|
private INonIncGroupByEvaluator _groupby_func;
|
||||||
|
private BlueRowHasher _row_hasher;
|
||||||
|
private string[] _groupby_cols;
|
||||||
|
private Stopwatch _timer = new();
|
||||||
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
|
|
||||||
|
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
||||||
|
_main_stream = main_stream;
|
||||||
|
_groupby_func = groupby_func;
|
||||||
|
_groupby_cols = _groupby_func.get_group_by_cols();
|
||||||
|
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
||||||
|
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
||||||
|
|
||||||
|
public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
|
||||||
|
: this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
|
||||||
|
|
||||||
|
// this is the main constructor that does everything
|
||||||
|
public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
|
||||||
|
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
|
||||||
|
Func<string[]> groupby_cols_func = () => {
|
||||||
|
return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
|
||||||
|
};
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
||||||
|
if (!String.IsNullOrEmpty(optional_sort_col))
|
||||||
|
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
|
||||||
|
// this si where the group by aggregation happens
|
||||||
|
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
|
||||||
|
return new IBlueRow[] { rv }.ToList();
|
||||||
|
};
|
||||||
|
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
|
||||||
|
this(parent_str, String.Empty, standard_group_by_text) {
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
||||||
|
|
||||||
|
private void _main_stream_onchange(object nv) {
|
||||||
|
DataPacket dp = nv as DataPacket;
|
||||||
|
///grouping together
|
||||||
|
_timer.Restart();
|
||||||
|
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
||||||
|
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
|
||||||
|
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
|
||||||
|
_timer.Stop();
|
||||||
|
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
|
if (groupElapsedMs >= 50) {
|
||||||
|
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
||||||
|
}
|
||||||
|
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||||
|
/// reducing
|
||||||
|
_timer.Restart();
|
||||||
|
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||||
|
_timer.Stop();
|
||||||
|
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
|
_n_total_fires++;
|
||||||
|
if (applyElapsedMs >= 50) {
|
||||||
|
_n_expensive_fires++;
|
||||||
|
Log.DataFlow.info(
|
||||||
|
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||||
|
}
|
||||||
|
// TODO,
|
||||||
|
if (total_state != null)
|
||||||
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
using RowHash = System.Int32;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Data.Entity.Core.Mapping;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("JoinStream")]
|
||||||
|
public class SimpleJoinStream : ViewStream {
|
||||||
|
public class JoinBookkeepingData {
|
||||||
|
// initialized in cons
|
||||||
|
private readonly List<string> _join_col_names;
|
||||||
|
private readonly bool _is_left;
|
||||||
|
private readonly BlueRowHasher _hasher;
|
||||||
|
// initialized on first udpate
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
private int[] _join_cols;
|
||||||
|
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
|
||||||
|
|
||||||
|
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
|
||||||
|
_join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
|
||||||
|
_is_left = is_left;
|
||||||
|
_hasher = row_hasher;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => _schema;
|
||||||
|
|
||||||
|
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
|
||||||
|
|
||||||
|
public List<IBlueRow> get_matching_rows (int hash) {
|
||||||
|
List<IBlueRow> rv = null;
|
||||||
|
_hash_row_map.TryGetValue(hash, out rv);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
private RowHash _get_hash(IBlueRow r) {
|
||||||
|
if (_join_cols == null || _join_cols.Length == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return _hasher.get_hash(r, _join_cols);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void update_data(DataPacket dp) {
|
||||||
|
if (!ready) {
|
||||||
|
_schema = dp.newRows.schema;
|
||||||
|
_join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
|
||||||
|
}
|
||||||
|
_hash_row_map.Clear();
|
||||||
|
foreach (IBlueRow r in dp.newRows) {
|
||||||
|
int hash = _get_hash(r);
|
||||||
|
List<IBlueRow> hash_rows = null;
|
||||||
|
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
|
||||||
|
hash_rows = new List<IBlueRow>();
|
||||||
|
_hash_row_map.Add(hash, hash_rows);
|
||||||
|
}
|
||||||
|
hash_rows.Add(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool ready {
|
||||||
|
get {
|
||||||
|
return _schema != null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private ViewStream _ls, _rs;
|
||||||
|
private JoinBookkeepingData _ldata, _rdata;
|
||||||
|
private ViewStreamDatabase _vsd;
|
||||||
|
private IBlueTableSchema _schema = null;
|
||||||
|
|
||||||
|
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
|
||||||
|
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
|
||||||
|
|
||||||
|
public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
|
||||||
|
_ls = ls;
|
||||||
|
_rs = rs;
|
||||||
|
List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
||||||
|
List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
||||||
|
Log.assert(lj_col_list.Count == rj_col_list.Count);
|
||||||
|
int n_cols = lj_col_list.Count;
|
||||||
|
BlueRowHasher hasher = new BlueRowHasher(n_cols);
|
||||||
|
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
|
||||||
|
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_ls.onChange += _ls_onchange;
|
||||||
|
_rs.onChange += _rs_onchange;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _ls_onchange(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
_ldata.update_data(dp);
|
||||||
|
_form_and_fire();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _rs_onchange(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket ;
|
||||||
|
_rdata.update_data(dp);
|
||||||
|
_form_and_fire();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_ls.onChange -= _ls_onchange;
|
||||||
|
_rs.onChange -= _rs_onchange;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _build_schema() {
|
||||||
|
if (_schema != null)
|
||||||
|
return;
|
||||||
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
|
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
|
||||||
|
foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
|
||||||
|
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
|
||||||
|
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
||||||
|
}
|
||||||
|
foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
|
||||||
|
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
|
||||||
|
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
||||||
|
}
|
||||||
|
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _form_and_fire() {
|
||||||
|
if (!_ldata.ready||!_rdata.ready)
|
||||||
|
return;
|
||||||
|
_build_schema();
|
||||||
|
List<object[]> res_rows = new List<object[]>();
|
||||||
|
|
||||||
|
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
|
||||||
|
int offset = 1;
|
||||||
|
JoinBookkeepingData sdata = _ldata;
|
||||||
|
if (is_left) {
|
||||||
|
offset = 1;
|
||||||
|
sdata = _ldata;
|
||||||
|
} else {
|
||||||
|
sdata = _rdata;
|
||||||
|
offset = 1 + _ldata.schema.columnInfo.Length;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
|
||||||
|
rval[offset + i] = r[i];
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
|
||||||
|
object[] rv = new object[_schema.columnNames.Length];
|
||||||
|
for (int i = 0; i < _schema.columnNames.Length; i++) {
|
||||||
|
rv[i] = DBNull.Value;
|
||||||
|
}
|
||||||
|
rv[0] = id;
|
||||||
|
copy_row(rv, l_row, true);
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
// this is where the join happens
|
||||||
|
foreach (var kv in _ldata.rows) {
|
||||||
|
RowHash hash = kv.Key;
|
||||||
|
List<IBlueRow> l_rows = kv.Value;
|
||||||
|
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
|
||||||
|
|
||||||
|
foreach (IBlueRow lr in l_rows) {
|
||||||
|
if (r_rows != null) {
|
||||||
|
foreach (IBlueRow r in r_rows) {
|
||||||
|
object[] rowvals = build_row_array(res_rows.Count, lr);
|
||||||
|
copy_row(rowvals, r, false);
|
||||||
|
res_rows.Add(rowvals);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
object[] rowvals = build_row_array(res_rows.Count, lr);
|
||||||
|
res_rows.Add(rowvals);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BlueTable rv = new BlueTable(_schema);
|
||||||
|
foreach (object[] rval in res_rows) {
|
||||||
|
rv.addRow(new BlueRow(_schema, rval));
|
||||||
|
}
|
||||||
|
var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
//namespace bts.Blueorb {
|
||||||
|
// using BCG;
|
||||||
|
|
||||||
|
// public class JsonSaverStream : ViewStream {
|
||||||
|
// private ViewStream _stream;
|
||||||
|
// private string _file_name;
|
||||||
|
// public JsonSaverStream(ViewStream vs, string file_name) {
|
||||||
|
// _stream = vs;
|
||||||
|
// _file_name = file_name;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnFirstSubscriber() {
|
||||||
|
// _stream.onChange += _on_data;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void _on_data(object dp_obj) {
|
||||||
|
// if (dp_obj == null) {
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
// DataPacket dp = dp_obj as DataPacket;
|
||||||
|
// dp.newRows.write_to_json_file(dp.newRows, _file_name);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnLastSubscriber() {
|
||||||
|
// _stream.onChange -= _on_data;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using Amazon.Runtime.Internal.Util;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
using BCG.Utilities;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
namespace bts.BlueOrb{
|
||||||
|
public class LiteralStream : ViewStream {
|
||||||
|
|
||||||
|
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
||||||
|
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
|
||||||
|
string[] pairs = key_value_pairs.Split(' ');
|
||||||
|
foreach (string p in pairs) {
|
||||||
|
string[] kvp = p.Split(':');
|
||||||
|
string col_name = kvp[0];
|
||||||
|
string val = kvp[1];
|
||||||
|
var tup = BlueDataUtility.guess_type(val);
|
||||||
|
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
|
||||||
|
rv.Add(Tuple.Create(ci, tup.Item2));
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
|
||||||
|
// col_name: col_value ... ; col_name: col_value
|
||||||
|
public LiteralStream(string table_name, string rows) {
|
||||||
|
Func<string, int, IBlueRow> get_row = (json_row, id) => {
|
||||||
|
Dictionary<string, object> map = new Dictionary<string, object>();
|
||||||
|
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
|
||||||
|
//Logger.assert(col_vals.Length == 2);
|
||||||
|
foreach (string t in toks) {
|
||||||
|
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
|
||||||
|
string col = col_vals[0];
|
||||||
|
string val = col_vals[1];
|
||||||
|
map.Add(col, val);
|
||||||
|
}
|
||||||
|
map["id"] = id;
|
||||||
|
return new BlueRow(map);
|
||||||
|
};
|
||||||
|
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
||||||
|
BlueTable table = null;
|
||||||
|
foreach (string json_string in row_arr) {
|
||||||
|
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
||||||
|
get_row(json_string, table == null ? 0 : table.Count);
|
||||||
|
table = table ?? new BlueTable(r.schema);
|
||||||
|
table.addRow(r);
|
||||||
|
}
|
||||||
|
ViewStream str = new BlueTableStream(table);
|
||||||
|
str = new SchemaGuesserStream(str);
|
||||||
|
mySetRealSource(str);
|
||||||
|
}
|
||||||
|
|
||||||
|
public LiteralStream(string tableName, string schema, string value) {
|
||||||
|
string[] columns = schema.Split(' ');
|
||||||
|
|
||||||
|
|
||||||
|
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
|
||||||
|
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
||||||
|
int n = 1;
|
||||||
|
foreach (string c in columns) {
|
||||||
|
string[] columnAndType = c.Split('/');
|
||||||
|
if (columnAndType.Length == 1)
|
||||||
|
ci[n] = new ColumnInfo(c, n, typeof(double));
|
||||||
|
else
|
||||||
|
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
||||||
|
n++;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
|
||||||
|
string[] rows = value.Split(';');
|
||||||
|
n = 0;
|
||||||
|
foreach (string r in rows)
|
||||||
|
try {
|
||||||
|
if (r == string.Empty)
|
||||||
|
continue;
|
||||||
|
t.addRow(
|
||||||
|
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
||||||
|
}
|
||||||
|
|
||||||
|
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
using System;
|
||||||
|
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using BCG;
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
namespace bts.BlueOrb {
|
||||||
|
[DataStreamUserRespresentation("MillisecondStream")]
|
||||||
|
public class MillisecondStream : ViewStream, IWorker {
|
||||||
|
private DateTime _last_fire_time = DateTime.MinValue;
|
||||||
|
private readonly TimeSpan _period;
|
||||||
|
private readonly int _size;
|
||||||
|
private readonly ViewStreamDatabase _vsd;
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
private bts.CircularBufferTable _table;
|
||||||
|
|
||||||
|
public MillisecondStream(int interval_msecs, int size)
|
||||||
|
: this(ViewStreamDatabase.instance, interval_msecs, size) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
|
||||||
|
_vsd = vsd;
|
||||||
|
_period = TimeSpan.FromMilliseconds(interval_msecs);
|
||||||
|
_size = size;
|
||||||
|
_schema = BlueDataUtility.form_schema("Milliseconds",
|
||||||
|
Tuple.Create("id", TypeCode.Int32),
|
||||||
|
Tuple.Create("time", TypeCode.DateTime),
|
||||||
|
Tuple.Create("display_time", TypeCode.String));
|
||||||
|
_table = new CircularBufferTable(_schema, _size);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_vsd.addWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_vsd.removeWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int _id = 0;
|
||||||
|
public void doWork() {
|
||||||
|
if (DateTime.Now - _last_fire_time >= _period) {
|
||||||
|
_last_fire_time = DateTime.Now;
|
||||||
|
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
|
||||||
|
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
|
||||||
|
_table.add_row(nr);
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
internal class MSecondStream : TimeUnitStream {
|
||||||
|
public MSecondStream(int intervalInms, int size)
|
||||||
|
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
|
protected override DateTime myRoundTime(DateTime dt) { return dt; }
|
||||||
|
}
|
||||||
|
|
||||||
|
internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
|
||||||
|
private IBlueTableSchema mySchema = new BlueTableSchema("ms",
|
||||||
|
new ColumnInfo("id", 0, typeof(int)),
|
||||||
|
new ColumnInfo("time", 1, typeof(string)));
|
||||||
|
public DisplayableMillisecondStream(int intervalInMs, int sz) {
|
||||||
|
mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
|
||||||
|
}
|
||||||
|
#region IStreamMapper Members
|
||||||
|
|
||||||
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||||
|
return mySchema;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region IDynamicStreamMapper Members
|
||||||
|
|
||||||
|
public IBlueRow map(IBlueRow row) {
|
||||||
|
DateTime time = row.get_datetime("time");
|
||||||
|
var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
||||||
|
return new BlueRow(mySchema, row["id"], time_txt);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
private string myToString(DateTime dt) {
|
||||||
|
string sep = ":";
|
||||||
|
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
//using MathNet.Numerics.Statistics;
|
||||||
|
|
||||||
|
|
||||||
|
public enum MultiRowOp {
|
||||||
|
Mean,
|
||||||
|
Median,
|
||||||
|
Mode,
|
||||||
|
Range,
|
||||||
|
StdDev,
|
||||||
|
Max,
|
||||||
|
Min,
|
||||||
|
First,
|
||||||
|
Last,
|
||||||
|
Sum,
|
||||||
|
EdgeDiff,
|
||||||
|
EdgeDiffPercentage,
|
||||||
|
}
|
||||||
|
public class MultiRowOpHelper {
|
||||||
|
// input + op
|
||||||
|
public readonly List<Tuple<string, MultiRowOp>> operations;
|
||||||
|
|
||||||
|
public static MultiRowOpHelper parse_ops(string text) {
|
||||||
|
// text is inp_col: op, inp_col: op, inp_col:op
|
||||||
|
// ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
|
||||||
|
List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
|
||||||
|
string[] calcs = text.Split(',');
|
||||||
|
foreach (string c in calcs) {
|
||||||
|
string[] toks = c.Split(':');
|
||||||
|
string inp_col = toks[0].Trim();
|
||||||
|
string op_txt = toks[1].Trim();
|
||||||
|
MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
|
||||||
|
rv.Add(Tuple.Create(inp_col, op));
|
||||||
|
}
|
||||||
|
return new MultiRowOpHelper(rv);
|
||||||
|
}
|
||||||
|
|
||||||
|
public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
|
||||||
|
operations = operations_;
|
||||||
|
}
|
||||||
|
|
||||||
|
// aggregates multiple rows into one - ie does group_by compute
|
||||||
|
public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
|
||||||
|
if (rows.Count() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
IBlueRow ir = rows.ElementAt(0);
|
||||||
|
IBlueTableSchema in_schema = ir.schema;
|
||||||
|
IBlueTableSchema out_schema = in_schema;
|
||||||
|
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
|
||||||
|
new Dictionary<string, Tuple<string, MultiRowOp>>();
|
||||||
|
foreach (var t in parameters.operations) {
|
||||||
|
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
||||||
|
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
||||||
|
out_col_map[out_col_name] = t;
|
||||||
|
}
|
||||||
|
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
||||||
|
foreach (var kv in out_col_map) {
|
||||||
|
string out_col_name = kv.Key;
|
||||||
|
string in_col_name = kv.Value.Item1;
|
||||||
|
MultiRowOp op = kv.Value.Item2;
|
||||||
|
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
|
||||||
|
double comp_res = compute_op(col_vals, op);
|
||||||
|
or.setValue(out_col_name, comp_res);
|
||||||
|
}
|
||||||
|
return or;
|
||||||
|
}
|
||||||
|
|
||||||
|
// aggregates multiple rows into one, used by MultiRowRollingMathStream
|
||||||
|
public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
|
||||||
|
if (rows.Count() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
IBlueRow ir = rows.ElementAt(start_idx);
|
||||||
|
IBlueTableSchema in_schema = ir.schema;
|
||||||
|
IBlueTableSchema out_schema = in_schema;
|
||||||
|
// out_col vs inp_cal + type
|
||||||
|
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
|
||||||
|
new Dictionary<string, Tuple<string, MultiRowOp>>();
|
||||||
|
foreach (var t in parameters.operations) {
|
||||||
|
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
||||||
|
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
||||||
|
out_col_map[out_col_name] = t;
|
||||||
|
}
|
||||||
|
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
||||||
|
foreach (var kv in out_col_map) {
|
||||||
|
string out_col_name = kv.Key;
|
||||||
|
string in_col_name = kv.Value.Item1;
|
||||||
|
MultiRowOp op = kv.Value.Item2;
|
||||||
|
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
|
||||||
|
double comp_res = compute_op(col_vals, op);
|
||||||
|
or.setValue(out_col_name, comp_res);
|
||||||
|
or.setValue("id", ir["id"]);
|
||||||
|
}
|
||||||
|
return or;
|
||||||
|
}
|
||||||
|
|
||||||
|
// rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
|
||||||
|
// calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
|
||||||
|
|
||||||
|
public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
|
||||||
|
if (rolling_rows.Count() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
IBlueRow ir = rolling_rows.ElementAt(0);
|
||||||
|
IBlueTableSchema in_schema = ir.schema;
|
||||||
|
IBlueTableSchema out_schema = in_schema;
|
||||||
|
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
|
||||||
|
new Dictionary<string, Tuple<string, MultiRowOp>>();
|
||||||
|
foreach (var t in operations) {
|
||||||
|
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
||||||
|
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
||||||
|
out_col_map[out_col_name] = t;
|
||||||
|
}
|
||||||
|
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
||||||
|
foreach (var kv in out_col_map) {
|
||||||
|
string out_col_name = kv.Key;
|
||||||
|
string in_col_name = kv.Value.Item1;
|
||||||
|
MultiRowOp op = kv.Value.Item2;
|
||||||
|
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
|
||||||
|
double comp_res = compute_op(col_vals, op);
|
||||||
|
or.setValue(out_col_name, comp_res);
|
||||||
|
}
|
||||||
|
return or;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
|
||||||
|
switch (op) {
|
||||||
|
case MultiRowOp.Max:
|
||||||
|
return col_vals.Max<double>();
|
||||||
|
case MultiRowOp.Min:
|
||||||
|
return col_vals.Min<double>();
|
||||||
|
case MultiRowOp.Mean:
|
||||||
|
return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
|
||||||
|
case MultiRowOp.Sum:
|
||||||
|
return col_vals.Sum();
|
||||||
|
case MultiRowOp.Median:
|
||||||
|
return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
|
||||||
|
//case MultiRowOp.Mode:
|
||||||
|
// return MathNet.Numerics.Statistics.Statistics.(col_vals);
|
||||||
|
case MultiRowOp.First:
|
||||||
|
return col_vals.First();
|
||||||
|
case MultiRowOp.Last:
|
||||||
|
return col_vals.Last();
|
||||||
|
case MultiRowOp.StdDev:
|
||||||
|
return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
|
||||||
|
default:
|
||||||
|
// TODO
|
||||||
|
Log.assert(false);
|
||||||
|
return Double.NaN;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Net;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("MultiRowRollingMathStream")]
|
||||||
|
public class MultiRowRollingMathStream : ViewStream {
|
||||||
|
private ViewStream _sorted_parent_str;
|
||||||
|
private MultiRowOpHelper _op;
|
||||||
|
private int _rolling_window_sz;
|
||||||
|
public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
|
||||||
|
_sorted_parent_str = new SortStream(parent_str, sort_exp);
|
||||||
|
_op = MultiRowOpHelper.parse_ops(op_txt);
|
||||||
|
_rolling_window_sz = rolling_window_sz;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_sorted_parent_str.onChange += _parent_str_onchange;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_sorted_parent_str.onChange -= _parent_str_onchange;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _parent_str_onchange(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
var rows = dp.newRows.to_list();
|
||||||
|
BlueTable out_table = null;
|
||||||
|
for (int i = 0; i < rows.Count; i++) {
|
||||||
|
var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
|
||||||
|
if (out_table == null)
|
||||||
|
out_table = new BlueTable(out_r.schema);
|
||||||
|
out_table.addRow(out_r);
|
||||||
|
}
|
||||||
|
var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
using System;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
#region mappings
|
||||||
|
internal class ProjectData {
|
||||||
|
// schema from the base stream
|
||||||
|
public readonly IBlueTableSchema in_schema;
|
||||||
|
// this is the projected schema
|
||||||
|
public readonly IBlueTableSchema out_schema;
|
||||||
|
// projected cols to the input cols, for cols who's names are changed
|
||||||
|
public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
|
||||||
|
public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
|
||||||
|
IDictionary<string, string> mapped_to_orig_col_map_) {
|
||||||
|
in_schema = in_schema_;
|
||||||
|
out_schema = out_schema_;
|
||||||
|
mapped_to_orig_col_map = mapped_to_orig_col_map_;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
|
||||||
|
#region Row with mappings build in
|
||||||
|
internal class NewProjectRow : IBlueRow {
|
||||||
|
#region IBlueRow members
|
||||||
|
private ProjectData _project_data;
|
||||||
|
private IBlueRow _row;
|
||||||
|
public NewProjectRow(ProjectData project_data, IBlueRow row) {
|
||||||
|
_project_data = project_data;
|
||||||
|
_row = row;
|
||||||
|
}
|
||||||
|
public IBlueTableSchema schema { get { return _project_data.out_schema; } }
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region ICollection members
|
||||||
|
public int Count { get { return schema.columnNames.Length; } }
|
||||||
|
|
||||||
|
public bool IsSynchronized { get { return false; } }
|
||||||
|
|
||||||
|
public object SyncRoot { get { return this; } }
|
||||||
|
|
||||||
|
public void CopyTo(Array array, int index) {
|
||||||
|
int i = index;
|
||||||
|
foreach (object o in this as IEnumerable)
|
||||||
|
array.SetValue(o, i++);
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region IDictionary members
|
||||||
|
public bool IsFixedSize { get { return true; } }
|
||||||
|
|
||||||
|
public bool IsReadOnly { get { return true; } }
|
||||||
|
|
||||||
|
public object this[object key] {
|
||||||
|
get {
|
||||||
|
if (key is string)
|
||||||
|
return this[(string)key];
|
||||||
|
return this[(int)key];
|
||||||
|
}
|
||||||
|
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
}
|
||||||
|
|
||||||
|
public object this[int index] {
|
||||||
|
get {
|
||||||
|
return this[schema.columnInfo[index].colName];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public object this[string col_name] {
|
||||||
|
get {
|
||||||
|
string mapped_to_col = null;
|
||||||
|
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
|
||||||
|
return _row[mapped_to_col];
|
||||||
|
}
|
||||||
|
return _row[col_name];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ICollection Keys { get { return schema.columnNames; } }
|
||||||
|
|
||||||
|
public ICollection Values {
|
||||||
|
get {
|
||||||
|
object[] retVal = new object[schema.columnNames.Length];
|
||||||
|
CopyTo(retVal, 0);
|
||||||
|
return retVal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
|
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
|
||||||
|
public bool Contains(object key) { return schema[(string)key] != null; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
||||||
|
/// iterate through the values, typecast this into an IEnumerable
|
||||||
|
/// </summary>
|
||||||
|
/// <returns></returns>
|
||||||
|
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||||
|
|
||||||
|
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
||||||
|
|
||||||
|
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("ProjectStream")]
|
||||||
|
public class NewProjectStream : ViewStream, IStreamMapper {
|
||||||
|
|
||||||
|
private ViewStream _stream;
|
||||||
|
private string _project_expression;
|
||||||
|
private ProjectData _project_data = null;
|
||||||
|
//public ProjectData
|
||||||
|
public NewProjectStream(ViewStream vs, string project_expression) {
|
||||||
|
_stream = vs;
|
||||||
|
_project_expression = project_expression;
|
||||||
|
_stream = new MapStream(vs, this);
|
||||||
|
mySetRealSource(_stream);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
|
||||||
|
_project_data = _create_project_data(input_schema, _project_expression);
|
||||||
|
return _project_data.out_schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow map(IBlueRow row) {
|
||||||
|
return new NewProjectRow(_project_data, row);
|
||||||
|
}
|
||||||
|
|
||||||
|
// this is where most of the logic is, in terms of translating cols
|
||||||
|
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
|
||||||
|
string project_expression) {
|
||||||
|
project_expression = project_expression.Trim();
|
||||||
|
string[] project_toks = project_expression.Split(' ', ',');
|
||||||
|
HashSet<string> removed_cols = new HashSet<string>();
|
||||||
|
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
|
||||||
|
// by default every col will show up, so if we have the projection as (a:a1)
|
||||||
|
// it will convert a to a1,
|
||||||
|
// modifications (6/30/2025):
|
||||||
|
// only the columns represented here, either mapped
|
||||||
|
// or not, will show
|
||||||
|
// examples: if a schema has cols a b c
|
||||||
|
// project(parent_str, "a:aa b") -> aa b
|
||||||
|
foreach (string o_p_tok in project_toks) {
|
||||||
|
string p_tok = o_p_tok.Trim();
|
||||||
|
if (p_tok.Contains(":")) {
|
||||||
|
// explicit mapping
|
||||||
|
string[] map_toks = p_tok.Split(":");
|
||||||
|
string orig_col = map_toks[0];
|
||||||
|
string mapped_col = map_toks[1];
|
||||||
|
orig_to_mapped.Add(orig_col, mapped_col);
|
||||||
|
} else {
|
||||||
|
// this is a col_name explicitely stated
|
||||||
|
if (input_schema.columnNames.Contains(p_tok)) {
|
||||||
|
string orig_col = p_tok;
|
||||||
|
string mapped_col = orig_col;
|
||||||
|
orig_to_mapped.Add(orig_col, mapped_col);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// make the new schema
|
||||||
|
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
||||||
|
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
||||||
|
if (ci.colName == "id") {
|
||||||
|
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||||
|
new_cols.Add(nc);
|
||||||
|
continue;
|
||||||
|
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
||||||
|
string mapped_name = orig_to_mapped[ci.colName];
|
||||||
|
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
||||||
|
new_cols.Add(nc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
||||||
|
var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
|
||||||
|
return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
using System.IO;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("PythonDataframeStream")]
|
||||||
|
public class PythonDataframeStream : ViewStream {
|
||||||
|
private ViewStream _parent_str;
|
||||||
|
private readonly string _in_csv_file_name;
|
||||||
|
private readonly string _out_csv_filename;
|
||||||
|
private readonly string _transform_function;
|
||||||
|
public PythonDataframeStream(ViewStream parent_str, string transform_function) {
|
||||||
|
var guid = System.Guid.NewGuid();
|
||||||
|
_in_csv_file_name = $"pdfs_in_{guid}.csv";
|
||||||
|
_out_csv_filename = $"pdfs_out_{guid}.csv";
|
||||||
|
_parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
|
||||||
|
_transform_function = transform_function;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_parent_str.onChange += _parent_str_onchange;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_parent_str.onChange -= _parent_str_onchange;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _parent_str_onchange(object dp_obj) {
|
||||||
|
Log.assert(dp_obj != null);
|
||||||
|
// the file _in_csv_file_name is now populated.
|
||||||
|
// call py_func_wrapper_main and pass this file name as well
|
||||||
|
// as the output file name
|
||||||
|
// py_func_wrapper_main reads _in_csv_file_name into the df and calls
|
||||||
|
// transform function, which returns the tranformed df. py_func_wrapper_main then
|
||||||
|
// writes it into the _out_csv_filename file.
|
||||||
|
// now this function should read _out_csv_filename into a dp
|
||||||
|
// and fire it
|
||||||
|
_compute_and_write();
|
||||||
|
ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
|
||||||
|
IBlueRowCollection nrows = null;
|
||||||
|
rs.onChange += (dp_obj) => {
|
||||||
|
nrows = (dp_obj as DataPacket).newRows;
|
||||||
|
};
|
||||||
|
var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _compute_and_write() {
|
||||||
|
File.WriteAllText("user_code.py", _transform_function);
|
||||||
|
// wrapper function
|
||||||
|
string py_func_wrapper_main = @"
|
||||||
|
import sys
|
||||||
|
from user_code import transform
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
input_file_name = sys.argv[1]
|
||||||
|
output_file_name = sys.argv[2]
|
||||||
|
# 1. load input_file_name into df
|
||||||
|
# 2. do some test calc
|
||||||
|
# 3. write the output of 2 into output_file_name
|
||||||
|
df = pd.read_csv(input_file_name)
|
||||||
|
df_out = transform(df)
|
||||||
|
df_out.to_csv(output_file_name, index=False)";
|
||||||
|
|
||||||
|
|
||||||
|
File.WriteAllText("runner.py", py_func_wrapper_main);
|
||||||
|
|
||||||
|
ProcessStartInfo psi = new ProcessStartInfo {
|
||||||
|
FileName = "python",
|
||||||
|
Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
|
||||||
|
RedirectStandardOutput = false,
|
||||||
|
UseShellExecute = false
|
||||||
|
};
|
||||||
|
var proc = Process.Start(psi);
|
||||||
|
proc.WaitForExit();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
#region Unused test code
|
||||||
|
private void _simple_test() {
|
||||||
|
|
||||||
|
// actual function that needs to be called ////////////////////////
|
||||||
|
string py_real_function = @"
|
||||||
|
def test(x, out_path):
|
||||||
|
rv = x * 10
|
||||||
|
with open(out_path, 'w') as f:
|
||||||
|
f.write(str(rv))
|
||||||
|
return rv";
|
||||||
|
// the function is in a string object.
|
||||||
|
File.WriteAllText("user_code.py", py_real_function);
|
||||||
|
|
||||||
|
|
||||||
|
// runner app (top level main function) //////////////////////////////
|
||||||
|
string py_func_wrapper_main = @"
|
||||||
|
import sys
|
||||||
|
from user_code import test
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
arg = int(sys.argv[1])
|
||||||
|
out_path = sys.argv[2]
|
||||||
|
rv = test(arg, out_path)
|
||||||
|
rv = rv + 12
|
||||||
|
with open(out_path, 'w') as f:to \
|
||||||
|
f.write(str(rv))";
|
||||||
|
|
||||||
|
File.WriteAllText("runner.py", py_func_wrapper_main);
|
||||||
|
|
||||||
|
string outputPath = "py_out.txt";
|
||||||
|
ProcessStartInfo psi = new ProcessStartInfo {
|
||||||
|
FileName = "python",
|
||||||
|
Arguments = $"runner.py 4 {outputPath}",
|
||||||
|
RedirectStandardOutput = false,
|
||||||
|
UseShellExecute = false
|
||||||
|
};
|
||||||
|
var proc = Process.Start(psi);
|
||||||
|
proc.WaitForExit();
|
||||||
|
|
||||||
|
// Now read the output written by the function
|
||||||
|
string result = File.ReadAllText(outputPath);
|
||||||
|
Console.WriteLine(result); // Should print "6"
|
||||||
|
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
using BCG;
|
||||||
|
|
||||||
|
namespace bts.BlueOrb {
|
||||||
|
public class SchemaGuesserStream : ViewStream {
|
||||||
|
|
||||||
|
private IBlueRowCollection _rows = null;
|
||||||
|
private ViewStream _vs;
|
||||||
|
public SchemaGuesserStream(ViewStream vs) {
|
||||||
|
_vs = vs;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_vs.onChange += _on_data;
|
||||||
|
}
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_vs.onChange -= _on_data;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _on_data(object obj) {
|
||||||
|
DataPacket dp = obj as DataPacket;
|
||||||
|
if (_rows == null) {
|
||||||
|
_rows = BlueDataUtility.get_typed_table(dp.newRows);
|
||||||
|
}
|
||||||
|
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
using Bts.ArithmeticExpressions;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
namespace bts.BlueOrb {
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("SingleRowMathStream")]
|
||||||
|
public class MathStream : ViewStream, IStreamMapper {
|
||||||
|
private IArithmeticProgram myExp;
|
||||||
|
private BooleanExpressionFilter _bool;
|
||||||
|
List<string> myExtraColumns = new List<string>();
|
||||||
|
BlueTableSchema mySchema;
|
||||||
|
|
||||||
|
|
||||||
|
public MathStream(ViewStream vs, string mathExpression)
|
||||||
|
: this(vs, mathExpression, string.Empty) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
|
||||||
|
myExp = ArithmeticProgramParser.Parse(mathExpression) ??
|
||||||
|
throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
|
||||||
|
myExtraColumns = myExp.OutputIdentifiers.ToList();
|
||||||
|
if (!string.IsNullOrEmpty(boolean_expression))
|
||||||
|
_bool = new BooleanExpressionFilter(boolean_expression);
|
||||||
|
mySetRealSource(new MapStream(vs, this));
|
||||||
|
}
|
||||||
|
|
||||||
|
#region IStreamMapper Members
|
||||||
|
|
||||||
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||||
|
if (mySchema == null) {
|
||||||
|
Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
|
||||||
|
foreach (ColumnInfo ci in inputSchema.columnInfo)
|
||||||
|
colIdx[ci.colName] = ci;
|
||||||
|
|
||||||
|
foreach (string ncn in myExtraColumns) {
|
||||||
|
ColumnInfo newC;
|
||||||
|
if (colIdx.ContainsKey(ncn))
|
||||||
|
newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
|
||||||
|
else
|
||||||
|
newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
|
||||||
|
colIdx[ncn] = newC;
|
||||||
|
}
|
||||||
|
mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
|
||||||
|
}
|
||||||
|
return mySchema;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region IDynamicStreamMapper Members
|
||||||
|
|
||||||
|
public IBlueRow map(IBlueRow row) {
|
||||||
|
MutableBlueRow result = new MutableBlueRow(mySchema, row);
|
||||||
|
if (_bool == null || _bool.filter(row)) {
|
||||||
|
myExp.Evaluate(result.AsDictionary());
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("SortStream")]
|
||||||
|
public class SortStream: ViewStream {
|
||||||
|
public SortStream (ViewStream parent_stream, string sort_expr) {
|
||||||
|
ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
[DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
|
||||||
|
public class TimeBasedRollingFilterStream : ViewStream {
|
||||||
|
public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
|
||||||
|
ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+3
-5
@@ -1,9 +1,8 @@
|
|||||||
using Google.Protobuf.WellKnownTypes;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
namespace BCG {
|
using BCG;
|
||||||
|
|
||||||
|
namespace bts.BlueOrb {
|
||||||
// Keeps adding changing rows to the stream, hence preserving the history
|
// Keeps adding changing rows to the stream, hence preserving the history
|
||||||
// and creating a time series, for example this can be used to make bbo updates
|
// and creating a time series, for example this can be used to make bbo updates
|
||||||
// into a time series of prices/qty
|
// into a time series of prices/qty
|
||||||
@@ -41,7 +40,6 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
_table.addRows(added_rows);
|
_table.addRows(added_rows);
|
||||||
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
|
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
[DataStreamUserRespresentation("TopStream")]
|
||||||
|
public class TopStream : ViewStream {
|
||||||
|
public TopStream(ViewStream vs, int n_rows, string sort_col) {
|
||||||
|
ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using mbts;
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("UnionStream")]
|
||||||
|
public class UnionStream : ViewStream {
|
||||||
|
public UnionStream(ViewStream left_stream, ViewStream right_stream) {
|
||||||
|
ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
@@ -18,8 +18,13 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
|
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
|
||||||
|
<PackageReference Include="IronPython" Version="3.4.2" />
|
||||||
|
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
|
||||||
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
||||||
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
||||||
|
<PackageReference Include="Parquet.Net" Version="5.0.1" />
|
||||||
|
<PackageReference Include="System.Data.SQLite" Version="1.0.118" />
|
||||||
|
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
@@ -2,15 +2,27 @@ namespace Bts.DataStream;
|
|||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
|
using bts.api.core;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
using System.Reactive.Linq;
|
using System.Reactive.Linq;
|
||||||
|
|
||||||
public static class DataStreamExtensions {
|
public static class DataStreamExtensions {
|
||||||
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
||||||
source as ViewStream ?? new ObservableViewStream(source);
|
source as ViewStream ?? new ObservableViewStream(source);
|
||||||
|
|
||||||
|
public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
|
||||||
|
source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
|
||||||
|
|
||||||
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
||||||
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
||||||
|
|
||||||
@@ -23,4 +35,100 @@ public static class DataStreamExtensions {
|
|||||||
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
|
||||||
|
source.Select(dp => {
|
||||||
|
dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
|
||||||
|
return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
|
||||||
|
|
||||||
|
IBlueRowCollection<T> CastRows(ICollection rows) =>
|
||||||
|
new BlueTableSelectorWrapper<T>(
|
||||||
|
rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
|
||||||
|
});
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
|
||||||
|
source.AsTables<IBlueRow, int>(r => r, r => r.Id());
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
|
||||||
|
source.AsTables<T, int>(valueSelector, r => r.Id());
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<T, int>> AsTables<T>(
|
||||||
|
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
|
||||||
|
source.AsTables<T, int>(valueSelector, keySelector);
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
|
||||||
|
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
|
||||||
|
rowKeySelector ??= r => keySelector(valueSelector(r));
|
||||||
|
return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
|
||||||
|
var bd = res.to_builder();
|
||||||
|
dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
|
||||||
|
foreach (IBlueRow r in addedRows) {
|
||||||
|
bd.add_or_update(valueSelector(r));
|
||||||
|
}
|
||||||
|
foreach (IBlueRow r in deletedRows) {
|
||||||
|
bd.remove(rowKeySelector(r));
|
||||||
|
}
|
||||||
|
return bd.to_table();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
||||||
|
source.Scan(
|
||||||
|
(Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
|
||||||
|
(res, table) => {
|
||||||
|
var bd = res.Table.ToDataPacketBuilder();
|
||||||
|
foreach (var r in table.removed) {
|
||||||
|
bd.Remove(r.Id());
|
||||||
|
}
|
||||||
|
foreach (var r in table.added) {
|
||||||
|
bd.AddOrUpdate(r);
|
||||||
|
}
|
||||||
|
var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
|
||||||
|
return (newRows, resDp);
|
||||||
|
})
|
||||||
|
.Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
|
||||||
|
.Select(t => t.Dp);
|
||||||
|
|
||||||
|
public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
||||||
|
source.AsDataPackets(schema).AsViewStream();
|
||||||
|
|
||||||
|
public static int ResolveIndices(this IBlueTableSchema schema, string name) {
|
||||||
|
var indices = ResolveIndices(schema, new[] { name });
|
||||||
|
return indices[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
|
||||||
|
var indices = ResolveIndices(schema, new[] { name1, name2 });
|
||||||
|
return (indices[0], indices[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
|
||||||
|
var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
|
||||||
|
return (indices[0], indices[1], indices[2]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
|
||||||
|
var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
|
||||||
|
return (indices[0], indices[1], indices[2], indices[3]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
|
||||||
|
var columns = schema.columnInfo;
|
||||||
|
var indices = new int[columnNames.Length];
|
||||||
|
int nResolved = 0;
|
||||||
|
for (int i = 0; i < columnNames.Length; i++) {
|
||||||
|
int origNResolved = nResolved;
|
||||||
|
for (int j = 0; j < columns.Length; j++) {
|
||||||
|
if (columns[j].colName == columnNames[i]) {
|
||||||
|
indices[i] = j;
|
||||||
|
nResolved++;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (nResolved == origNResolved) {
|
||||||
|
throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return indices;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using System;
|
||||||
|
using BCG;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using Bts.DataStreamExpressions;
|
||||||
|
|
||||||
|
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
|
||||||
|
// So for example, if an expresison is passed
|
||||||
|
[DataStreamUserRespresentation("dsl")]
|
||||||
|
public class DSLStream : ViewStream {
|
||||||
|
private readonly string _dsl;
|
||||||
|
public DSLStream(string dsl) {
|
||||||
|
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
|
||||||
|
_dsl = dsl;
|
||||||
|
ViewStream rs = build_stream(dsl);
|
||||||
|
rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
|
||||||
|
static ViewStream? build_stream(string stream_expression) {
|
||||||
|
try {
|
||||||
|
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||||
|
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||||
|
return rv;
|
||||||
|
|
||||||
|
} catch (Exception ex) {
|
||||||
|
Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -39,8 +39,8 @@ namespace BCG {
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
||||||
var server_address = new BlutoSubAddress(_ip, _port);
|
var server_rci = new RemoteConnectionInfo(_ip, _port);
|
||||||
bts.AsyncTCPConnector_APM.connect_to(server_address, this);
|
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
|
|||||||
@@ -4,14 +4,21 @@ using BCG;
|
|||||||
|
|
||||||
using bts;
|
using bts;
|
||||||
|
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
|
||||||
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
|
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
|
||||||
readonly IObservable<DataPacket> source;
|
readonly IObservable<DataPacket> source;
|
||||||
|
readonly string? name;
|
||||||
|
readonly ILogger? logger;
|
||||||
IDisposable? subscription;
|
IDisposable? subscription;
|
||||||
|
|
||||||
public ObservableViewStream(IObservable<DataPacket> source) =>
|
public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
|
||||||
this.source = source;
|
this.source = source;
|
||||||
|
this.name = name;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
public void Dispose() {
|
public void Dispose() {
|
||||||
GC.SuppressFinalize(this);
|
GC.SuppressFinalize(this);
|
||||||
@@ -23,7 +30,13 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
|
|||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
Log.isnull(subscription);
|
Log.isnull(subscription);
|
||||||
subscription = source.Subscribe(myFireEvent);
|
subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
|
||||||
|
|
||||||
|
void OnError(Exception exception) =>
|
||||||
|
logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
|
||||||
|
|
||||||
|
void OnCompleted() =>
|
||||||
|
logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
|
|||||||
@@ -1,33 +1,51 @@
|
|||||||
using System;
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public class AddColumnsStream : ViewStream {
|
public class AddColumnsStream : ViewStream {
|
||||||
public static Tuple<string, Type, object> parse_exp(string exp) {
|
public static Tuple<string, Type, object> parse_single_tok(string exp) {
|
||||||
string[] toks = exp.Split(':');
|
string[] toks = exp.Split(':');
|
||||||
string col_name = toks[0];
|
string col_name = toks[0].Trim();
|
||||||
string col_value = toks[1];
|
string col_value = toks[1].Trim();
|
||||||
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
|
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
|
||||||
return Tuple.Create(col_name, tup.Item1, tup.Item2);
|
return Tuple.Create(col_name, tup.Item1, tup.Item2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static List<Tuple<string, Type, object>> parse_exp(string exp) {
|
||||||
|
string[] toks = exp.Split(",");
|
||||||
|
List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
|
||||||
|
foreach (string s in toks) {
|
||||||
|
var tup = parse_single_tok(s.Trim());
|
||||||
|
rv.Add(tup);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
private Tuple<string, Type, object> _new_col;
|
private Tuple<string, Type, object> _new_col;
|
||||||
|
private List<Tuple<string, Type, object>> _new_col_map;
|
||||||
private IBlueTableSchema _out_schema;
|
private IBlueTableSchema _out_schema;
|
||||||
|
|
||||||
public AddColumnsStream(ViewStream vs, string col_def) {
|
public AddColumnsStream(ViewStream vs, string col_def) {
|
||||||
_new_col = parse_exp(col_def);
|
_new_col_map = parse_exp(col_def);
|
||||||
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
|
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
|
||||||
mySetRealSource(rs);
|
mySetRealSource(rs);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
|
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
|
||||||
if (_out_schema == null)
|
if (_out_schema == null) {
|
||||||
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
|
_out_schema = input_schema;
|
||||||
|
foreach (var m in _new_col_map) {
|
||||||
|
_out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
|
||||||
|
}
|
||||||
|
}
|
||||||
return _out_schema;
|
return _out_schema;
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRow _map(IBlueRow r) {
|
private IBlueRow _map(IBlueRow r) {
|
||||||
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
|
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
|
||||||
rv.setValue(_new_col.Item1, _new_col.Item3);
|
foreach (var m in _new_col_map) {
|
||||||
|
rv.setValue(m.Item1, m.Item3);
|
||||||
|
}
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,52 +1,47 @@
|
|||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace BCG
|
namespace bts.BlueOrb {
|
||||||
{
|
using BCG;
|
||||||
[DataStreamUserRespresentation ("count")]
|
[DataStreamUserRespresentation("CountStream")]
|
||||||
public class CountStream : ViewStream
|
public class CountStream : ViewStream {
|
||||||
{
|
|
||||||
private DataPacket myLastFiredDataPacket;
|
private DataPacket myLastFiredDataPacket;
|
||||||
private IBlueTableSchema mySchema = new BlueTableSchema ("count",
|
private IBlueTableSchema mySchema = new BlueTableSchema("count",
|
||||||
new ColumnInfo ("id", 0, typeof(int)),
|
new ColumnInfo("id", 0, typeof(int)),
|
||||||
new ColumnInfo ("count", 1, typeof(int)));
|
new ColumnInfo("count", 1, typeof(int)));
|
||||||
private int myId = 0;
|
private int myId = 0;
|
||||||
private ViewStream myVS;
|
private ViewStream myVS;
|
||||||
|
|
||||||
public CountStream (ViewStream vs)
|
public CountStream(ViewStream vs) {
|
||||||
{
|
|
||||||
myVS = vs;
|
myVS = vs;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CountStream (string tableName)
|
public CountStream(string tableName)
|
||||||
: this (new TableStream (tableName))
|
: this(new TableStream(tableName)) { }
|
||||||
{}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber () {
|
protected override void myOnFirstSubscriber() {
|
||||||
myVS.onChange += new ChangeDelegate(myHandleChanges);
|
myVS.onChange += new ChangeDelegate(myHandleChanges);
|
||||||
}
|
}
|
||||||
protected override void myOnLastSubscriber () {
|
protected override void myOnLastSubscriber() {
|
||||||
myVS.onChange -= new ChangeDelegate(myHandleChanges);
|
myVS.onChange -= new ChangeDelegate(myHandleChanges);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void myHandleChanges(object newValue)
|
private void myHandleChanges(object newValue) {
|
||||||
{
|
myFireEvent(
|
||||||
myFireEvent (
|
|
||||||
myLastFiredDataPacket =
|
myLastFiredDataPacket =
|
||||||
myLastFiredDataPacket == null ?
|
myLastFiredDataPacket == null ?
|
||||||
new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId),
|
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
||||||
new BlueTable (mySchema))
|
new BlueTable(mySchema))
|
||||||
:
|
:
|
||||||
new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId),
|
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
||||||
myLastFiredDataPacket.newRows)
|
myLastFiredDataPacket.newRows)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRowCollection myGetRows (object dp, int id)
|
private IBlueRowCollection myGetRows(object dp, int id) {
|
||||||
{
|
BlueTable retVal = new BlueTable(mySchema);
|
||||||
BlueTable retVal = new BlueTable (mySchema);
|
|
||||||
retVal.addRow
|
retVal.addRow
|
||||||
(new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
|
(new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,23 +1,24 @@
|
|||||||
|
using bts;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Provides a stream that with distinct rows with respect to the columns
|
/// Provides a stream that with distinct rows with respect to the columns
|
||||||
/// the user specifies in the argument
|
/// the user specifies in the argument
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[DataStreamUserRespresentation ("distinct")]
|
[DataStreamUserRespresentation("distinct")]
|
||||||
public class DistinctRowStream : ViewStream
|
public class DistinctRowStream : ViewStream {
|
||||||
{
|
public DistinctRowStream(ViewStream stream, params string[] cols) {
|
||||||
public DistinctRowStream (ViewStream stream, params string[] cols)
|
var g_by_eval = new DistinctRowEvaluator(cols);
|
||||||
{
|
ViewStream r = new GroupByStream(stream, g_by_eval);
|
||||||
ArrayList groupByCols = new ArrayList ();
|
var proj_exp = String.Empty;
|
||||||
foreach (string col in cols)
|
foreach (var c in cols)
|
||||||
foreach (string c in col.Split (' '))
|
proj_exp += $"{c} ";
|
||||||
groupByCols.Add (c);
|
proj_exp = proj_exp.Trim();
|
||||||
mySetRealSource (new GroupByStream (stream,
|
r = new NewProjectStream(r, proj_exp);
|
||||||
new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
|
mySetRealSource(r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
// using System;
|
|
||||||
// using System.Collections.Generic;
|
|
||||||
// using System.Text;
|
|
||||||
// using System.IO;
|
|
||||||
|
|
||||||
// using System.Runtime.Serialization.Formatters.Binary;
|
|
||||||
|
|
||||||
// namespace BCG
|
|
||||||
// {
|
|
||||||
// public class DumpLoaderStream : ViewStream, ISideEffectStream
|
|
||||||
// {
|
|
||||||
// //wtf
|
|
||||||
// //public class Exception : System.Exception
|
|
||||||
// //{public Exception(string msg) : base(msg) { }}
|
|
||||||
|
|
||||||
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
|
|
||||||
// {
|
|
||||||
// mySetRealSource(
|
|
||||||
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
|
||||||
// ViewStreamDatabase.instance));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public DumpLoaderStream(string fileName)
|
|
||||||
// {
|
|
||||||
// if (fileName == null)
|
|
||||||
// throw new LoaderStream.Exception("File name cannot be null");
|
|
||||||
|
|
||||||
// for (int i = 0; i < 5; i++)
|
|
||||||
// try
|
|
||||||
// {
|
|
||||||
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
|
||||||
// {
|
|
||||||
// IClientDeserializer deser = new ClientUniversalDeserializer();
|
|
||||||
// byte[] the_ops;
|
|
||||||
// Array the_rows;
|
|
||||||
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
|
|
||||||
|
|
||||||
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
|
|
||||||
// }
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// catch (IOException)
|
|
||||||
// {
|
|
||||||
// System.Threading.Thread.Sleep(50);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
|
// This is the other side of the SaverStream, its "loads" the stream and fires once
|
||||||
[DataStreamUserRespresentation("loader")]
|
[DataStreamUserRespresentation("loader")]
|
||||||
public class LoaderStream : ViewStream {
|
public class LoaderStream : ViewStream {
|
||||||
|
|
||||||
|
|||||||
@@ -5,10 +5,11 @@ using System.Text;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using bts;
|
using BCG;
|
||||||
|
|
||||||
namespace BCG {
|
namespace bts {
|
||||||
// Writes all the udpates to a stream in a file.
|
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
|
||||||
|
// which just writes one update, as a create)
|
||||||
// File naming:
|
// File naming:
|
||||||
// token_epoch_yyyyMMdd_HHmmss
|
// token_epoch_yyyyMMdd_HHmmss
|
||||||
// File location: directory(passed)\token\
|
// File location: directory(passed)\token\
|
||||||
@@ -94,10 +95,10 @@ namespace BCG {
|
|||||||
void _on_change(object new_value) {
|
void _on_change(object new_value) {
|
||||||
DataPacket dp = new_value as DataPacket;
|
DataPacket dp = new_value as DataPacket;
|
||||||
byte[] serialized_update = null;
|
byte[] serialized_update = null;
|
||||||
bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
|
bool first_write = _serializer == null || _n_updates_in_current_file == 0;
|
||||||
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
||||||
if (_write_create) {
|
if (first_write) {
|
||||||
// new files always starts with create
|
// first file always starts with create
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
||||||
} else
|
} else
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
||||||
@@ -105,8 +106,8 @@ namespace BCG {
|
|||||||
long utc_epoch = epoch_now();
|
long utc_epoch = epoch_now();
|
||||||
|
|
||||||
//time, length of data, data
|
//time, length of data, data
|
||||||
bw.Write(utc_epoch);
|
//bw.Write(utc_epoch);
|
||||||
bw.Write(serialized_update.Length);
|
//bw.Write(serialized_update.Length);
|
||||||
bw.Write(serialized_update);
|
bw.Write(serialized_update);
|
||||||
_out_fs.Flush();
|
_out_fs.Flush();
|
||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
|
|||||||
@@ -1,190 +0,0 @@
|
|||||||
// using System;
|
|
||||||
// using System.Collections.Generic;
|
|
||||||
// using System.Linq;
|
|
||||||
// using System.Text;
|
|
||||||
// using System.IO;
|
|
||||||
// using System.Runtime.Serialization.Formatters.Binary;
|
|
||||||
// using BCG.Utilities.General;
|
|
||||||
// using bts.collections;
|
|
||||||
// using System.Reflection.Metadata;
|
|
||||||
|
|
||||||
|
|
||||||
// namespace BCG {
|
|
||||||
|
|
||||||
// public class PlaybackStream : ViewStream, IWorker {
|
|
||||||
// private readonly PlaybackType _type;
|
|
||||||
// private BlueTable _current_table;
|
|
||||||
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
|
||||||
// // these are the packets already downloaded from the current file
|
|
||||||
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
|
|
||||||
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
|
|
||||||
// private IEnumerable<string> _file_names;
|
|
||||||
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
|
|
||||||
// private DateTime _playback_t0, _playback_t1;
|
|
||||||
// private bool _all_files_read = false;
|
|
||||||
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
|
|
||||||
|
|
||||||
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
|
|
||||||
// _type = PlaybackType.Streaming;
|
|
||||||
// _playback_t0 = PersistorStream.parse_datetime(datetime_start);
|
|
||||||
// _playback_t1 = PersistorStream.parse_datetime(datetime_end);
|
|
||||||
// string token_directory_name = $"{directory_name}\\{token}";
|
|
||||||
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
|
|
||||||
// List<string> files_in_range = new List<string>();
|
|
||||||
// if (Directory.Exists(token_directory_name)) {
|
|
||||||
// _file_names =
|
|
||||||
// Directory.GetFiles(token_directory_name).Where(f => {
|
|
||||||
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
|
|
||||||
// return file_time > _playback_t0 && file_time < _playback_t1;
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
|
||||||
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
|
|
||||||
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
|
|
||||||
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
|
|
||||||
// if (snapshot_packet != null) {
|
|
||||||
// foreach (PlaybackPacket p in _snapshot_data.packets) {
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
|
||||||
// byte op = ops[i];
|
|
||||||
// if (op == StreamOpCode.create) {
|
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
// if (p.idx == snapshot_packet.idx) {
|
|
||||||
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
|
||||||
// private void _load_packets_from_current_file() {
|
|
||||||
// int curr_idx = _current_fs.Item1;
|
|
||||||
// FileStream curr_fs = _current_fs.Item2;
|
|
||||||
// _packets = new List<PlaybackPacket>();
|
|
||||||
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private bool current_file_all_replayed {
|
|
||||||
// get {
|
|
||||||
// bool rv = true;
|
|
||||||
// _packets.ForEach(p => rv = rv && p.replayed);
|
|
||||||
// return rv;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
|
||||||
// var fs = _current_fs.Item2;
|
|
||||||
// if (fs != null) {
|
|
||||||
// fs.Close();
|
|
||||||
// }
|
|
||||||
// ViewStreamDatabase.instance.removeWorker(this);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// // public event PlaybackStreamProgress onProgress;
|
|
||||||
// // public event Action onFileExhausted;
|
|
||||||
// private PlaybackPacket _locate_packet_to_replay() {
|
|
||||||
// foreach (var p in _packets) {
|
|
||||||
// // returns the first non replayed packet
|
|
||||||
// if (!p.replayed)
|
|
||||||
// return p;
|
|
||||||
// }
|
|
||||||
// //else
|
|
||||||
// {
|
|
||||||
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
|
|
||||||
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
|
|
||||||
// if (onProgress != null)
|
|
||||||
// onProgress(myFileReader.Position, myFileReader.Length);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// #region IWorker Members
|
|
||||||
|
|
||||||
// public void doWork() {
|
|
||||||
// if (_all_files_read)
|
|
||||||
// return;
|
|
||||||
// if (current_file_all_replayed) {
|
|
||||||
// _load_next_file();
|
|
||||||
// }
|
|
||||||
// PlaybackPacket curr_packet = _locate_packet_to_replay();
|
|
||||||
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// // what a stupid interface!
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
|
||||||
// _fire_packet(ops, row_array);
|
|
||||||
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
|
||||||
// } else {
|
|
||||||
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
|
|
||||||
// // this is the time we need to fire (replay) this packet
|
|
||||||
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
|
|
||||||
// long now_epoch = PersistorStream.epoch_now();
|
|
||||||
// // fire when we are past the
|
|
||||||
// if (now_epoch >= time_to_fire) {
|
|
||||||
// byte[] ops;
|
|
||||||
// Array row_array;
|
|
||||||
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
|
||||||
// _fire_packet(ops, row_array);
|
|
||||||
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _fire_packet(byte[] ops, Array row_array) {
|
|
||||||
// for (int i = 0; i < ops.Length; i++) {
|
|
||||||
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
|
||||||
// byte op = ops[i];
|
|
||||||
// if (op == StreamOpCode.create) {
|
|
||||||
// IBlueRowCollection full_table = rows_args[0];
|
|
||||||
// _current_table = new BlueTable(full_table);
|
|
||||||
// //_current_table.addRows(full_table);
|
|
||||||
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
|
||||||
// (myLastValue as DataPacket).newRows));
|
|
||||||
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
|
||||||
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
|
||||||
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
|
||||||
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
|
||||||
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
|
||||||
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
|
||||||
// } else {
|
|
||||||
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
|
||||||
// }
|
|
||||||
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _load_next_file() {
|
|
||||||
// var new_idx = _current_fs.Item1 + 1;
|
|
||||||
// if (new_idx >= _file_names.Count()) {
|
|
||||||
// _all_files_read = true;
|
|
||||||
// bts.Log.Gen.info("All files read");
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
// var prev_fs = _current_fs.Item2;
|
|
||||||
// if (prev_fs != null)
|
|
||||||
// prev_fs.Close();
|
|
||||||
// var new_filename = _file_names.ElementAt(new_idx);
|
|
||||||
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
|
|
||||||
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
|
|
||||||
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
|
|
||||||
// _load_packets_from_current_file();
|
|
||||||
// }
|
|
||||||
|
|
||||||
// #endregion
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
// using System;
|
|
||||||
// using System.Collections.Generic;
|
|
||||||
// using System.Linq;
|
|
||||||
// using System.Text;
|
|
||||||
// using System.IO;
|
|
||||||
|
|
||||||
// namespace BCG
|
|
||||||
// {
|
|
||||||
// public class RecorderStream : ViewStream
|
|
||||||
// {
|
|
||||||
// private ViewStream mySource;
|
|
||||||
// private DateTime myStartTime;
|
|
||||||
// private FileStream myOutFile;
|
|
||||||
// private string myFileName;
|
|
||||||
// private IBlueDataSerializer mySerializer;
|
|
||||||
|
|
||||||
// public RecorderStream(ViewStream src, string recordFile)
|
|
||||||
// {
|
|
||||||
// mySource = src;
|
|
||||||
// myFileName = recordFile;
|
|
||||||
|
|
||||||
// myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber()
|
|
||||||
// {
|
|
||||||
// myStartTime = DateTime.Now;
|
|
||||||
// mySource.onChange += new ChangeDelegate(mySource_onChange);
|
|
||||||
// myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber()
|
|
||||||
// {
|
|
||||||
// mySource.onChange -= mySource_onChange;
|
|
||||||
// myOutFile.Close();
|
|
||||||
// }
|
|
||||||
|
|
||||||
// void mySource_onChange(object newValue)
|
|
||||||
// {
|
|
||||||
// DataPacket dp = (DataPacket)newValue;
|
|
||||||
|
|
||||||
// BlueTable toWrite = new BlueTable(dp.schema);
|
|
||||||
|
|
||||||
// if (mySerializer == null)
|
|
||||||
// {
|
|
||||||
// mySerializer = new UniversalSerializer(dp.schema);
|
|
||||||
// }
|
|
||||||
// foreach (IBlueRow r in dp.newRows)
|
|
||||||
// toWrite.addRow(r);
|
|
||||||
|
|
||||||
// byte[] result;
|
|
||||||
|
|
||||||
// if (myLastValue == null)
|
|
||||||
// mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
|
|
||||||
// else
|
|
||||||
// mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
|
|
||||||
|
|
||||||
// BinaryWriter bw = new BinaryWriter(myOutFile);
|
|
||||||
// //write down what time this happened
|
|
||||||
// bw.Write((DateTime.Now - myStartTime).TotalSeconds);
|
|
||||||
// bw.Write(result.Length);
|
|
||||||
// bw.Write(result);
|
|
||||||
// myOutFile.Flush();
|
|
||||||
|
|
||||||
// myFireEvent(dp);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -5,7 +5,8 @@ using System.IO;
|
|||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
// this saves the total state in the filr provided in the cons
|
// this saves the updates as a Create on the same file, each update calls -save, which
|
||||||
|
// reqrites the whole data to the packet. The other side of this stream is the Loader
|
||||||
[DataStreamUserRespresentation("saver")]
|
[DataStreamUserRespresentation("saver")]
|
||||||
public class SaverStream : ViewStream {
|
public class SaverStream : ViewStream {
|
||||||
private ViewStream _stream;
|
private ViewStream _stream;
|
||||||
@@ -31,7 +32,6 @@ namespace bts {
|
|||||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||||
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||||
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
|
|||||||
@@ -5,25 +5,20 @@ using mbts.config;
|
|||||||
using System;
|
using System;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
namespace BCG.StreamWorld.Addition
|
namespace BCG.StreamWorld.Addition {
|
||||||
{
|
[DataStreamUserRespresentation(false)]
|
||||||
[DataStreamUserRespresentation (false)]
|
public class FilterStream : ViewStream {
|
||||||
public class FilterStream : ViewStream
|
|
||||||
{
|
|
||||||
private ViewStream myBaseStream;
|
private ViewStream myBaseStream;
|
||||||
private BlueTable myBaseTable, myStateTable;
|
private BlueTable myBaseTable, myStateTable;
|
||||||
private IStreamFilter myFilter;
|
private IStreamFilter myFilter;
|
||||||
|
|
||||||
public FilterStream(ViewStream vs)
|
public FilterStream(ViewStream vs)
|
||||||
: this(vs, "1=1")
|
: this(vs, "1=1") { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, string filterExp)
|
public FilterStream(ViewStream vs, string filterExp)
|
||||||
: this(vs, new BooleanExpressionFilter(filterExp))
|
: this(vs, new BooleanExpressionFilter(filterExp)) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, IStreamFilter filter)
|
public FilterStream(ViewStream vs, IStreamFilter filter) {
|
||||||
{
|
|
||||||
myBaseStream = vs;
|
myBaseStream = vs;
|
||||||
myFilter = filter;
|
myFilter = filter;
|
||||||
if (filter is IDynamicStreamFilter)
|
if (filter is IDynamicStreamFilter)
|
||||||
@@ -31,43 +26,35 @@ namespace BCG.StreamWorld.Addition
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
|
||||||
myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
|
myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
|
||||||
myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
|
myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
|
||||||
myBaseTable.clear();
|
myBaseTable.clear();
|
||||||
myStateTable.clear();
|
myStateTable.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleChanges(object newValue)
|
private void myHandleChanges(object newValue) {
|
||||||
{
|
|
||||||
DataPacket dp = newValue as DataPacket;
|
DataPacket dp = newValue as DataPacket;
|
||||||
if (myStateTable == null)
|
if (myStateTable == null) {
|
||||||
{
|
|
||||||
bts.Log.assert(myBaseTable == null);
|
bts.Log.assert(myBaseTable == null);
|
||||||
myStateTable = new BlueTable(dp.schema);
|
myStateTable = new BlueTable(dp.schema);
|
||||||
myBaseTable = new BlueTable(dp.schema);
|
myBaseTable = new BlueTable(dp.schema);
|
||||||
}
|
}
|
||||||
BlueTable dels = new BlueTable(dp.schema),
|
BlueTable dels = new BlueTable(dp.schema),
|
||||||
adds = new BlueTable(dp.schema);
|
adds = new BlueTable(dp.schema);
|
||||||
foreach (IBlueRow r in dp.deletedRows)
|
foreach (IBlueRow r in dp.deletedRows) {
|
||||||
{
|
|
||||||
myBaseTable.deleteRow(r);
|
myBaseTable.deleteRow(r);
|
||||||
if (myStateTable.containsID((int)r["id"]))
|
if (myStateTable.containsID((int)r["id"])) {
|
||||||
{
|
|
||||||
dels.addRow(r);
|
dels.addRow(r);
|
||||||
myStateTable.deleteRow(r);
|
myStateTable.deleteRow(r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
foreach (IBlueRow r in dp.addedRows)
|
foreach (IBlueRow r in dp.addedRows) {
|
||||||
{
|
|
||||||
myBaseTable.addRow(r);
|
myBaseTable.addRow(r);
|
||||||
if (myFilter.filter(r))
|
if (myFilter.filter(r)) {
|
||||||
{
|
|
||||||
adds.addRow(r);
|
adds.addRow(r);
|
||||||
myStateTable.addRow(r);
|
myStateTable.addRow(r);
|
||||||
}
|
}
|
||||||
@@ -76,13 +63,10 @@ namespace BCG.StreamWorld.Addition
|
|||||||
myFireEvent(new DataPacket(myStateTable, adds, dels));
|
myFireEvent(new DataPacket(myStateTable, adds, dels));
|
||||||
}
|
}
|
||||||
|
|
||||||
public IStreamFilter filter
|
public IStreamFilter filter {
|
||||||
{
|
set {
|
||||||
set
|
|
||||||
{
|
|
||||||
myFilter = value;
|
myFilter = value;
|
||||||
if (myBaseTable != null)
|
if (myBaseTable != null) {
|
||||||
{
|
|
||||||
IBlueRowCollection oldState = new BlueTable(myStateTable);
|
IBlueRowCollection oldState = new BlueTable(myStateTable);
|
||||||
myStateTable.clear();
|
myStateTable.clear();
|
||||||
foreach (IBlueRow r in myBaseTable)
|
foreach (IBlueRow r in myBaseTable)
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class NullFilterStream : ViewStream {
|
||||||
|
public NullFilterStream(ViewStream parent_str, string col_name)
|
||||||
|
: this(parent_str, col_name, true) { }
|
||||||
|
public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
|
||||||
|
ViewStream rs = new FilterStream(parent_str,
|
||||||
|
r => {
|
||||||
|
bool is_dbnull = r.is_null(col_name);
|
||||||
|
if (is_dbnull) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (r.schema[col_name].type_code() == TypeCode.Double) {
|
||||||
|
return !Double.IsNaN(r.get<double>(col_name));
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -52,14 +52,14 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
||||||
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
|
: this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
|
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
|
||||||
myVsd = vsd;
|
myVsd = vsd;
|
||||||
_time_col = rowTimeColumnName;
|
_time_col = rowTimeColumnName;
|
||||||
_ttl = rowTimeToLive;
|
_ttl = rowTimeToLive;
|
||||||
myFilterFrequency = filterFrequency;
|
_filter_period = filter_period;
|
||||||
myNextFilter = DateTime.MinValue;
|
myNextFilter = DateTime.MinValue;
|
||||||
|
|
||||||
// add worker
|
// add worker
|
||||||
@@ -92,7 +92,7 @@ namespace BCG {
|
|||||||
// fire event
|
// fire event
|
||||||
onFilterChange();
|
onFilterChange();
|
||||||
|
|
||||||
myNextFilter = now.Add(myFilterFrequency);
|
myNextFilter = now.Add(_filter_period);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
@@ -108,7 +108,7 @@ namespace BCG {
|
|||||||
private ViewStreamDatabase myVsd;
|
private ViewStreamDatabase myVsd;
|
||||||
private string _time_col;
|
private string _time_col;
|
||||||
private TimeSpan _ttl;
|
private TimeSpan _ttl;
|
||||||
private TimeSpan myFilterFrequency;
|
private TimeSpan _filter_period;
|
||||||
private DateTime myNextFilter;
|
private DateTime myNextFilter;
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,102 +1,102 @@
|
|||||||
using System;
|
// using System;
|
||||||
using System.Collections.Generic;
|
// using System.Collections.Generic;
|
||||||
using System.Linq;
|
// using System.Linq;
|
||||||
using System.Text;
|
// using System.Text;
|
||||||
|
|
||||||
namespace BCG
|
// namespace BCG
|
||||||
{
|
// {
|
||||||
|
|
||||||
public class AverageGroupsStream : ViewStream
|
// public class AverageGroupsStream : ViewStream
|
||||||
{
|
// {
|
||||||
public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
|
// public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
|
||||||
{
|
// {
|
||||||
|
|
||||||
myColToAverage = colToAverage;
|
// myColToAverage = colToAverage;
|
||||||
myInput = input;
|
// myInput = input;
|
||||||
myKeyColumn = keyColumn;
|
// myKeyColumn = keyColumn;
|
||||||
}
|
// }
|
||||||
|
|
||||||
private ViewStream myInput;
|
// private ViewStream myInput;
|
||||||
private string myColToAverage;
|
// private string myColToAverage;
|
||||||
private BlueTableSchema mySchema;
|
// private BlueTableSchema mySchema;
|
||||||
private string myKeyColumn;
|
// private string myKeyColumn;
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
// protected override void myOnFirstSubscriber()
|
||||||
{
|
// {
|
||||||
myInput.onChange += new ChangeDelegate(myInput_onChange);
|
// myInput.onChange += new ChangeDelegate(myInput_onChange);
|
||||||
}
|
// }
|
||||||
protected override void myOnLastSubscriber()
|
// protected override void myOnLastSubscriber()
|
||||||
{
|
// {
|
||||||
myInput.onChange -= myInput_onChange;
|
// myInput.onChange -= myInput_onChange;
|
||||||
}
|
// }
|
||||||
private void myInput_onChange(object newValue)
|
// private void myInput_onChange(object newValue)
|
||||||
{
|
// {
|
||||||
if (mySchema == null)
|
// if (mySchema == null)
|
||||||
myBuildSchema(((DataPacket)newValue).schema);
|
// myBuildSchema(((DataPacket)newValue).schema);
|
||||||
|
|
||||||
IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
|
// IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
|
||||||
|
|
||||||
if (myLastValue == null)
|
// if (myLastValue == null)
|
||||||
myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
|
// myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
|
||||||
else
|
// else
|
||||||
{
|
// {
|
||||||
IBlueRowCollection added, deleted;
|
// IBlueRowCollection added, deleted;
|
||||||
BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
|
// BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
|
||||||
myFireEvent(new DataPacket(newRows, added, deleted));
|
// myFireEvent(new DataPacket(newRows, added, deleted));
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
private void myBuildSchema(IBlueTableSchema schema)
|
// private void myBuildSchema(IBlueTableSchema schema)
|
||||||
{
|
// {
|
||||||
List<ColumnInfo> columns = new List<ColumnInfo>();
|
// List<ColumnInfo> columns = new List<ColumnInfo>();
|
||||||
|
|
||||||
foreach (ColumnInfo ci in schema.columnInfo)
|
// foreach (ColumnInfo ci in schema.columnInfo)
|
||||||
columns.Add(ci);
|
// columns.Add(ci);
|
||||||
|
|
||||||
columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
|
// columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
|
||||||
mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
|
// mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private IBlueRowCollection myEvaluate(DataPacket packet)
|
// private IBlueRowCollection myEvaluate(DataPacket packet)
|
||||||
{
|
// {
|
||||||
|
|
||||||
IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
|
// IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
|
||||||
IDictionary<string, int> colToCount = new Dictionary<string, int>();
|
// IDictionary<string, int> colToCount = new Dictionary<string, int>();
|
||||||
IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
|
// IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
|
||||||
|
|
||||||
foreach (IBlueRow r in packet.newRows)
|
// foreach (IBlueRow r in packet.newRows)
|
||||||
{
|
// {
|
||||||
double thisVal = Convert.ToDouble(r[myColToAverage]);
|
// double thisVal = Convert.ToDouble(r[myColToAverage]);
|
||||||
|
|
||||||
string keyVal = r[myKeyColumn].ToString();
|
// string keyVal = r[myKeyColumn].ToString();
|
||||||
if (!colToCount.ContainsKey(keyVal))
|
// if (!colToCount.ContainsKey(keyVal))
|
||||||
{
|
// {
|
||||||
colToRunningTotal[keyVal] = thisVal;
|
// colToRunningTotal[keyVal] = thisVal;
|
||||||
colToCount[keyVal] = 1;
|
// colToCount[keyVal] = 1;
|
||||||
}
|
// }
|
||||||
else
|
// else
|
||||||
{
|
// {
|
||||||
colToRunningTotal[keyVal] += thisVal;
|
// colToRunningTotal[keyVal] += thisVal;
|
||||||
colToCount[keyVal] += 1;
|
// colToCount[keyVal] += 1;
|
||||||
}
|
// }
|
||||||
colToLastRow[keyVal] = r;
|
// colToLastRow[keyVal] = r;
|
||||||
}
|
// }
|
||||||
|
|
||||||
BlueTable result = new BlueTable(mySchema);
|
// BlueTable result = new BlueTable(mySchema);
|
||||||
foreach (string key in colToCount.Keys)
|
// foreach (string key in colToCount.Keys)
|
||||||
{
|
// {
|
||||||
object[] values = new object[mySchema.columnInfo.Length];
|
// object[] values = new object[mySchema.columnInfo.Length];
|
||||||
|
|
||||||
colToLastRow[key].CopyTo(values, 0);
|
// colToLastRow[key].CopyTo(values, 0);
|
||||||
values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
|
// values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
|
||||||
result.addRow(new BlueRow(mySchema, values));
|
// result.addRow(new BlueRow(mySchema, values));
|
||||||
}
|
// }
|
||||||
|
|
||||||
return result;
|
// return result;
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("Average")]
|
||||||
|
public class AverageNonIncGroupsStream : ViewStream {
|
||||||
|
public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
|
||||||
|
NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
|
||||||
|
(g_rows) => _find_mean(inp_col_name, g_rows));
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
|
||||||
|
IBlueRow inp_row = g_rows.ElementAt(0);
|
||||||
|
IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
|
||||||
|
double sum = 0.0;
|
||||||
|
inp_vals.ToList().ForEach(d => sum += d);
|
||||||
|
string out_col_name = $"average_{inp_col_name}";
|
||||||
|
double average = sum / inp_vals.Count();
|
||||||
|
IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
|
||||||
|
MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
|
||||||
|
out_row.setValue(out_col_name, average);
|
||||||
|
List<IBlueRow> rv = new List<IBlueRow>();
|
||||||
|
rv.Add(out_row);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
//using MathNet.Numerics.Statistics;
|
||||||
|
|
||||||
|
|
||||||
|
public enum MultiRowOp {
|
||||||
|
Mean,
|
||||||
|
Median,
|
||||||
|
Mode,
|
||||||
|
Range,
|
||||||
|
StdDev,
|
||||||
|
Max,
|
||||||
|
Min,
|
||||||
|
First,
|
||||||
|
Last,
|
||||||
|
Sum,
|
||||||
|
EdgeDiff,
|
||||||
|
EdgeDiffPercentage,
|
||||||
|
//IsEdgeDiff,
|
||||||
|
}
|
||||||
|
public class MultiRowOpHelper {
|
||||||
|
public readonly List<Tuple<string, MultiRowOp>> operations;
|
||||||
|
|
||||||
|
public static MultiRowOpHelper parse(string text) {
|
||||||
|
// text is inp_col: op, inp_col: op, inp_col:op
|
||||||
|
List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
|
||||||
|
string[] calcs = text.Split(',');
|
||||||
|
foreach (string c in calcs) {
|
||||||
|
string[] toks = c.Split(':');
|
||||||
|
string inp_col = toks[0].Trim();
|
||||||
|
string op_txt = toks[1].Trim();
|
||||||
|
MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
|
||||||
|
rv.Add(Tuple.Create(inp_col, op));
|
||||||
|
}
|
||||||
|
return new MultiRowOpHelper(rv);
|
||||||
|
}
|
||||||
|
|
||||||
|
public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
|
||||||
|
operations = operations_;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
|
||||||
|
if (rows.Count() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
IBlueRow ir = rows.ElementAt(0);
|
||||||
|
IBlueTableSchema in_schema = ir.schema;
|
||||||
|
IBlueTableSchema out_schema = in_schema;
|
||||||
|
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
|
||||||
|
new Dictionary<string, Tuple<string, MultiRowOp>>();
|
||||||
|
foreach (var t in parameters.operations) {
|
||||||
|
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
||||||
|
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
||||||
|
out_col_map[out_col_name] = t;
|
||||||
|
}
|
||||||
|
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
||||||
|
foreach (var kv in out_col_map) {
|
||||||
|
string out_col_name = kv.Key;
|
||||||
|
string in_col_name = kv.Value.Item1;
|
||||||
|
MultiRowOp op = kv.Value.Item2;
|
||||||
|
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
|
||||||
|
double comp_res = compute_op(col_vals, op);
|
||||||
|
or.setValue(out_col_name, comp_res);
|
||||||
|
}
|
||||||
|
return or;
|
||||||
|
}
|
||||||
|
|
||||||
|
// rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
|
||||||
|
// calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
|
||||||
|
|
||||||
|
public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
|
||||||
|
if (rolling_rows.Count() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
IBlueRow ir = rolling_rows.ElementAt(0);
|
||||||
|
IBlueTableSchema in_schema = ir.schema;
|
||||||
|
IBlueTableSchema out_schema = in_schema;
|
||||||
|
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
|
||||||
|
new Dictionary<string, Tuple<string, MultiRowOp>>();
|
||||||
|
foreach (var t in operations) {
|
||||||
|
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
||||||
|
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
||||||
|
out_col_map[out_col_name] = t;
|
||||||
|
}
|
||||||
|
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
||||||
|
foreach (var kv in out_col_map) {
|
||||||
|
string out_col_name = kv.Key;
|
||||||
|
string in_col_name = kv.Value.Item1;
|
||||||
|
MultiRowOp op = kv.Value.Item2;
|
||||||
|
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
|
||||||
|
double comp_res = compute_op(col_vals, op);
|
||||||
|
or.setValue(out_col_name, comp_res);
|
||||||
|
}
|
||||||
|
return or;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
|
||||||
|
switch (op) {
|
||||||
|
case MultiRowOp.Max:
|
||||||
|
return col_vals.Max<double>();
|
||||||
|
case MultiRowOp.Min:
|
||||||
|
return col_vals.Min<double>();
|
||||||
|
case MultiRowOp.Mean:
|
||||||
|
return col_vals.Sum() / col_vals.Count();
|
||||||
|
case MultiRowOp.Sum:
|
||||||
|
return col_vals.Sum();
|
||||||
|
case MultiRowOp.First:
|
||||||
|
return col_vals.First();
|
||||||
|
case MultiRowOp.Last:
|
||||||
|
return col_vals.Last();
|
||||||
|
case MultiRowOp.StdDev:
|
||||||
|
return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
|
||||||
|
default:
|
||||||
|
// TODO
|
||||||
|
Log.assert(false);
|
||||||
|
return Double.NaN;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,7 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public class GroupByStream : MethodDrivenStream {
|
internal class GroupByStream : MethodDrivenStream {
|
||||||
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
||||||
private string[] _groupby_cols;
|
private string[] _groupby_cols;
|
||||||
private Func<DataPacket, IBlueRow> _groupby_func;
|
private Func<DataPacket, IBlueRow> _groupby_func;
|
||||||
|
|||||||
@@ -10,6 +10,6 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public MaxStream(ViewStream stream, string outputCol)
|
public MaxStream(ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
: this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("AggregationStream")]
|
||||||
|
public class AgregationStream : ViewStream {
|
||||||
|
|
||||||
|
public AgregationStream(ViewStream parent_str, string operation_txt) {
|
||||||
|
ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
|
||||||
|
public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
|
||||||
|
ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+69
-15
@@ -1,13 +1,40 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
using bts;
|
using bts;
|
||||||
|
using bts.SilverPine;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
//[DataStreamUserRespresentation("AggregateStream")]
|
||||||
public class NonIncGroupByStream : ViewStream
|
public class NonIncGroupByStream : ViewStream {
|
||||||
{
|
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
||||||
|
Func<string[]> _groupby_cols_func;
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
||||||
|
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
||||||
|
_groupby_cols_func = groupby_cols_func;
|
||||||
|
_evaluate_func = evaluate_func;
|
||||||
|
}
|
||||||
|
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||||
|
BlueTable rv = null;
|
||||||
|
foreach (var kv in grouped_rows) {
|
||||||
|
List<IBlueRow> g_rows = kv.Value;
|
||||||
|
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
||||||
|
if (out_rows != null && out_rows.Count > 0) {
|
||||||
|
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
|
||||||
|
}
|
||||||
|
rv.addRows(out_rows);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
public string[] get_group_by_cols() {
|
||||||
|
return _groupby_cols_func();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
private ViewStream _main_stream;
|
private ViewStream _main_stream;
|
||||||
private INonIncGroupByEvaluator _groupby_func;
|
private INonIncGroupByEvaluator _groupby_func;
|
||||||
private BlueRowHasher _row_hasher;
|
private BlueRowHasher _row_hasher;
|
||||||
@@ -15,24 +42,50 @@ namespace BCG
|
|||||||
private Stopwatch _timer = new();
|
private Stopwatch _timer = new();
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
|
|
||||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
||||||
{
|
|
||||||
_main_stream = main_stream;
|
_main_stream = main_stream;
|
||||||
_groupby_func = groupby_func;
|
_groupby_func = groupby_func;
|
||||||
_groupby_cols = _groupby_func.get_group_by_cols();
|
_groupby_cols = _groupby_func.get_group_by_cols();
|
||||||
_row_hasher = new BlueRowHasher(_groupby_cols.Length);
|
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
||||||
|
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
||||||
|
|
||||||
|
public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
|
||||||
|
: this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
|
||||||
|
|
||||||
|
public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
|
||||||
|
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
|
||||||
|
Func<string[]> groupby_cols_func = () => {
|
||||||
|
return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
|
||||||
|
};
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
||||||
|
if (!String.IsNullOrEmpty(optional_sort_col))
|
||||||
|
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
|
||||||
|
IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
|
||||||
|
return new IBlueRow[] { rv }.ToList();
|
||||||
|
};
|
||||||
|
ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
|
||||||
|
this(parent_str, String.Empty, standard_group_by_text) {
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
||||||
|
|
||||||
private void _main_stream_onchange(object nv)
|
private void _main_stream_onchange(object nv) {
|
||||||
{
|
|
||||||
DataPacket dp = nv as DataPacket;
|
DataPacket dp = nv as DataPacket;
|
||||||
///grouping together
|
///grouping together
|
||||||
_timer.Restart();
|
_timer.Restart();
|
||||||
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
||||||
|
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
|
||||||
|
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
|
||||||
_timer.Stop();
|
_timer.Stop();
|
||||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
if (groupElapsedMs >= 50) {
|
if (groupElapsedMs >= 50) {
|
||||||
@@ -44,14 +97,15 @@ namespace BCG
|
|||||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||||
_timer.Stop();
|
_timer.Stop();
|
||||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (applyElapsedMs >= 50)
|
if (applyElapsedMs >= 50) {
|
||||||
{
|
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
Log.DataFlow.info(
|
Log.DataFlow.info(
|
||||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||||
}
|
}
|
||||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
// TODO,
|
||||||
|
if (total_state != null)
|
||||||
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,31 +1,31 @@
|
|||||||
using System;
|
//using System;
|
||||||
using System.Collections.Generic;
|
//using System.Collections.Generic;
|
||||||
using System.Linq;
|
//using System.Linq;
|
||||||
using System.Text;
|
//using System.Text;
|
||||||
|
|
||||||
namespace BCG
|
//namespace BCG
|
||||||
{
|
//{
|
||||||
public class FullJoinStream : ViewStream
|
// public class FullJoinStream : ViewStream
|
||||||
{
|
// {
|
||||||
private ViewStream myMainStream;
|
// private ViewStream myMainStream;
|
||||||
public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
|
// public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
|
||||||
ViewStream v2, string joinCol2, string cmExp2, string empty_col)
|
// ViewStream v2, string joinCol2, string cmExp2, string empty_col)
|
||||||
{
|
// {
|
||||||
myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
|
// myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
|
||||||
ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
|
// ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
|
||||||
filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
|
// filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
|
||||||
myMainStream = new AppendStream(myMainStream, filtered);
|
// myMainStream = new AppendStream(myMainStream, filtered);
|
||||||
}
|
// }
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
// protected override void myOnFirstSubscriber()
|
||||||
{
|
// {
|
||||||
myMainStream.onChange += myHandleMainStreamChange;
|
// myMainStream.onChange += myHandleMainStreamChange;
|
||||||
}
|
// }
|
||||||
|
|
||||||
private void myHandleMainStreamChange(object newData)
|
// private void myHandleMainStreamChange(object newData)
|
||||||
{
|
// {
|
||||||
IBlueRowCollection result = (newData as DataPacket).newRows;
|
// IBlueRowCollection result = (newData as DataPacket).newRows;
|
||||||
myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
|
// myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
//}
|
||||||
+56
-32
@@ -1,15 +1,16 @@
|
|||||||
using bts;
|
using bts;
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
using mbts;
|
using mbts;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
public class NonIncJoinBookkeeper : IDisposable {
|
||||||
public class NonIncJoinBookkeeper : IDisposable
|
|
||||||
{
|
|
||||||
public Action on_new_data;
|
public Action on_new_data;
|
||||||
private readonly NonIncJoinSpec _join_spec;
|
private readonly NonIncJoinSpec _join_spec;
|
||||||
private readonly int _join_position;
|
private readonly int _join_position;
|
||||||
@@ -26,41 +27,71 @@ namespace BCG
|
|||||||
|
|
||||||
private int[] _join_col_indices = null,
|
private int[] _join_col_indices = null,
|
||||||
_data_col_indices = null;
|
_data_col_indices = null;
|
||||||
|
|
||||||
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
|
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
||||||
{
|
|
||||||
_join_spec = the_join_spec;
|
_join_spec = the_join_spec;
|
||||||
_join_position = join_position;
|
_join_position = join_position;
|
||||||
_start_index_in_output = start_index_in_output;
|
_start_index_in_output = start_index_in_output;
|
||||||
_row_hasher = row_hasher;
|
_row_hasher = row_hasher;
|
||||||
_is_left_most = _join_position == 0;
|
_is_left_most = _join_position == 0;
|
||||||
_is_tag_along = _join_spec.join_col_names.Length == 0;
|
_is_tag_along = _join_spec.join_col_names.Length == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool _filled_missing_data = false;
|
||||||
|
private void _fill_missing_data(DataPacket dp) {
|
||||||
|
if (_filled_missing_data)
|
||||||
|
return;
|
||||||
|
Func<string[], bool> null_or_empty = (str_arr) => {
|
||||||
|
if (str_arr == null || str_arr.Length == 0)
|
||||||
|
return true;
|
||||||
|
foreach (string s in str_arr) {
|
||||||
|
if (!String.IsNullOrEmpty(s)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
Func<string[], string[]> remove_bad_cols = (col_names) => {
|
||||||
|
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
|
||||||
|
List<string> cols_list = new List<string>(col_names);
|
||||||
|
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
|
||||||
|
return cols_list.ToArray();
|
||||||
|
};
|
||||||
|
if (null_or_empty(_join_spec.data_col_names)) {
|
||||||
|
_join_spec.data_col_names = dp.schema.columnNames;
|
||||||
|
}
|
||||||
|
_join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
|
||||||
|
if (null_or_empty(_join_spec.data_col_names)) {
|
||||||
|
_join_spec.data_col_names = dp.schema.columnNames;
|
||||||
|
}
|
||||||
|
if (null_or_empty(_join_spec.projected_col_names)) {
|
||||||
|
_join_spec.projected_col_names = _join_spec.data_col_names;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void init() {
|
||||||
_join_spec.stream.onChange += _on_stream_cb;
|
_join_spec.stream.onChange += _on_stream_cb;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void _on_stream_cb(object dp_obj)
|
public void _on_stream_cb(object dp_obj) {
|
||||||
{
|
//Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
DataPacket dp = (DataPacket)dp_obj;
|
DataPacket dp = (DataPacket)dp_obj;
|
||||||
if (_schema == null)
|
_fill_missing_data(dp);
|
||||||
{
|
if (_schema == null) {
|
||||||
_schema = dp.schema;
|
_schema = dp.schema;
|
||||||
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
||||||
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (_is_left_most)
|
if (_is_left_most) {
|
||||||
{
|
|
||||||
_row_list.Clear();
|
_row_list.Clear();
|
||||||
foreach (IBlueRow r in dp.newRows)
|
foreach (IBlueRow r in dp.newRows)
|
||||||
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
|
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
if (_is_tag_along)
|
if (_is_tag_along)
|
||||||
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
|
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
|
||||||
else
|
else {
|
||||||
{
|
|
||||||
_row_map.Clear();
|
_row_map.Clear();
|
||||||
foreach (IBlueRow r in dp.newRows)
|
foreach (IBlueRow r in dp.newRows)
|
||||||
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
||||||
@@ -79,31 +110,25 @@ namespace BCG
|
|||||||
|
|
||||||
public bool is_ready { get { return _schema != null; } }
|
public bool is_ready { get { return _schema != null; } }
|
||||||
|
|
||||||
public void constribute_cols_to_schema(List<ColumnInfo> cis)
|
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
|
||||||
{
|
|
||||||
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
||||||
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
||||||
|
|
||||||
public void copy_master_row_to_output_row(int row_index, object[] output_row)
|
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
|
||||||
{
|
|
||||||
IBlueRow this_row = _row_list[row_index].second;
|
IBlueRow this_row = _row_list[row_index].second;
|
||||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
|
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
|
||||||
{
|
if (_is_tag_along) {
|
||||||
if (_is_tag_along)
|
|
||||||
{
|
|
||||||
if (_solitary_row != null)
|
if (_solitary_row != null)
|
||||||
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
else
|
else
|
||||||
copy_dbnull_to_output_row(output_row);
|
copy_dbnull_to_output_row(output_row);
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
IBlueRow this_row = null;
|
IBlueRow this_row = null;
|
||||||
if (_row_map.TryGetValue(row_hash, out this_row))
|
if (_row_map.TryGetValue(row_hash, out this_row))
|
||||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
@@ -112,8 +137,7 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void copy_dbnull_to_output_row(object[] output_row)
|
private void copy_dbnull_to_output_row(object[] output_row) {
|
||||||
{
|
|
||||||
for (int i = 0; i < _data_col_indices.Length; i++)
|
for (int i = 0; i < _data_col_indices.Length; i++)
|
||||||
output_row[i + _start_index_in_output] = DBNull.Value;
|
output_row[i + _start_index_in_output] = DBNull.Value;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,63 +3,63 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
using mbts;
|
using mbts;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
// this a collection of data, theres no joining logic in here
|
||||||
public class NonIncJoinSpec
|
public class NonIncJoinSpec {
|
||||||
{
|
|
||||||
public ViewStream stream;
|
public ViewStream stream;
|
||||||
|
// join col names are the names on which the streams are joined (for equality),
|
||||||
|
// data col names are the names in the original stream
|
||||||
|
// projected col names are their new names, ie if its x:l_x, then x is in the
|
||||||
|
// data_col_names and l_x in projected cols
|
||||||
public string[] join_col_names, data_col_names, projected_col_names;
|
public string[] join_col_names, data_col_names, projected_col_names;
|
||||||
|
|
||||||
public NonIncJoinSpec() { }
|
public NonIncJoinSpec() { }
|
||||||
|
|
||||||
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
|
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
|
||||||
: this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
|
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
|
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
|
||||||
{
|
|
||||||
stream = the_stream;
|
stream = the_stream;
|
||||||
join_col_names = the_join_col_names;
|
join_col_names = the_join_col_names;
|
||||||
data_col_names = the_data_col_names;
|
data_col_names = the_data_col_names;
|
||||||
projected_col_names = the_projected_col_names;
|
projected_col_names = the_projected_col_names;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
|
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
|
||||||
{
|
|
||||||
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
||||||
foreach (Triplet<ViewStream, string, string> r in join_specs)
|
foreach (Triplet<ViewStream, string, string> r in join_specs) {
|
||||||
{
|
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
|
||||||
string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
|
|
||||||
string[] visible_cols = r.third.Split(' ');
|
string[] visible_cols = r.third.Split(' ');
|
||||||
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
||||||
foreach (string s in visible_cols)
|
foreach (string s in visible_cols) {
|
||||||
{
|
|
||||||
string[] toks = s.Split(':');
|
string[] toks = s.Split(':');
|
||||||
if (toks.Length == 1)
|
if (toks.Length == 1) {
|
||||||
{
|
|
||||||
data_col_names.Add(toks[0]);
|
data_col_names.Add(toks[0]);
|
||||||
mapped_col_names.Add(toks[0]);
|
mapped_col_names.Add(toks[0]);
|
||||||
}
|
} else if (toks.Length == 2) {
|
||||||
else if (toks.Length == 2)
|
|
||||||
{
|
|
||||||
data_col_names.Add(toks[0]);
|
data_col_names.Add(toks[0]);
|
||||||
mapped_col_names.Add(toks[1]);
|
mapped_col_names.Add(toks[1]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
|
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
||||||
}
|
}
|
||||||
return rv.ToArray ();
|
return rv.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
public override string ToString() {
|
||||||
|
Func<string[], string> arr_txt = (a) => {
|
||||||
|
string rv = "";
|
||||||
|
foreach (string s in a) {
|
||||||
|
rv += "{s} ";
|
||||||
|
}
|
||||||
|
rv = rv.Trim();
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
|
||||||
|
return rv;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//[DataStreamUserRespresentation("ninctest")]
|
|
||||||
//public class NonIncJoinTestStream : ViewStream
|
|
||||||
//{
|
|
||||||
// public NonIncJoinTestStream (string key_cols, string data_cols)
|
|
||||||
// {
|
|
||||||
// NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
}
|
}
|
||||||
@@ -6,11 +6,9 @@ using System;
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG {
|
||||||
{
|
|
||||||
[DataStreamUserRespresentation("fastJoin")]
|
[DataStreamUserRespresentation("fastJoin")]
|
||||||
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
|
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
|
||||||
{
|
|
||||||
private readonly NonIncJoinSpec[] _join_specs;
|
private readonly NonIncJoinSpec[] _join_specs;
|
||||||
private readonly INonIncJoinInlineMapper _output_mapper;
|
private readonly INonIncJoinInlineMapper _output_mapper;
|
||||||
private readonly BlueRowHasher _row_hasher;
|
private readonly BlueRowHasher _row_hasher;
|
||||||
@@ -28,24 +26,19 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
||||||
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
||||||
})
|
}) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
||||||
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
|
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
||||||
params Triplet<ViewStream, string, string>[] join_specs)
|
params Triplet<ViewStream, string, string>[] join_specs)
|
||||||
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
|
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
||||||
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
|
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
|
||||||
{
|
|
||||||
_join_specs = join_specs;
|
_join_specs = join_specs;
|
||||||
_output_mapper = output_mapper;
|
_output_mapper = output_mapper;
|
||||||
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
|
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
|
||||||
@@ -53,12 +46,11 @@ namespace BCG
|
|||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
|
||||||
int data_col_no = 1;
|
int data_col_no = 1;
|
||||||
for (int jp = 0; jp < _join_specs.Length; jp++)
|
for (int jp = 0; jp < _join_specs.Length; jp++) {
|
||||||
{
|
|
||||||
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
|
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
|
||||||
|
//Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
|
||||||
data_col_no += _join_specs[jp].data_col_names.Length;
|
data_col_no += _join_specs[jp].data_col_names.Length;
|
||||||
s.on_new_data += _stream_callback;
|
s.on_new_data += _stream_callback;
|
||||||
_book_keeping_data.Add(s);
|
_book_keeping_data.Add(s);
|
||||||
@@ -66,10 +58,11 @@ namespace BCG
|
|||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
_output_mapper.onMapperChange += _on_mapper_change;
|
_output_mapper.onMapperChange += _on_mapper_change;
|
||||||
_vsd.addWorker(this);
|
_vsd.addWorker(this);
|
||||||
|
foreach (var bkd in _book_keeping_data)
|
||||||
|
bkd.init();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
d.Dispose();
|
d.Dispose();
|
||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
@@ -77,30 +70,26 @@ namespace BCG
|
|||||||
_vsd.removeWorker(this);
|
_vsd.removeWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _stream_callback()
|
private void _stream_callback() {
|
||||||
{
|
//Log.Gen.info("NonIncJoinStream._stream_callback received callback");
|
||||||
_is_dirty = true;
|
_is_dirty = true;
|
||||||
doWork();
|
doWork();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _on_mapper_change()
|
private void _on_mapper_change() {
|
||||||
{
|
|
||||||
_is_dirty = true;
|
_is_dirty = true;
|
||||||
doWork();
|
doWork();
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool _is_ready()
|
private bool _is_ready() {
|
||||||
{
|
|
||||||
bool rv = true;
|
bool rv = true;
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
rv &= d.is_ready;
|
rv &= d.is_ready;
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _construct_schema()
|
private void _construct_schema() {
|
||||||
{
|
if (_output_schema == null) {
|
||||||
if (_output_schema == null)
|
|
||||||
{
|
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
d.constribute_cols_to_schema(cis);
|
d.constribute_cols_to_schema(cis);
|
||||||
@@ -109,27 +98,23 @@ namespace BCG
|
|||||||
}
|
}
|
||||||
|
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
private void _form_and_fire()
|
private void _form_and_fire() {
|
||||||
{
|
//Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
|
||||||
if (_is_ready())
|
if (_is_ready()) {
|
||||||
{
|
|
||||||
_construct_schema();
|
_construct_schema();
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
||||||
List<object[]> output_rows = new List<object[]>();
|
List<object[]> output_rows = new List<object[]>();
|
||||||
|
|
||||||
for (int i = 0; i < left_row_list.Count; i++)
|
for (int i = 0; i < left_row_list.Count; i++) {
|
||||||
{
|
|
||||||
object[] or = new object[_output_schema.columnInfo.Length];
|
object[] or = new object[_output_schema.columnInfo.Length];
|
||||||
or[0] = i;
|
or[0] = i;
|
||||||
output_rows.Add(or);
|
output_rows.Add(or);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int ji = 0; ji < _book_keeping_data.Count; ji++)
|
for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
|
||||||
{
|
|
||||||
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
|
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
|
||||||
for (int i = 0; i < left_row_list.Count; i++)
|
for (int i = 0; i < left_row_list.Count; i++) {
|
||||||
{
|
|
||||||
if (ji == 0)
|
if (ji == 0)
|
||||||
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
|
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
|
||||||
else
|
else
|
||||||
@@ -139,19 +124,20 @@ namespace BCG
|
|||||||
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
||||||
timer.Stop();
|
timer.Stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (timer.ElapsedMilliseconds >= 250) {
|
//if (timer.ElapsedMilliseconds >= 250) {
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
//string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
||||||
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
|
// + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
|
||||||
}
|
//Log.Gen.info(log_msg);
|
||||||
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
//}
|
||||||
|
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
|
||||||
|
myFireEvent(dp_out);
|
||||||
|
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
|
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
|
||||||
{
|
if (_output_mapper != null) {
|
||||||
if (_output_mapper != null)
|
|
||||||
{
|
|
||||||
IBlueRowCollection output_rows;
|
IBlueRowCollection output_rows;
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
output_rows = _output_mapper.map_rows(unmapped_rows);
|
output_rows = _output_mapper.map_rows(unmapped_rows);
|
||||||
@@ -160,17 +146,14 @@ namespace BCG
|
|||||||
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
||||||
}
|
}
|
||||||
return output_rows;
|
return output_rows;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
return unmapped_rows;
|
return unmapped_rows;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void doWork()
|
public void doWork() {
|
||||||
{
|
//Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
|
||||||
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
|
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
|
||||||
{
|
|
||||||
_is_dirty = false;
|
_is_dirty = false;
|
||||||
_last_fire_time = DateTime.Now;
|
_last_fire_time = DateTime.Now;
|
||||||
_form_and_fire();
|
_form_and_fire();
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
namespace bts {
|
||||||
|
using BCG;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Diagnostics;
|
||||||
|
|
||||||
|
using ZstdSharp.Unsafe;
|
||||||
|
|
||||||
|
// functon that joins the 2 rows and returns the outputput row. The function
|
||||||
|
// can return a null which means the 2 rows dont need to be joined
|
||||||
|
using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
|
||||||
|
public class ProximityJoinStream : ViewStream, IWorker {
|
||||||
|
private ViewStream _lvs, _rvs;
|
||||||
|
private JoinFunc _join_func;
|
||||||
|
private bool _dirty = false;
|
||||||
|
private Stopwatch _stopwatch = new Stopwatch();
|
||||||
|
private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
|
||||||
|
public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
|
||||||
|
_lvs = lvs;
|
||||||
|
_rvs = rvs;
|
||||||
|
_join_func = join_func;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_lvs.onChange += _handle_data;
|
||||||
|
_rvs.onChange += _handle_data;
|
||||||
|
_vsd.addWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_lvs.onChange -= _handle_data;
|
||||||
|
_rvs.onChange -= _handle_data;
|
||||||
|
_vsd.removeWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _handle_data(object dp_obj) {
|
||||||
|
_dirty = true;
|
||||||
|
doWork();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
Func<bool> all_data_revcd = () => {
|
||||||
|
return _lvs.rows != null && _rvs.rows != null;
|
||||||
|
};
|
||||||
|
_stopwatch.Start();
|
||||||
|
if (_dirty && all_data_revcd()) {
|
||||||
|
_dirty = false;
|
||||||
|
BlueTable rv = null;
|
||||||
|
foreach (IBlueRow lr in _lvs.rows) {
|
||||||
|
foreach (IBlueRow rr in _rvs.rows) {
|
||||||
|
IBlueRow resr = _join_func(lr, rr);
|
||||||
|
if (resr == null)
|
||||||
|
continue;
|
||||||
|
rv = rv ?? new BlueTable(resr.schema);
|
||||||
|
rv.addRow(resr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_stopwatch.Stop();
|
||||||
|
Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
namespace bts.SilverPine {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.CodeDom;
|
||||||
|
using System.Configuration;
|
||||||
|
using System.Reactive.Disposables;
|
||||||
|
|
||||||
|
public class TimeProximityJoinStream : ViewStream {
|
||||||
|
private int _id = 0;
|
||||||
|
|
||||||
|
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
|
||||||
|
string pre_join_period, string post_join_period) :
|
||||||
|
this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
|
||||||
|
TimeBasedFilter.parse_duration(post_join_period)) { }
|
||||||
|
|
||||||
|
|
||||||
|
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
|
||||||
|
TimeSpan pre_join_period, TimeSpan post_join_period) {
|
||||||
|
Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
|
||||||
|
var id = 0;
|
||||||
|
lock (this) {
|
||||||
|
id = _id++;
|
||||||
|
}
|
||||||
|
IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
|
||||||
|
DateTime ldt = lr.get_datetime(l_time_col);
|
||||||
|
DateTime rdt = rr.get_datetime(r_time_col);
|
||||||
|
TimeSpan diff = ldt - rdt;
|
||||||
|
if (ldt < rdt) {
|
||||||
|
if (diff.Duration() > pre_join_period) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (diff > post_join_period)
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return create_row(lr, rr);
|
||||||
|
};
|
||||||
|
|
||||||
|
ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
|
||||||
|
mySetRealSource(rstr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,9 +2,13 @@ using System;
|
|||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using Amazon.Runtime.Internal.Util;
|
||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public class LiteralStream : ViewStream {
|
public class LiteralStream : ViewStream {
|
||||||
|
|
||||||
@@ -22,6 +26,34 @@ namespace BCG {
|
|||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
|
||||||
|
public LiteralStream(string table_name, string rows) {
|
||||||
|
Func<string, int, IBlueRow> get_row = (json_row, id) => {
|
||||||
|
Dictionary<string, object> map = new Dictionary<string, object>();
|
||||||
|
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
|
||||||
|
//Logger.assert(col_vals.Length == 2);
|
||||||
|
foreach (string t in toks) {
|
||||||
|
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
|
||||||
|
string col = col_vals[0];
|
||||||
|
string val = col_vals[1];
|
||||||
|
map.Add(col, val);
|
||||||
|
}
|
||||||
|
map["id"] = id;
|
||||||
|
return new BlueRow(map);
|
||||||
|
};
|
||||||
|
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
||||||
|
BlueTable table = null;
|
||||||
|
foreach (string json_string in row_arr) {
|
||||||
|
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
||||||
|
get_row(json_string, table == null ? 0 : table.Count);
|
||||||
|
table = table ?? new BlueTable(r.schema);
|
||||||
|
table.addRow(r);
|
||||||
|
}
|
||||||
|
ViewStream str = new BlueTableStream(table);
|
||||||
|
str = new SchemaGuesserStream(str);
|
||||||
|
mySetRealSource(str);
|
||||||
|
}
|
||||||
|
|
||||||
public LiteralStream(string tableName, string schema, string value) {
|
public LiteralStream(string tableName, string schema, string value) {
|
||||||
string[] columns = schema.Split(' ');
|
string[] columns = schema.Split(' ');
|
||||||
|
|
||||||
@@ -55,7 +87,7 @@ namespace BCG {
|
|||||||
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
||||||
}
|
}
|
||||||
|
|
||||||
public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
// public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
||||||
|
|
||||||
// public LiteralStream(string value) {
|
// public LiteralStream(string value) {
|
||||||
|
|
||||||
|
|||||||
Loaded 100 of 2768 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user