Compare commits
1046
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f685c7e95 | ||
|
|
5568c4051d | ||
|
|
389550e5cd | ||
|
|
d73d12ec31 | ||
|
|
387891521b | ||
|
|
1d329760f0 | ||
|
|
f6fe4194d2 | ||
|
|
f93f13048a | ||
|
|
5995a00d69 | ||
|
|
226999ae5d | ||
|
|
7b88b35b03 | ||
|
|
fdf2cacd1d | ||
|
|
865940daf3 | ||
|
|
10770c64e4 | ||
|
|
c6a4b91c27 | ||
|
|
c0b7b39091 | ||
|
|
4e198183eb | ||
|
|
eda34e0e3b | ||
|
|
6532c329ac | ||
|
|
588af742bf | ||
|
|
fe5fab1fa0 | ||
|
|
b8944da8ed | ||
|
|
fca10887ce | ||
|
|
8f53ff5ff9 | ||
|
|
1178aaf410 | ||
|
|
62b790a156 | ||
|
|
1fc77869e9 | ||
|
|
3b91a697e9 | ||
|
|
8908cc300a | ||
|
|
5d039b54f5 | ||
|
|
21f044cc3b | ||
|
|
55c132a033 | ||
|
|
ddd0de8eca | ||
|
|
f4451ad2f4 | ||
|
|
12d9185b6e | ||
|
|
84c3759d17 | ||
|
|
618a18b927 | ||
|
|
1e25099c62 | ||
|
|
afeb5babd4 | ||
|
|
0ff372a63e | ||
|
|
197a81443c | ||
|
|
ef98a7eda6 | ||
|
|
94e9c8c3f2 | ||
|
|
0f9e515d01 | ||
|
|
342ecd9a52 | ||
|
|
d0c316b3fc | ||
|
|
44db6190a1 | ||
|
|
9856e10eb6 | ||
|
|
deda37b057 | ||
|
|
ce5095523b | ||
|
|
7d374346f2 | ||
|
|
ad0e9fa3c9 | ||
|
|
befa17d29d | ||
|
|
3165ba1235 | ||
|
|
53a9e6a62d | ||
|
|
ff816befb0 | ||
|
|
a69c79b6a9 | ||
|
|
7219ff5a95 | ||
|
|
603f01699c | ||
|
|
0ca318b222 | ||
|
|
4addb58743 | ||
|
|
c365330b08 | ||
|
|
4bb4008ac6 | ||
|
|
001892ab34 | ||
|
|
e77ee859a3 | ||
|
|
c1a8702a47 | ||
|
|
6045c602dc | ||
|
|
4a2a814af0 | ||
|
|
08a655b1d5 | ||
|
|
2830af5644 | ||
|
|
659eb83f4d | ||
|
|
5d9d364de3 | ||
|
|
07c1ad3204 | ||
|
|
e61829f8e0 | ||
|
|
0cf44fcc40 | ||
|
|
2549e34a17 | ||
|
|
693f2deb67 | ||
|
|
63665e9b10 | ||
|
|
a79f5ff860 | ||
|
|
dae75c507e | ||
|
|
4db9cdd357 | ||
|
|
18b19ac787 | ||
|
|
b5ea918378 | ||
|
|
9df4ecf6c8 | ||
|
|
957f6cb33f | ||
|
|
03294ffe37 | ||
|
|
0550078de0 | ||
|
|
89129f726e | ||
|
|
97eefb1a1e | ||
|
|
805693e170 | ||
|
|
3a7f3ac608 | ||
|
|
b82cc31d2d | ||
|
|
34859d7506 | ||
|
|
8a9bc1872e | ||
|
|
6b4ff90a56 | ||
|
|
d2a1b67717 | ||
|
|
144b29c728 | ||
|
|
e7625341e3 | ||
|
|
fb15b3fb24 | ||
|
|
ea0246929a | ||
|
|
52f35cc421 | ||
|
|
7e968fec76 | ||
|
|
188acc08ca | ||
|
|
29be8c3ff5 | ||
|
|
eb28e04bf1 | ||
|
|
4ef2dbe4d7 | ||
|
|
979051f147 | ||
|
|
35a17b0fd0 | ||
|
|
d93ce336f6 | ||
|
|
0d10c2ba5d | ||
|
|
2a4d62a1e9 | ||
|
|
b9e191eeb9 | ||
|
|
75f2678df6 | ||
|
|
8bbe850549 | ||
|
|
25860af7bf | ||
|
|
68198a058e | ||
|
|
7297d667a2 | ||
|
|
ace1b5f04d | ||
|
|
bd252c251c | ||
|
|
57b82cf10d | ||
|
|
f07d83e7bb | ||
|
|
619be37379 | ||
|
|
9348106585 | ||
|
|
760cdc0fba | ||
|
|
fcd1b0e799 | ||
|
|
eb102a5898 | ||
|
|
fb3262edea | ||
|
|
6e1e5d4bbf | ||
|
|
256f7025a2 | ||
|
|
24892fb1a8 | ||
|
|
e3c2efcfcb | ||
|
|
29080795f7 | ||
|
|
cd291b7a0d | ||
|
|
8af9d13802 | ||
|
|
318d8fb275 | ||
|
|
f33d19c185 | ||
|
|
fb43d214f3 | ||
|
|
c4f12be3e4 | ||
|
|
54d7279832 | ||
|
|
03df578a81 | ||
|
|
7a50ad2cc5 | ||
|
|
0cc4b83417 | ||
|
|
4f1efea066 | ||
|
|
ef4bcf4c44 | ||
|
|
5a6139f3c5 | ||
|
|
e2816aaa33 | ||
|
|
fc702db7fa | ||
|
|
787cf0becd | ||
|
|
b4e26d2fbe | ||
|
|
e2a101eb20 | ||
|
|
6300f8f7c2 | ||
|
|
1c6f4669be | ||
|
|
45fcb2e20b | ||
|
|
75dee2a05a | ||
|
|
dcb02feb90 | ||
|
|
81444c1045 | ||
|
|
934c0ae70a | ||
|
|
efdd91935e | ||
|
|
da96744f7f | ||
|
|
6ed6a36ee1 | ||
|
|
e0fc22121f | ||
|
|
3671e4d713 | ||
|
|
dd69842a3c | ||
|
|
15883e42b6 | ||
|
|
34caf54015 | ||
|
|
84a4182077 | ||
|
|
e8197a81cc | ||
|
|
17d2f6d701 | ||
|
|
b728a8cb46 | ||
|
|
6766e48706 | ||
|
|
e7246b22bc | ||
|
|
5550166d03 | ||
|
|
489c683d2e | ||
|
|
ff18d39d58 | ||
|
|
05644ed6cf | ||
|
|
d6a88a7ac9 | ||
|
|
f7e620db3e | ||
|
|
c745407ae7 | ||
|
|
1bb7d6eca2 | ||
|
|
519ef8b0af | ||
|
|
6b93c07271 | ||
|
|
c3288c721d | ||
|
|
2622626eed | ||
|
|
d9470bdb6e | ||
|
|
f32e9bba10 | ||
|
|
f119f30b62 | ||
|
|
4877355fee | ||
|
|
ada7d4bed1 | ||
|
|
f65b6b82f2 | ||
|
|
1155fd6997 | ||
|
|
f423ea518e | ||
|
|
088b543e48 | ||
|
|
2163821b43 | ||
|
|
abf4cdacf8 | ||
|
|
70fc4f9ae7 | ||
|
|
797ba111ec | ||
|
|
a51b015907 | ||
|
|
ec057605a5 | ||
|
|
e0903d1c82 | ||
|
|
1616e1a8b9 | ||
|
|
cfd5579703 | ||
|
|
849737b2ef | ||
|
|
85a5b72ec5 | ||
|
|
e9ac78b438 | ||
|
|
e5288f1be2 | ||
|
|
50c2a57c5c | ||
|
|
8f293f73e3 | ||
|
|
ac42732a19 | ||
|
|
97246e7ce7 | ||
|
|
0815f9316d | ||
|
|
cc00ff2a0a | ||
|
|
28545b5099 | ||
|
|
04c1ad7be8 | ||
|
|
2441a5267d | ||
|
|
12f44f63aa | ||
|
|
c2fe578ccb | ||
|
|
a01fc6cc15 | ||
|
|
d80cd9c0fd | ||
|
|
d707847079 | ||
|
|
96d98a239f | ||
|
|
118e2a9f2e | ||
|
|
1481a3fd07 | ||
|
|
a5e58c26ce | ||
|
|
8637bb9254 | ||
|
|
7c61c2ef6d | ||
|
|
0668e0cd8c | ||
|
|
3ae88248cc | ||
|
|
1367cea9ca | ||
|
|
b384d89820 | ||
|
|
d669d24aea | ||
|
|
e5bc6667e7 | ||
|
|
41512cb811 | ||
|
|
01d600b821 | ||
|
|
d6197f7d1f | ||
|
|
9e1cb14523 | ||
|
|
cd7c04dce5 | ||
|
|
6aee76590f | ||
|
|
fc79c451b2 | ||
|
|
b916f1b836 | ||
|
|
826e03af94 | ||
|
|
57217c9adf | ||
|
|
deae8c7fb0 | ||
|
|
81d3b10c2b | ||
|
|
f2cc576c51 | ||
|
|
b25038fac6 | ||
|
|
9b95caa8e6 | ||
|
|
b4cb6ac856 | ||
|
|
83cbb0f47f | ||
|
|
4e6d1a2335 | ||
|
|
41cd59e6f9 | ||
|
|
f2ace4ae32 | ||
|
|
8db5655151 | ||
|
|
135db27733 | ||
|
|
f240f005f4 | ||
|
|
67b1a213b2 | ||
|
|
e8a16b11bb | ||
|
|
d3f6ca8afb | ||
|
|
8e42eb5d18 | ||
|
|
70d4a33b2b | ||
|
|
f289c3d2b0 | ||
|
|
6e77eda16c | ||
|
|
41be31a7b0 | ||
|
|
96976b3fcd | ||
|
|
58d6a4534e | ||
|
|
bf913f2b99 | ||
|
|
003b0a3fa0 | ||
|
|
de7c94014e | ||
|
|
714cf3572e | ||
|
|
3a6bdbc27e | ||
|
|
bf1ce9467d | ||
|
|
906e48743d | ||
|
|
33764aaa28 | ||
|
|
bdfef3d9b1 | ||
|
|
1df508a16c | ||
|
|
57423e449d | ||
|
|
934ea76f5a | ||
|
|
97efe8396e | ||
|
|
cc80785d8a | ||
|
|
a9267394ef | ||
|
|
6939934399 | ||
|
|
b6b7be7f25 | ||
|
|
66c0baa150 | ||
|
|
5305b32c04 | ||
|
|
e6304609f8 | ||
|
|
91d6f80b45 | ||
|
|
21d84d811d | ||
|
|
77e542efa9 | ||
|
|
6b28fc4779 | ||
|
|
c0602ceb93 | ||
|
|
6f5d89cceb | ||
|
|
7baaafbeeb | ||
|
|
8569b02f5c | ||
|
|
8f0784e6c2 | ||
|
|
c15968e17e | ||
|
|
0e331229d4 | ||
|
|
45c85310e7 | ||
|
|
7b41dbb19a | ||
|
|
e6a3116327 | ||
|
|
86ce16790a | ||
|
|
897cf10be6 | ||
|
|
ca27903f63 | ||
|
|
1b5f1bfb27 | ||
|
|
217db51cbc | ||
|
|
c08a893ecf | ||
|
|
a022f08b49 | ||
|
|
5af98658d8 | ||
|
|
933511f430 | ||
|
|
63291d7374 | ||
|
|
30276066f8 | ||
|
|
809a66323d | ||
|
|
5c4ee1a262 | ||
|
|
504f441d2e | ||
|
|
fd51025b00 | ||
|
|
f4afd92db2 | ||
|
|
b1fe5b52f6 | ||
|
|
164ca306f5 | ||
|
|
a6d23ef4ef | ||
|
|
c0f4090d70 | ||
|
|
63c94e09e7 | ||
|
|
fc94ec4d46 | ||
|
|
e2fdb569c6 | ||
|
|
804c27eb49 | ||
|
|
6448b4cf90 | ||
|
|
e01a825aff | ||
|
|
00649e0c14 | ||
|
|
ae1e9ecb29 | ||
|
|
ff65acba05 | ||
|
|
dc08945719 | ||
|
|
a739ca9a75 | ||
|
|
ed7b187876 | ||
|
|
53e1e92a6a | ||
|
|
eee7af494a | ||
|
|
91a7c35edf | ||
|
|
3eaef1a1b6 | ||
|
|
4a70b4910d | ||
|
|
7697f73ada | ||
|
|
029875e0a0 | ||
|
|
9dc7611c0a | ||
|
|
782cdd7430 | ||
|
|
4184278d94 | ||
|
|
0b3fa813f1 | ||
|
|
e60f2159e1 | ||
|
|
610861be20 | ||
|
|
1787460c35 | ||
|
|
588b86710f | ||
|
|
3022a25a10 | ||
|
|
cf30649dd2 | ||
|
|
afb8444815 | ||
|
|
fc59247958 | ||
|
|
14c417eeec | ||
|
|
62a6e4340f | ||
|
|
cce85e45b7 | ||
|
|
ff584ee19a | ||
|
|
f146859677 | ||
|
|
cf55e4bc1d | ||
|
|
a0b1078572 | ||
|
|
afd229b7c7 | ||
|
|
bfc7615c9a | ||
|
|
0023574905 | ||
|
|
7e79a21cd4 | ||
|
|
6b0030559b | ||
|
|
77caef2640 | ||
|
|
edb8a9d516 | ||
|
|
153631191a | ||
|
|
026a8af893 | ||
|
|
3739532947 | ||
|
|
0cb3cadc11 | ||
|
|
36db69dc67 | ||
|
|
c7c755a964 | ||
|
|
0e460353a7 | ||
|
|
5198236b45 | ||
|
|
767fa25d61 | ||
|
|
0858eaac23 | ||
|
|
593601551d | ||
|
|
036df46b1c | ||
|
|
84697a6d65 | ||
|
|
bb550d3f88 | ||
|
|
85bd2da414 | ||
|
|
cc55fe1acf | ||
|
|
bb64760c62 | ||
|
|
8b9525f503 | ||
|
|
eccfd1ee30 | ||
|
|
7b933026bb | ||
|
|
ffe859703a | ||
|
|
4756e243fa | ||
|
|
0133159296 | ||
|
|
38afd3055f | ||
|
|
4f184c2566 | ||
|
|
f608497094 | ||
|
|
9035b95612 | ||
|
|
7483665b5a | ||
|
|
5f141a8030 | ||
|
|
83f1b5b567 | ||
|
|
06ad3a1d9e | ||
|
|
01de13bb43 | ||
|
|
155a396af5 | ||
|
|
399f9e61e2 | ||
|
|
1f67456cc5 | ||
|
|
9ff7821b59 | ||
|
|
c598f831d0 | ||
|
|
3ca4cedf54 | ||
|
|
d919ec5474 | ||
|
|
98c2d738b2 | ||
|
|
cf3f7e6862 | ||
|
|
ae868ab74d | ||
|
|
bd22f4779d | ||
|
|
27cbd51b1c | ||
|
|
02626ce35c | ||
|
|
d219804caf | ||
|
|
1e0ec5bb46 | ||
|
|
c281bc49b5 | ||
|
|
b3566155a3 | ||
|
|
a4e39afbb9 | ||
|
|
efce01d28e | ||
|
|
d362022b89 | ||
|
|
e7d751fb68 | ||
|
|
50eefc65d4 | ||
|
|
8770de4a87 | ||
|
|
e799b6e24b | ||
|
|
69e338c1ca | ||
|
|
e8bb64a80f | ||
|
|
8e2ede596f | ||
|
|
9772d0eb77 | ||
|
|
687e2477c0 | ||
|
|
1bd0ba9950 | ||
|
|
ddabc744f2 | ||
|
|
74b88b003e | ||
|
|
aaf1be09c2 | ||
|
|
e3bd9a457c | ||
|
|
59f62b6ec9 | ||
|
|
b46e75f7d7 | ||
|
|
3f100feaca | ||
|
|
093da2a938 | ||
|
|
2f17c2b8c5 | ||
|
|
33bfdc517a | ||
|
|
d17be48e6d | ||
|
|
cbaa1df641 | ||
|
|
8c23665d65 | ||
|
|
c7db105533 | ||
|
|
3f9923239e | ||
|
|
9ae9451359 | ||
|
|
c33e5ddb02 | ||
|
|
54388d3b92 | ||
|
|
17ada3c57c | ||
|
|
046f412ec4 | ||
|
|
c94d11f525 | ||
|
|
2b69846381 | ||
|
|
bdd5e08b60 | ||
|
|
7c9330650d | ||
|
|
98fca5ad7f | ||
|
|
8a0fcce90d | ||
|
|
112527f9ed | ||
|
|
fc1e20538d | ||
|
|
83acbecf55 | ||
|
|
06c494291c | ||
|
|
fe74aed048 | ||
|
|
b94cdb561e | ||
|
|
06bd8791c0 | ||
|
|
b5850cdb02 | ||
|
|
545d76142f | ||
|
|
edc2cea60e | ||
|
|
e6466f8309 | ||
|
|
501be8d6a5 | ||
|
|
7dea9b3792 | ||
|
|
a3f49b1ce8 | ||
|
|
765237d701 | ||
|
|
97c52410a2 | ||
|
|
8b43d93e26 | ||
|
|
14baa7d62e | ||
|
|
a054f4cc63 | ||
|
|
5311128607 | ||
|
|
075acdc8eb | ||
|
|
e1fdf6306b | ||
|
|
3b9fb2759b | ||
|
|
3f89d3b7f7 | ||
|
|
a6dd0d842d | ||
|
|
d050d2d19a | ||
|
|
942d00a8b1 | ||
|
|
31cb25ac2e | ||
|
|
cbb43a22de | ||
|
|
9e7b4849f0 | ||
|
|
6e72890153 | ||
|
|
c238fce057 | ||
|
|
b8f5db4ee2 | ||
|
|
87c8b9862a | ||
|
|
fbc2d8633d | ||
|
|
d12d798fa5 | ||
|
|
e75be2da2f | ||
|
|
3ae3a7094e | ||
|
|
bafa2f71ba | ||
|
|
63bd29a49e | ||
|
|
5fdc9ca48b | ||
|
|
a19e7bc7b4 | ||
|
|
04b0a89644 | ||
|
|
363a43cb8b | ||
|
|
c9f9c5f0f6 | ||
|
|
ab84a05644 | ||
|
|
4681f22e71 | ||
|
|
a25e055e35 | ||
|
|
2386612b51 | ||
|
|
3606dac145 | ||
|
|
bf936c9d13 | ||
|
|
1aeeb7170e | ||
|
|
78ad8a3aed | ||
|
|
5ca8dbafaf | ||
|
|
b464b9466e | ||
|
|
7a5c676e99 | ||
|
|
1e8d699dd1 | ||
|
|
aadd2dd594 | ||
|
|
5163ca4cae | ||
|
|
b5d520d568 | ||
|
|
83e4465a3b | ||
|
|
54cfb343a8 | ||
|
|
4e9b360e23 | ||
|
|
16bc93b1a0 | ||
|
|
f56d4d9062 | ||
|
|
36cc186626 | ||
|
|
2e78af1756 | ||
|
|
70fae06a30 | ||
|
|
07b59559a2 | ||
|
|
cd1709563b | ||
|
|
94f22ddf40 | ||
|
|
7f6c08f956 | ||
|
|
1d737cdf82 | ||
|
|
b645192355 | ||
|
|
5ad7755b28 | ||
|
|
df4ca124c3 | ||
|
|
2c8b9ee288 | ||
|
|
f02f28e347 | ||
|
|
c8a89c1e8d | ||
|
|
7ae9e07a30 | ||
|
|
a130e47218 | ||
|
|
a377f6de7f | ||
|
|
855373c9e6 | ||
|
|
e1b535b633 | ||
|
|
3b9ac71460 | ||
|
|
9a5075daaf | ||
|
|
4e7fe811a8 | ||
|
|
0a6733180d | ||
|
|
f2169c6bd4 | ||
|
|
23ef478391 | ||
|
|
236a5a9362 | ||
|
|
cb560bbe33 | ||
|
|
248f2644ca | ||
|
|
b87d86f4d6 | ||
|
|
8515196dcb | ||
|
|
e5e042f016 | ||
|
|
da3593326d | ||
|
|
313d3a2377 | ||
|
|
22f0e126d0 | ||
|
|
c76b061e20 | ||
|
|
0107b65281 | ||
|
|
6b5496b799 | ||
|
|
42f9daf9cb | ||
|
|
bc5ddade7f | ||
|
|
d8d198f7f4 | ||
|
|
af9eb670c1 | ||
|
|
d91837c377 | ||
|
|
7725ae33ef | ||
|
|
9f56abaa38 | ||
|
|
95444d4a4c | ||
|
|
a7e1c18fe2 | ||
|
|
d11fd23cb0 | ||
|
|
8499ac0dfb | ||
|
|
f4afe348e6 | ||
|
|
096424b60b | ||
|
|
b4f5e66466 | ||
|
|
69a0bcc9cd | ||
|
|
d72ebdf421 | ||
|
|
8e93856767 | ||
|
|
19dae6cd34 | ||
|
|
24f42550dd | ||
|
|
c2150ae450 | ||
|
|
40dbfaa068 | ||
|
|
3756a8a8b7 | ||
|
|
46c8dcb18f | ||
|
|
024480f32e | ||
|
|
ba525372bb | ||
|
|
8aa0f1b674 | ||
|
|
fc14215d9b | ||
|
|
091ed5a327 | ||
|
|
30bb9d0837 | ||
|
|
9cb2c580d6 | ||
|
|
2e2150ac1a | ||
|
|
21d9435a67 | ||
|
|
e7e334fb5a | ||
|
|
61f04c4605 | ||
|
|
24fb2e1e39 | ||
|
|
ece141c448 | ||
|
|
617bac8e74 | ||
|
|
ab217401eb | ||
|
|
77e5c1481b | ||
|
|
a0f77f92a3 | ||
|
|
0acdcb150d | ||
|
|
ac2f3b9bc6 | ||
|
|
0e522f90e5 | ||
|
|
40ad1f1b48 | ||
|
|
1cb75d399f | ||
|
|
cddecc0543 | ||
|
|
484e36b330 | ||
|
|
8a9c22fa2a | ||
|
|
a6e8853cb9 | ||
|
|
dafa193283 | ||
|
|
23651970c9 | ||
|
|
043efab5a2 | ||
|
|
5476466d55 | ||
|
|
54cb5a2ab2 | ||
|
|
29bf7d8d7e | ||
|
|
855d2fd790 | ||
|
|
8a6a652882 | ||
|
|
3f288704bf | ||
|
|
98bc78463d | ||
|
|
6d271ad7cd | ||
|
|
2a17d656a5 | ||
|
|
10436f0f3b | ||
|
|
86b19f0ef0 | ||
|
|
502024dd10 | ||
|
|
623012b0b0 | ||
|
|
5859f0f2d1 | ||
|
|
eeffb735ae | ||
|
|
b602997f3a | ||
|
|
3530a77b38 | ||
|
|
afdf635b31 | ||
|
|
4d4aefc0a7 | ||
|
|
960a8af77a | ||
|
|
89828659a4 | ||
|
|
76f83fd7f5 | ||
|
|
d4e40948d2 | ||
|
|
912ab512d2 | ||
|
|
5d3eb92175 | ||
|
|
924386084b | ||
|
|
acab3ac392 | ||
|
|
d06a62f6ee | ||
|
|
14e3eee0bc | ||
|
|
7277bf52f5 | ||
|
|
b81dc7f5cf | ||
|
|
b1a4635345 | ||
|
|
7c7c689170 | ||
|
|
116506c8da | ||
|
|
3feda49ef1 | ||
|
|
10318be7bc | ||
|
|
f7d525a527 | ||
|
|
d2fe9f167a | ||
|
|
696d91a5af | ||
|
|
d575ba0ea8 | ||
|
|
058b131760 | ||
|
|
57f8f3ff58 | ||
|
|
775df2850d | ||
|
|
cb2b15f10b | ||
|
|
60362a4c95 | ||
|
|
c6d7a8f38c | ||
|
|
7931c62b46 | ||
|
|
58518451c8 | ||
|
|
e3bad78aef | ||
|
|
8e9ab8f2eb | ||
|
|
8512f414d6 | ||
|
|
513fc353cf | ||
|
|
4a04beafd2 | ||
|
|
79a3251f26 | ||
|
|
8785e0fecd | ||
|
|
04251c96b6 | ||
|
|
c2d400af1a | ||
|
|
133cc16628 | ||
|
|
16f7aede93 | ||
|
|
2b9488faff | ||
|
|
51b6d592d3 | ||
|
|
1eb4ceba0a | ||
|
|
dbf5cadb44 | ||
|
|
90f1bb0851 | ||
|
|
547d83e1b8 | ||
|
|
e888149c41 | ||
|
|
b944c3980f | ||
|
|
c2a8266106 | ||
|
|
4277ba61b9 | ||
|
|
35cfe9533b | ||
|
|
d49efef93d | ||
|
|
502ed13456 | ||
|
|
716c4dd674 | ||
|
|
4a2b189a07 | ||
|
|
a42edaf9fa | ||
|
|
cbc142ab15 | ||
|
|
3980f2d030 | ||
|
|
00f6d269c9 | ||
|
|
10b2d38de8 | ||
|
|
6d4d8f6ac1 | ||
|
|
5db89e1e8d | ||
|
|
379a46afa2 | ||
|
|
d9eff37184 | ||
|
|
3d055de139 | ||
|
|
648be7dd97 | ||
|
|
2cac9affe4 | ||
|
|
30ff3ba707 | ||
|
|
9d3afd68aa | ||
|
|
8c3de1789b | ||
|
|
b251e1d47e | ||
|
|
6ccd48a8ba | ||
|
|
41d84f7f73 | ||
|
|
795062864d | ||
|
|
1ff72e9444 | ||
|
|
e13b5066a3 | ||
|
|
5ba7901258 | ||
|
|
bc6eb59110 | ||
|
|
5ae5a5dcbf | ||
|
|
7966bcce77 | ||
|
|
ff86b10895 | ||
|
|
ea3c0717d8 | ||
|
|
d0abca3fd9 | ||
|
|
7895fe7357 | ||
|
|
6f5f4f5cc2 | ||
|
|
066a0ba693 | ||
|
|
c2b67b4813 | ||
|
|
ecad456a2d | ||
|
|
2cb3cfdcdd | ||
|
|
0ebc8e5b4b | ||
|
|
2072105c0f | ||
|
|
d663c86aa0 | ||
|
|
4a2dfa46a1 | ||
|
|
6e35db6a24 | ||
|
|
b45dc3bc2d | ||
|
|
a1912907b8 | ||
|
|
b1b3dcdeb8 | ||
|
|
03dc8ef767 | ||
|
|
975bf56c2a | ||
|
|
3f58c2ce84 | ||
|
|
96f28acf91 | ||
|
|
c2c2658e21 | ||
|
|
d6b8e359cd | ||
|
|
84a0850d95 | ||
|
|
4effdc4160 | ||
|
|
b9fe39f6e3 | ||
|
|
b555850015 | ||
|
|
23794fa30d | ||
|
|
90b5a309a3 | ||
|
|
07beda3fc6 | ||
|
|
e3c1ea70f2 | ||
|
|
93b93969f5 | ||
|
|
0a10111846 | ||
|
|
eee00cc407 | ||
|
|
0eb58eaba8 | ||
|
|
12f6733d1e | ||
|
|
93a0984ed1 | ||
|
|
b7e9c393a0 | ||
|
|
aa602b3fd6 | ||
|
|
b4b8cabdae | ||
|
|
cfe06869b3 | ||
|
|
b177ed1cc0 | ||
|
|
ff17a64933 | ||
|
|
81a1fe11f6 | ||
|
|
ad5bf88fcf | ||
|
|
e15f155ac9 | ||
|
|
6d6822cd65 | ||
|
|
ace0c70516 | ||
|
|
e9e5059bc2 | ||
|
|
cde72f51a0 | ||
|
|
e1fd49ae32 | ||
|
|
274f3f98e4 | ||
|
|
a4b7d84318 | ||
|
|
be17205094 | ||
|
|
6ef9bee8e4 | ||
|
|
2c8b3563f1 | ||
|
|
71f7e64ccb | ||
|
|
c31b44ecfb | ||
|
|
19efb77e07 | ||
|
|
76de640421 | ||
|
|
fb030742d3 | ||
|
|
73608f1b45 | ||
|
|
58aebe7002 | ||
|
|
299cb0b29e | ||
|
|
06663f711e | ||
|
|
51479f4885 | ||
|
|
5b520ba241 | ||
|
|
f0651a5341 | ||
|
|
897e0990a2 | ||
|
|
30e1ae5c3c | ||
|
|
efe83f81ec | ||
|
|
5fb21ada97 | ||
|
|
58edd07f6b | ||
|
|
9d81354c3a | ||
|
|
0f37be89a4 | ||
|
|
f6586fe636 | ||
|
|
7827ccc0e7 | ||
|
|
af6a2acc0a | ||
|
|
84abf59a55 | ||
|
|
2fc1d30bf1 | ||
|
|
4dcd5d63a1 | ||
|
|
63e5697b20 | ||
|
|
8e5cb100a7 | ||
|
|
938bee1817 | ||
|
|
d48af15480 | ||
|
|
e3b2e98cd9 | ||
|
|
ee53fb854c | ||
|
|
d36e2d61f6 | ||
|
|
bfff3ea5fe | ||
|
|
059c03d289 | ||
|
|
8c561f7886 | ||
|
|
288567d1eb | ||
|
|
ecfc5405fe | ||
|
|
8db34a2a11 | ||
|
|
91094c4d84 | ||
|
|
dae8c85d73 | ||
|
|
abdd98082f | ||
|
|
2f6aa785ab | ||
|
|
7acf1f7043 | ||
|
|
1c3c7a078a | ||
|
|
93ace52bfa | ||
|
|
b5d97c0b1b | ||
|
|
284aa53ad4 | ||
|
|
8a9952f52d | ||
|
|
748aafaab3 | ||
|
|
3ba30c70de | ||
|
|
e72f465999 | ||
|
|
f3d93f7749 | ||
|
|
8d142b8a74 | ||
|
|
8210b9f35f | ||
|
|
12b20b05ce | ||
|
|
72efbccdbd | ||
|
|
33643e9eab | ||
|
|
0f6dc6f40b | ||
|
|
8c778f5e41 | ||
|
|
f308654200 | ||
|
|
bdf5b980fc | ||
|
|
914ac597cd | ||
|
|
4aa6a77680 | ||
|
|
6b59dd84ee | ||
|
|
4e2c5f2d17 | ||
|
|
9a1b94a1b0 | ||
|
|
13b3c55771 | ||
|
|
d0a6fcdb9c | ||
|
|
1eb13028c1 | ||
|
|
424e5ff16b | ||
|
|
bed389885b | ||
|
|
85cf25a57a | ||
|
|
61471d4675 | ||
|
|
d83435d49d | ||
|
|
582fc10dce | ||
|
|
5969985d24 | ||
|
|
fabe7d665c | ||
|
|
7ce75bb8e7 | ||
|
|
b8e1c3cd60 | ||
|
|
4909944694 | ||
|
|
7258af09b4 | ||
|
|
7f2ce6c1a9 | ||
|
|
4835725e1c | ||
|
|
6a9e8de475 | ||
|
|
4639951acc | ||
|
|
d8de211aac | ||
|
|
87394bc60b | ||
|
|
beba30f65c | ||
|
|
2d58424d8f | ||
|
|
d1bbfa3791 | ||
|
|
aab17f460f | ||
|
|
461ef53322 | ||
|
|
2486bc90b2 | ||
|
|
1c5cbaf60b | ||
|
|
b8de447cfb | ||
|
|
8bfc2d40f9 | ||
|
|
c09fd71c94 | ||
|
|
f3b037553e | ||
|
|
d386d93166 | ||
|
|
5217742c71 | ||
|
|
c59ec95460 | ||
|
|
70f01583ec | ||
|
|
66d2f9eb63 | ||
|
|
4625bed8b8 | ||
|
|
f18fb332b5 | ||
|
|
8746a6f5ca | ||
|
|
702d47abb5 | ||
|
|
bf2a54717c | ||
|
|
9db779fb25 | ||
|
|
f55ddec708 | ||
|
|
6e942dd2d4 | ||
|
|
142310a17a | ||
|
|
7f99a89a95 | ||
|
|
bfce663b4b | ||
|
|
33c743643d | ||
|
|
858e0a389b | ||
|
|
9c3bb85674 | ||
|
|
f4930e8cd4 | ||
|
|
f21e7f3602 | ||
|
|
b56c35aca4 | ||
|
|
e6ac83196e | ||
|
|
7b6f74fc4a | ||
|
|
f754953fac | ||
|
|
2dd1143d9f | ||
|
|
e1ee3275d8 | ||
|
|
2549e406f3 | ||
|
|
d5024f1858 | ||
|
|
3623ac96db | ||
|
|
19834625f9 | ||
|
|
c8fc649b6b | ||
|
|
6b1fc1c413 | ||
|
|
6f086532ac | ||
|
|
dc66282225 | ||
|
|
e2c74c0bb3 | ||
|
|
e89d50eec1 | ||
|
|
8bebbc278d | ||
|
|
2e6d12c9c1 | ||
|
|
102a5e23ea | ||
|
|
582dc791db | ||
|
|
afb35d8c25 | ||
|
|
3a32ed5fe6 | ||
|
|
4e01697327 | ||
|
|
22032c1742 | ||
|
|
0a9bce317c | ||
|
|
40e44db597 | ||
|
|
4a447e44e1 | ||
|
|
b1f4654b3c | ||
|
|
9d350334c0 | ||
|
|
ac597f505f | ||
|
|
147d9ceb1f | ||
|
|
377804dfb9 | ||
|
|
0d26401961 | ||
|
|
889e226151 | ||
|
|
986d53f4e0 | ||
|
|
d13ba2968a | ||
|
|
1a192c8117 | ||
|
|
5e36d4801c | ||
|
|
2cf6411ca7 | ||
|
|
e99df129d5 | ||
|
|
d07c01bd17 | ||
|
|
d69673c524 | ||
|
|
4ee595cdaf | ||
|
|
6fcaf07df0 | ||
|
|
492d48d886 | ||
|
|
b6a56e768b | ||
|
|
3e2b5976a8 | ||
|
|
edd3f6cbd6 | ||
|
|
2e9347341b | ||
|
|
2061e38349 | ||
|
|
650dd57b6d | ||
|
|
82747f7dc0 | ||
|
|
04b26904a3 | ||
|
|
f123e4c72c | ||
|
|
b887cbebab | ||
|
|
14e975bc7e | ||
|
|
064f557057 | ||
|
|
4d106b5e6c | ||
|
|
9fbeaf0d68 | ||
|
|
4018ac8b78 | ||
|
|
81c9503300 | ||
|
|
477210926d | ||
|
|
0aa48a79a9 | ||
|
|
9e71c7af02 | ||
|
|
4647751409 | ||
|
|
a3087b874b | ||
|
|
5b2f5c5c8f | ||
|
|
479227090e | ||
|
|
2dfcf320f5 | ||
|
|
b4c7213cb2 | ||
|
|
c7dfd69b90 | ||
|
|
4fdc866485 | ||
|
|
f5d5ca9a1c | ||
|
|
cfafcf77b9 | ||
|
|
8ebe783139 | ||
|
|
1851a96a58 | ||
|
|
0b334ef870 | ||
|
|
ee811a729a | ||
|
|
cb15be276d | ||
|
|
7d5af6729b | ||
|
|
e5679cf082 | ||
|
|
98f89b9ad4 | ||
|
|
ba4f8f67e3 | ||
|
|
9c78380490 | ||
|
|
f6166f0674 | ||
|
|
6334a902e7 | ||
|
|
082ca62fff | ||
|
|
9dbd8dd7c6 | ||
|
|
8f7a4d0df6 | ||
|
|
f372dd56b3 | ||
|
|
86eba20d61 | ||
|
|
35a985280d | ||
|
|
f5e1b3319c | ||
|
|
65c3948e4f | ||
|
|
002bdff9a3 | ||
|
|
bb6e5dd548 | ||
|
|
7c1aed3339 | ||
|
|
22ae51fb99 | ||
|
|
a53d347f81 | ||
|
|
838dbcd886 | ||
|
|
5ed2309d22 | ||
|
|
4d41704dd2 | ||
|
|
bd1000e518 | ||
|
|
dd4aa3c2ed | ||
|
|
fee9d3f597 | ||
|
|
416f9aef31 | ||
|
|
48f5e4c201 | ||
|
|
4438b4ca8d | ||
|
|
73356e0d61 | ||
|
|
4a285fa581 | ||
|
|
cfa82c2177 | ||
|
|
0410a0431a | ||
|
|
02843f14ff | ||
|
|
f1f1d4259c | ||
|
|
71fc43af68 | ||
|
|
350144ffd5 | ||
|
|
da227a5550 | ||
|
|
0c92cf3a87 | ||
|
|
1188af391e | ||
|
|
0bb85cf915 | ||
|
|
d77f5996fe | ||
|
|
940c41a274 | ||
|
|
e671960dc3 | ||
|
|
8fa34b6b39 | ||
|
|
f9d5429b5b | ||
|
|
2c342482eb | ||
|
|
3aefac26c2 | ||
|
|
1554594cdd | ||
|
|
727a0f38b5 | ||
|
|
7a0e8cdb26 | ||
|
|
a4e97b04d7 | ||
|
|
463980a6ef | ||
|
|
9b25857042 | ||
|
|
ef19e7d76b | ||
|
|
471faec2cb | ||
|
|
ce96d42db8 | ||
|
|
187894ca84 | ||
|
|
63492c4575 | ||
|
|
e150c611f6 | ||
|
|
bb837d6c26 | ||
|
|
965e541ac8 | ||
|
|
1b232aa610 | ||
|
|
25439b094a | ||
|
|
f927ef2c5e | ||
|
|
d42389b75a | ||
|
|
671cbcf03a | ||
|
|
e589e23f41 | ||
|
|
11fadc0a3e | ||
|
|
b8c2d93866 | ||
|
|
67a23dd196 | ||
|
|
ff22c4eb5f | ||
|
|
5addff4c50 | ||
|
|
142e1bd2da | ||
|
|
d05d91bbe7 | ||
|
|
3a90ca9cb5 | ||
|
|
85a46727be | ||
|
|
624fd2c976 | ||
|
|
16d599d2d6 | ||
|
|
c00e4fcb6b | ||
|
|
03e19afe1d | ||
|
|
058e11eadc | ||
|
|
6ff52e7107 | ||
|
|
ee4c28ed41 | ||
|
|
50d26f0060 | ||
|
|
36378e6074 | ||
|
|
420b46741e | ||
|
|
e8f65f95bd |
No files matched your search
@@ -9,7 +9,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"fable": {
|
"fable": {
|
||||||
"version": "4.0.6",
|
"version": "4.24.0",
|
||||||
"commands": [
|
"commands": [
|
||||||
"fable"
|
"fable"
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ RUN \
|
|||||||
&& apt-get install -y cmake
|
&& apt-get install -y cmake
|
||||||
|
|
||||||
# Dotnet
|
# Dotnet
|
||||||
RUN apt-get update && apt-get install -y dotnet7
|
RUN apt-get update && apt-get install -y dotnet8
|
||||||
|
|
||||||
# Node
|
# Node
|
||||||
ARG node_major=18
|
ARG node_major=18
|
||||||
@@ -129,5 +129,3 @@ RUN \
|
|||||||
-G sudo \
|
-G sudo \
|
||||||
--password "$(openssl passwd -1 "$docker_pass")" \
|
--password "$(openssl passwd -1 "$docker_pass")" \
|
||||||
"$docker_user"
|
"$docker_user"
|
||||||
|
|
||||||
RUN usermod -aG docker "$docker_user"
|
|
||||||
@@ -1,15 +1,10 @@
|
|||||||
{
|
{
|
||||||
"dockerFile": "Dockerfile",
|
"dockerFile": "Dockerfile",
|
||||||
// Creates local dir to mount to HOME in docker
|
|
||||||
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
|
|
||||||
"hostRequirements": {
|
"hostRequirements": {
|
||||||
"memory": "4gb",
|
"memory": "8gb",
|
||||||
"storage": "6gb"
|
"storage": "10gb"
|
||||||
},
|
},
|
||||||
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
|
|
||||||
"containerUser": "bts",
|
"containerUser": "bts",
|
||||||
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
|
|
||||||
"workspaceFolder": "/home/bts/spark",
|
|
||||||
"updateRemoteUserUID": true,
|
"updateRemoteUserUID": true,
|
||||||
"shutdownAction": "none",
|
"shutdownAction": "none",
|
||||||
// Note: If running multiple containers on the same host, each must use a
|
// Note: If running multiple containers on the same host, each must use a
|
||||||
@@ -17,26 +12,20 @@
|
|||||||
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
||||||
"runArgs": [
|
"runArgs": [
|
||||||
"--shm-size",
|
"--shm-size",
|
||||||
"2gb",
|
"8gb"
|
||||||
"--network",
|
|
||||||
"host"
|
|
||||||
],
|
],
|
||||||
"mounts": [
|
"mounts": [
|
||||||
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
|
|
||||||
"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
|
"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
|
||||||
],
|
],
|
||||||
"customizations": {
|
"customizations": {
|
||||||
"vscode": {
|
"vscode": {
|
||||||
"settings": {
|
|
||||||
"clangd.arguments": [
|
|
||||||
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"extensions": [
|
"extensions": [
|
||||||
"eamodio.gitlens",
|
"eamodio.gitlens",
|
||||||
"llvm-vs-code-extensions.vscode-clangd",
|
"llvm-vs-code-extensions.vscode-clangd",
|
||||||
|
"ms-python.python",
|
||||||
"ms-vscode.cmake-tools",
|
"ms-vscode.cmake-tools",
|
||||||
"ms-vscode.cpptools"
|
"ms-vscode.cpptools",
|
||||||
|
"xaver.clang-format"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,10 +7,10 @@
|
|||||||
# We use Docker Compose to build images and launch containers for various services
|
# We use Docker Compose to build images and launch containers for various services
|
||||||
# that make up our software suite:
|
# that make up our software suite:
|
||||||
# - sparkd
|
# - sparkd
|
||||||
# - pubsub
|
# - TODO: pubsub
|
||||||
# - md-publisher
|
# - TODO: md-publisher
|
||||||
# - webapp (DSQA on the web)
|
# - TODO: webapp (DSQA on the web)
|
||||||
# - api-server (websocket based)
|
# - TODO: api-server (websocket based)
|
||||||
# From inside a devcontainer, you can launch all services in separate containers
|
# From inside a devcontainer, you can launch all services in separate containers
|
||||||
# on your machine by running:
|
# on your machine by running:
|
||||||
#
|
#
|
||||||
@@ -31,51 +31,19 @@
|
|||||||
|
|
||||||
### Sparkd ###
|
### Sparkd ###
|
||||||
# The internal image for sparkd to use. For more info, view:
|
# The internal image for sparkd to use. For more info, view:
|
||||||
# src/app_sparkd/Dockerfile
|
# src/server/sparkd/Dockerfile
|
||||||
SPARKD__IMAGE=sparkd
|
SPARKD__IMAGE=sparkd
|
||||||
# The 'spark.conf' for this service to use
|
# The config dir for this service to use
|
||||||
SPARKD__CONF_FILE=src/app_sparkd/spark.conf
|
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
|
||||||
# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
|
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
|
||||||
# SPARKD__IMAGE=sparkd_md_replay
|
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
|
||||||
# SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
|
# SPARKD__DATA_DIR=/home/bts/sparkdata/data
|
||||||
# (configure which md to replay in the new SPARKD__CONF_FILE)
|
|
||||||
SPARKD__MD_RECORDING_DIR=.md_recording
|
|
||||||
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
|
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
|
||||||
|
SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
|
||||||
SPARKD__CONFIG_PORT=8000
|
SPARKD__CONFIG_PORT=8000
|
||||||
SPARKD__QUICKBUS_PORT=8001
|
SPARKD__QUICKBUS_PORT=8001
|
||||||
SPARKD__SIM_FIX_PORT=8002
|
SPARKD__SIM_FIX_PORT=8002
|
||||||
|
SPARKD__SHM_SIZE=4gb
|
||||||
|
SPARKD__PYTHON_REQUIREMENTS=requirements.txt
|
||||||
|
|
||||||
### Pubsub ###
|
# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
|
||||||
# The 'spark.conf' for this service to use
|
|
||||||
PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
|
|
||||||
PUBSUB__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
|
||||||
PUBSUB__CPP_SOURCE_DIR=install/data-stream-suite-linux
|
|
||||||
# The appsettings to load at runtime. Expects a file named:
|
|
||||||
# src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
|
|
||||||
PUBSUB__DOTNET_ENVIRONMENT=Staging
|
|
||||||
|
|
||||||
### Md Publisher ###
|
|
||||||
# The 'spark.conf' for this service to use
|
|
||||||
MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
|
|
||||||
MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
|
||||||
# Determines which appsettings to load at runtime. Expects a file named:
|
|
||||||
# src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
|
|
||||||
MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
|
|
||||||
|
|
||||||
### Webapp (DSQA on the web) ###
|
|
||||||
# The 'spark.conf' for this service to use
|
|
||||||
WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
|
|
||||||
WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
|
||||||
WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
|
|
||||||
# Determines which appsettings to load at runtime. Expects a file named:
|
|
||||||
# src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
|
|
||||||
WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
|
|
||||||
|
|
||||||
### Api Server ###
|
|
||||||
# The 'spark.conf' for this service to use
|
|
||||||
API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
|
|
||||||
API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
|
||||||
API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
|
|
||||||
# Determines which appsettings to load at runtime. Expects a file named:
|
|
||||||
# src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
|
|
||||||
API_SERVER__ASPNETCORE_ENVIRONMENT=Staging
|
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
*.cmd text=auto eol=crlf
|
*.cmd text=auto eol=crlf
|
||||||
*.cs text=auto eol=crlf
|
*.cs text=auto eol=crlf
|
||||||
*.sh text=auto eol=lf
|
*.sh text=auto eol=lf
|
||||||
*.py text=auto eol=lf
|
|
||||||
+4
-2
@@ -11,6 +11,7 @@ cmake-build-debug/
|
|||||||
**.vscode/*
|
**.vscode/*
|
||||||
!.vscode/launch.json
|
!.vscode/launch.json
|
||||||
!.vscode/tasks.json
|
!.vscode/tasks.json
|
||||||
|
!.vscode/*.env
|
||||||
.vs
|
.vs
|
||||||
obj/
|
obj/
|
||||||
*.csproj.user
|
*.csproj.user
|
||||||
@@ -54,11 +55,12 @@ trace_data/
|
|||||||
.cache/
|
.cache/
|
||||||
/src/Spark.Admin.WebApp/tsconfig.*.g.json
|
/src/Spark.Admin.WebApp/tsconfig.*.g.json
|
||||||
/src/Spark.Admin.WebApp/src/bin/
|
/src/Spark.Admin.WebApp/src/bin/
|
||||||
|
/src/client/**/*appsettings.Local.json
|
||||||
|
/src/DataStreamSuite/**/*appsettings.Local.json
|
||||||
**/CMakeDeps.cmake
|
**/CMakeDeps.cmake
|
||||||
**/CMakeDeps_win.cmake
|
**/CMakeDeps_win.cmake
|
||||||
.devcontainer/
|
.devcontainer/
|
||||||
*.Generated.cs
|
*.Generated.cs
|
||||||
*.lst
|
*.lst
|
||||||
src/app_sparkd/spark.local.conf
|
|
||||||
.env.local
|
.env.local
|
||||||
.md_recording
|
.secrets
|
||||||
Vendored
+157
-73
@@ -1,6 +1,14 @@
|
|||||||
{
|
{
|
||||||
"version": "0.2.0",
|
"version": "0.2.0",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": "Python Debugger: Current File with Arguments",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "${file}",
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"args": "${command:pickArgs}"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": ".NET Core Attach",
|
"name": ".NET Core Attach",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
@@ -16,19 +24,10 @@
|
|||||||
"program": "spark_api_server.exe",
|
"program": "spark_api_server.exe",
|
||||||
"args": [
|
"args": [
|
||||||
"run",
|
"run",
|
||||||
|
"--sparkd",
|
||||||
|
"http://127.0.0.1:8000/config"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"preLaunchTask": "Build Api Server (Debug)",
|
|
||||||
"program": "spark_api_server",
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
|
||||||
},
|
|
||||||
"env": {
|
|
||||||
"ASPNETCORE_ENVIRONMENT": "Development",
|
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -42,18 +41,10 @@
|
|||||||
"program": "spark_api_server.exe",
|
"program": "spark_api_server.exe",
|
||||||
"args": [
|
"args": [
|
||||||
"run",
|
"run",
|
||||||
|
"--sparkd",
|
||||||
|
"http://127.0.0.1:8000/config"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"program": "spark_api_server",
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
|
||||||
},
|
|
||||||
"env": {
|
|
||||||
"ASPNETCORE_ENVIRONMENT": "Development",
|
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -62,9 +53,9 @@
|
|||||||
"name": "Bts.DataStreamWebApp.Server",
|
"name": "Bts.DataStreamWebApp.Server",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"preLaunchTask": "Build DataStream (Debug)",
|
"preLaunchTask": "Build DataStream WebApp (Debug)",
|
||||||
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
|
"program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
||||||
"env": {
|
"env": {
|
||||||
"ASPNETCORE_HTTP_PORT": "8080",
|
"ASPNETCORE_HTTP_PORT": "8080",
|
||||||
"ASPNETCORE_HTTPS_PORT": "8443",
|
"ASPNETCORE_HTTPS_PORT": "8443",
|
||||||
@@ -76,8 +67,8 @@
|
|||||||
"name": "Bts.DataStreamWebApp.Server (Skip Build)",
|
"name": "Bts.DataStreamWebApp.Server (Skip Build)",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
|
"program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
||||||
"env": {
|
"env": {
|
||||||
"ASPNETCORE_HTTP_PORT": "8080",
|
"ASPNETCORE_HTTP_PORT": "8080",
|
||||||
"ASPNETCORE_HTTPS_PORT": "8443",
|
"ASPNETCORE_HTTPS_PORT": "8443",
|
||||||
@@ -97,10 +88,10 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamPublisher.exe",
|
"program": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
||||||
},
|
},
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -115,10 +106,10 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamPublisher.exe",
|
"program": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
||||||
},
|
},
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -127,7 +118,39 @@
|
|||||||
"type": "cppdbg",
|
"type": "cppdbg",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
"args": ["run"],
|
"args": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
|
"environment": [],
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "spreader tests2",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"[spreader2]"
|
||||||
|
],
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
|
"environment": [],
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "spreader tests",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"[spreader]"
|
||||||
|
],
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
"environment": [],
|
"environment": [],
|
||||||
@@ -142,12 +165,25 @@
|
|||||||
"program": "",
|
"program": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
"program": "DataStreamQueryAnalyzer.exe",
|
||||||
"args": [
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
|
||||||
"env": {
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
|
||||||
|
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "DataStreamQueryAnalyzer (Skip Build)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"program": "DataStreamQueryAnalyzer.exe",
|
||||||
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -166,7 +202,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "PubSubServer.exe",
|
"program": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
},
|
},
|
||||||
@@ -184,7 +220,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "PubSubServer.exe",
|
"program": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
},
|
},
|
||||||
@@ -199,11 +235,25 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
"preLaunchTask": "Build Spark (Debug)",
|
||||||
"program": "Spark.exe",
|
"program": "Spark.exe",
|
||||||
"args": [
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
"--sparkd",
|
"env": {
|
||||||
"http://127.0.0.1:8000/config"
|
"DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
|
||||||
],
|
}
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Spark (Skip Build)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"program": "Spark.exe",
|
||||||
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -220,7 +270,40 @@
|
|||||||
"--sparkd",
|
"--sparkd",
|
||||||
"${input:sparkConfigServer}"
|
"${input:sparkConfigServer}"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Spark Test",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"preLaunchTask": "Build Spark (Debug)",
|
||||||
|
"program": "Spark.Console.exe",
|
||||||
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkTest}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Spark Test (Skip Build)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"windows": {
|
||||||
|
"program": "Spark.Console.exe",
|
||||||
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkTest}"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -229,7 +312,6 @@
|
|||||||
"name": "Sparkd",
|
"name": "Sparkd",
|
||||||
"type": "cppdbg",
|
"type": "cppdbg",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"preLaunchTask": "Build Sparkd (Debug | Clang)",
|
|
||||||
"program": "",
|
"program": "",
|
||||||
"linux": {
|
"linux": {
|
||||||
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
||||||
@@ -282,34 +364,6 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"name": "Sparkd Web (Skip Build)",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"web"
|
|
||||||
],
|
|
||||||
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
"setupCommands": [
|
|
||||||
{
|
|
||||||
"description": "Enable pretty-printing for gdb",
|
|
||||||
"text": "-enable-pretty-printing",
|
|
||||||
"ignoreFailures": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"description": "Set Disassembly Flavor to Intel",
|
|
||||||
"text": "-gdb-set disassembly-flavor intel",
|
|
||||||
"ignoreFailures": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
],
|
],
|
||||||
"inputs": [
|
"inputs": [
|
||||||
{
|
{
|
||||||
@@ -349,12 +403,42 @@
|
|||||||
"Staging",
|
"Staging",
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "dataStreamQueryAnalyzerEnvironment",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "DataStreamQueryAnalyzer Configuration",
|
||||||
|
"options": [
|
||||||
|
"Development",
|
||||||
|
"Local",
|
||||||
|
],
|
||||||
|
"default": "Development",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkEnvironment",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "Spark Configuration",
|
||||||
|
"options": [
|
||||||
|
"Development",
|
||||||
|
"Local",
|
||||||
|
],
|
||||||
|
"default": "Development",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "sparkConfigServer",
|
"id": "sparkConfigServer",
|
||||||
"type": "promptString",
|
"type": "promptString",
|
||||||
"description": "Config Server URL",
|
"description": "Config Server URL",
|
||||||
"default": "http://localhost:8000/config",
|
"default": "http://localhost:8000/config",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkTest",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Test to run",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkdTest",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Test to run",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "isSparkApiEnabled",
|
"id": "isSparkApiEnabled",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
|
|||||||
Vendored
+1
@@ -0,0 +1 @@
|
|||||||
|
PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
|
||||||
Vendored
+169
-61
@@ -485,12 +485,12 @@
|
|||||||
"command": "",
|
"command": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamQueryAnalyzer.exe",
|
"command": "DataStreamQueryAnalyzer.exe",
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
|
||||||
|
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -510,7 +510,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -530,7 +530,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -549,12 +549,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -572,12 +571,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -595,12 +593,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -701,12 +698,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -724,12 +720,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -746,12 +741,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -769,12 +763,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -791,12 +784,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"args": [
|
|
||||||
"--config",
|
|
||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
|
||||||
],
|
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
||||||
|
"env": {
|
||||||
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -811,7 +803,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -827,7 +819,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -843,7 +835,22 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Spark (Skip Build)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "Spark.exe",
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -868,7 +875,7 @@
|
|||||||
"${input:sparkConfigQuery}",
|
"${input:sparkConfigQuery}",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -894,6 +901,24 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"label": "Run Sparkd (Skip Build)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"run"
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "run",
|
||||||
|
"color": "terminal.ansiGreen",
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"label": "Run Sparkd Config Query",
|
"label": "Run Sparkd Config Query",
|
||||||
"dependsOn": "Build Sparkd",
|
"dependsOn": "Build Sparkd",
|
||||||
@@ -915,6 +940,65 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"label": "Test Spark (Debug)",
|
||||||
|
"dependsOn": "Build Spark (Debug)",
|
||||||
|
"type": "process",
|
||||||
|
"icon": {
|
||||||
|
"id": "coverage",
|
||||||
|
"color": "terminal.ansiYellow",
|
||||||
|
},
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "Spark.Console.exe",
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkTest}"
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Test Sparkd",
|
||||||
|
"dependsOn": "Build Sparkd",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkdTest}",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "coverage",
|
||||||
|
"color": "terminal.ansiYellow",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Test Sparkd (Skip Build)",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"test",
|
||||||
|
"${input:sparkdTest}",
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "coverage",
|
||||||
|
"color": "terminal.ansiYellow",
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
@@ -1003,26 +1087,40 @@
|
|||||||
"default": "sparkd-debug-clang",
|
"default": "sparkd-debug-clang",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "dataStreamPublisherConfig",
|
"id": "dataStreamPublisherEnvironment",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
"description": "Publisher Configuration",
|
"description": "Publisher Configuration",
|
||||||
"options": [
|
"options": [
|
||||||
"MarketDataPublisher",
|
"Development",
|
||||||
"SimpleEchoClient",
|
"MarketData",
|
||||||
"SimpleTestDataPublisher",
|
"Production",
|
||||||
"TestDataPublisher",
|
"ProductionAws",
|
||||||
|
"Staging",
|
||||||
],
|
],
|
||||||
"default": "MarketDataPublisher",
|
"default": "MarketDataPublisher",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "pubSubServerConfig",
|
"id": "dataStreamQueryAnalyzerEnvironment",
|
||||||
|
"type": "pickString",
|
||||||
|
"description": "DataStreamQueryAnalyzer Configuration",
|
||||||
|
"options": [
|
||||||
|
"Development",
|
||||||
|
"Local",
|
||||||
|
],
|
||||||
|
"default": "Development",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "pubSubServerEnvironment",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
"description": "PubSub Configuration",
|
"description": "PubSub Configuration",
|
||||||
"options": [
|
"options": [
|
||||||
"PubSubServer",
|
"Development",
|
||||||
"SimpleEchoClient",
|
"EchoServer",
|
||||||
|
"Production",
|
||||||
|
"ProductionAws",
|
||||||
|
"Staging",
|
||||||
],
|
],
|
||||||
"default": "PubSubServer",
|
"default": "Development",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "sparkConfigQuery",
|
"id": "sparkConfigQuery",
|
||||||
@@ -1033,6 +1131,16 @@
|
|||||||
"id": "sparkdConfigQuery",
|
"id": "sparkdConfigQuery",
|
||||||
"type": "promptString",
|
"type": "promptString",
|
||||||
"description": "Config Type(s)"
|
"description": "Config Type(s)"
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkTest",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Test to run",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sparkdTest",
|
||||||
|
"type": "promptString",
|
||||||
|
"description": "Test to run",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+82
-66
@@ -1,7 +1,10 @@
|
|||||||
cmake_minimum_required(VERSION 3.12)
|
cmake_minimum_required(VERSION 3.12)
|
||||||
project(spark)
|
project(spark)
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
|
||||||
set(CMAKE_CXX_STANDARD 20)
|
set(CMAKE_CXX_STANDARD 20)
|
||||||
|
|
||||||
|
get_version_from_git()
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
set(platform "windows")
|
set(platform "windows")
|
||||||
else()
|
else()
|
||||||
@@ -33,85 +36,98 @@ else()
|
|||||||
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
|
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
||||||
|
link_directories(${EXTERNALS_LIB})
|
||||||
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
||||||
if(DEFINED APP_DATA_DIR)
|
if(DEFINED APP_DATA_DIR)
|
||||||
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
||||||
endif()
|
endif()
|
||||||
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
||||||
|
if(WIN32)
|
||||||
|
# subdirs
|
||||||
|
add_subdirectory(src/lib_spark)
|
||||||
|
else()
|
||||||
|
if(BUILD_SPARKD)
|
||||||
|
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||||
|
# read by their code). We override some below.
|
||||||
|
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
||||||
|
|
||||||
if(BUILD_SPARKD)
|
# This allows us to override hardcoded exchange-urls with ones read from
|
||||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
# config (i.e., resolved at runtime rather than compile-time)
|
||||||
# read by their code). We override some below.
|
add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
|
||||||
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
|
||||||
|
|
||||||
# This allows us to override hardcoded exchange-urls with ones read from
|
# Enable ccapi services
|
||||||
# config (i.e., resolved at runtime rather than compile-time)
|
add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
|
||||||
add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
|
add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
|
||||||
|
# Enable ccapi exchanges
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
|
||||||
|
|
||||||
# Enable ccapi services
|
set(CCAPI_LOG_LEVEL
|
||||||
add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
|
"Warn"
|
||||||
add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
|
CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
|
||||||
# Enable ccapi exchanges
|
string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
|
add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
|
|
||||||
|
|
||||||
set(CCAPI_LOG_LEVEL
|
# ** Binance **
|
||||||
"Warn"
|
add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
|
||||||
CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
|
add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
||||||
string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
|
add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
||||||
add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
|
|
||||||
|
|
||||||
# ** Binance **
|
# ** Binance US **
|
||||||
add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
|
add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
||||||
add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
||||||
add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
|
||||||
|
|
||||||
# ** Binance US **
|
# ** Binance USDS Futures **
|
||||||
add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
|
||||||
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
|
||||||
|
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
|
||||||
|
add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
|
||||||
|
|
||||||
# ** Binance USDS Futures **
|
# ** Binance Coin Futures **
|
||||||
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
||||||
|
|
||||||
# ** Deribit **
|
# ** Deribit **
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
|
add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
|
||||||
add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
|
add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
|
||||||
add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
|
add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
|
||||||
add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
|
add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
|
add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
|
add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
|
||||||
add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
|
add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
|
||||||
add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
|
add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
|
||||||
|
|
||||||
# ** Okx **
|
# ** Okx **
|
||||||
add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
|
add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
|
||||||
add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
|
add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
# subdirs
|
# subdirs
|
||||||
add_subdirectory(src/lib_system)
|
add_subdirectory(src/lib_spark)
|
||||||
add_subdirectory(src/lib_domain)
|
if (BUILD_SPARKD)
|
||||||
add_subdirectory(src/lib_http)
|
add_subdirectory(src/lib_python_interop)
|
||||||
if (BUILD_SPARKD)
|
add_subdirectory(src/lib_onixs_fix)
|
||||||
add_subdirectory(src/lib_python_md)
|
add_subdirectory(src/lib_binance)
|
||||||
add_subdirectory(src/app_sparkd)
|
add_subdirectory(src/lib_okx_execution)
|
||||||
add_subdirectory(src/data_files)
|
add_subdirectory(src/app_sparkd)
|
||||||
endif()
|
add_subdirectory(src/data_files)
|
||||||
add_subdirectory(src/lib_spark)
|
endif()
|
||||||
if (BUILD_ADMIN_WEB_APP)
|
if (BUILD_ADMIN_WEB_APP)
|
||||||
add_subdirectory(src/Spark.Admin.WebApp)
|
add_subdirectory(src/Spark.Admin.WebApp)
|
||||||
endif()
|
endif()
|
||||||
if(BUILD_DOCS)
|
if(BUILD_DOCS)
|
||||||
add_subdirectory(doc/user)
|
add_subdirectory(doc/user)
|
||||||
endif()
|
endif()
|
||||||
if(BUILD_SDK)
|
if(BUILD_SDK)
|
||||||
add_subdirectory(src/sdk)
|
add_subdirectory(src/sdk)
|
||||||
|
endif()
|
||||||
endif()
|
endif()
|
||||||
+15
-12
@@ -24,12 +24,13 @@
|
|||||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||||
"CMAKE_CXX_STANDARD": "20",
|
"CMAKE_CXX_STANDARD": "20",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
|
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
|
||||||
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
||||||
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -49,11 +50,20 @@
|
|||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"platform": "",
|
"platform": "",
|
||||||
|
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "OFF"
|
||||||
|
},
|
||||||
"CMAKE_CXX_STANDARD": "20",
|
"CMAKE_CXX_STANDARD": "20",
|
||||||
|
"CMAKE_INSTALL_RPATH": ".",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
|
"CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
|
||||||
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
||||||
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
|
"DATABENTO_USE_EXTERNAL_JSON": {
|
||||||
|
"type": "BOOL",
|
||||||
|
"value": "ON"
|
||||||
|
},
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
||||||
"BUILD_SDK": {
|
"BUILD_SDK": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
@@ -164,12 +174,13 @@
|
|||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CMAKE_EXPORT_COMPILE_COMMANDS": {
|
"CMAKE_EXPORT_COMPILE_COMMANDS": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
},
|
},
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -208,7 +219,7 @@
|
|||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -241,7 +252,7 @@
|
|||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -294,14 +305,6 @@
|
|||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
|
||||||
"name": "sparkd-debug-clang-j6",
|
|
||||||
"configurePreset": "sparkd-debug-clang",
|
|
||||||
"displayName": "Sparkd (Debug | Clang | jobs=6)",
|
|
||||||
"inherits": "sparkd",
|
|
||||||
"jobs": 6
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-debug-gcc",
|
"name": "sparkd-debug-gcc",
|
||||||
"configurePreset": "sparkd-debug-gcc",
|
"configurePreset": "sparkd-debug-gcc",
|
||||||
|
|||||||
@@ -42,8 +42,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
|
|
||||||
EndProject
|
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|x64 = Debug|x64
|
Debug|x64 = Debug|x64
|
||||||
@@ -196,14 +194,6 @@ Global
|
|||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
|
function(get_version_from_git)
|
||||||
|
find_package(Git QUIET)
|
||||||
|
if(NOT Git_FOUND)
|
||||||
|
message(WARNING "Git not found")
|
||||||
|
return()
|
||||||
|
endif()
|
||||||
|
|
||||||
|
execute_process(
|
||||||
|
COMMAND ${GIT_EXECUTABLE} describe --tags --always
|
||||||
|
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
||||||
|
OUTPUT_VARIABLE GIT_TAG
|
||||||
|
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||||
|
RESULT_VARIABLE GIT_RESULT
|
||||||
|
)
|
||||||
|
|
||||||
|
if(NOT GIT_RESULT EQUAL 0)
|
||||||
|
message(WARNING "Failed to get git tag")
|
||||||
|
return()
|
||||||
|
endif()
|
||||||
|
|
||||||
|
execute_process(
|
||||||
|
COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
|
||||||
|
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
||||||
|
OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
|
||||||
|
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||||
|
)
|
||||||
|
set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
|
||||||
|
set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
|
||||||
|
|
||||||
|
string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
|
||||||
|
if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
|
||||||
|
set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
|
||||||
|
set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||||
|
set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
|
||||||
|
set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
|
||||||
|
set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
|
||||||
|
set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
|
||||||
|
|
||||||
|
set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}")
|
||||||
|
set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
|
||||||
|
set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
|
||||||
|
set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
|
||||||
|
else()
|
||||||
|
message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
||||||
|
|
||||||
|
function(init_python_module)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args
|
||||||
|
NAME
|
||||||
|
)
|
||||||
|
set(_multi_value_args
|
||||||
|
SOURCES
|
||||||
|
)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
|
||||||
|
|
||||||
|
set(module_proj ${_name}_module)
|
||||||
|
set(${module_proj}_SOURCES ${_sources})
|
||||||
|
pybind11_add_module(${module_proj} SHARED ${_sources})
|
||||||
|
target_link_libraries(
|
||||||
|
${module_proj}
|
||||||
|
PUBLIC
|
||||||
|
pybind11::embed
|
||||||
|
spark
|
||||||
|
${_name}
|
||||||
|
)
|
||||||
|
target_include_directories(
|
||||||
|
${module_proj}
|
||||||
|
PRIVATE
|
||||||
|
$<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
|
||||||
|
)
|
||||||
|
target_include_directories(
|
||||||
|
${module_proj}
|
||||||
|
PUBLIC
|
||||||
|
${Python3_INCLUDE_DIRS}
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
||||||
|
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
||||||
|
)
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
||||||
|
DEPENDS ${_name} ${module_proj}
|
||||||
|
COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
|
||||||
|
COMMENT "Generating py stubs"
|
||||||
|
)
|
||||||
|
add_custom_target(
|
||||||
|
"${_name}_py_stubs"
|
||||||
|
ALL
|
||||||
|
DEPENDS
|
||||||
|
${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
||||||
|
)
|
||||||
|
install(
|
||||||
|
TARGETS ${module_proj}
|
||||||
|
EXPORT ${module_proj}Targets
|
||||||
|
RUNTIME
|
||||||
|
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||||
|
LIBRARY
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||||
|
ARCHIVE
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||||
|
)
|
||||||
|
foreach(dir ${pybind11_INCLUDE_DIRS})
|
||||||
|
install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
|
||||||
|
endforeach()
|
||||||
|
install(
|
||||||
|
EXPORT ${module_proj}Targets
|
||||||
|
FILE ${module_proj}Targets.cmake
|
||||||
|
NAMESPACE ${_name}::
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
||||||
|
)
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(GNUInstallDirs)
|
||||||
|
|
||||||
|
function(init_python_plugin)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args NAME PLUGIN_NAME)
|
||||||
|
set(_multi_value_args SOURCES)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
||||||
|
set(_plugin_name ${_ARGS_PLUGIN_NAME})
|
||||||
|
if (NOT DEFINED _ARGS_PLUGIN_NAME)
|
||||||
|
set(_plugin_name ${_name}_plugins)
|
||||||
|
endif()
|
||||||
|
set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
|
||||||
|
if (NOT DEFINED _ARGS_SOURCES)
|
||||||
|
file(
|
||||||
|
GLOB_RECURSE ${_plugin_name}_SOURCES
|
||||||
|
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
${_plugin_name}/*.py)
|
||||||
|
endif()
|
||||||
|
set(src_outs)
|
||||||
|
foreach(_src ${${_plugin_name}_SOURCES})
|
||||||
|
set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
|
||||||
|
set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT ${_src_out}
|
||||||
|
COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
|
||||||
|
DEPENDS ${_src_in}
|
||||||
|
COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
|
||||||
|
)
|
||||||
|
list(APPEND src_outs ${_src_out})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
|
||||||
|
|
||||||
|
add_library(${_plugin_name} INTERFACE)
|
||||||
|
add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
|
||||||
|
|
||||||
|
set_target_properties(
|
||||||
|
${_plugin_name}
|
||||||
|
PROPERTIES
|
||||||
|
INTERFACE_PYTHON_SCRIPTS_DIR
|
||||||
|
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
|
||||||
|
)
|
||||||
|
|
||||||
|
set_target_properties(
|
||||||
|
${_plugin_name}
|
||||||
|
PROPERTIES
|
||||||
|
INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
|
||||||
|
)
|
||||||
|
|
||||||
|
install(
|
||||||
|
TARGETS ${_plugin_name}
|
||||||
|
EXPORT ${_plugin_name}Targets
|
||||||
|
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||||
|
)
|
||||||
|
|
||||||
|
install(
|
||||||
|
EXPORT ${_plugin_name}Targets
|
||||||
|
FILE ${_plugin_name}Targets.cmake
|
||||||
|
NAMESPACE ${_name}::
|
||||||
|
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
||||||
|
)
|
||||||
|
install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
|
||||||
|
set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
|
||||||
|
set(PROTOBUF_STATIC_LIB "libprotobuf")
|
||||||
|
set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
|
||||||
|
set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
|
||||||
|
|
||||||
|
function(Protobuf_Generate_Cpp)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
|
||||||
|
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set(${_ARGS_CXX_SOURCES})
|
||||||
|
set(${_ARGS_CXX_HEADERS})
|
||||||
|
|
||||||
|
set(_proto_include_cli_args)
|
||||||
|
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
||||||
|
list(APPEND _proto_include_cli_args -I ${_dir})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
foreach(FIL ${_ARGS_PROTO_SOURCES})
|
||||||
|
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
|
||||||
|
get_filename_component(FIL_WE ${FIL} NAME_WE)
|
||||||
|
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
|
||||||
|
if(FIL_DIR)
|
||||||
|
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
|
||||||
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
|
||||||
|
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
|
COMMAND
|
||||||
|
${Protobuf_PROTOC_EXECUTABLE}
|
||||||
|
ARGS
|
||||||
|
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
|
||||||
|
DEPENDS ${ABS_FIL}
|
||||||
|
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
set_source_files_properties(
|
||||||
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
${${_ARGS_CXX_HEADERS}}
|
||||||
|
PROPERTIES
|
||||||
|
GENERATED TRUE
|
||||||
|
)
|
||||||
|
|
||||||
|
# remove some warnings from protobuf generated files
|
||||||
|
if(WIN32)
|
||||||
|
set_source_files_properties(
|
||||||
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
PROPERTIES
|
||||||
|
COMPILE_FLAGS "/wd4309 /wd4838"
|
||||||
|
)
|
||||||
|
else()
|
||||||
|
set_source_files_properties(
|
||||||
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
PROPERTIES
|
||||||
|
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
||||||
|
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
||||||
|
endfunction()
|
||||||
|
|
||||||
|
function(Service_Compiler_Generate_Cpp)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
|
||||||
|
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
||||||
|
set(${_ARGS_CXX_SOURCES})
|
||||||
|
set(${_ARGS_CXX_HEADERS})
|
||||||
|
|
||||||
|
set(_proto_include_cli_args)
|
||||||
|
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
||||||
|
list(APPEND _proto_include_cli_args -I ${_dir})
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
foreach(FIL ${_ARGS_PROTO_SOURCES})
|
||||||
|
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
|
||||||
|
get_filename_component(FIL_WE ${FIL} NAME_WE)
|
||||||
|
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
|
||||||
|
if(FIL_DIR)
|
||||||
|
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
|
||||||
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
|
||||||
|
|
||||||
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
|
||||||
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
|
||||||
|
|
||||||
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
|
||||||
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
|
||||||
|
|
||||||
|
set(TEMPLATES
|
||||||
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
|
||||||
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
|
||||||
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
|
||||||
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
|
||||||
|
)
|
||||||
|
|
||||||
|
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
|
||||||
|
if(WIN32)
|
||||||
|
set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
||||||
|
COMMAND SET PYTHONPATH=${SPARK_PYTHON}
|
||||||
|
COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
|
||||||
|
COMMAND
|
||||||
|
${Protobuf_PROTOC_EXECUTABLE}
|
||||||
|
ARGS
|
||||||
|
--cpp_out=${_ARGS_CXX_OUT_DIR}
|
||||||
|
--plugin=protoc-gen-service=${_plugin_cmd}
|
||||||
|
--service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
|
||||||
|
-I ${PROTOBUF_INCLUDE_DIRS}
|
||||||
|
-I ${_protobuf_src_dir}
|
||||||
|
${ABS_FIL}
|
||||||
|
DEPENDS
|
||||||
|
${ABS_FIL}
|
||||||
|
${TEMPLATES}
|
||||||
|
${_plugin_cmd}
|
||||||
|
${_plugin_py}
|
||||||
|
COMMENT "Running C++ service compiler on ${FIL}"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
else()
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
||||||
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
||||||
|
COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
|
||||||
|
ARGS
|
||||||
|
${Protobuf_PROTOC_EXECUTABLE}
|
||||||
|
--cpp_out=${_ARGS_CXX_OUT_DIR}
|
||||||
|
--plugin=protoc-gen-service=${_plugin_py}
|
||||||
|
--service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
|
||||||
|
${_proto_include_cli_args} ${ABS_FIL}
|
||||||
|
DEPENDS
|
||||||
|
${ABS_FIL}
|
||||||
|
${TEMPLATES}
|
||||||
|
${_plugin_py}
|
||||||
|
COMMENT "Running C++ service compiler on ${FIL}"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
endforeach()
|
||||||
|
|
||||||
|
# remove some warnings from protobuf generated files
|
||||||
|
if(WIN32)
|
||||||
|
set_source_files_properties(
|
||||||
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
PROPERTIES
|
||||||
|
COMPILE_FLAGS "/wd4309 /wd4838"
|
||||||
|
)
|
||||||
|
else()
|
||||||
|
set_source_files_properties(
|
||||||
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
PROPERTIES
|
||||||
|
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
||||||
|
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
||||||
|
endfunction()
|
||||||
|
|
||||||
@@ -39,6 +39,8 @@ function(protobuf_generate_cpp)
|
|||||||
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
|
||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
||||||
message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
||||||
|
# TODO: Pass proto sources by relative path (so this fn can be called from
|
||||||
|
# arbitrary subdirs).
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT
|
OUTPUT
|
||||||
"${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
|
||||||
@@ -46,7 +48,7 @@ function(protobuf_generate_cpp)
|
|||||||
COMMAND
|
COMMAND
|
||||||
${_ARGS_PROTOC}
|
${_ARGS_PROTOC}
|
||||||
ARGS
|
ARGS
|
||||||
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
|
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
|
||||||
DEPENDS ${_abs_fil}
|
DEPENDS ${_abs_fil}
|
||||||
COMMENT "Running C++ protocol buffer compiler on ${_fil}"
|
COMMENT "Running C++ protocol buffer compiler on ${_fil}"
|
||||||
VERBATIM
|
VERBATIM
|
||||||
@@ -78,3 +80,4 @@ function(protobuf_generate_cpp)
|
|||||||
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
||||||
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
include_guard(GLOBAL)
|
||||||
|
|
||||||
function(protobuf_generate_cpp_config)
|
function(protobuf_generate_cpp_config)
|
||||||
set(_options)
|
set(_options)
|
||||||
set(_one_value_args
|
set(_one_value_args
|
||||||
@@ -56,3 +58,4 @@ function(protobuf_generate_cpp_config)
|
|||||||
file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
|
file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
|
||||||
endif()
|
endif()
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
include_guard(DIRECTORY)
|
||||||
|
include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
|
||||||
|
|
||||||
|
function(sym_link_python_module)
|
||||||
|
set(_options)
|
||||||
|
set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
|
||||||
|
set(_multi_value_args)
|
||||||
|
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
|
||||||
|
"${_multi_value_args}" ${ARGN})
|
||||||
|
if(NOT DEFINED _ARGS_LIB_NAME)
|
||||||
|
message(
|
||||||
|
FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
|
||||||
|
endif()
|
||||||
|
set(_lib_name ${_ARGS_LIB_NAME})
|
||||||
|
set(_module_name ${_ARGS_MODULE_NAME})
|
||||||
|
if(NOT DEFINED _ARGS_MODULE_NAME)
|
||||||
|
set(_module_name ${_lib_name}_module)
|
||||||
|
endif()
|
||||||
|
set(_plugin_name ${_ARGS_PLUGIN_NAME})
|
||||||
|
set(_deps ${_lib_name} ${_module_name})
|
||||||
|
if(DEFINED _ARGS_PLUGIN_NAME)
|
||||||
|
list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
|
||||||
|
endif()
|
||||||
|
add_dependencies(${PROJECT_NAME} ${_deps})
|
||||||
|
sym_link_local_lib(${_lib_name})
|
||||||
|
set(_module_suffix cpython-310-x86_64-linux-gnu.so)
|
||||||
|
sym_link_file(
|
||||||
|
SOURCE
|
||||||
|
${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
|
||||||
|
DEST
|
||||||
|
${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
|
||||||
|
INSTALL_DEST
|
||||||
|
${CMAKE_INSTALL_BINDIR}/)
|
||||||
|
if(DEFINED _ARGS_PLUGIN_NAME)
|
||||||
|
get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
|
||||||
|
sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
|
||||||
|
${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
#ifndef BTS_SPARK_VERSION_H
|
||||||
|
#define BTS_SPARK_VERSION_H
|
||||||
|
|
||||||
|
#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
|
||||||
|
#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
|
||||||
|
#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
|
||||||
|
#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
|
||||||
|
#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
|
||||||
|
#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
cmake_minimum_required(VERSION 3.5)
|
|
||||||
|
|
||||||
# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
|
|
||||||
# setting it explicitly prevents a compile-warning)
|
|
||||||
cmake_policy(SET CMP0076 NEW)
|
|
||||||
|
|
||||||
# add protos to the library to facilitate source_group
|
|
||||||
target_sources(spark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
|
|
||||||
|
|
||||||
function(assign_source_group)
|
|
||||||
foreach(_source IN ITEMS ${ARGN})
|
|
||||||
if(IS_ABSOLUTE "${_source}")
|
|
||||||
file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
|
|
||||||
else()
|
|
||||||
set(_source_rel "${_source}")
|
|
||||||
endif()
|
|
||||||
get_filename_component(_source_path "${_source_rel}" PATH)
|
|
||||||
string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
|
|
||||||
source_group("${_source_path_msvc}" FILES "${_source}")
|
|
||||||
set(SOURCE_FILES "${_source}")
|
|
||||||
endforeach()
|
|
||||||
endfunction(assign_source_group)
|
|
||||||
|
|
||||||
assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
services:
|
|
||||||
sparkd:
|
|
||||||
build:
|
|
||||||
dockerfile: Dockerfile
|
|
||||||
context: ./sparkd
|
|
||||||
command: [ "./sparkd", "run" ]
|
|
||||||
ports: [ "8000:8000", "8001:8001", "8002:8002" ]
|
|
||||||
|
|
||||||
pubsub:
|
|
||||||
build:
|
|
||||||
dockerfile: pubsub.Dockerfile
|
|
||||||
context: ./DataStream
|
|
||||||
command: [ "./PubSubServer" ]
|
|
||||||
environment:
|
|
||||||
- DOTNET_ENVIRONMENT=Production
|
|
||||||
ports:
|
|
||||||
- 22019:22019
|
|
||||||
|
|
||||||
md-publisher:
|
|
||||||
build:
|
|
||||||
dockerfile: md-publisher.Dockerfile
|
|
||||||
context: ./DataStream
|
|
||||||
depends_on:
|
|
||||||
- pubsub
|
|
||||||
- sparkd
|
|
||||||
command:
|
|
||||||
[
|
|
||||||
"dockerize",
|
|
||||||
"-wait",
|
|
||||||
"tcp://pubsub:22019",
|
|
||||||
"-wait",
|
|
||||||
"tcp://sparkd:8000",
|
|
||||||
"-timeout",
|
|
||||||
"10s",
|
|
||||||
"./DataStreamPublisher"
|
|
||||||
]
|
|
||||||
environment:
|
|
||||||
- DOTNET_ENVIRONMENT=MarketData
|
|
||||||
|
|
||||||
webapp:
|
|
||||||
ports:
|
|
||||||
- 8080:8080
|
|
||||||
- 8443:8443
|
|
||||||
build:
|
|
||||||
dockerfile: webapp.Dockerfile
|
|
||||||
context: ./DataStream
|
|
||||||
command: [ "./Bts.DataStreamWebApp.Server" ]
|
|
||||||
environment:
|
|
||||||
- ASPNETCORE_HTTP_PORT=8080
|
|
||||||
- ASPNETCORE_HTTPS_PORT=8443
|
|
||||||
- ASPNETCORE_ENVIRONMENT=Staging
|
|
||||||
+137
-53
@@ -1,69 +1,100 @@
|
|||||||
services:
|
services:
|
||||||
sparkd:
|
sparkd:
|
||||||
|
volumes:
|
||||||
|
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
||||||
build:
|
build:
|
||||||
dockerfile: src/app_sparkd/Dockerfile
|
dockerfile: src/app_sparkd/Dockerfile
|
||||||
args:
|
args:
|
||||||
- IMAGE=${SPARKD__IMAGE}
|
- IMAGE=${SPARKD__IMAGE}
|
||||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||||
- CONF_FILE=${SPARKD__CONF_FILE}
|
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||||
- MD_RECORDING_DIR=${SPARKD__MD_RECORDING_DIR}
|
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||||
command: [ "./sparkd", "run" ]
|
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||||
|
command: >
|
||||||
|
sh -c "
|
||||||
|
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
|
||||||
|
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
||||||
|
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||||
|
fi
|
||||||
|
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
||||||
|
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
||||||
|
set -a
|
||||||
|
. /home/bts/sparkdata/conf/.env.local
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
if [ -d /home/bts/sparkdata/conf/plugins ]; then
|
||||||
|
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
|
||||||
|
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
|
||||||
|
fi
|
||||||
|
./sparkd run"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
- ${SPARKD__CONFIG_DIR}/.env
|
||||||
ports:
|
ports:
|
||||||
- "8000:8000"
|
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
||||||
- "8001:8001"
|
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
||||||
- "8002:8002"
|
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: "${SPARKD__MEMORY:-4gb}"
|
||||||
|
|
||||||
pubsub:
|
sparkd-symbols:
|
||||||
environment:
|
volumes:
|
||||||
- DOTNET_ENVIRONMENT=${PUBSUB__DOTNET_ENVIRONMENT}
|
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
||||||
|
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
|
||||||
build:
|
build:
|
||||||
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
|
dockerfile: src/app_sparkd/Dockerfile
|
||||||
args:
|
args:
|
||||||
- CS_SOURCE_DIR=${PUBSUB__CS_SOURCE_DIR}
|
- IMAGE=${SPARKD__IMAGE}
|
||||||
- CPP_SOURCE_DIR=${PUBSUB__CPP_SOURCE_DIR}
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||||
- CONF_FILE=${PUBSUB__CONF_FILE}
|
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||||
|
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||||
|
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||||
|
command: >
|
||||||
|
sh -c "
|
||||||
|
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
|
||||||
|
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
||||||
|
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||||
|
fi
|
||||||
|
ls -alh /home/bts/sparkdata/conf/
|
||||||
|
cat /home/bts/sparkdata/conf/.env.local
|
||||||
|
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
||||||
|
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
||||||
|
set -a
|
||||||
|
. /home/bts/sparkdata/conf/.env.local
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
if [ -d /home/bts/sparkdata/conf/plugins ]; then
|
||||||
|
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
|
||||||
|
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
|
||||||
|
fi
|
||||||
|
./sparkd run"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
- ${SPARKD__CONFIG_DIR}/.env
|
||||||
ports:
|
ports:
|
||||||
- "22019:22019"
|
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
||||||
command: [ "./PubSubServer" ]
|
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
||||||
|
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: any
|
||||||
|
delay: 3s
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: "${SPARKD__MEMORY:-4gb}"
|
||||||
|
|
||||||
md-publisher:
|
sparkd-tail:
|
||||||
build:
|
image: bluetradingsystems/linux_dotnet7
|
||||||
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
|
volumes:
|
||||||
args:
|
- type: volume
|
||||||
- SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
|
read_only: true
|
||||||
- CONF_FILE=${MD_PUBLISHER__CONF_FILE}
|
source: sparkd_data
|
||||||
depends_on:
|
target: /home/bts/sparkdata/data/logging.sparkd
|
||||||
- pubsub
|
volume:
|
||||||
- sparkd
|
subpath: logging.sparkd
|
||||||
command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
|
command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
|
||||||
environment:
|
|
||||||
- DOTNET_ENVIRONMENT=${MD_PUBLISHER__DOTNET_ENVIRONMENT}
|
|
||||||
|
|
||||||
webapp:
|
|
||||||
ports:
|
|
||||||
- "8080:8080"
|
|
||||||
- "8443:8443"
|
|
||||||
build:
|
|
||||||
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
|
|
||||||
args:
|
|
||||||
- SOURCE_DIR=${WEBAPP__SOURCE_DIR}
|
|
||||||
- WEB_ROOT=${WEBAPP__WEB_ROOT}
|
|
||||||
- CONF_FILE=${WEBAPP__CONF_FILE}
|
|
||||||
command: [ "./Bts.DataStreamWebApp.Server" ]
|
|
||||||
environment:
|
|
||||||
- ASPNETCORE_ENVIRONMENT=${WEBAPP__ASPNETCORE_ENVIRONMENT}
|
|
||||||
|
|
||||||
api-server:
|
|
||||||
depends_on:
|
|
||||||
- sparkd
|
|
||||||
ports:
|
|
||||||
- 8008:8008
|
|
||||||
- 9443:9443
|
|
||||||
build:
|
|
||||||
dockerfile: src/client/spark_api_server/Dockerfile
|
|
||||||
command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
|
|
||||||
environment:
|
|
||||||
- ASPNETCORE_ENVIRONMENT=${API_SERVER__ASPNETCORE_ENVIRONMENT}
|
|
||||||
|
|
||||||
sparkd-ping-test:
|
sparkd-ping-test:
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -77,3 +108,56 @@ services:
|
|||||||
USER "root"
|
USER "root"
|
||||||
RUN apt-get update && apt-get -y install iputils-ping
|
RUN apt-get update && apt-get -y install iputils-ping
|
||||||
command: [ "ping", "sparkd" ]
|
command: [ "ping", "sparkd" ]
|
||||||
|
|
||||||
|
pubsub:
|
||||||
|
environment:
|
||||||
|
- DOTNET_ENVIRONMENT=Staging
|
||||||
|
build:
|
||||||
|
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
|
||||||
|
args:
|
||||||
|
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
||||||
|
- CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
|
||||||
|
ports:
|
||||||
|
- 22019:22019
|
||||||
|
command: [ "./PubSubServer" ]
|
||||||
|
|
||||||
|
md-publisher:
|
||||||
|
build:
|
||||||
|
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
|
||||||
|
args:
|
||||||
|
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
||||||
|
- CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
|
||||||
|
depends_on:
|
||||||
|
- pubsub
|
||||||
|
- sparkd
|
||||||
|
command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
|
||||||
|
environment:
|
||||||
|
- DOTNET_ENVIRONMENT=MarketData
|
||||||
|
|
||||||
|
api-server:
|
||||||
|
depends_on:
|
||||||
|
- sparkd
|
||||||
|
ports:
|
||||||
|
- 8008:8008
|
||||||
|
- 9443:9443
|
||||||
|
build:
|
||||||
|
dockerfile: src/client/spark_api_server/Dockerfile
|
||||||
|
command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
|
||||||
|
|
||||||
|
webapp:
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
- 8443:8443
|
||||||
|
build:
|
||||||
|
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
|
||||||
|
args:
|
||||||
|
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
||||||
|
- WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
|
||||||
|
- CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
|
||||||
|
command: [ "./Bts.DataStreamWebApp.Server" ]
|
||||||
|
environment:
|
||||||
|
- ASPNETCORE_ENVIRONMENT=Staging
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
sparkd_data:
|
||||||
|
name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
|
||||||
@@ -124,8 +124,8 @@ service OrderService {
|
|||||||
```
|
```
|
||||||
### C++ Example
|
### C++ Example
|
||||||
```cpp
|
```cpp
|
||||||
#include "lib/api/public/ServiceContext.h"
|
#include "spark/api/public/ServiceContext.h"
|
||||||
#include "lib/api/services/OrderService.client.pb.h"
|
#include "spark/api/services/OrderService.client.pb.h"
|
||||||
using namespace bts::api;
|
using namespace bts::api;
|
||||||
ServiceContext ctx;
|
ServiceContext ctx;
|
||||||
auto orders = ctx.locate<orderman::OrderService>();
|
auto orders = ctx.locate<orderman::OrderService>();
|
||||||
|
|||||||
@@ -29,9 +29,9 @@ namespace sample {
|
|||||||
```
|
```
|
||||||
|
|
||||||
```cpp
|
```cpp
|
||||||
#include "lib/api/public/ServiceContext.h"
|
#include "spark/api/public/ServiceContext.h"
|
||||||
#include "spark/lib/domain_model/Security.h"
|
#include "spark/domain_model/Security.h"
|
||||||
#include "lib/api/services/MarketDataService.client.pb.h"
|
#include "spark/api/services/MarketDataService.client.pb.h"
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
using namespace bts::api;
|
using namespace bts::api;
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
|
|||||||
// Subscribe to security trading status
|
// Subscribe to security trading status
|
||||||
void IAlgoServices::subscribe_status(SecurityKey);
|
void IAlgoServices::subscribe_status(SecurityKey);
|
||||||
// You implement
|
// You implement
|
||||||
virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
|
virtual void on_status(SecurityKey, co2::TradingStatus) override;
|
||||||
```
|
```
|
||||||
|
|
||||||
# Child Orders
|
# Child Orders
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
FROM ubuntu:22.04
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y pkg-config
|
||||||
|
RUN apt-get update && apt-get install -y locales
|
||||||
|
RUN apt-get update && apt-get install -y ca-certificates
|
||||||
|
RUN apt-get update && apt-get install -y libssh2-1-dev
|
||||||
|
RUN apt-get update && apt-get install -y software-properties-common
|
||||||
|
RUN apt-get update && apt-get install -y sudo
|
||||||
|
|
||||||
|
# GCC
|
||||||
|
ARG gcc_major=13
|
||||||
|
RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
|
||||||
|
&& apt-get update \
|
||||||
|
&& apt-get install -y g++-$gcc_major
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y dotnet7
|
||||||
|
|
||||||
|
RUN locale-gen en_US.UTF-8
|
||||||
|
|
||||||
|
ARG docker_user="bts"
|
||||||
|
ARG docker_uid="1000"
|
||||||
|
ARG docker_pass="bts"
|
||||||
|
|
||||||
|
RUN \
|
||||||
|
groupadd -g "$docker_uid" "$docker_user" \
|
||||||
|
&& useradd \
|
||||||
|
-m \
|
||||||
|
-u "$docker_uid" \
|
||||||
|
-g "$docker_uid" \
|
||||||
|
-G sudo \
|
||||||
|
--password "$(openssl passwd -1 "$docker_pass")" \
|
||||||
|
"$docker_user"
|
||||||
|
|
||||||
|
USER bts
|
||||||
|
WORKDIR /home/bts
|
||||||
Submodule
+1
Submodule docker/linux added at 6537812262.
Submodule
+1
Submodule docker/windows added at de7d40c09d.
+1
-1
Submodule externals updated: 4f5e385556...4f6734703c.
@@ -1,6 +0,0 @@
|
|||||||
{
|
|
||||||
"sdk": {
|
|
||||||
"version": "6.0.200",
|
|
||||||
"rollForward": "latestMajor"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,22 +1,42 @@
|
|||||||
astroid==2.7.3
|
astroid==2.7.3
|
||||||
autopep8==1.5.7
|
autopep8==1.5.7
|
||||||
|
blinker==1.9.0
|
||||||
|
boto3==1.37.9
|
||||||
|
botocore==1.37.9
|
||||||
certifi==2023.5.7
|
certifi==2023.5.7
|
||||||
charset-normalizer==3.1.0
|
charset-normalizer==3.1.0
|
||||||
|
click==8.1.8
|
||||||
colorama==0.4.4
|
colorama==0.4.4
|
||||||
|
Flask==3.1.0
|
||||||
idna==3.4
|
idna==3.4
|
||||||
isort==5.9.3
|
isort==5.9.3
|
||||||
|
itsdangerous==2.2.0
|
||||||
|
Jinja2==3.1.6
|
||||||
|
jmespath==1.0.1
|
||||||
lazy-object-proxy==1.6.0
|
lazy-object-proxy==1.6.0
|
||||||
|
MarkupSafe==3.0.2
|
||||||
mccabe==0.6.1
|
mccabe==0.6.1
|
||||||
|
numpy==2.2.6
|
||||||
platformdirs==2.3.0
|
platformdirs==2.3.0
|
||||||
protobuf==4.21.12
|
protobuf==4.21.12
|
||||||
pybars3==0.9.7
|
pybars3==0.9.7
|
||||||
|
pybind11==2.13.6
|
||||||
|
pybind11-stubgen==2.5.3
|
||||||
|
pybit==5.11.0
|
||||||
pycodestyle==2.7.0
|
pycodestyle==2.7.0
|
||||||
pycparser==2.20
|
pycparser==2.20
|
||||||
|
pycryptodome==3.23.0
|
||||||
pylint==2.10.2
|
pylint==2.10.2
|
||||||
PyMeta3==0.5.1
|
PyMeta3==0.5.1
|
||||||
|
python-dateutil==2.9.0.post0
|
||||||
|
python-dotenv==1.0.1
|
||||||
requests==2.31.0
|
requests==2.31.0
|
||||||
|
s3transfer==0.11.4
|
||||||
|
six==1.17.0
|
||||||
termcolor==1.1.0
|
termcolor==1.1.0
|
||||||
toml==0.10.2
|
toml==0.10.2
|
||||||
urllib3==2.0.3
|
urllib3==2.0.3
|
||||||
|
websocket-client==1.8.0
|
||||||
|
Werkzeug==3.1.3
|
||||||
wrapt==1.12.1
|
wrapt==1.12.1
|
||||||
yapf==0.31.0
|
yapf==0.31.0
|
||||||
+164
-103
@@ -1,103 +1,164 @@
|
|||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio Version 17
|
# Visual Studio Version 17
|
||||||
VisualStudioVersion = 17.3.32922.545
|
VisualStudioVersion = 17.3.32922.545
|
||||||
MinimumVisualStudioVersion = 15.0.26124.0
|
MinimumVisualStudioVersion = 15.0.26124.0
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
||||||
EndProject
|
EndProject
|
||||||
Global
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
EndProject
|
||||||
Debug|x64 = Debug|x64
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
|
||||||
MinSizeRel|x64 = MinSizeRel|x64
|
EndProject
|
||||||
Release|x64 = Release|x64
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
|
||||||
RelWithDebInfo|x64 = RelWithDebInfo|x64
|
EndProject
|
||||||
EndGlobalSection
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
|
||||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
EndProject
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
|
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
|
EndProject
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
|
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
|
EndProject
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
|
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
|
EndProject
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
|
EndProject
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
|
Global
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
|
Debug|x64 = Debug|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
|
MinSizeRel|x64 = MinSizeRel|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
|
Release|x64 = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
|
RelWithDebInfo|x64 = RelWithDebInfo|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
EndGlobalSection
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
EndGlobalSection
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
|
||||||
HideSolutionNode = FALSE
|
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
EndGlobalSection
|
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
|
||||||
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
|
||||||
EndGlobalSection
|
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
GlobalSection(Performance) = preSolution
|
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
HasPerformanceSessions = true
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
EndGlobalSection
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
|
||||||
EndGlobal
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
|
||||||
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
||||||
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
|
HideSolutionNode = FALSE
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||||
|
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(Performance) = preSolution
|
||||||
|
HasPerformanceSessions = true
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(NestedProjects) = preSolution
|
||||||
|
{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
|
||||||
|
{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
|
||||||
|
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
|
||||||
|
EndGlobalSection
|
||||||
|
EndGlobal
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -6,23 +6,80 @@ using Spark.Core.Collections;
|
|||||||
|
|
||||||
static class ArgumentComparer {
|
static class ArgumentComparer {
|
||||||
public static int Compare(
|
public static int Compare(
|
||||||
IArgument value1, IArgument value2, IDictionary<string,object?> env) {
|
IArgument value1, IArgument value2, IDictionary<string, object?> env) {
|
||||||
if (value1 is null) {
|
if (value1 is null) {
|
||||||
throw new ArgumentNullException(nameof(value1));
|
throw new ArgumentNullException(nameof(value1));
|
||||||
}
|
}
|
||||||
if (value2 is null) {
|
if (value2 is null) {
|
||||||
throw new ArgumentNullException(nameof(value2));
|
throw new ArgumentNullException(nameof(value2));
|
||||||
}
|
}
|
||||||
var cmp1 = value1.Cast<IComparable>();
|
var obj1 = value1.GetValue(env);
|
||||||
var cmp2 = value2.Cast<IComparable>();
|
var obj2 = value2.GetValue(env);
|
||||||
|
if (obj1 is null) {
|
||||||
|
if (obj2 is null) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (obj2 is null) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
return compareObjs(obj1, obj2);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int compareObjs(object obj1, object obj2) {
|
||||||
try {
|
try {
|
||||||
var v1 = cmp1.GetValue(env);
|
var typ1 = obj1.GetType();
|
||||||
var v2 = cmp2.GetValue(env);
|
var typ2 = obj2.GetType();
|
||||||
return v1.GetType().IsNumeric() ?
|
bool areSameTypes = typ1 == typ2;
|
||||||
new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
|
if (areSameTypes) {
|
||||||
|
return ((IComparable)obj1).CompareTo(obj2);
|
||||||
|
}
|
||||||
|
if (typ1.IsNumeric()) {
|
||||||
|
return tryCompareNumeric(obj1, obj2);
|
||||||
|
} else if (typ2.IsNumeric()) {
|
||||||
|
return -tryCompareNumeric(obj2, obj1);
|
||||||
|
} else if (typ1.IsEnum) {
|
||||||
|
if (typ2 == typeof(string)) {
|
||||||
|
return -((string)obj2).CompareTo(obj1.ToString());
|
||||||
|
}
|
||||||
|
} else if (typ1 == typeof(string)) {
|
||||||
|
return ((string)obj1).CompareTo(obj2.ToString());
|
||||||
|
} else if (typ1 == typeof(DateTime)) {
|
||||||
|
if (typ2 == typeof(TimeSpan)) {
|
||||||
|
return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
|
||||||
|
}
|
||||||
|
} else if (typ1 == typeof(TimeSpan)) {
|
||||||
|
if (typ2 == typeof(DateTime)) {
|
||||||
|
return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
|
||||||
|
}
|
||||||
|
}
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
Log.Ui.error($"Comparison failed: {ex}");
|
Log.Ui.error($"Comparison failed: {ex}");
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
|
throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
|
||||||
|
}
|
||||||
|
|
||||||
|
static int tryCompareNumeric(object numericObj1, object obj2) {
|
||||||
|
var typ2 = obj2.GetType();
|
||||||
|
if (typ2.IsNumeric()) {
|
||||||
|
return new NumericComparer().Compare(numericObj1, obj2);
|
||||||
|
} else if (typ2.IsEnum) {
|
||||||
|
var undTyp2 = typ2.GetEnumUnderlyingType();
|
||||||
|
if (undTyp2 == typeof(byte)) {
|
||||||
|
return new NumericComparer().Compare(numericObj1, (byte)obj2);
|
||||||
|
}
|
||||||
|
if (undTyp2 == typeof(short)) {
|
||||||
|
return new NumericComparer().Compare(numericObj1, (short)obj2);
|
||||||
|
}
|
||||||
|
if (undTyp2 == typeof(int)) {
|
||||||
|
return new NumericComparer().Compare(numericObj1, (int)obj2);
|
||||||
|
}
|
||||||
|
if (undTyp2 == typeof(long)) {
|
||||||
|
return new NumericComparer().Compare(numericObj1, (long)obj2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -41,7 +41,6 @@ Identifier {Letter}({Letter}*[0-9]*_*)*
|
|||||||
OpenParenthesis \(
|
OpenParenthesis \(
|
||||||
CloseParenthesis \)
|
CloseParenthesis \)
|
||||||
Comma ,
|
Comma ,
|
||||||
Negate '!'
|
|
||||||
MapsTo ':'
|
MapsTo ':'
|
||||||
IsOfType '/'
|
IsOfType '/'
|
||||||
|
|
||||||
@@ -94,10 +93,6 @@ IsOfType '/'
|
|||||||
return (int)Token.COMMA;
|
return (int)Token.COMMA;
|
||||||
}
|
}
|
||||||
|
|
||||||
{Negate} {
|
|
||||||
return (int)Token.NEGATE;
|
|
||||||
}
|
|
||||||
|
|
||||||
{MapsTo} {
|
{MapsTo} {
|
||||||
return (int)Token.MAPS_TO;
|
return (int)Token.MAPS_TO;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,14 +1,14 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.0" />
|
<PackageReference Include="YaccLexTools" Version="1.2.3" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ namespace Bts.BooleanExpressions;
|
|||||||
|
|
||||||
using Bts.BooleanExpressions.Arguments;
|
using Bts.BooleanExpressions.Arguments;
|
||||||
|
|
||||||
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
class BoundBooleanExpression : IBooleanExpression {
|
class BoundBooleanExpression : IBooleanExpression {
|
||||||
readonly IBooleanExpression expression;
|
readonly IBooleanExpression expression;
|
||||||
readonly IDictionary<string, FunctionDefinition> definitions;
|
readonly IDictionary<string, FunctionDefinition> definitions;
|
||||||
@@ -17,10 +19,10 @@ class BoundBooleanExpression : IBooleanExpression {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public bool Evaluate(IDictionary<string, object?> env) {
|
public bool Evaluate(IDictionary<string, object?> env) {
|
||||||
foreach (var kv in env) {
|
if (identifiers.Count > 0) {
|
||||||
identifiers[kv.Key] = kv.Value;
|
env = AggregateDictionary.Create(env, identifiers);
|
||||||
}
|
}
|
||||||
env = new EvaluationEnvironment(identifiers, definitions);
|
env = new EvaluationEnvironment(env, definitions);
|
||||||
return expression.Evaluate(env);
|
return expression.Evaluate(env);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,8 +24,6 @@ namespace QUT.GplexBuffers
|
|||||||
public BufferException(string message) : base(message) { }
|
public BufferException(string message) : base(message) { }
|
||||||
public BufferException(string message, Exception innerException)
|
public BufferException(string message, Exception innerException)
|
||||||
: base(message, innerException) { }
|
: base(message, innerException) { }
|
||||||
protected BufferException(SerializationInfo info, StreamingContext context)
|
|
||||||
: base(info, context) { }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
internal abstract class ScanBuff
|
internal abstract class ScanBuff
|
||||||
@@ -60,12 +58,10 @@ namespace QUT.GplexBuffers
|
|||||||
return new BuildBuffer(source);
|
return new BuildBuffer(source);
|
||||||
}
|
}
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
||||||
{
|
{
|
||||||
return new BuildBuffer(source, fallbackCodePage);
|
return new BuildBuffer(source, fallbackCodePage);
|
||||||
}
|
}
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -353,17 +349,15 @@ namespace QUT.GplexBuffers
|
|||||||
{
|
{
|
||||||
FileStream fStrm = (stream as FileStream);
|
FileStream fStrm = (stream as FileStream);
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
if (fStrm != null) FileName = fStrm.Name;
|
||||||
NextBlk = BlockReaderFactory.Raw(stream);
|
NextBlk = BlockReaderFactory.Get(stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
public BuildBuffer(Stream stream, int fallbackCodePage)
|
public BuildBuffer(Stream stream, int fallbackCodePage)
|
||||||
{
|
{
|
||||||
FileStream fStrm = (stream as FileStream);
|
FileStream fStrm = (stream as FileStream);
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
if (fStrm != null) FileName = fStrm.Name;
|
||||||
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Marks a conservative lower bound for the buffer,
|
/// Marks a conservative lower bound for the buffer,
|
||||||
@@ -434,43 +428,19 @@ namespace QUT.GplexBuffers
|
|||||||
//
|
//
|
||||||
internal static class BlockReaderFactory
|
internal static class BlockReaderFactory
|
||||||
{
|
{
|
||||||
public static BlockReader Raw(Stream stream)
|
|
||||||
{
|
|
||||||
return delegate(char[] block, int index, int number)
|
|
||||||
{
|
|
||||||
byte[] b = new byte[number];
|
|
||||||
int count = stream.Read(b, 0, number);
|
|
||||||
int i = 0;
|
|
||||||
int j = index;
|
|
||||||
for (; i < count; i++, j++)
|
|
||||||
block[j] = (char)b[i];
|
|
||||||
return count;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#if (!BYTEMODE)
|
public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
|
||||||
public static BlockReader Get(Stream stream, int fallbackCodePage)
|
|
||||||
{
|
{
|
||||||
Encoding encoding;
|
Encoding encoding;
|
||||||
int preamble = Preamble(stream);
|
int preamble = Preamble(stream);
|
||||||
|
|
||||||
if (preamble != 0) // There is a valid BOM here!
|
if (preamble != 0) // There is a valid BOM here!
|
||||||
encoding = Encoding.GetEncoding(preamble);
|
encoding = Encoding.GetEncoding(preamble);
|
||||||
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
|
else if (fallbackCodePage >= 0) // Anything but "guess"
|
||||||
return Raw(stream);
|
|
||||||
else if (fallbackCodePage != -2) // Anything but "guess"
|
|
||||||
encoding = Encoding.GetEncoding(fallbackCodePage);
|
encoding = Encoding.GetEncoding(fallbackCodePage);
|
||||||
else // This is the "guess" option
|
else // This is the "guess" option
|
||||||
{
|
encoding = GuessEncoding(stream);
|
||||||
int guess = new Guesser(stream).GuessCodePage();
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (guess == -1) // ==> this is a 7-bit file
|
|
||||||
encoding = Encoding.ASCII;
|
|
||||||
else if (guess == 65001)
|
|
||||||
encoding = Encoding.UTF8;
|
|
||||||
else // ==> use the machine default
|
|
||||||
encoding = Encoding.Default;
|
|
||||||
}
|
|
||||||
StreamReader reader = new StreamReader(stream, encoding);
|
StreamReader reader = new StreamReader(stream, encoding);
|
||||||
return reader.Read;
|
return reader.Read;
|
||||||
}
|
}
|
||||||
@@ -488,6 +458,14 @@ namespace QUT.GplexBuffers
|
|||||||
int b2 = stream.ReadByte();
|
int b2 = stream.ReadByte();
|
||||||
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
||||||
return 65001; // UTF8
|
return 65001; // UTF8
|
||||||
|
|
||||||
|
int b3 = stream.ReadByte();
|
||||||
|
// Check for UTF-32 BOM (Byte Order Mark)
|
||||||
|
if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
|
||||||
|
return 12001; // UTF-32BE
|
||||||
|
if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
|
||||||
|
return 12000; // UTF-32LE
|
||||||
|
|
||||||
//
|
//
|
||||||
// There is no unicode preamble, so we
|
// There is no unicode preamble, so we
|
||||||
// return denoter for the machine default.
|
// return denoter for the machine default.
|
||||||
@@ -495,7 +473,150 @@ namespace QUT.GplexBuffers
|
|||||||
stream.Seek(0, SeekOrigin.Begin);
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
#endif // !BYTEMODE
|
|
||||||
|
private static Encoding GuessEncoding(Stream stream)
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (IsUtf16(stream, out Encoding encoding))
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return encoding;
|
||||||
|
}
|
||||||
|
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (IsUtf8(stream))
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return Encoding.UTF8;
|
||||||
|
}
|
||||||
|
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (IsAscii(stream))
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return Encoding.ASCII;
|
||||||
|
}
|
||||||
|
|
||||||
|
#if !NET462_OR_GREATER
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (IsLatin1(stream))
|
||||||
|
{
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return Encoding.Latin1;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
return Encoding.Default;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsUtf8(Stream stream)
|
||||||
|
{
|
||||||
|
int b;
|
||||||
|
while ((b = stream.ReadByte()) != -1)
|
||||||
|
{
|
||||||
|
if ((b & 0x80) == 0) // 0xxxxxxx
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
|
||||||
|
{
|
||||||
|
if (!IsValidUtf8ContinuationByte(stream)) return false;
|
||||||
|
}
|
||||||
|
else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
|
||||||
|
{
|
||||||
|
if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
|
||||||
|
}
|
||||||
|
else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
||||||
|
{
|
||||||
|
if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsValidUtf8ContinuationByte(Stream stream)
|
||||||
|
{
|
||||||
|
int b = stream.ReadByte();
|
||||||
|
return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsUtf16(Stream stream, out Encoding encoding)
|
||||||
|
{
|
||||||
|
encoding = Encoding.Default;
|
||||||
|
|
||||||
|
if (stream.Length >= 2)
|
||||||
|
{
|
||||||
|
// Read the first two bytes
|
||||||
|
int b0 = stream.ReadByte();
|
||||||
|
int b1 = stream.ReadByte();
|
||||||
|
|
||||||
|
// Check for UTF-16 BOM (Byte Order Mark)
|
||||||
|
if (b0 == 0xFF && b1 == 0xFE)
|
||||||
|
{
|
||||||
|
encoding = Encoding.Unicode;
|
||||||
|
return true; // UTF-16LE
|
||||||
|
}
|
||||||
|
if (b0 == 0xFE && b1 == 0xFF)
|
||||||
|
{
|
||||||
|
encoding = Encoding.BigEndianUnicode;
|
||||||
|
return true; // UTF-16BE
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset the stream position
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
|
||||||
|
// Check for common UTF-16 patterns
|
||||||
|
byte[] buffer = new byte[2];
|
||||||
|
while (stream.Read(buffer, 0, 2) == 2)
|
||||||
|
{
|
||||||
|
if (buffer[0] == 0x00 && buffer[1] != 0x00)
|
||||||
|
{
|
||||||
|
encoding = Encoding.BigEndianUnicode;
|
||||||
|
return true; // Likely UTF-16BE
|
||||||
|
}
|
||||||
|
if (buffer[0] != 0x00 && buffer[1] == 0x00)
|
||||||
|
{
|
||||||
|
encoding = Encoding.Unicode;
|
||||||
|
return true; // Likely UTF-16LE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsAscii(Stream stream)
|
||||||
|
{
|
||||||
|
int b;
|
||||||
|
while ((b = stream.ReadByte()) != -1)
|
||||||
|
{
|
||||||
|
// Latin1 includes byte values from 0x00 to 0xFF
|
||||||
|
if (b > 0x7F)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsLatin1(Stream stream)
|
||||||
|
{
|
||||||
|
int b;
|
||||||
|
while ((b = stream.ReadByte()) != -1)
|
||||||
|
{
|
||||||
|
// Latin1 includes byte values from 0x00 to 0xFF
|
||||||
|
if (b > 0xFF)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
#endregion Buffer classes
|
#endregion Buffer classes
|
||||||
@@ -540,180 +661,6 @@ namespace QUT.GplexBuffers
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#region guesser
|
|
||||||
#if (!BYTEMODE)
|
|
||||||
// ==============================================================
|
|
||||||
// ============ Encoding Guesser =============
|
|
||||||
// ==============================================================
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class provides a simple finite state automaton that
|
|
||||||
/// scans the file looking for (1) valid UTF-8 byte patterns,
|
|
||||||
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
|
|
||||||
/// The method then guesses whether it is UTF-8 or maybe some
|
|
||||||
/// local machine default encoding. This works well for the
|
|
||||||
/// various Latin encodings.
|
|
||||||
/// </summary>
|
|
||||||
internal class Guesser
|
|
||||||
{
|
|
||||||
ScanBuff buffer;
|
|
||||||
|
|
||||||
public int GuessCodePage() { return Scan(); }
|
|
||||||
|
|
||||||
const int maxAccept = 10;
|
|
||||||
const int initial = 0;
|
|
||||||
const int eofNum = 0;
|
|
||||||
const int goStart = -1;
|
|
||||||
const int INITIAL = 0;
|
|
||||||
const int EndToken = 0;
|
|
||||||
|
|
||||||
#region user code
|
|
||||||
/*
|
|
||||||
* Reads the bytes of a file to determine if it is
|
|
||||||
* UTF-8 or a single-byte code page file.
|
|
||||||
*/
|
|
||||||
public long utfX;
|
|
||||||
public long uppr;
|
|
||||||
#endregion user code
|
|
||||||
|
|
||||||
int state;
|
|
||||||
int currentStart = startState[0];
|
|
||||||
int code;
|
|
||||||
|
|
||||||
#region ScannerTables
|
|
||||||
static int[] startState = new int[] { 11, 0 };
|
|
||||||
|
|
||||||
#region CharacterMap
|
|
||||||
static sbyte[] map = new sbyte[256] {
|
|
||||||
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
||||||
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
||||||
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
||||||
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
||||||
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
static sbyte[][] nextState = new sbyte[][] {
|
|
||||||
new sbyte[] {0, 0, 0, 0, 0, 0},
|
|
||||||
new sbyte[] {-1, -1, 10, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, -1, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 8, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 5, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 6, -1, -1, -1},
|
|
||||||
new sbyte[] {-1, -1, 7, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, -1, 9, -1, -1, -1},
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
new sbyte[] {-1, 1, 2, 3, 4, 2}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
|
|
||||||
// Reason for suppression: cannot have self-reference in array initializer.
|
|
||||||
static Guesser()
|
|
||||||
{
|
|
||||||
nextState[7] = nextState[2];
|
|
||||||
nextState[9] = nextState[2];
|
|
||||||
nextState[10] = nextState[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
int NextState()
|
|
||||||
{
|
|
||||||
if (code == ScanBuff.EndOfFile)
|
|
||||||
return eofNum;
|
|
||||||
else
|
|
||||||
return nextState[state][map[code]];
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
public Guesser(System.IO.Stream file) { SetSource(file); }
|
|
||||||
|
|
||||||
public void SetSource(System.IO.Stream source)
|
|
||||||
{
|
|
||||||
this.buffer = new BuildBuffer(source);
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
|
|
||||||
int Scan()
|
|
||||||
{
|
|
||||||
for (; ; )
|
|
||||||
{
|
|
||||||
int next;
|
|
||||||
state = currentStart;
|
|
||||||
while ((next = NextState()) == goStart)
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
|
|
||||||
while ((next = NextState()) > eofNum)
|
|
||||||
{
|
|
||||||
state = next;
|
|
||||||
code = buffer.Read();
|
|
||||||
}
|
|
||||||
if (state <= maxAccept)
|
|
||||||
{
|
|
||||||
#region ActionSwitch
|
|
||||||
#pragma warning disable 162
|
|
||||||
switch (state)
|
|
||||||
{
|
|
||||||
case eofNum:
|
|
||||||
switch (currentStart)
|
|
||||||
{
|
|
||||||
case 11:
|
|
||||||
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
|
|
||||||
else if (uppr * 10 > utfX) return 0; /* default code page */
|
|
||||||
else return 65001; /* UTF-8 encoding */
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return EndToken;
|
|
||||||
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
|
|
||||||
case 2: // Recognized '{Upper128}', Shortest string "\x80"
|
|
||||||
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
|
|
||||||
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
|
|
||||||
uppr++;
|
|
||||||
break;
|
|
||||||
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
|
|
||||||
uppr += 3;
|
|
||||||
break;
|
|
||||||
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
|
|
||||||
utfX += 3;
|
|
||||||
break;
|
|
||||||
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
|
|
||||||
uppr += 2;
|
|
||||||
break;
|
|
||||||
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
|
|
||||||
utfX += 2;
|
|
||||||
break;
|
|
||||||
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
|
|
||||||
utfX++;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
#pragma warning restore 162
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // end class Guesser
|
|
||||||
|
|
||||||
#endif // !BYTEMODE
|
|
||||||
#endregion
|
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
|
|
||||||
// End of code copied from embedded resource
|
// End of code copied from embedded resource
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
module.exports = {
|
||||||
|
root: true,
|
||||||
|
env: { browser: true, es2020: true },
|
||||||
|
extends: [
|
||||||
|
'eslint:recommended',
|
||||||
|
'plugin:react/recommended',
|
||||||
|
'plugin:react/jsx-runtime',
|
||||||
|
'plugin:react-hooks/recommended',
|
||||||
|
],
|
||||||
|
ignorePatterns: ['dist', '.eslintrc.cjs'],
|
||||||
|
parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
|
||||||
|
settings: { react: { version: '18.3.1' } },
|
||||||
|
plugins: ['react-refresh'],
|
||||||
|
rules: {
|
||||||
|
'react-refresh/only-export-components': [
|
||||||
|
'warn',
|
||||||
|
{ allowConstantExport: true },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
import importPlugin from "eslint-plugin-import";
|
|
||||||
import reactPkg from "eslint-plugin-react";
|
|
||||||
const { configs: reactConfigs } = reactPkg;
|
|
||||||
import js from "@eslint/js";
|
|
||||||
import defineConfig from "./vite.config.js";
|
|
||||||
|
|
||||||
export default [
|
|
||||||
js.configs.recommended,
|
|
||||||
importPlugin.flatConfigs.recommended,
|
|
||||||
reactConfigs.flat.recommended,
|
|
||||||
reactConfigs.flat["jsx-runtime"],
|
|
||||||
{
|
|
||||||
files: ["**/*.{js,jsx}"],
|
|
||||||
languageOptions: {
|
|
||||||
ecmaVersion: "latest",
|
|
||||||
sourceType: "module",
|
|
||||||
globals: {
|
|
||||||
console: "readonly"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
settings: {
|
|
||||||
"import/resolver": {
|
|
||||||
vite: {
|
|
||||||
viteConfig: defineConfig({ mode: "development" })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
rules: {
|
|
||||||
semi: "error",
|
|
||||||
"prefer-const": "error",
|
|
||||||
"no-unused-vars": "warn",
|
|
||||||
"import/no-dynamic-require": "warn",
|
|
||||||
"import/no-nodejs-modules": "warn",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
];
|
|
||||||
+1885
-1431
File diff suppressed because it is too large.
Load diff
@@ -10,37 +10,34 @@
|
|||||||
"preview": "vite preview"
|
"preview": "vite preview"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@angular/material": "^18.2.8",
|
"@angular/material": "^18.0.4",
|
||||||
"@emotion/react": "^11.13.3",
|
"@emotion/react": "^11.11.4",
|
||||||
"@emotion/styled": "^11.13.0",
|
"@emotion/styled": "^11.11.5",
|
||||||
"@fontsource/noto-sans": "^5.1.0",
|
"@fontsource/noto-sans": "^5.0.22",
|
||||||
"@material/data-table": "^15.0.0-canary.423edc3dc.0",
|
"@material/data-table": "^15.0.0-canary.7f224ddd4.0",
|
||||||
"@mui/icons-material": "^6.1.4",
|
"@mui/icons-material": "^5.15.20",
|
||||||
"@mui/material": "^6.1.4",
|
"@mui/material": "^5.15.20",
|
||||||
"@mui/x-data-grid": "^7.20.0",
|
"@mui/x-data-grid": "^7.7.1",
|
||||||
"bulma": "^1.0.2",
|
"bulma": "^1.0.1",
|
||||||
"classnames": "^2.5.1",
|
"classnames": "^2.5.1",
|
||||||
"react": "^18.3.1",
|
"react": "^18.3.1",
|
||||||
"react-data-grid": "^7.0.0-beta.47",
|
"react-data-grid": "^7.0.0-beta.44",
|
||||||
"react-data-table-component": "^7.6.2",
|
"react-data-table-component": "^7.6.2",
|
||||||
"react-dom": "^18.3.1",
|
"react-dom": "^18.3.1",
|
||||||
"react-json-view-lite": "^1.5.0",
|
"react-json-view-lite": "^1.4.0",
|
||||||
"react-pro-sidebar": "^1.1.0",
|
"react-pro-sidebar": "^1.1.0",
|
||||||
"react-use-websocket": "^4.9.0",
|
"react-use-websocket": "^4.8.1",
|
||||||
"rpc-websockets": "^9.0.4"
|
"rpc-websockets": "^7.11.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/react": "^18.3.11",
|
"@types/react": "^18.3.3",
|
||||||
"@types/react-dom": "^18.3.1",
|
"@types/react-dom": "^18.3.0",
|
||||||
"@vitejs/plugin-react": "^4.3.2",
|
"@vitejs/plugin-react": "^4.3.1",
|
||||||
"eslint": "^9.12.0",
|
"eslint": "^8.57.0",
|
||||||
"@eslint/js": "^9.12.0",
|
"eslint-plugin-react": "^7.34.2",
|
||||||
"eslint-plugin-import": "^2.31.0",
|
"eslint-plugin-react-hooks": "^4.6.2",
|
||||||
"eslint-plugin-react": "^7.37.1",
|
"eslint-plugin-react-refresh": "^0.4.7",
|
||||||
"eslint-plugin-react-hooks": "^5.0.0",
|
"react-mui-sidebar": "^1.3.5",
|
||||||
"eslint-plugin-react-refresh": "^0.4.12",
|
"vite": "^5.2.13"
|
||||||
"eslint-import-resolver-vite": "^2.0.1",
|
|
||||||
"react-mui-sidebar": "^1.3.8",
|
|
||||||
"vite": "^5.4.9"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -12,7 +12,7 @@ const QueryStatusView = ({ queryState, details, error }) => {
|
|||||||
|
|
||||||
let header = queryState;
|
let header = queryState;
|
||||||
let headerContent = null;
|
let headerContent = null;
|
||||||
const detailElements = [];
|
let detailElements = [];
|
||||||
let severity = queryState == "Subscribed" ? "success" : "info";
|
let severity = queryState == "Subscribed" ? "success" : "info";
|
||||||
let cursor = details == null ? "default" : "pointer";
|
let cursor = details == null ? "default" : "pointer";
|
||||||
if (error != null) {
|
if (error != null) {
|
||||||
@@ -20,14 +20,14 @@ const QueryStatusView = ({ queryState, details, error }) => {
|
|||||||
cursor = "pointer";
|
cursor = "pointer";
|
||||||
header = `Error Code: ${error.code}`;
|
header = `Error Code: ${error.code}`;
|
||||||
headerContent = error.message;
|
headerContent = error.message;
|
||||||
detailElements.push(<Typography key="error-msg" variant="body1">{error.data.error_message}</Typography>);
|
detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
|
||||||
detailElements.push(<Typography key="error-details" variant="subtitle1">Stack Trace:</Typography>);
|
detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
|
||||||
(error.data.stack_trace ?? []).forEach((line, idx) => detailElements.push(
|
(error.data.stack_trace ?? []).forEach(line => detailElements.push(
|
||||||
<Typography key={`stacktrace-${idx}`} variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
|
<Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
|
||||||
{line}
|
{line}
|
||||||
</Typography>));
|
</Typography>));
|
||||||
} else if (details != null) {
|
} else if (details != null) {
|
||||||
detailElements.push(<Typography key="status-msg" variant="body1">{details}</Typography>);
|
detailElements.push(<Typography variant="body1">{details}</Typography>);
|
||||||
}
|
}
|
||||||
return (
|
return (
|
||||||
<Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
|
<Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
|
||||||
|
|||||||
@@ -29,12 +29,13 @@ if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const target = env.ASPNETCORE_HTTPS_PORT
|
||||||
|
? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
|
||||||
|
: env.ASPNETCORE_URLS
|
||||||
|
? env.ASPNETCORE_URLS.split(';')[0]
|
||||||
|
: 'https://localhost:8443';
|
||||||
|
|
||||||
export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
|
export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
|
||||||
const target = env.ASPNETCORE_HTTPS_PORT
|
|
||||||
? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
|
|
||||||
: env.ASPNETCORE_URLS
|
|
||||||
? env.ASPNETCORE_URLS.split(';')[0]
|
|
||||||
: 'https://localhost:8443';
|
|
||||||
return {
|
return {
|
||||||
plugins: [plugin()],
|
plugins: [plugin()],
|
||||||
resolve: {
|
resolve: {
|
||||||
@@ -67,10 +68,6 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
|
|||||||
'^/ws': {
|
'^/ws': {
|
||||||
target,
|
target,
|
||||||
secure: false
|
secure: false
|
||||||
},
|
|
||||||
'^/browse': {
|
|
||||||
target,
|
|
||||||
secure: false
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
port: 5173,
|
port: 5173,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<InvariantGlobalization>true</InvariantGlobalization>
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
@@ -12,10 +12,10 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="CommandLineParser" Version="2.9.1" />
|
<PackageReference Include="CommandLineParser" Version="2.9.1" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
|
<PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="8.0.18" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
|
<PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="8.0.18" />
|
||||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
|
<PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.3" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
+34
-37
@@ -1,37 +1,34 @@
|
|||||||
namespace Bts.DataStreamWebApp.Server.Controllers;
|
namespace Bts.DataStreamWebApp.Server.Controllers;
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
|
||||||
[Route("/ws")]
|
[Route("/ws")]
|
||||||
public class DataStreamSubscriptionController : ControllerBase {
|
public class DataStreamSubscriptionController : ControllerBase {
|
||||||
readonly ILogger logger;
|
readonly ILogger logger;
|
||||||
readonly IDataStreamFactory streamFactory;
|
readonly IDataStreamFactory streamFactory;
|
||||||
|
|
||||||
public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
|
public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
|
||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
logger.LogInformation("Creating {ControllerType}", GetType().Name);
|
logger.LogInformation("Creating {ControllerType}", GetType().Name);
|
||||||
var streamFactoryList = streamFactories.ToList();
|
streamFactory = DataStreamFactory.Create(streamFactories);
|
||||||
streamFactory = streamFactoryList.Count == 1
|
}
|
||||||
? streamFactoryList[0]
|
|
||||||
: new CompositeDataStreamFactory(streamFactoryList);
|
[Route("/ws")]
|
||||||
}
|
public async Task Get(CancellationToken cancellationToken) {
|
||||||
|
if (!HttpContext.WebSockets.IsWebSocketRequest) {
|
||||||
[Route("/ws")]
|
logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
|
||||||
public async Task Get(CancellationToken cancellationToken) {
|
return;
|
||||||
if (!HttpContext.WebSockets.IsWebSocketRequest) {
|
}
|
||||||
logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
|
logger.LogInformation("Establishing websocket connection...");
|
||||||
return;
|
try {
|
||||||
}
|
using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
|
||||||
logger.LogInformation("Establishing websocket connection...");
|
logger.LogInformation("Websocket connection established.");
|
||||||
try {
|
using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
|
||||||
using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
|
await server.Run(cancellationToken);
|
||||||
logger.LogInformation("Websocket connection established.");
|
} catch (Exception ex) {
|
||||||
using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
|
logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
|
||||||
await server.Run(cancellationToken);
|
}
|
||||||
} catch (Exception ex) {
|
}
|
||||||
logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,7 +2,7 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
|
|||||||
|
|
||||||
using bts.websocket_core.json_rpc;
|
using bts.websocket_core.json_rpc;
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStreamCore;
|
||||||
using Bts.DataStreamWebApp.Server.Messages;
|
using Bts.DataStreamWebApp.Server.Messages;
|
||||||
|
|
||||||
using System.Net.WebSockets;
|
using System.Net.WebSockets;
|
||||||
|
|||||||
@@ -1,13 +1,11 @@
|
|||||||
FROM bluetradingsystems/linux_dotnet7
|
FROM bluetradingsystems/linux_dotnet7
|
||||||
|
|
||||||
ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
|
ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
|
||||||
ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
|
|
||||||
ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
|
ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
|
||||||
ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
|
ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
|
||||||
|
|
||||||
RUN dotnet dev-certs https --trust
|
RUN dotnet dev-certs https --trust
|
||||||
COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
|
COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
|
||||||
COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
|
|
||||||
COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
|
COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
|
||||||
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
|
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
|
||||||
WORKDIR "/home/bts/bin"
|
WORKDIR "/home/bts/bin"
|
||||||
@@ -4,7 +4,7 @@ using BCG;
|
|||||||
|
|
||||||
using bts.websocket_core.json_rpc;
|
using bts.websocket_core.json_rpc;
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStreamCore;
|
||||||
using Bts.DataStreamExpressions;
|
using Bts.DataStreamExpressions;
|
||||||
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
using System.Diagnostics.CodeAnalysis;
|
||||||
@@ -60,7 +60,7 @@ public class SubscribeRequest {
|
|||||||
}
|
}
|
||||||
Exception? exception = null;
|
Exception? exception = null;
|
||||||
try {
|
try {
|
||||||
stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
|
stream = exp.Evaluate(streamFactory) as ViewStream;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
exception = ex;
|
exception = ex;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,8 +79,8 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
|
|||||||
.AddSparkApi(config.GetSection("SparkApi"))
|
.AddSparkApi(config.GetSection("SparkApi"))
|
||||||
.AddMasterWorker(config.GetSection("MasterWorker"))
|
.AddMasterWorker(config.GetSection("MasterWorker"))
|
||||||
.AddViewStreamDatabase()
|
.AddViewStreamDatabase()
|
||||||
.AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams"))
|
.AddPrimitiveStreams(config.GetSection("PrimitiveStreams"))
|
||||||
.AddOhlcvStreams(config.GetSection("StreamDatabase"))
|
.AddMarketDataStreams()
|
||||||
.AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
|
.AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
|
||||||
.AddCompositeDataStreamFactory()
|
.AddCompositeDataStreamFactory()
|
||||||
.AddPubSubClient(config.GetSection("PubSub"));
|
.AddPubSubClient(config.GetSection("PubSub"));
|
||||||
|
|||||||
@@ -116,7 +116,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
|
"natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
|
||||||
"dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\")"
|
"dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", exclusive_end: true)"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"natural_language": "Retrieve NVIDIA L1 books from Oct 3",
|
"natural_language": "Retrieve NVIDIA L1 books from Oct 3",
|
||||||
@@ -147,10 +147,10 @@
|
|||||||
"interval (sample frequency of the data), ",
|
"interval (sample frequency of the data), ",
|
||||||
"levels (the number of price levels to examine)",
|
"levels (the number of price levels to examine)",
|
||||||
"begin and end (inclusive beginning and end of the time window to query, respectively), ",
|
"begin and end (inclusive beginning and end of the time window to query, respectively), ",
|
||||||
|
"inclusive_end (a bool flag that, when true, makes the end time inclusive). ",
|
||||||
"Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
|
"Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
|
||||||
"Depending on the wording of the request, some arguments may be omitted. ",
|
"Depending on the wording of the request, some arguments may be omitted. ",
|
||||||
"The begin, end, and interval args should be expressed in iso 8601 format. ",
|
"The begin, end, and interval args should be expressed in iso 8601 format. ",
|
||||||
"If the prompt doesn't contain an explicit interval, please choose a reasonable value depending on the duration between begin and end.",
|
|
||||||
"Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
|
"Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
|
||||||
"In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
|
"In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
|
||||||
"which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
|
"which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
|
<PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.7" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
@@ -3,15 +3,12 @@
|
|||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
|
|
||||||
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
|
||||||
public static class HostBuilderExtensions {
|
public static class HostBuilderExtensions {
|
||||||
public static IHostBuilder CreateBtsBuilder(string[] args) {
|
public static IHostBuilder CreateBtsBuilder(string[] args) {
|
||||||
// Obtain calling assembly name here, rather than in the lambda where it is actually needed,
|
// The calling assembly's name serves as the "appsettings" json file prefix.
|
||||||
// because want the name of the assembly that called this function, not the assembly that will eventually invoke
|
|
||||||
// the lambda...that will be the Microsoft.Extensions.Hosting assembly.
|
|
||||||
// Get the name of the assembly that called this function. That will act as the "appsettings"
|
|
||||||
// json file prefix
|
|
||||||
var exeName = GetCallingAssemblyName();
|
var exeName = GetCallingAssemblyName();
|
||||||
return Host.CreateDefaultBuilder(args)
|
return Host.CreateDefaultBuilder(args)
|
||||||
.ConfigureAppConfiguration((context, configBuilder) =>
|
.ConfigureAppConfiguration((context, configBuilder) =>
|
||||||
@@ -56,4 +53,21 @@ public static class HostBuilderExtensions {
|
|||||||
?.GetName()
|
?.GetName()
|
||||||
?.Name
|
?.Name
|
||||||
?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
|
?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Separates command line args from config args, using '--' as the delimiter. Args
|
||||||
|
/// before the delimiter are considered command line args, while those after it are
|
||||||
|
/// config args.
|
||||||
|
/// </summary>
|
||||||
|
public static (string[] Args, string[] ConfigArgs) SplitConfigArgs(string[] args) {
|
||||||
|
int splitIdx = Array.IndexOf(args, "--");
|
||||||
|
if (splitIdx == -1) {
|
||||||
|
return (args, Array.Empty<string>());
|
||||||
|
}
|
||||||
|
var cfgArgs = new string[args.Length - splitIdx - 1];
|
||||||
|
Array.Copy(args, splitIdx + 1, cfgArgs, 0, cfgArgs.Length);
|
||||||
|
var nonCfgArgs = new string[args.Length - cfgArgs.Length - 1];
|
||||||
|
Array.Copy(args, 0, nonCfgArgs, 0, nonCfgArgs.Length);
|
||||||
|
return (nonCfgArgs, cfgArgs);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net7.0</TargetFramework>
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
public class AddColumnRow : IBlueRow {
|
||||||
|
readonly IBlueRow inputRow;
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
readonly object? value;
|
||||||
|
readonly int valueIndex;
|
||||||
|
|
||||||
|
public AddColumnRow(IBlueRow inputRow, IBlueTableSchema outputSchema, object? value, int valueIndex) {
|
||||||
|
this.inputRow = inputRow;
|
||||||
|
this.outputSchema = outputSchema;
|
||||||
|
this.value = value;
|
||||||
|
this.valueIndex = valueIndex;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object? this[int index] {
|
||||||
|
get {
|
||||||
|
if (index == valueIndex) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
if (index >= valueIndex) {
|
||||||
|
index--;
|
||||||
|
}
|
||||||
|
return inputRow[index];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRow.schema => outputSchema;
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
public class AddColumnTable : IBlueRowCollection {
|
||||||
|
readonly IBlueRowCollection inputRows;
|
||||||
|
readonly Func<IBlueRow, object?> getValue;
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
readonly int valueIndex;
|
||||||
|
|
||||||
|
public AddColumnTable(IBlueRowCollection inputRows, string columnName, Type columnType, Func<IBlueRow, object?> getValue) {
|
||||||
|
this.inputRows = inputRows;
|
||||||
|
this.getValue = getValue;
|
||||||
|
var inputCols = inputRows.schema.columnInfo;
|
||||||
|
var outputCols = new ColumnInfo[inputCols.Length + 1];
|
||||||
|
Array.Copy(inputCols, outputCols, inputCols.Length);
|
||||||
|
outputCols[^1] = new(columnName, inputCols.Length, columnType);
|
||||||
|
outputSchema = new BlueTableSchema(outputCols, "AddColumnTable");
|
||||||
|
valueIndex = inputCols.Length;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] {
|
||||||
|
get {
|
||||||
|
var r = inputRows[id];
|
||||||
|
return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => outputSchema;
|
||||||
|
|
||||||
|
public int Count => inputRows.Count;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
inputRows.containsID(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
foreach (var r in inputRows) {
|
||||||
|
yield return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) {
|
||||||
|
var r = inputRows.GetRow(index);
|
||||||
|
return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
if (!inputRows.containsID(id)) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
var r = inputRows[id];
|
||||||
|
return BlueRowEqualityComparer.Equals(new AddColumnRow(r, outputSchema, getValue(r), valueIndex), item)
|
||||||
|
? inputRows.IndexOf(r)
|
||||||
|
: -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("AddColumnsStream")]
|
||||||
|
public class AddColumnsStream: ViewStream {
|
||||||
|
public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
|
||||||
|
ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
#nullable enable
|
||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class ApplyStream : ViewStream {
|
||||||
|
readonly ViewStream source;
|
||||||
|
readonly Func<IBlueRow, object?> apply;
|
||||||
|
IBlueTableSchema? outputSchema;
|
||||||
|
BlueTableImmutable? lastFired;
|
||||||
|
IDisposable? sub;
|
||||||
|
|
||||||
|
public ApplyStream(ViewStream source, IFunctionEvaluator applyEvaluator) {
|
||||||
|
this.source = source;
|
||||||
|
apply = Apply;
|
||||||
|
|
||||||
|
object? Apply(IBlueRow r) {
|
||||||
|
var res = applyEvaluator.Evaluate(string.Empty, new StreamArgument[] { (null, 0, r) });
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApplyStream(ViewStream source, Func<IBlueRow, object?> apply) {
|
||||||
|
this.source = source;
|
||||||
|
this.apply = apply;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
sub = source.Subscribe(OnSourceChanged);
|
||||||
|
|
||||||
|
void OnSourceChanged(IDataPacket dp) {
|
||||||
|
Lazy<BlueTableImmutable> lastFired = new(() => this.lastFired ??= new(Log.notnull(outputSchema)));
|
||||||
|
Lazy<BlueTableImmutable.DataPacketBuilder> bd = new(() => lastFired.Value.ToDataPacketBuilder());
|
||||||
|
|
||||||
|
foreach (var r in dp.deletedRows) {
|
||||||
|
var res = apply(r);
|
||||||
|
if (res is IBlueRow blueRow) {
|
||||||
|
bd.Value.Remove(blueRow.Id());
|
||||||
|
} else if (res is IBlueRowCollection blueRows) {
|
||||||
|
foreach (var br in blueRows) {
|
||||||
|
bd.Value.Remove(br.Id());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foreach (var r in dp.addedRows) {
|
||||||
|
var res = apply(r);
|
||||||
|
if (res is IBlueRow blueRow) {
|
||||||
|
outputSchema ??= blueRow.schema;
|
||||||
|
bd.Value.AddOrUpdate(blueRow);
|
||||||
|
} else if (res is IBlueRowCollection blueRows) {
|
||||||
|
outputSchema ??= blueRows.schema;
|
||||||
|
foreach (var br in blueRows) {
|
||||||
|
bd.Value.AddOrUpdate(br);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (outputSchema is null) {
|
||||||
|
// We get the outputSchema from the result of the first apply, if it's null, we
|
||||||
|
// cannot determine the output data type.
|
||||||
|
// TODO: maybe support specifying return type in DSL?
|
||||||
|
Log.notnull(res);
|
||||||
|
}
|
||||||
|
var resType = res?.GetType();
|
||||||
|
if (outputSchema is null) {
|
||||||
|
Log.notnull(resType);
|
||||||
|
Log.assert(ColumnInfo.isAllowedType(resType));
|
||||||
|
outputSchema = new BlueTableSchema(new ColumnInfo[] {
|
||||||
|
new("id", 0, typeof(int)),
|
||||||
|
new("result", 1, resType),
|
||||||
|
}, dp.newRows.schema.tableName + "_Apply");
|
||||||
|
}
|
||||||
|
bd.Value.AddOrUpdate(new BlueRow(outputSchema, new object?[] { r.Id(), res }));
|
||||||
|
// Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (bd.IsValueCreated) {
|
||||||
|
myFireEvent(bd.Value.ToDataPacket(out this.lastFired));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
public class ApplyTable : IBlueRowCollection {
|
||||||
|
readonly IBlueRowCollection source;
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
readonly List<IBlueRow> outputRows;
|
||||||
|
readonly Dictionary<int, int> idToIndex = new();
|
||||||
|
readonly Func<IBlueRow, IBlueRow> applyFn;
|
||||||
|
|
||||||
|
public ApplyTable(IBlueRowCollection source, IBlueTableSchema? outputSchema, Func<IBlueRow, IBlueRow> applyFn) {
|
||||||
|
this.source = source;
|
||||||
|
this.applyFn = applyFn;
|
||||||
|
outputRows = new(source.Count);
|
||||||
|
foreach (var r in source) {
|
||||||
|
outputRows.Add(applyFn(r));
|
||||||
|
}
|
||||||
|
if (outputRows.Count > 0) {
|
||||||
|
outputSchema ??= outputRows[0].schema;
|
||||||
|
}
|
||||||
|
this.outputSchema = outputSchema ?? source.schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] => outputRows[idToIndex[id]];
|
||||||
|
|
||||||
|
IBlueRow IList<IBlueRow>.this[int index] {
|
||||||
|
get => GetRow(index);
|
||||||
|
set => throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => outputSchema;
|
||||||
|
|
||||||
|
public int Count => source.Count;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
source.containsID(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() =>
|
||||||
|
outputRows.GetEnumerator();
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
outputRows[index];
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
if (!source.containsID(id)) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
var inputRow = source[id];
|
||||||
|
int idx = source.IndexOf(inputRow);
|
||||||
|
if (BlueRowEqualityComparer.Equals(outputRows[idx], item)) {
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
namespace Bts.DataStream.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
using Bts.PubSub;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
public static class AsyncViewStream {
|
||||||
|
public static ViewStream Create(IWorkScheduler workScheduler, Func<Task<ViewStream?>> factory) {
|
||||||
|
return ViewStreamFactory.Create(Subscribe);
|
||||||
|
|
||||||
|
IDisposable Subscribe(Action<IDataPacket> fire) {
|
||||||
|
var cts = new CancellationTokenSource();
|
||||||
|
var tcs = new TaskCompletionSource<IDisposable>();
|
||||||
|
var syncRoot = new object();
|
||||||
|
IDisposable? streamSub = null;
|
||||||
|
workScheduler.Post(Init);
|
||||||
|
return Disposable.Create(Dispose);
|
||||||
|
|
||||||
|
async void Init() {
|
||||||
|
lock (syncRoot) {
|
||||||
|
if (cts.IsCancellationRequested) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var stream = await factory();
|
||||||
|
if (stream is null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
var tempStreamSub = stream.Subscribe(fire);
|
||||||
|
lock (syncRoot) {
|
||||||
|
if (cts.IsCancellationRequested) {
|
||||||
|
tempStreamSub.Dispose();
|
||||||
|
} else {
|
||||||
|
streamSub = tempStreamSub;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void Dispose() {
|
||||||
|
IDisposable? tempStreamSub;
|
||||||
|
lock (syncRoot) {
|
||||||
|
tempStreamSub = streamSub;
|
||||||
|
if (tempStreamSub is null) {
|
||||||
|
cts.Cancel();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
streamSub = null;
|
||||||
|
}
|
||||||
|
tempStreamSub?.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
using bts.utils;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class BlueAddIndexColumnTable : IBlueRowCollection {
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
readonly IList<object?>[] columnValues;
|
||||||
|
readonly int count;
|
||||||
|
|
||||||
|
BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
|
||||||
|
this.schema = schema;
|
||||||
|
this.columnValues = columnValues;
|
||||||
|
this.count = count ?? columnValues[0].Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRowCollection.schema => schema;
|
||||||
|
|
||||||
|
public int Count => count;
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
id > 0 && id <= count
|
||||||
|
? GetRow(id - 1)
|
||||||
|
: throw new KeyNotFoundException();
|
||||||
|
|
||||||
|
public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
|
||||||
|
if (columnInfos.Any(c => c.colName == "id")) {
|
||||||
|
throw new ArgumentException("'id' column already exists", nameof(columnInfos));
|
||||||
|
}
|
||||||
|
if (columnInfos.Length != columnValues.Length) {
|
||||||
|
throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
|
||||||
|
}
|
||||||
|
int count = columnValues.FirstOrDefault()?.Count ?? 0;
|
||||||
|
var origCols = columnInfos;
|
||||||
|
var origColValues = columnValues;
|
||||||
|
columnInfos = new ColumnInfo[origCols.Length + 1];
|
||||||
|
columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
|
||||||
|
columnValues = new IList<object?>[origCols.Length + 1];
|
||||||
|
columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
|
||||||
|
for (int i = 1; i < columnInfos.Length; i++) {
|
||||||
|
columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
|
||||||
|
columnValues[i] = origColValues[i - 1];
|
||||||
|
}
|
||||||
|
var schema = new BlueTableSchema(columnInfos, tableName);
|
||||||
|
return new(schema, columnValues, count);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
|
||||||
|
int count = columnValues.FirstOrDefault()?.Count ?? 0;
|
||||||
|
int? idColIdx = schema.TryResolveIndices("id");
|
||||||
|
var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
|
||||||
|
var columnInfos = schema.columnInfo;
|
||||||
|
if (idColIdx.HasValue) {
|
||||||
|
// Replace existing id column. columnValues may or may not contain the old ids (if its
|
||||||
|
// length is one less column than that of columnInfos, it doesn't contain them).
|
||||||
|
switch (columnValues.Length - schema.columnInfo.Length) {
|
||||||
|
case 0:
|
||||||
|
columnValues[idColIdx.Value] = idColValues;
|
||||||
|
break;
|
||||||
|
case -1:
|
||||||
|
columnValues = ArrayUtils.InsertExact(columnValues, idColIdx.Value, true, idColValues);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (columnValues.Length != columnInfos.Length) {
|
||||||
|
throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
|
||||||
|
}
|
||||||
|
var origColInfos = columnInfos;
|
||||||
|
columnInfos = new ColumnInfo[columnInfos.Length + 1];
|
||||||
|
columnInfos[0] = new("id", 0, typeof(int));
|
||||||
|
for (int i = 1; i < columnInfos.Length; i++) {
|
||||||
|
var c = origColInfos[i - 1];
|
||||||
|
columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
|
||||||
|
}
|
||||||
|
columnValues = ArrayUtils.InsertExact(columnValues, 0, true, idColValues);
|
||||||
|
}
|
||||||
|
return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
id > 0 && id <= count;
|
||||||
|
|
||||||
|
public IList<object?> GetColumn(int index) =>
|
||||||
|
columnValues[index];
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
new BlueRowColumnWrapper(schema, index, columnValues);
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
|
||||||
|
? id - 1
|
||||||
|
: -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
int count = Count;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
yield return GetRow(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
public class BlueRowColumnWrapper : IBlueRow {
|
||||||
|
readonly int index;
|
||||||
|
readonly IList<object?>[] columns;
|
||||||
|
|
||||||
|
public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
|
||||||
|
this.schema = schema;
|
||||||
|
this.index = index;
|
||||||
|
this.columns = columns;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object? this[int index] =>
|
||||||
|
columns[index][this.index];
|
||||||
|
|
||||||
|
public IBlueTableSchema schema { get; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using mbts.utils;
|
||||||
|
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class BlueIndexedColumnTable : IBlueRowCollection {
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
readonly IList<object?>[] columnValues;
|
||||||
|
readonly Dictionary<int, int> rowIdToIndex;
|
||||||
|
|
||||||
|
public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
|
||||||
|
this.schema = schema;
|
||||||
|
this.columnValues = columnValues;
|
||||||
|
int idColumnIndex = schema.ResolveIndices("id");
|
||||||
|
rowIdToIndex = columnValues[idColumnIndex]
|
||||||
|
.Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
|
||||||
|
.ToDictionary();
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
rowIdToIndex.TryGetValue(id, out var idx)
|
||||||
|
? GetRow(idx)
|
||||||
|
: throw new KeyNotFoundException();
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRowCollection.schema => schema;
|
||||||
|
|
||||||
|
public int Count => rowIdToIndex.Count;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
rowIdToIndex.ContainsKey(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
int count = Count;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
yield return GetRow(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
new BlueRowColumnWrapper(schema, index, columnValues);
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) =>
|
||||||
|
rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
|
||||||
|
? idx
|
||||||
|
: -1;
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using System;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("CSVReaderStream")]
|
||||||
|
public class CSVReaderStream : ViewStream {
|
||||||
|
public CSVReaderStream(string file_name)
|
||||||
|
: this(file_name, false) { }
|
||||||
|
public CSVReaderStream(string file_name, bool guess_schema) {
|
||||||
|
ViewStream rs = new BCG.CSVReaderStream(file_name);
|
||||||
|
if (guess_schema) {
|
||||||
|
rs = new SchemaGuesserStream(rs);
|
||||||
|
}
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("CSVSaverStream")]
|
||||||
|
public class CSVSaverStream: ViewStream {
|
||||||
|
public CSVSaverStream(ViewStream parent_str, string file_name) {
|
||||||
|
BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class CastStream : ViewStream, IDisposable {
|
||||||
|
readonly ViewStream inputStream;
|
||||||
|
readonly (string, Type)[] casts;
|
||||||
|
IDisposable? sub;
|
||||||
|
|
||||||
|
public CastStream(ViewStream inputStream, params (string, Type)[] casts) {
|
||||||
|
this.inputStream = inputStream;
|
||||||
|
this.casts = casts;
|
||||||
|
}
|
||||||
|
|
||||||
|
~CastStream() =>
|
||||||
|
Dispose();
|
||||||
|
|
||||||
|
public void Dispose() {
|
||||||
|
GC.SuppressFinalize(this);
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
inputStream.Subscribe(OnInputChanged);
|
||||||
|
|
||||||
|
void OnInputChanged(IDataPacket dp) {
|
||||||
|
var dpOut = new DataPacket(
|
||||||
|
new CastTable(dp.newRows, casts),
|
||||||
|
new CastTable(dp.addedRows, casts),
|
||||||
|
new CastTable(dp.deletedRows, casts));
|
||||||
|
myFireEvent(dpOut);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
public class CastTable : IBlueRowCollection {
|
||||||
|
readonly IBlueRowCollection rows;
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
readonly (int Index, Func<IBlueRow, object?> GetValue)[] casts;
|
||||||
|
readonly IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer =
|
||||||
|
TupleComparer.Comparer<int, Func<IBlueRow, object?>>(item1Ordinal: 0);
|
||||||
|
|
||||||
|
public CastTable(IBlueRowCollection rows, params (string, Type)[] casts) {
|
||||||
|
this.rows = rows;
|
||||||
|
var inputSchema = rows.schema;
|
||||||
|
var inputCols = inputSchema.columnInfo;
|
||||||
|
var castIndices = inputSchema.ResolveIndices(Array.ConvertAll(casts, c => c.Item1));
|
||||||
|
this.casts = new (int Index, Func<IBlueRow, object?> GetValue)[casts.Length];
|
||||||
|
var outputCols = inputCols.ToArray();
|
||||||
|
for (int i = 0; i < casts.Length; i++) {
|
||||||
|
var colIdx = castIndices[i];
|
||||||
|
this.casts[i] = (colIdx, GetCaster(casts[i].Item2));
|
||||||
|
var inputCol = inputCols[colIdx];
|
||||||
|
Log.assert(inputCol.colNo == colIdx);
|
||||||
|
outputCols[colIdx] = new(inputCol.colName, inputCol.colNo, casts[i].Item2);
|
||||||
|
|
||||||
|
Func<IBlueRow, object?> GetCaster(Type to) {
|
||||||
|
if (to == typeof(short)) {
|
||||||
|
return r => r.GetInt16(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(int)) {
|
||||||
|
return r => r.GetInt32(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(long)) {
|
||||||
|
return r => r.GetInt64(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(ushort)) {
|
||||||
|
return r => r.GetUInt16(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(uint)) {
|
||||||
|
return r => r.GetUInt32(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(ulong)) {
|
||||||
|
return r => r.GetUInt64(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(double)) {
|
||||||
|
return r => r.GetFloat64(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(float)) {
|
||||||
|
return r => r.GetFloat32(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(DateTime)) {
|
||||||
|
return r => r.GetDateTime(colIdx);
|
||||||
|
}
|
||||||
|
if (to == typeof(TimeSpan)) {
|
||||||
|
return r => r.GetTimeSpan(colIdx);
|
||||||
|
}
|
||||||
|
throw new NotImplementedException($"CastTable cast not implemented for (out) type {to}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
outputSchema = new BlueTableSchema(inputSchema.tableName + "_Cast", outputCols);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
new Row(rows[id], outputSchema, casts, castComparer);
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => outputSchema;
|
||||||
|
|
||||||
|
public int Count => rows.Count;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
rows.containsID(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
foreach (var r in rows) {
|
||||||
|
yield return new Row(r, outputSchema, casts, castComparer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
new Row(rows.GetRow(index), outputSchema, casts, castComparer);
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
var inputRow = rows[item.Id()];
|
||||||
|
return BlueRowEqualityComparer.Equals(item, new Row(inputRow, outputSchema, casts, castComparer))
|
||||||
|
? rows.IndexOf(inputRow)
|
||||||
|
: -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
class Row : IBlueRow {
|
||||||
|
readonly IBlueRow source;
|
||||||
|
readonly (int Index, Func<IBlueRow, object?> GetValue)[] casts;
|
||||||
|
readonly IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer;
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
|
||||||
|
public Row(IBlueRow source, IBlueTableSchema outputSchema, (int Index, Func<IBlueRow, object?> GetValue)[] casts, IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer) {
|
||||||
|
this.source = source;
|
||||||
|
this.casts = casts;
|
||||||
|
this.castComparer = castComparer;
|
||||||
|
this.outputSchema = outputSchema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object? this[int index] {
|
||||||
|
get {
|
||||||
|
var castIdx = casts.BinarySearch((index, null!), castComparer);
|
||||||
|
return castIdx > -1
|
||||||
|
? casts[castIdx].GetValue(source)
|
||||||
|
: source[index];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRow.schema => outputSchema;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
|
||||||
|
using System.Reactive.Linq;
|
||||||
|
|
||||||
|
public class ColumnInjectionStream : ViewStream {
|
||||||
|
public ColumnInjectionStream(ViewStream inputStream, IInjectedColumn[] outputColumns) {
|
||||||
|
IBlueTableSchema? outputSchema = null;
|
||||||
|
mySetRealSource(inputStream.AsObservable().Select(Transform).AsViewStream());
|
||||||
|
|
||||||
|
IDataPacket Transform(IDataPacket dp) {
|
||||||
|
outputSchema ??= ColumnInjectionTable.GetOutputSchema(dp.Schema, outputColumns);
|
||||||
|
var newRows = new ColumnInjectionTable(dp.newRows, outputSchema, outputColumns);
|
||||||
|
var addedRows = new ColumnInjectionTable(dp.addedRows, outputSchema, outputColumns);
|
||||||
|
var deletedRows = new ColumnInjectionTable(dp.deletedRows, outputSchema, outputColumns);
|
||||||
|
return new DataPacket(newRows, addedRows, deletedRows);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
public interface IInjectedColumn {
|
||||||
|
string Name { get; }
|
||||||
|
Type Type { get; }
|
||||||
|
int Ordinal { get; }
|
||||||
|
ColumnInfo ColumnInfo => new(Name, Ordinal, Type);
|
||||||
|
object? GetValue(IBlueRow row);
|
||||||
|
}
|
||||||
|
|
||||||
|
public interface IInjectedColumn<T> : IInjectedColumn {
|
||||||
|
Type IInjectedColumn.Type => typeof(T);
|
||||||
|
|
||||||
|
new T? GetValue(IBlueRow row);
|
||||||
|
|
||||||
|
object? IInjectedColumn.GetValue(IBlueRow row) =>
|
||||||
|
GetValue(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class InjectedColumn : IInjectedColumn {
|
||||||
|
readonly Func<IBlueRow, object?> getValue;
|
||||||
|
|
||||||
|
public InjectedColumn(string name, int index, Type type, Func<IBlueRow, object?> getValue) {
|
||||||
|
Name = name;
|
||||||
|
Ordinal = index;
|
||||||
|
Type = type;
|
||||||
|
this.getValue = getValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public string Name { get; }
|
||||||
|
public int Ordinal { get; }
|
||||||
|
public Type Type { get; }
|
||||||
|
|
||||||
|
public object? GetValue(IBlueRow row) =>
|
||||||
|
getValue(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class InjectedColumn<T> : IInjectedColumn<T> {
|
||||||
|
readonly Func<IBlueRow, T?> getValue;
|
||||||
|
|
||||||
|
public InjectedColumn(string name, int index, Func<IBlueRow, T?> getValue) {
|
||||||
|
Name = name;
|
||||||
|
Ordinal = index;
|
||||||
|
this.getValue = getValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public string Name { get; }
|
||||||
|
public int Ordinal { get; }
|
||||||
|
|
||||||
|
public T? GetValue(IBlueRow row) =>
|
||||||
|
getValue(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ColumnInjectionTable : IBlueRowCollection {
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
readonly IInjectedColumn[] outputColumns;
|
||||||
|
readonly IBlueRowCollection source;
|
||||||
|
|
||||||
|
public ColumnInjectionTable(IBlueRowCollection source, IBlueTableSchema outputSchema, IInjectedColumn[] outputColumns) {
|
||||||
|
this.source = source;
|
||||||
|
this.outputSchema = outputSchema;
|
||||||
|
this.outputColumns = outputColumns;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ColumnInjectionTable Create(IBlueRowCollection source, IEnumerable<IInjectedColumn> injectedColumns) {
|
||||||
|
var inputCols = source.schema.columnInfo;
|
||||||
|
var outputColumns = new List<IInjectedColumn>(inputCols.Length + 1);
|
||||||
|
var outputColInfos = new List<ColumnInfo>(inputCols.Length + 1);
|
||||||
|
int injectionIdx = 0;
|
||||||
|
using var en = injectedColumns.GetEnumerator();
|
||||||
|
bool hasMoreInjections = en.MoveNext();
|
||||||
|
for (int i = 0; i < inputCols.Length;) {
|
||||||
|
if (hasMoreInjections && en.Current.Ordinal <= i) {
|
||||||
|
outputColInfos.Add(new(en.Current.Name, outputColumns.Count, en.Current.Type));
|
||||||
|
outputColumns.Add(en.Current);
|
||||||
|
injectionIdx++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
var ci = inputCols[i];
|
||||||
|
outputColInfos.Add(new(ci.colName, outputColumns.Count, ci.colType));
|
||||||
|
outputColumns.Add(new InjectedColumn(ci.colName, outputColumns.Count, ci.colType, r => r[ci.colNo]));
|
||||||
|
}
|
||||||
|
var outputSchema = new BlueTableSchema(outputColInfos.ToArray(), $"ColumnInjection({source.schema.tableName})");
|
||||||
|
return new(source, outputSchema, outputColumns.ToArray());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, IEnumerable<IInjectedColumn> injectedColumns) {
|
||||||
|
var inputCols = inputSchema.columnInfo;
|
||||||
|
var outputColumns = new List<IInjectedColumn>(inputCols.Length + 1);
|
||||||
|
var outputColInfos = new List<ColumnInfo>(inputCols.Length + 1);
|
||||||
|
using var en = injectedColumns.GetEnumerator();
|
||||||
|
bool hasMoreInjections = en.MoveNext();
|
||||||
|
for (int i = 0; i < inputCols.Length;) {
|
||||||
|
if (hasMoreInjections && en.Current.Ordinal <= i) {
|
||||||
|
outputColInfos.Add(new(en.Current.Name, outputColumns.Count, en.Current.Type));
|
||||||
|
outputColumns.Add(en.Current);
|
||||||
|
hasMoreInjections = en.MoveNext();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
var ci = inputCols[i];
|
||||||
|
outputColInfos.Add(new(ci.colName, outputColumns.Count, ci.colType));
|
||||||
|
outputColumns.Add(new InjectedColumn(ci.colName, outputColumns.Count, ci.colType, r => r[ci.colNo]));
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
return new BlueTableSchema(outputColInfos.ToArray(), $"ColumnInjection({inputSchema.tableName})");
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
new ColumnInjectionRow(source[id], outputSchema, outputColumns);
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => throw new System.NotImplementedException();
|
||||||
|
|
||||||
|
public int Count => source.Count;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
source.containsID(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
foreach (var r in source) {
|
||||||
|
yield return new ColumnInjectionRow(r, outputSchema, outputColumns);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
new ColumnInjectionRow(source.GetRow(index), outputSchema, outputColumns);
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
return source.containsID(id) && BlueRowEqualityComparer.Equals(item, this[id])
|
||||||
|
? source.IndexOf(source[id])
|
||||||
|
: -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
class ColumnInjectionRow : IBlueRow {
|
||||||
|
readonly IBlueRow row;
|
||||||
|
readonly IInjectedColumn[] columns;
|
||||||
|
|
||||||
|
public ColumnInjectionRow(IBlueRow row, IBlueTableSchema schema, IInjectedColumn[] columns) {
|
||||||
|
this.row = row;
|
||||||
|
this.schema = schema;
|
||||||
|
this.columns = columns;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema { get; }
|
||||||
|
|
||||||
|
public object? this[int index] =>
|
||||||
|
columns[index].GetValue(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
public class ColumnMaskRow : IBlueRow {
|
||||||
|
readonly IBlueRow inputRow;
|
||||||
|
readonly (string? Name, int Index)[] columnMap;
|
||||||
|
|
||||||
|
public ColumnMaskRow(IBlueRow inputRow, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
|
||||||
|
this.inputRow = inputRow;
|
||||||
|
this.columnMap = columnMap;
|
||||||
|
this.schema = schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema { get; }
|
||||||
|
|
||||||
|
public object? this[int index] =>
|
||||||
|
inputRow[columnMap[index].Index];
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
|
||||||
|
public class ColumnMaskStream : ViewStream {
|
||||||
|
readonly ViewStream source;
|
||||||
|
readonly string? columnMaskExpression;
|
||||||
|
|
||||||
|
IDisposable? sub;
|
||||||
|
IBlueTableSchema? outputSchema;
|
||||||
|
// Each item in this array represents an output column as a tuple of
|
||||||
|
// new name, and corresponding index in the input schema.
|
||||||
|
(string? OutputName, int InputIndex)[]? columnMappings;
|
||||||
|
(string From, string To)[]? columnNameMappings;
|
||||||
|
readonly bool includeUnmappedColumns;
|
||||||
|
|
||||||
|
public ColumnMaskStream(ViewStream source, string columnMaskExpression) {
|
||||||
|
this.source = source;
|
||||||
|
this.columnMaskExpression = columnMaskExpression;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ColumnMaskStream(ViewStream source, bool includeUnmappedColumns, params (string From, string To)[] columnNameMappings) {
|
||||||
|
this.source = source;
|
||||||
|
this.columnNameMappings = columnNameMappings;
|
||||||
|
this.includeUnmappedColumns = includeUnmappedColumns;
|
||||||
|
}
|
||||||
|
|
||||||
|
~ColumnMaskStream() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
Log.isnull(sub);
|
||||||
|
sub = source.Subscribe(OnSourceChanged);
|
||||||
|
|
||||||
|
void OnSourceChanged(IDataPacket dp) {
|
||||||
|
ensureOutputSchema(dp.Schema);
|
||||||
|
var addedRows = new ColumnMaskTable(dp.addedRows, columnMappings, outputSchema);
|
||||||
|
var deletedRows = new ColumnMaskTable(dp.deletedRows, columnMappings, outputSchema);
|
||||||
|
var allRows = new ColumnMaskTable(dp.newRows, columnMappings, outputSchema);
|
||||||
|
myFireEvent(new DataPacket(allRows, addedRows, deletedRows));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
|
||||||
|
[MemberNotNull(nameof(outputSchema), nameof(columnMappings))]
|
||||||
|
void ensureOutputSchema(IBlueTableSchema inputSchema) {
|
||||||
|
if (outputSchema is not null) {
|
||||||
|
Log.notnull(columnMappings);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (columnMaskExpression is not null) {
|
||||||
|
outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, columnMaskExpression, out columnMappings);
|
||||||
|
} else if (columnNameMappings is not null) {
|
||||||
|
outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, includeUnmappedColumns, columnNameMappings, out columnMappings);
|
||||||
|
} else {
|
||||||
|
throw new InvalidOperationException("Cannot build output schema");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
using bts.utils;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class ColumnMaskTable : IBlueRowCollection {
|
||||||
|
readonly IBlueRowCollection inputRows;
|
||||||
|
readonly (string? Name, int Index)[] columnMap;
|
||||||
|
|
||||||
|
public ColumnMaskTable(IBlueRowCollection inputRows, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
|
||||||
|
this.inputRows = inputRows;
|
||||||
|
this.columnMap = columnMap;
|
||||||
|
this.schema = schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema { get; }
|
||||||
|
|
||||||
|
public int Count => inputRows.Count;
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
new ColumnMaskRow(inputRows[id], columnMap, schema);
|
||||||
|
|
||||||
|
public static ColumnMaskTable Create(IBlueRowCollection table, string columnMaskExpression) {
|
||||||
|
var outputSchema = GetOutputSchema(table.schema, columnMaskExpression, out var columnMappings);
|
||||||
|
return new ColumnMaskTable(table, columnMappings, outputSchema);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, string columnMaskExpression, out (string? Name, int Index)[] columnMappings) {
|
||||||
|
bool includeOthers;
|
||||||
|
if (columnMaskExpression.StartsWith('!')) {
|
||||||
|
columnMaskExpression = columnMaskExpression[1..].TrimStart();
|
||||||
|
includeOthers = false;
|
||||||
|
} else {
|
||||||
|
includeOthers = true;
|
||||||
|
}
|
||||||
|
var map = columnMaskExpression.Split(' ').Select(MapColumn).ToOrderedDictionary(tup => tup.Index, tup => tup.Name);
|
||||||
|
var inputCols = inputSchema.columnInfo;
|
||||||
|
if (includeOthers) {
|
||||||
|
var implicitCols = inputCols.Where(c => !map.ContainsKey(c.colNo)).ToList();
|
||||||
|
map.AddRange(implicitCols.Select(c => new KeyValuePair<int, string?>(c.colNo, null)));
|
||||||
|
} else if (!map.Any(c => inputCols[c.Key].colName == "id") && inputCols.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
|
||||||
|
// Add 'id' col if missing
|
||||||
|
map.Insert(0, new(idColInfo.colNo, "id"));
|
||||||
|
}
|
||||||
|
var outputCols = new ColumnInfo[map.Count];
|
||||||
|
for (int i = 0; i < outputCols.Length; i++) {
|
||||||
|
var inputColInfo = inputCols[map.Keys[i]];
|
||||||
|
outputCols[i] = new(map.Values[i] ?? inputColInfo.colName, i, inputColInfo.colType);
|
||||||
|
}
|
||||||
|
columnMappings = map.WrapSelector(kv => (kv.Value, kv.Key), null).ToArray();
|
||||||
|
return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
|
||||||
|
|
||||||
|
(string? Name, int Index) MapColumn(string columnStr) {
|
||||||
|
var parts = columnStr.Split(':');
|
||||||
|
return parts.Length switch {
|
||||||
|
1 => ((string?)null, inputSchema[parts[0]].colNo),
|
||||||
|
2 => (parts[1], inputSchema[parts[0]].colNo),
|
||||||
|
_ => throw new InvalidOperationException("Unable to get output schema due to invalid column expression"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, bool includeUnmapped, (string From, string To)[] columnNameMappings, out (string? OutputName, int InputIndex)[] columnMappings) {
|
||||||
|
var inputCols = inputSchema.columnInfo;
|
||||||
|
ColumnInfo[]? outputCols;
|
||||||
|
if (includeUnmapped) {
|
||||||
|
columnMappings = Array.ConvertAll(inputCols, c => ((string?)c.colName, c.colNo));
|
||||||
|
outputCols = ArrayUtils.Clone(inputCols);
|
||||||
|
foreach (var (from, to) in columnNameMappings) {
|
||||||
|
var idx = inputCols.ResolveIndices(from);
|
||||||
|
var c = outputCols[idx];
|
||||||
|
Log.assert(c.colNo == idx);
|
||||||
|
outputCols[idx] = new(to, c.colNo, c.colType);
|
||||||
|
columnMappings[idx] = (to, idx);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
outputCols = new ColumnInfo[columnNameMappings.Length];
|
||||||
|
columnMappings = Array.ConvertAll(columnNameMappings, m => ((string?)m.To, inputCols.ResolveIndices(m.From)));
|
||||||
|
Array.Copy(inputCols, outputCols, inputCols.Length);
|
||||||
|
for (int i = 0; i < columnNameMappings.Length; i++) {
|
||||||
|
var (from, to) = columnNameMappings[i];
|
||||||
|
var idx = inputCols.ResolveIndices(from);
|
||||||
|
outputCols[idx] = new(to, i, inputCols[columnMappings[i].InputIndex].colType);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
inputRows.containsID(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
foreach (var r in inputRows) {
|
||||||
|
yield return new ColumnMaskRow(r, columnMap, schema);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
new ColumnMaskRow(inputRows.GetRow(index), columnMap, schema);
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
if (inputRows.containsID(id)) {
|
||||||
|
var inputRow = inputRows[id];
|
||||||
|
var outputRow = new ColumnMaskRow(inputRow, columnMap, schema);
|
||||||
|
if (BlueRowEqualityComparer.Equals(item, outputRow)) {
|
||||||
|
return inputRows.IndexOf(inputRow);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
// namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
// using BCG;
|
||||||
|
|
||||||
|
// using System;
|
||||||
|
// using System.Linq;
|
||||||
|
|
||||||
|
// /// <summary>
|
||||||
|
// /// Represents a row transformation table (all columns except 'id' can be changed).
|
||||||
|
// /// </summary>
|
||||||
|
// public class ColumnSelectorRow : IBlueRow {
|
||||||
|
// readonly IBlueRow source;
|
||||||
|
// readonly IBlueTableSchema outputSchema;
|
||||||
|
// readonly Func<object?, object?>[] columnSelectors;
|
||||||
|
|
||||||
|
// public ColumnSelectorRow(IBlueRow source, params (int Index, Type Type, Func<object?, object?> GetValue)[] columnSelectors) {
|
||||||
|
// this.source = source;
|
||||||
|
// this.columnSelectors = Array.ConvertAll(columnSelectors, c => (c.Index, c.GetValue));
|
||||||
|
// outputSchema = getSchema(source.schema, columnSelectors);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// public ColumnSelectorRow(IBlueRow source, IBlueTableSchema outputSchema, params (int Index, Func<object?, object?> GetValue)[] columnSelectors) {
|
||||||
|
// this.source = source;
|
||||||
|
// this.outputSchema = outputSchema;
|
||||||
|
// this.columnSelectors = columnSelectors;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// public object? this[int index] {
|
||||||
|
// get {
|
||||||
|
// for (int i = 0; i < columnSelectors.Length; i++) {
|
||||||
|
// if (columnSelectors[i].Index == index) {
|
||||||
|
// return columnSelectors[i].GetValue(source[index]);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// return source[index];
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// public IBlueTableSchema schema => outputSchema;
|
||||||
|
|
||||||
|
// public static Func<object?, object?>[] BuildColumnSelectors(IBlueTableSchema schema, params (int Index, Func<object?, object?> GetValue)[] columnSelectors) {
|
||||||
|
// var cols = schema.columnInfo;
|
||||||
|
// var rv = Array.ConvertAll(schema.columnInfo, c => new Func<object?, object?>(_ => _));
|
||||||
|
// for (int i = 0; i < columnSelectors.Length; i++) {
|
||||||
|
// var cs = columnSelectors[i];
|
||||||
|
// rv[cs.Index] = cs.GetValue;
|
||||||
|
// }
|
||||||
|
// return rv;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// static IBlueTableSchema getSchema(IBlueTableSchema inputSchema, (int Index, Type Type, Func<object?, object?> GetValue)[] columnSelectors) {
|
||||||
|
// var inputCols = inputSchema.columnInfo;
|
||||||
|
// var outputCols = inputCols.ToArray();
|
||||||
|
// for (int i = 0; i < columnSelectors.Length; i++) {
|
||||||
|
// var (idx, typ, _) = columnSelectors[i];
|
||||||
|
// var col = inputCols[idx];
|
||||||
|
// outputCols[idx] = new(col.colName, col.colNo, typ);
|
||||||
|
// }
|
||||||
|
// return new BlueTableSchema(outputCols, "ColumnSelector");
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class ColumnSelectorRowFactory {
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
readonly Func<object?, object?>[] columnSelectors;
|
||||||
|
|
||||||
|
public ColumnSelectorRowFactory(IBlueTableSchema outputSchema, Func<object?, object?>[] columnSelectors) {
|
||||||
|
this.outputSchema = outputSchema;
|
||||||
|
this.columnSelectors = columnSelectors;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema OutputSchema => outputSchema;
|
||||||
|
|
||||||
|
public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, params (string Name, Type Type, Func<object?, object?> GetValue)[] columnOverrides) =>
|
||||||
|
Create(inputSchema, columnOverrides.AsEnumerable());
|
||||||
|
|
||||||
|
public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, params (int Index, Type Type, Func<object?, object?> GetValue)[] columnOverrides) =>
|
||||||
|
Create(inputSchema, columnOverrides.AsEnumerable());
|
||||||
|
|
||||||
|
public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, IEnumerable<(string Name, Type Type, Func<object?, object?> GetValue)> columnOverrides) =>
|
||||||
|
Create(inputSchema, columnOverrides.Select(c => (inputSchema.ResolveIndices(c.Name), c.Type, c.GetValue)));
|
||||||
|
|
||||||
|
public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, IEnumerable<(int Index, Type Type, Func<object?, object?> GetValue)> columnOverrides) {
|
||||||
|
var bd = new Builder(inputSchema);
|
||||||
|
foreach (var c in columnOverrides) {
|
||||||
|
bd.AddOrUpdate(c);
|
||||||
|
}
|
||||||
|
return bd.ToImmutable();
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow Create(IBlueRow row) =>
|
||||||
|
new Row(row, outputSchema, columnSelectors);
|
||||||
|
|
||||||
|
public Builder ToBuilder() =>
|
||||||
|
new(outputSchema);
|
||||||
|
|
||||||
|
class Row : IBlueRow {
|
||||||
|
readonly IBlueRow source;
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
readonly Func<object?, object?>[] columnSelectors;
|
||||||
|
|
||||||
|
public Row(IBlueRow source, IBlueTableSchema schema, Func<object?, object?>[] columnSelectors) {
|
||||||
|
this.source = source;
|
||||||
|
this.schema = schema;
|
||||||
|
this.columnSelectors = columnSelectors;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object? this[int index] =>
|
||||||
|
columnSelectors[index](source[index]);
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRow.schema => schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public class Builder {
|
||||||
|
readonly IBlueTableSchema inputSchema;
|
||||||
|
readonly List<(int Index, Type Type, Func<object?, object?> GetValue)> columnOverrides = new();
|
||||||
|
|
||||||
|
public Builder(IBlueTableSchema inputSchema) {
|
||||||
|
this.inputSchema = inputSchema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema ToSchema() =>
|
||||||
|
ToSchema(inputSchema.tableName + "_ColumnSelector");
|
||||||
|
|
||||||
|
public IBlueTableSchema ToSchema(string tableName) {
|
||||||
|
var inputCols = inputSchema.columnInfo;
|
||||||
|
var outputCols = inputCols.ToArray();
|
||||||
|
foreach (var c in columnOverrides) {
|
||||||
|
var inputCol = inputCols[c.Index];
|
||||||
|
outputCols[c.Index] = new(inputCol.colName, c.Index, c.Type);
|
||||||
|
}
|
||||||
|
return new BlueTableSchema(outputCols, tableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ColumnSelectorRowFactory ToImmutable() {
|
||||||
|
var outputSchema = ToSchema();
|
||||||
|
var columnSelectors = Array.ConvertAll(outputSchema.columnInfo, c => new Func<object?, object?>(_ => _));
|
||||||
|
foreach (var cs in columnOverrides) {
|
||||||
|
columnSelectors[cs.Index] = cs.GetValue;
|
||||||
|
}
|
||||||
|
return new(outputSchema, columnSelectors);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void AddOrUpdate((int Index, Type Type, Func<object?, object?> GetValue) item) =>
|
||||||
|
columnOverrides.Add(item);
|
||||||
|
|
||||||
|
public void AddOrUpdate(int index, Type type, Func<object?, object?> getValue) =>
|
||||||
|
columnOverrides.Add((index, type, getValue));
|
||||||
|
|
||||||
|
public void AddOrUpdate<T>(int index, Func<object?, T> getValue) {
|
||||||
|
columnOverrides.Add((index, typeof(T), GetValueWrapper));
|
||||||
|
|
||||||
|
object? GetValueWrapper(object? inputVal) =>
|
||||||
|
getValue(inputVal);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
using System;
|
||||||
|
|
||||||
|
namespace BCG {
|
||||||
|
[DataStreamUserRespresentation("CountStream")]
|
||||||
|
public class CountStream : ViewStream {
|
||||||
|
private DataPacket myLastFiredDataPacket;
|
||||||
|
private IBlueTableSchema mySchema = new BlueTableSchema("count",
|
||||||
|
new ColumnInfo("id", 0, typeof(int)),
|
||||||
|
new ColumnInfo("count", 1, typeof(int)));
|
||||||
|
private int myId = 0;
|
||||||
|
private ViewStream myVS;
|
||||||
|
|
||||||
|
public CountStream(ViewStream vs) {
|
||||||
|
myVS = vs;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
myVS.onChange += myHandleChanges;
|
||||||
|
}
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
myVS.onChange -= myHandleChanges;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void myHandleChanges(object newValue) {
|
||||||
|
myFireEvent(
|
||||||
|
myLastFiredDataPacket =
|
||||||
|
myLastFiredDataPacket == null ?
|
||||||
|
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
||||||
|
new BlueTable(mySchema))
|
||||||
|
:
|
||||||
|
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
||||||
|
myLastFiredDataPacket.newRows)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private IBlueRowCollection myGetRows(object dp, int id) {
|
||||||
|
BlueTable retVal = new BlueTable(mySchema);
|
||||||
|
retVal.addRow
|
||||||
|
(new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
|
||||||
|
return retVal;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using MathNet.Numerics.Statistics;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class CovarianceTable : IBlueRowCollection {
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
readonly Lazy<IBlueRow> row;
|
||||||
|
|
||||||
|
public CovarianceTable(IBlueRowCollection rows1, IBlueRowCollection rows2) {
|
||||||
|
var inputSchema = rows1.schema;
|
||||||
|
var inputCols = inputSchema.columnInfo;
|
||||||
|
var outputCols = Array.ConvertAll(rows1.schema.columnInfo, ColumnFactory);
|
||||||
|
schema = new BlueTableSchema(outputCols, inputSchema.tableName + "_" + rows2.schema.tableName + "_Covariance");
|
||||||
|
row = new(RowFactory);
|
||||||
|
|
||||||
|
static ColumnInfo ColumnFactory(ColumnInfo inputCol) {
|
||||||
|
var t = inputCol.colType;
|
||||||
|
if (inputCol.colName != "id") {
|
||||||
|
if (t == typeof(short) || t == typeof(int) || t == typeof(long)
|
||||||
|
|| t == typeof(ushort) || t == typeof(uint) || t == typeof(ulong)
|
||||||
|
|| t == typeof(double) || t == typeof(float)) {
|
||||||
|
t = typeof(double);
|
||||||
|
} else if (t == typeof(DateTime)) {
|
||||||
|
t = typeof(TimeSpan);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new(inputCol.colName, inputCol.colNo, t);
|
||||||
|
}
|
||||||
|
|
||||||
|
IBlueRow RowFactory() {
|
||||||
|
var rowVals = new object[inputCols.Length];
|
||||||
|
for (int i = 0; i < inputCols.Length; i++) {
|
||||||
|
var col = inputCols[i];
|
||||||
|
var colType = col.colType;
|
||||||
|
if (col.colName == "id") {
|
||||||
|
rowVals[i] = 1;
|
||||||
|
} else if (colType == typeof(short)) {
|
||||||
|
rowVals[i] = GetValue(v => v as short?);
|
||||||
|
} else if (colType == typeof(int)) {
|
||||||
|
rowVals[i] = GetValue(v => v as int?);
|
||||||
|
} else if (colType == typeof(long)) {
|
||||||
|
rowVals[i] = GetValue(v => v as long?);
|
||||||
|
} else if (colType == typeof(ushort)) {
|
||||||
|
rowVals[i] = GetValue(v => v as ushort?);
|
||||||
|
} else if (colType == typeof(uint)) {
|
||||||
|
rowVals[i] = GetValue(v => v as uint?);
|
||||||
|
} else if (colType == typeof(ulong)) {
|
||||||
|
rowVals[i] = GetValue(v => v as ulong?);
|
||||||
|
} else if (colType == typeof(double)) {
|
||||||
|
rowVals[i] = GetValue(v => v as double?);
|
||||||
|
} else if (colType == typeof(float)) {
|
||||||
|
rowVals[i] = GetValue(v => v as float?);
|
||||||
|
} else if (colType == typeof(DateTime)) {
|
||||||
|
rowVals[i] = TimeSpan.FromSeconds(GetValue(v => (v as DateTime?)?.Ticks / 10_000_000));
|
||||||
|
}
|
||||||
|
|
||||||
|
double GetValue(Func<object?, double?> convert) {
|
||||||
|
var vals1 = rows1.GetColumn(i)
|
||||||
|
.Select(convert)
|
||||||
|
.SelectWhereValues();
|
||||||
|
var vals2 = rows2.GetColumn(i)
|
||||||
|
.Select(convert)
|
||||||
|
.SelectWhereValues();
|
||||||
|
return vals1.Covariance(vals2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new Row(schema, rowVals);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
id == 1 ? row.Value : throw new KeyNotFoundException();
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRowCollection.schema => schema;
|
||||||
|
|
||||||
|
public int Count => 1;
|
||||||
|
|
||||||
|
IBlueRow IList<IBlueRow>.this[int index] {
|
||||||
|
get => GetRow(index);
|
||||||
|
set => throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
id == 1;
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
yield return row.Value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
index == 0 ? row.Value : throw new IndexOutOfRangeException();
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) =>
|
||||||
|
BlueRowEqualityComparer.Equals(item, row.Value) ? 0 : -1;
|
||||||
|
|
||||||
|
class Row : IBlueRow {
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
readonly object?[] values;
|
||||||
|
|
||||||
|
public Row(IBlueTableSchema schema, object?[] values) {
|
||||||
|
this.schema = schema;
|
||||||
|
this.values = values;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object? this[int index] => values[index];
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRow.schema => schema;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class CummulativeSumStream : ViewStream, IDisposable {
|
||||||
|
readonly ViewStream inputStream;
|
||||||
|
readonly CummulativeSumTable.StringBehavior stringBehavior;
|
||||||
|
IBlueTableSchema? schema;
|
||||||
|
IDisposable? sub;
|
||||||
|
SingleRowTable? lastFired;
|
||||||
|
object?[]? lastFiredValues;
|
||||||
|
Func<object?, object?, object?>[]? columnOperators;
|
||||||
|
int idColumnIdx = -1;
|
||||||
|
|
||||||
|
public CummulativeSumStream(ViewStream inputStream, CummulativeSumTable.StringBehavior stringBehavior = CummulativeSumTable.StringBehavior.LastValue) {
|
||||||
|
this.inputStream = inputStream;
|
||||||
|
this.stringBehavior = stringBehavior;
|
||||||
|
}
|
||||||
|
|
||||||
|
~CummulativeSumStream() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
|
||||||
|
public void Dispose() {
|
||||||
|
GC.SuppressFinalize(this);
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
sub = inputStream.Subscribe(OnInputStreamChanged);
|
||||||
|
|
||||||
|
void OnInputStreamChanged(IDataPacket dp) {
|
||||||
|
ensureInit(dp);
|
||||||
|
var values = lastFiredValues?.ToArray() ?? new object?[schema.columnInfo.Length];
|
||||||
|
foreach (var r in dp.addedRows) {
|
||||||
|
for (int i = 0; i < values.Length; i++) {
|
||||||
|
values[i] = columnOperators[i](values[i], r[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var table = new SingleRowTable(new BlueRow(schema, values));
|
||||||
|
DataPacket dpOut = new(table, table, lastFired as IBlueRowCollection ?? new BlueTable(schema));
|
||||||
|
lastFired = table;
|
||||||
|
lastFiredValues = values;
|
||||||
|
myFireEvent(dpOut);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
|
||||||
|
[MemberNotNull(nameof(schema), nameof(columnOperators))]
|
||||||
|
void ensureInit(IDataPacket dp) {
|
||||||
|
if (schema is not null) {
|
||||||
|
Log.notnull(columnOperators);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
schema = dp.Schema;
|
||||||
|
idColumnIdx = schema.ResolveIndices("id");
|
||||||
|
var colInfos = schema.columnInfo;
|
||||||
|
columnOperators = new Func<object?, object?, object?>[schema.columnInfo.Length];
|
||||||
|
for (int i = 0; i < colInfos.Length; i++) {
|
||||||
|
if (i == idColumnIdx) {
|
||||||
|
columnOperators[i] = (x, y) => 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
columnOperators[i] = schema.columnInfo[i].type_code() switch {
|
||||||
|
TypeCode.Int32 => (x, y) => (x as int? ?? 0) + (BlueDataUtility.GetInt32(y) ?? 0),
|
||||||
|
TypeCode.Int64 => (x, y) => (x as long? ?? 0) + (BlueDataUtility.GetInt64(y) ?? 0),
|
||||||
|
TypeCode.UInt32 => (x, y) => (x as uint? ?? 0) + (BlueDataUtility.GetUInt32(y) ?? 0),
|
||||||
|
TypeCode.UInt64 => (x, y) => (x as ulong? ?? 0) + (BlueDataUtility.GetUInt64(y) ?? 0),
|
||||||
|
TypeCode.Single => (x, y) => (x as float? ?? 0) + (BlueDataUtility.GetFloat32(y) ?? 0),
|
||||||
|
TypeCode.Double => (x, y) => (x as double? ?? 0) + (BlueDataUtility.GetFloat64(y) ?? 0),
|
||||||
|
TypeCode.String => stringBehavior switch {
|
||||||
|
CummulativeSumTable.StringBehavior.Concatenate => (x, y) => (x as string ?? string.Empty) + (y as string ?? string.Empty),
|
||||||
|
_ => (accum, next) => next,
|
||||||
|
},
|
||||||
|
TypeCode.DateTime => (x, y) => y,
|
||||||
|
_ => throw new InvalidOperationException($"Failed to add row values for col '{schema.columnInfo[i]}'"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class CummulativeSumTable : IBlueRowCollection {
|
||||||
|
readonly IBlueRowCollection input;
|
||||||
|
readonly IList<object?>[] columns;
|
||||||
|
readonly StringBehavior stringBehavior;
|
||||||
|
|
||||||
|
public CummulativeSumTable(IBlueRowCollection input, StringBehavior stringBehavior = StringBehavior.LastValue) {
|
||||||
|
this.input = input;
|
||||||
|
var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
|
||||||
|
if (colInfos.Length > 0) {
|
||||||
|
Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
|
||||||
|
}
|
||||||
|
columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// By default, we take the last value for strings intstead of adding (concatenating) them.
|
||||||
|
/// Otherwise it would be too easy to create massive concatenated strings by mistake.
|
||||||
|
/// </summary>
|
||||||
|
public enum StringBehavior {
|
||||||
|
LastValue,
|
||||||
|
Concatenate
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
id == 1 ? GetRow(0) : throw new KeyNotFoundException();
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => input.schema;
|
||||||
|
|
||||||
|
public int Count => 1;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
id == 1;
|
||||||
|
|
||||||
|
public IList<object?> GetColumn(int colIndex) {
|
||||||
|
var count = input.Count;
|
||||||
|
var colInfo = input.schema.columnInfo[colIndex];
|
||||||
|
if (colInfo.colName == "id") {
|
||||||
|
return new object?[] { 1 };
|
||||||
|
}
|
||||||
|
object? res = null;
|
||||||
|
var colVals = input.GetColumn(colIndex);
|
||||||
|
if (colInfo.colType == typeof(TimeSpan)) {
|
||||||
|
// TimeSpan is a special case because it doesn't have a TypeCode.
|
||||||
|
res = TimeSpanImpl(colVals);
|
||||||
|
} else {
|
||||||
|
var typeCode = colInfo.type_code();
|
||||||
|
const int MAX_STRING_LENGTH = 1024;
|
||||||
|
res = typeCode switch {
|
||||||
|
TypeCode.Int32 => colVals.Cast<int>().Sum(),
|
||||||
|
TypeCode.Int64 => colVals.Cast<long>().Sum(),
|
||||||
|
TypeCode.UInt32 => colVals.Cast<uint>().Sum(),
|
||||||
|
TypeCode.UInt64 => colVals.Cast<ulong>().Aggregate(0UL, (s, v) => s + v),
|
||||||
|
TypeCode.Single => colVals.Cast<float>().Sum(),
|
||||||
|
TypeCode.Double => colVals.Cast<double>().Sum(),
|
||||||
|
TypeCode.DateTime => colVals.Cast<DateTime>().Aggregate(DateTime.MinValue, (s, v) => new DateTime(s.Ticks + v.Ticks)),
|
||||||
|
TypeCode.String => stringBehavior switch {
|
||||||
|
StringBehavior.Concatenate => colVals.Cast<string>().Aggregate(string.Empty, (s, v) => {
|
||||||
|
if (s.Length + v.Length > MAX_STRING_LENGTH) {
|
||||||
|
throw new InvalidOperationException("Cannot sum strings because the result is too long");
|
||||||
|
}
|
||||||
|
return s + v;
|
||||||
|
}),
|
||||||
|
_ => count == 0 ? null : colVals[^1] as string,
|
||||||
|
},
|
||||||
|
_ => null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return new object?[] { res };
|
||||||
|
|
||||||
|
static object? TimeSpanImpl(IList<object?> colVals) {
|
||||||
|
var ticks = colVals.Cast<TimeSpan>().Select(ts => ts.Ticks).Sum();
|
||||||
|
return TimeSpan.FromTicks(ticks);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
yield return GetRow(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
new BlueRowColumnWrapper(input.schema, index, columns);
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
var inputRow = input[id];
|
||||||
|
int idx = input.IndexOf(inputRow);
|
||||||
|
if (idx != -1) {
|
||||||
|
var row = GetRow(idx);
|
||||||
|
if (BlueRowEqualityComparer.Equals(item, row)) {
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class CummulativeSumTableEvaluator : IFunctionEvaluator {
|
||||||
|
public object? Evaluate(string functionId, IEnumerable<StreamArgument> args) {
|
||||||
|
if (args.TryGetFirst(out var a)) {
|
||||||
|
var table = a.Value as IBlueRowCollection;
|
||||||
|
if (table is not null) {
|
||||||
|
return new CummulativeSumTable(table);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
+7
-7
@@ -1,19 +1,19 @@
|
|||||||
namespace bts.SilverPine {
|
namespace bts.BlueOrb {
|
||||||
using bts;
|
using bts;
|
||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
[DataStreamUserRespresentation("DateTimeExpanderStream")]
|
||||||
public class DateTimeExpanstionStream : ViewStream {
|
public class DateTimeExpanstionStream : ViewStream {
|
||||||
public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
|
public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
|
||||||
Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
|
Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
|
||||||
IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
|
IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
|
||||||
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
|
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
|
||||||
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
|
||||||
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
|
||||||
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
|
||||||
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
|
||||||
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
|
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
|
||||||
return rv;
|
return rv;
|
||||||
};
|
};
|
||||||
@@ -24,15 +24,15 @@ namespace bts.SilverPine {
|
|||||||
IBlueTableSchema s = get_schema(r.schema);
|
IBlueTableSchema s = get_schema(r.schema);
|
||||||
MutableBlueRow rv = new MutableBlueRow(s, r);
|
MutableBlueRow rv = new MutableBlueRow(s, r);
|
||||||
rv.setValue($"{inp_col}_date", dt.Date);
|
rv.setValue($"{inp_col}_date", dt.Date);
|
||||||
rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
|
rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
|
||||||
rv.setValue($"{inp_col}_day_of_month", dt.Day);
|
rv.setValue($"{inp_col}_day_of_month", dt.Day);
|
||||||
rv.setValue($"{inp_col}_month", dt.Date.Month);
|
rv.setValue($"{inp_col}_month", dt.Date.Month);
|
||||||
rv.setValue($"{inp_col}_hour", dt.Hour);
|
rv.setValue($"{inp_col}_hour", dt.Hour);
|
||||||
rv.setValue($"{inp_col}_min", dt.Minute);
|
rv.setValue($"{inp_col}_minute", dt.Minute);
|
||||||
rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
|
rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
|
||||||
return rv;
|
return rv;
|
||||||
});
|
});
|
||||||
mySetRealSource(rs);
|
mySetRealSource(rs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Collections;
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq.Expressions;
|
||||||
|
|
||||||
|
public class DiffStream : ViewStream, IDisposable {
|
||||||
|
readonly ViewStream source;
|
||||||
|
readonly int period;
|
||||||
|
IBlueTableSchema? outputSchema;
|
||||||
|
IDisposable? sub;
|
||||||
|
BlueTableImmutable? lastFired;
|
||||||
|
|
||||||
|
public DiffStream(ViewStream source, int period) {
|
||||||
|
this.source = source;
|
||||||
|
this.period = period;
|
||||||
|
}
|
||||||
|
|
||||||
|
~DiffStream() =>
|
||||||
|
Dispose();
|
||||||
|
|
||||||
|
public void Dispose() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
Log.isnull(sub);
|
||||||
|
sub = source.Subscribe(OnSourceChange);
|
||||||
|
|
||||||
|
void OnSourceChange(IDataPacket dp) {
|
||||||
|
if (outputSchema is null) {
|
||||||
|
var inputSchema = dp.Schema;
|
||||||
|
var outputCols = new ColumnInfo[inputSchema.columnInfo.Length];
|
||||||
|
for (int i = 0; i < inputSchema.columnInfo.Length; i++) {
|
||||||
|
outputCols[i] = GetOutputColumn(inputSchema.columnInfo[i]);
|
||||||
|
}
|
||||||
|
outputSchema = new BlueTableSchema($"{inputSchema.tableName}_Diff{period}", outputCols);
|
||||||
|
}
|
||||||
|
lastFired ??= new(outputSchema);
|
||||||
|
var newTable = new DiffTable(dp.newRows, period);
|
||||||
|
var allRows = new List<object?[]>();
|
||||||
|
foreach (IBlueRow r in newTable) {
|
||||||
|
var rvals = new object?[r.schema.columnInfo.Length];
|
||||||
|
int x = 0;
|
||||||
|
foreach (var col in r.schema.columnInfo) {
|
||||||
|
var val = r[col.colNo];
|
||||||
|
rvals[x++] = val;
|
||||||
|
}
|
||||||
|
allRows.Add(rvals);
|
||||||
|
}
|
||||||
|
var bd = lastFired.ToDataPacketBuilder();
|
||||||
|
var removedIndices = new List<int>();
|
||||||
|
var addedIndices = new List<int>();
|
||||||
|
var ops = new List<(int Index, bool IsAdd)>();
|
||||||
|
foreach (var r in dp.deletedRows) {
|
||||||
|
int id = r.Id();
|
||||||
|
var rOut = lastFired[id];
|
||||||
|
var idx = lastFired.IndexOf(rOut);
|
||||||
|
Log.assert(idx >= 0 && idx < lastFired.Count);
|
||||||
|
removedIndices.Add(idx);
|
||||||
|
ops.Add((idx, false));
|
||||||
|
}
|
||||||
|
foreach (var r in dp.addedRows) {
|
||||||
|
var idx = dp.newRows.IndexOf(r);
|
||||||
|
Log.assert(idx >= 0 && idx < dp.newRows.Count);
|
||||||
|
addedIndices.Add(idx);
|
||||||
|
ops.Add((idx, true));
|
||||||
|
}
|
||||||
|
ops.Sort(Comparer<(int Index, bool IsAdd)>.Create((a, b) => a.Index.CompareTo(b.Index)));
|
||||||
|
int rmCount = 0;
|
||||||
|
var removedRows = new Dictionary<int, IBlueRow>();
|
||||||
|
for (int i = 0; i < ops.Count; i++) {
|
||||||
|
var op = ops[i];
|
||||||
|
if (op.IsAdd) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rmCount++;
|
||||||
|
int j = op.Index/* - rmCount*/;
|
||||||
|
var stop = Math.Min(j + period, lastFired.Count/* - rmCount*/);
|
||||||
|
// Remove the row that was removed in the incoming DataPacket.
|
||||||
|
bd.Remove(lastFired.GetRow(j).Id());
|
||||||
|
for (j++; j < stop; j++) {
|
||||||
|
// Remove rows that changed as a result. We will add the updated versions
|
||||||
|
// of these rows in the next pass.
|
||||||
|
var r = lastFired.GetRow(j);
|
||||||
|
bd.Remove(r.Id());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int addCount = ops.Count - rmCount;
|
||||||
|
for (int i = ops.Count - 1; i >= 0; i--) {
|
||||||
|
var op = ops[i];
|
||||||
|
if (op.IsAdd) {
|
||||||
|
addCount--;
|
||||||
|
int j = op.Index;
|
||||||
|
var stop = Math.Min(j + period, newTable.Count);
|
||||||
|
for (; j < stop; j++) {
|
||||||
|
var r = newTable.GetRow(j);
|
||||||
|
bd.AddOrUpdate(r);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
rmCount--;
|
||||||
|
int j = op.Index - rmCount + addCount;
|
||||||
|
var stop = Math.Min(j + period, newTable.Count);
|
||||||
|
for (; j < stop; j++) {
|
||||||
|
// Add rows that changed due to the removed row (the row with which they compute
|
||||||
|
// the diff has changed).
|
||||||
|
var r = newTable.GetRow(j);
|
||||||
|
bd.AddOrUpdate(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var dpOut = bd.ToDataPacket(out lastFired, false);
|
||||||
|
myFireEvent(dpOut);
|
||||||
|
}
|
||||||
|
|
||||||
|
ColumnInfo GetOutputColumn(ColumnInfo inputColumn) {
|
||||||
|
return inputColumn.type_code() switch {
|
||||||
|
TypeCode.Boolean => inputColumn.Clone(),
|
||||||
|
TypeCode.Byte or TypeCode.SByte or TypeCode.Int16 => WithType(TypeCode.Int32),
|
||||||
|
TypeCode.UInt32 or TypeCode.UInt64 => WithType(TypeCode.Int64),
|
||||||
|
_ => inputColumn.Clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
ColumnInfo WithType(TypeCode typeCode) =>
|
||||||
|
new(inputColumn.colName, inputColumn.colNo, typeCode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
|
||||||
|
static class ExpressionBuilder {
|
||||||
|
public static Func<IBlueRow, IBlueRow, int?> Int32DiffExpression(int colIndex) {
|
||||||
|
var colIndexParam = Expression.Constant(colIndex, typeof(int));
|
||||||
|
var rowParam = Expression.Parameter(typeof(IBlueRow), "row");
|
||||||
|
var getValMethod = typeof(ExpressionBuilder).GetMethod(nameof(getInt32Value))
|
||||||
|
?? throw new InvalidOperationException("Failed to find int32 value getter via reflection");
|
||||||
|
var getVal = Expression.Lambda<Func<IBlueRow, int?>>(Expression.Call(null, getValMethod, rowParam, colIndexParam), rowParam).Compile();
|
||||||
|
return ComputeDiff;
|
||||||
|
|
||||||
|
int? ComputeDiff(IBlueRow leftRow, IBlueRow rightRow) {
|
||||||
|
var leftVal = getVal(leftRow);
|
||||||
|
var rightVal = getVal(rightRow);
|
||||||
|
if (leftVal.HasValue && rightVal.HasValue) {
|
||||||
|
return leftVal.Value - rightVal.Value;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static int? getInt32Value(IBlueRow row, int colIndex) {
|
||||||
|
var obj = row[colIndex];
|
||||||
|
return obj as int?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class DiffTable : IBlueRowCollection {
|
||||||
|
readonly IBlueRowCollection input;
|
||||||
|
readonly int period;
|
||||||
|
readonly IList<object?>[] columns;
|
||||||
|
|
||||||
|
public DiffTable(IBlueRowCollection input, int period) {
|
||||||
|
this.input = input;
|
||||||
|
this.period = period;
|
||||||
|
var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
|
||||||
|
if (colInfos.Length > 0) {
|
||||||
|
Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
|
||||||
|
}
|
||||||
|
columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
GetRow(input.IndexOf(input[id]));
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => input.schema;
|
||||||
|
|
||||||
|
public int Count => input.Count;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
input.containsID(id);
|
||||||
|
|
||||||
|
public IList<object?> GetColumn(int colIndex) {
|
||||||
|
var count = input.Count;
|
||||||
|
var colInfo = input.schema.columnInfo[colIndex];
|
||||||
|
if (colInfo.colName == "id") {
|
||||||
|
return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetId, null);
|
||||||
|
}
|
||||||
|
var typeCode = colInfo.type_code();
|
||||||
|
return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetValue, null);
|
||||||
|
|
||||||
|
object? GetId(int rowIndex) {
|
||||||
|
var row = input.GetRow(rowIndex);
|
||||||
|
return row.Id();
|
||||||
|
}
|
||||||
|
|
||||||
|
object? GetValue(int rowIndex) {
|
||||||
|
var rightIndex = rowIndex - period;
|
||||||
|
if (rowIndex < 0 || rowIndex >= count || rightIndex < 0 || rightIndex >= count) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
var leftRow = input.GetRow(rowIndex);
|
||||||
|
var rightRow = input.GetRow(rightIndex);
|
||||||
|
return GetDiff(leftRow[colIndex], rightRow[colIndex], typeCode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
var count = input.Count;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
yield return GetRow(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
new BlueRowColumnWrapper(input.schema, index, columns);
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
var inputRow = input[id];
|
||||||
|
int idx = input.IndexOf(inputRow);
|
||||||
|
if (idx != -1) {
|
||||||
|
var row = GetRow(idx);
|
||||||
|
if (BlueRowEqualityComparer.Equals(item, row)) {
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static object? GetDiff(object? leftVal, object? rightVal, TypeCode colType) {
|
||||||
|
if (leftVal is null || rightVal is null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return colType switch {
|
||||||
|
TypeCode.Int32 => (int)leftVal - (int)rightVal,
|
||||||
|
TypeCode.Int64 => (long)leftVal - (long)rightVal,
|
||||||
|
TypeCode.Double => (double)leftVal - (double)rightVal,
|
||||||
|
TypeCode.String => ((string)leftVal).Replace((string)rightVal, string.Empty),
|
||||||
|
TypeCode.DateTime => new DateTime(((DateTime)leftVal).Ticks - ((DateTime)rightVal).Ticks),
|
||||||
|
_ => null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,506 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using Apache.Arrow;
|
||||||
|
using Apache.Arrow.Ipc;
|
||||||
|
using Apache.Arrow.Types;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.collections;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
using Spark.Core.Threading;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.Immutable;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("featherReader")]
|
||||||
|
public class FeatherReaderStream : ViewStream {
|
||||||
|
readonly string file;
|
||||||
|
FileStream fileStream;
|
||||||
|
// Let the table leak in case some dangling references linger around after
|
||||||
|
// myOnLastSubscriber is called. This should be fine since it disposes itself
|
||||||
|
// in its finalizer.
|
||||||
|
FeatherTable? table;
|
||||||
|
|
||||||
|
public FeatherReaderStream(string file) =>
|
||||||
|
this.file = file;
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
if (!Path.Exists(file)) {
|
||||||
|
throw new InvalidOperationException("Feather file does not exist");
|
||||||
|
}
|
||||||
|
fileStream = File.OpenRead(file);
|
||||||
|
table = new(new ArrowStreamReader(fileStream));
|
||||||
|
myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
|
||||||
|
}
|
||||||
|
|
||||||
|
interface IFeatherColumn {
|
||||||
|
int Count { get; }
|
||||||
|
|
||||||
|
object? GetValue(int i);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface IFeatherColumn<T> : IFeatherColumn {
|
||||||
|
new T? GetValue(int i);
|
||||||
|
|
||||||
|
object? IFeatherColumn.GetValue(int i) =>
|
||||||
|
GetValue(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherColumn : IFeatherColumn {
|
||||||
|
readonly IArrowArray array;
|
||||||
|
|
||||||
|
public FeatherColumn(IArrowArray array) {
|
||||||
|
this.array = array;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public static IFeatherColumn Create(IArrowArray array) =>
|
||||||
|
array switch {
|
||||||
|
Int32Array a => new FeatherInt32Column(a),
|
||||||
|
Int64Array a => new FeatherInt64Column(a),
|
||||||
|
DoubleArray a => new FeatherFloat64Column(a),
|
||||||
|
StringArray a => new FeatherStringColumn(a),
|
||||||
|
BooleanArray a => new FeatherBooleanColumn(a),
|
||||||
|
Date64Array a => new FeatherDateTimeColumn(a),
|
||||||
|
Time64Array a => new FeatherTimeStampColumn(a),
|
||||||
|
_ => throw new NotSupportedException($"Unsupported type: {array.GetType()}")
|
||||||
|
};
|
||||||
|
|
||||||
|
public object? GetValue(int i) =>
|
||||||
|
array switch {
|
||||||
|
Int32Array a => a.GetValue(i),
|
||||||
|
Int64Array a => a.GetValue(i),
|
||||||
|
DoubleArray a => a.GetValue(i),
|
||||||
|
StringArray a => a.GetString(i),
|
||||||
|
BooleanArray a => a.GetValue(i),
|
||||||
|
Date64Array a => a.GetValue(i),
|
||||||
|
Time64Array a => a.GetValue(i),
|
||||||
|
_ => throw new NotSupportedException($"Unsupported type: {array.GetType()}")
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherInt32Column : IFeatherColumn<int?> {
|
||||||
|
readonly Int32Array array;
|
||||||
|
|
||||||
|
public FeatherInt32Column(Int32Array array) =>
|
||||||
|
this.array = array;
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public int? GetValue(int i) => array.GetValue(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherInt64Column : IFeatherColumn<long?> {
|
||||||
|
readonly Int64Array array;
|
||||||
|
|
||||||
|
public FeatherInt64Column(Int64Array array) =>
|
||||||
|
this.array = array;
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public long? GetValue(int i) => array.GetValue(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherFloat64Column : IFeatherColumn<double?> {
|
||||||
|
readonly DoubleArray array;
|
||||||
|
|
||||||
|
public FeatherFloat64Column(DoubleArray array) =>
|
||||||
|
this.array = array;
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public double? GetValue(int i) => array.GetValue(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherFloat32Column : IFeatherColumn<float?> {
|
||||||
|
readonly FloatArray array;
|
||||||
|
|
||||||
|
public FeatherFloat32Column(FloatArray array) =>
|
||||||
|
this.array = array;
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public float? GetValue(int i) => array.GetValue(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherStringColumn : IFeatherColumn<string?> {
|
||||||
|
readonly StringArray array;
|
||||||
|
|
||||||
|
public FeatherStringColumn(StringArray array) =>
|
||||||
|
this.array = array;
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public string? GetValue(int i) => array.GetString(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherBooleanColumn : IFeatherColumn<bool?> {
|
||||||
|
readonly BooleanArray array;
|
||||||
|
|
||||||
|
public FeatherBooleanColumn(BooleanArray array) =>
|
||||||
|
this.array = array;
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public bool? GetValue(int i) => array.GetValue(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherDateTimeColumn : IFeatherColumn<DateTime?> {
|
||||||
|
readonly Date64Array array;
|
||||||
|
|
||||||
|
public FeatherDateTimeColumn(Date64Array array) =>
|
||||||
|
this.array = array;
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public DateTime? GetValue(int i) {
|
||||||
|
var nanos = array.GetValue(i);
|
||||||
|
if (!nanos.HasValue) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new(nanos.Value / 100);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherTimeStampColumn : IFeatherColumn<DateTime?> {
|
||||||
|
readonly Time64Array array;
|
||||||
|
|
||||||
|
public FeatherTimeStampColumn(Time64Array array) =>
|
||||||
|
this.array = array;
|
||||||
|
|
||||||
|
public int Count => array.Length;
|
||||||
|
|
||||||
|
public DateTime? GetValue(int i) {
|
||||||
|
var nanos = array.GetValue(i);
|
||||||
|
if (!nanos.HasValue) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new(nanos.Value / 100);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherBlueRow : IBlueRow {
|
||||||
|
readonly int id;
|
||||||
|
readonly int rowIndex;
|
||||||
|
readonly IFeatherColumn[] data;
|
||||||
|
|
||||||
|
public FeatherBlueRow(int id, int rowIndex, IFeatherColumn[] data, IBlueTableSchema schema) {
|
||||||
|
this.id = id;
|
||||||
|
this.rowIndex = rowIndex;
|
||||||
|
this.data = data;
|
||||||
|
Schema = schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object? this[int index] =>
|
||||||
|
index == 0
|
||||||
|
? id
|
||||||
|
: data[index - 1].GetValue(rowIndex);
|
||||||
|
|
||||||
|
public IBlueTableSchema Schema { get; }
|
||||||
|
IBlueTableSchema IBlueRow.schema => Schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherColumnDataList : IReadOnlyList<object?> {
|
||||||
|
// Represents 1 column split into batches (hence it being an array)
|
||||||
|
readonly IReadOnlyList<IFeatherColumn> columns;
|
||||||
|
int? count;
|
||||||
|
|
||||||
|
public FeatherColumnDataList(IReadOnlyList<IFeatherColumn> columns) {
|
||||||
|
this.columns = columns;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object? this[int index] {
|
||||||
|
get {
|
||||||
|
for (int i = 0; i < columns.Count; i++) {
|
||||||
|
var col = columns[i];
|
||||||
|
if (index < col.Count) {
|
||||||
|
return col.GetValue(index);
|
||||||
|
}
|
||||||
|
index -= col.Count;
|
||||||
|
}
|
||||||
|
throw new IndexOutOfRangeException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Count =>
|
||||||
|
count ??= columns.Sum(c => c.Count);
|
||||||
|
|
||||||
|
public IEnumerator<object?> GetEnumerator() {
|
||||||
|
for (int i = 0; i < columns.Count; i++) {
|
||||||
|
var col = columns[i];
|
||||||
|
for (int j = 0; j < col.Count; j++) {
|
||||||
|
yield return col.GetValue(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
IEnumerator IEnumerable.GetEnumerator() =>
|
||||||
|
GetEnumerator();
|
||||||
|
}
|
||||||
|
|
||||||
|
class FeatherTable : IImmutableRowCollection, IDisposable {
|
||||||
|
ImmutableList<(RecordBatch Records, IFeatherColumn[] Columns)> recordBatches = ImmutableList<(RecordBatch Records, IFeatherColumn[] Columns)>.Empty;
|
||||||
|
readonly IReadOnlyDictionary<string, Field> fields;
|
||||||
|
readonly AtomicGate gate = new();
|
||||||
|
int count;
|
||||||
|
ArrowStreamReader? reader;
|
||||||
|
|
||||||
|
public FeatherTable(ArrowStreamReader reader) {
|
||||||
|
this.reader = reader;
|
||||||
|
if (reader.Schema is null) {
|
||||||
|
var batch = reader.ReadNextRecordBatch();
|
||||||
|
Log.notnull(batch);
|
||||||
|
Log.notnull(reader.Schema);
|
||||||
|
var fcols = new IFeatherColumn[batch.ColumnCount];
|
||||||
|
for (int i = 0; i < fcols.Length; i++) {
|
||||||
|
fcols[i] = FeatherColumn.Create(batch.Column(i));
|
||||||
|
}
|
||||||
|
recordBatches = recordBatches.Add((batch, fcols));
|
||||||
|
count = batch.Length;
|
||||||
|
}
|
||||||
|
fields = reader.Schema.Fields;
|
||||||
|
var columns = new ColumnInfo[fields.Count + 1];
|
||||||
|
columns[0] = new("id", 0, typeof(int));
|
||||||
|
for (int i = 0; i < fields.Count; i++) {
|
||||||
|
var f = reader.Schema.GetFieldByIndex(i);
|
||||||
|
Log.assert(f.Name != "id");
|
||||||
|
columns[i + 1] = new(f.Name, i + 1, getFieldType(f));
|
||||||
|
}
|
||||||
|
schema = new BlueTableSchema("FeatherReader", columns);
|
||||||
|
}
|
||||||
|
|
||||||
|
~FeatherTable() =>
|
||||||
|
Disposable.DisposeThreadSafe(ref reader);
|
||||||
|
|
||||||
|
public IBlueTableSchema schema { get; }
|
||||||
|
|
||||||
|
public int Count {
|
||||||
|
get {
|
||||||
|
if (reader is not null) {
|
||||||
|
readBatches().Wait();
|
||||||
|
Log.isnull(reader);
|
||||||
|
}
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] => GetRow(id - 1);
|
||||||
|
|
||||||
|
public bool containsID(int id) {
|
||||||
|
while (id > count && reader is not null) {
|
||||||
|
readBatches(1).Wait();
|
||||||
|
}
|
||||||
|
return id <= count;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
while (id > count && reader is not null) {
|
||||||
|
readBatches(1).Wait();
|
||||||
|
}
|
||||||
|
return id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
|
||||||
|
? id - 1
|
||||||
|
: -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) {
|
||||||
|
if (index < 0) {
|
||||||
|
throw new IndexOutOfRangeException();
|
||||||
|
}
|
||||||
|
if (count <= index) {
|
||||||
|
readRows(index + 1).Wait();
|
||||||
|
}
|
||||||
|
int id = index + 1;
|
||||||
|
foreach (var (records, columns) in recordBatches) {
|
||||||
|
if (index < records.Length) {
|
||||||
|
return new FeatherBlueRow(id, index, columns, schema);
|
||||||
|
}
|
||||||
|
index -= records.Length;
|
||||||
|
}
|
||||||
|
throw new IndexOutOfRangeException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public IList<object?> GetColumn(int index) =>
|
||||||
|
new FeatherColumnDataList(new MyColList(this, index)).as_mutable();
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
int i = 0;
|
||||||
|
foreach (var (records, columns) in recordBatches) {
|
||||||
|
yield return new FeatherBlueRow(i + 1, i, columns, schema);
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
while (reader is not null) {
|
||||||
|
var task = readBatches(1);
|
||||||
|
task.Wait();
|
||||||
|
if (!task.Result) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
yield return new FeatherBlueRow(i + 1, i, recordBatches[i].Columns, schema);
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose() =>
|
||||||
|
Disposable.DisposeThreadSafe(ref reader);
|
||||||
|
|
||||||
|
static Type getFieldType(Field f) =>
|
||||||
|
f.DataType.TypeId switch {
|
||||||
|
ArrowTypeId.Null => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Boolean => typeof(bool),
|
||||||
|
ArrowTypeId.UInt8 => typeof(byte),
|
||||||
|
ArrowTypeId.Int8 => typeof(sbyte),
|
||||||
|
ArrowTypeId.UInt16 => typeof(ushort),
|
||||||
|
ArrowTypeId.Int16 => typeof(short),
|
||||||
|
ArrowTypeId.UInt32 => typeof(uint),
|
||||||
|
ArrowTypeId.Int32 => typeof(int),
|
||||||
|
ArrowTypeId.UInt64 => typeof(ulong),
|
||||||
|
ArrowTypeId.Int64 => typeof(long),
|
||||||
|
ArrowTypeId.HalfFloat => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Float => typeof(float),
|
||||||
|
ArrowTypeId.Double => typeof(double),
|
||||||
|
ArrowTypeId.String => typeof(string),
|
||||||
|
ArrowTypeId.Binary => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.FixedSizedBinary => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Date32 => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Date64 => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Timestamp => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Time32 => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Time64 => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Interval => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Decimal128 => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Decimal256 => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.List => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Struct => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Union => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Dictionary => throw new NotImplementedException(),
|
||||||
|
ArrowTypeId.Map => throw new NotImplementedException(),
|
||||||
|
_ => throw new ArgumentException("Unsupported field type", nameof(f)),
|
||||||
|
};
|
||||||
|
|
||||||
|
async Task<bool> readRows(int minCount) {
|
||||||
|
if (reader is null) {
|
||||||
|
return count >= minCount;
|
||||||
|
}
|
||||||
|
int n = 0;
|
||||||
|
using (gate.BeginTransaction()) {
|
||||||
|
if (reader is not null) {
|
||||||
|
int origCount = count;
|
||||||
|
while ((count + n) < minCount) {
|
||||||
|
Log.assert(count == origCount);
|
||||||
|
var batch = await reader.ReadNextRecordBatchAsync();
|
||||||
|
if (batch is null) {
|
||||||
|
Disposable.dispose(ref reader);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
var columns = new IFeatherColumn[batch.ColumnCount];
|
||||||
|
for (int i = 0; i < columns.Length; i++) {
|
||||||
|
columns[i] = FeatherColumn.Create(batch.Column(i));
|
||||||
|
}
|
||||||
|
recordBatches.Add((batch, columns));
|
||||||
|
n += batch.Length;
|
||||||
|
}
|
||||||
|
count += n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count >= minCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task<bool> readBatches(int? count = null) {
|
||||||
|
if (reader is null) {
|
||||||
|
return !count.HasValue || count.Value == 0;
|
||||||
|
}
|
||||||
|
if (count.HasValue && count.Value <= recordBatches.Count) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
int n = 0;
|
||||||
|
using (gate.BeginTransaction()) {
|
||||||
|
if (reader is not null) {
|
||||||
|
int origCount = this.count;
|
||||||
|
var recordBatchesBd = recordBatches.ToBuilder();
|
||||||
|
while (!count.HasValue || count-- > recordBatchesBd.Count) {
|
||||||
|
Log.assert(count == origCount);
|
||||||
|
var batch = await reader.ReadNextRecordBatchAsync();
|
||||||
|
if (batch is null) {
|
||||||
|
Disposable.dispose(ref reader);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
var columns = new IFeatherColumn[batch.ColumnCount];
|
||||||
|
for (int i = 0; i < columns.Length; i++) {
|
||||||
|
columns[i] = FeatherColumn.Create(batch.Column(i));
|
||||||
|
}
|
||||||
|
recordBatchesBd.Add((batch, columns));
|
||||||
|
n += batch.Length;
|
||||||
|
}
|
||||||
|
recordBatches = recordBatchesBd.ToImmutable();
|
||||||
|
this.count += n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return !count.HasValue || count.Value <= recordBatches.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
class MyColList : IReadOnlyList<IFeatherColumn> {
|
||||||
|
readonly FeatherTable owner;
|
||||||
|
readonly int columnIndex;
|
||||||
|
|
||||||
|
public MyColList(FeatherTable owner, int columnIndex) {
|
||||||
|
this.owner = owner;
|
||||||
|
this.columnIndex = columnIndex;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IFeatherColumn this[int index] {
|
||||||
|
get {
|
||||||
|
if (index < 0) {
|
||||||
|
throw new IndexOutOfRangeException();
|
||||||
|
}
|
||||||
|
if (index >= owner.recordBatches.Count) {
|
||||||
|
var task = owner.readBatches(index + 1);
|
||||||
|
task.Wait();
|
||||||
|
if (!task.Result) {
|
||||||
|
throw new IndexOutOfRangeException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return owner.recordBatches[index].Columns[columnIndex];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Count {
|
||||||
|
get {
|
||||||
|
if (owner.reader is not null) {
|
||||||
|
owner.readBatches().Wait();
|
||||||
|
}
|
||||||
|
return owner.recordBatches.Count;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IEnumerator<IFeatherColumn> GetEnumerator() {
|
||||||
|
int i = 0;
|
||||||
|
foreach (var batch in owner.recordBatches) {
|
||||||
|
yield return batch.Columns[columnIndex];
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
while (owner.reader is not null) {
|
||||||
|
var task = owner.readBatches(1);
|
||||||
|
task.Wait();
|
||||||
|
if (!task.Result) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
yield return owner.recordBatches[i].Columns[columnIndex];
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
IEnumerator IEnumerable.GetEnumerator() =>
|
||||||
|
GetEnumerator();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using mbts.utils;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("FilterStream")]
|
||||||
|
public class FilterStream : ViewStream {
|
||||||
|
public class RegExpFilter : IStreamFilter {
|
||||||
|
private Regex[] myRegExps;
|
||||||
|
private string[] myCols;
|
||||||
|
|
||||||
|
public RegExpFilter(string col, string regExp)
|
||||||
|
: this(new string[] { col }, new string[] { regExp }) { }
|
||||||
|
|
||||||
|
public RegExpFilter(string[] cols, string[] regExps) {
|
||||||
|
myCols = cols;
|
||||||
|
myRegExps = new Regex[regExps.Length];
|
||||||
|
for (int i = 0; i < regExps.Length; i++)
|
||||||
|
myRegExps[i] = new Regex(regExps[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#region IStreamFilter Members
|
||||||
|
public bool filter(IBlueRow dr) {
|
||||||
|
bool rv = true;
|
||||||
|
for (int i = 0; i < myCols.Length; i++) {
|
||||||
|
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]]?.ToString() ?? string.Empty);
|
||||||
|
if (!rv)
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
BlueTableImmutable? myLastFiredRows;
|
||||||
|
IBlueRowCollection? myLastReceivedRows;
|
||||||
|
private IStreamFilter myFilter;
|
||||||
|
private ViewStream myParentStream;
|
||||||
|
private string myFilterString;
|
||||||
|
private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
|
||||||
|
|
||||||
|
#region constructors
|
||||||
|
public delegate bool OnFilter(IBlueRow r);
|
||||||
|
private class OnFilterWrapper : IStreamFilter {
|
||||||
|
public OnFilterWrapper(OnFilter f) { this.f = f; }
|
||||||
|
|
||||||
|
public bool filter(IBlueRow dr) { return f(dr); }
|
||||||
|
|
||||||
|
private OnFilter f;
|
||||||
|
}
|
||||||
|
public FilterStream(ViewStream vs, OnFilter f)
|
||||||
|
: this(vs, new OnFilterWrapper(f)) { }
|
||||||
|
|
||||||
|
|
||||||
|
public FilterStream(ViewStream vs, IStreamFilter filter) {
|
||||||
|
myParentStream = vs;
|
||||||
|
myFilter = filter;
|
||||||
|
myFilterString = filter.ToString() ?? string.Empty;
|
||||||
|
if (filter is IDynamicStreamFilter dynFilter) {
|
||||||
|
dynFilter.onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public FilterStream(ViewStream vs, string filterExpression)
|
||||||
|
: this(vs, new BooleanExpressionFilter(filterExpression)) {
|
||||||
|
myFilterString = filterExpression;
|
||||||
|
}
|
||||||
|
|
||||||
|
public FilterStream(ViewStream vs, string col, string regExp)
|
||||||
|
: this(vs, new RegExpFilter(col, regExp)) {
|
||||||
|
myFilterString = col + " , " + regExp;
|
||||||
|
}
|
||||||
|
|
||||||
|
public FilterStream(ViewStream vs)
|
||||||
|
: this(vs, new BinaryFilter(true)) {
|
||||||
|
myFilterString = true.ToString();
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region overridden methods
|
||||||
|
protected override void myOnFirstSubscriber() =>
|
||||||
|
myParentStream.onChange += myHandleCallbacks;
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
myLastFiredRows = null;
|
||||||
|
myParentStream.onChange -= myHandleCallbacks;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myFireEvent(object o) {
|
||||||
|
var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
|
||||||
|
if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
|
||||||
|
base.myFireEvent(o);
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
private void myHandleCallbacks(object newValue) {
|
||||||
|
var arrivedPacket = ExceptionAssert.Argument.Is<IDataPacket>(newValue);
|
||||||
|
myLastReceivedRows = arrivedPacket.newRows;
|
||||||
|
var bd = (myLastFiredRows ?? new BlueTableImmutable(arrivedPacket.Schema)).ToDataPacketBuilder();
|
||||||
|
foreach (IBlueRow r in arrivedPacket.deletedRows) {
|
||||||
|
// Don't bother evaluating the filter, just try to remove the row. The latter is
|
||||||
|
// cheaper than the former (it's a noop if this row wasn't in the filtered table).
|
||||||
|
bd.Remove(r.Id());
|
||||||
|
}
|
||||||
|
foreach (IBlueRow r in arrivedPacket.newRows) {
|
||||||
|
if (myFilter.filter(r)) {
|
||||||
|
bd.AddOrUpdate(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var dpOut = bd.ToDataPacket(out myLastFiredRows);
|
||||||
|
myFireEvent(dpOut);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IStreamFilter filter {
|
||||||
|
set {
|
||||||
|
{
|
||||||
|
myFilter = value;
|
||||||
|
if (numberOfSubscribers > 0) {
|
||||||
|
if (myLastReceivedRows != null) {
|
||||||
|
var bd = (myLastFiredRows ?? new BlueTableImmutable(myLastReceivedRows.schema)).ToDataPacketBuilder();
|
||||||
|
foreach (IBlueRow r in myLastReceivedRows) {
|
||||||
|
int id = r.Id();
|
||||||
|
bool includeNow = myFilter.filter(r);
|
||||||
|
bool includeBefore = myLastFiredRows?.containsID(id) ?? false;
|
||||||
|
if (includeNow == includeBefore) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (includeNow) {
|
||||||
|
bd.AddOrUpdate(r);
|
||||||
|
} else {
|
||||||
|
bd.Remove(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var dpOut = bd.ToDataPacket(out myLastFiredRows);
|
||||||
|
myFireEvent(dpOut);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
get {
|
||||||
|
return myFilter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//public string filterString { set { filter = new BooleanExpressionFilter(value); } }
|
||||||
|
|
||||||
|
public override string ToString() {
|
||||||
|
return GetType().ToString() + " " + myFilterString;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void myHandleFilterChange() {
|
||||||
|
filter = myFilter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
public class FilterTable : IBlueRowCollection {
|
||||||
|
readonly IBlueRowCollection source;
|
||||||
|
readonly HashSet<int> excludedIndices = new();
|
||||||
|
readonly Dictionary<int, (int Index, int NumSkips, bool Included, IBlueRow Row)> rowMap = new();
|
||||||
|
readonly List<int> filteredIndexToId = new();
|
||||||
|
|
||||||
|
public FilterTable(IBlueRowCollection source, Func<IBlueRow, bool> filter) {
|
||||||
|
this.source = source;
|
||||||
|
int idx = 0;
|
||||||
|
int numSkips = 0;
|
||||||
|
foreach (var r in source) {
|
||||||
|
int id = r.Id();
|
||||||
|
bool included = filter(r);
|
||||||
|
rowMap.Add(id, (idx, numSkips, included, r));
|
||||||
|
if (!included) {
|
||||||
|
excludedIndices.Add(idx);
|
||||||
|
numSkips++;
|
||||||
|
} else {
|
||||||
|
filteredIndexToId.Add(id);
|
||||||
|
}
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] {
|
||||||
|
get {
|
||||||
|
var r = rowMap[id];
|
||||||
|
return r.Included ? r.Row : throw new KeyNotFoundException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => source.schema;
|
||||||
|
|
||||||
|
public int Count => filteredIndexToId.Count;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
rowMap.TryGetValue(id, out var r) && r.Included;
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
int idx = 0;
|
||||||
|
foreach (var r in source) {
|
||||||
|
if (!excludedIndices.Contains(idx)) {
|
||||||
|
yield return r;
|
||||||
|
}
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) {
|
||||||
|
if (index < 0 || index >= source.Count) {
|
||||||
|
throw new IndexOutOfRangeException();
|
||||||
|
}
|
||||||
|
var id = filteredIndexToId[index];
|
||||||
|
return rowMap[id].Row;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) =>
|
||||||
|
rowMap.TryGetValue(item.Id(), out var r)
|
||||||
|
&& r.Included
|
||||||
|
&& BlueRowEqualityComparer.Equals(r.Row, item)
|
||||||
|
? r.Index - r.NumSkips
|
||||||
|
: -1;
|
||||||
|
}
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
namespace Bts.SharedStreams.StreamFactories;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
using bts.api.core;
|
||||||
|
using bts.BlueOrb;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.Immutable;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Reactive.Linq;
|
||||||
|
using System.Threading;
|
||||||
|
|
||||||
|
public class GroupByApplyStream : ViewStream {
|
||||||
|
readonly ViewStream inputStream;
|
||||||
|
readonly IFunctionEvaluator groupBy;
|
||||||
|
readonly IFunctionEvaluator apply;
|
||||||
|
|
||||||
|
public GroupByApplyStream(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) {
|
||||||
|
this.inputStream = inputStream;
|
||||||
|
this.groupBy = groupBy;
|
||||||
|
this.apply = apply;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<IBlueRow, int>> CreateGroupByApplyTable(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) =>
|
||||||
|
CreateApplyTable(CreateGroupByTable(inputStream, groupBy), apply);
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<IBlueRow, int>> CreateApplyTable<TGroupKey>(IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey>> inputStream, Func<TGroupKey, IBlueRowCollection, IBlueRowCollection?> apply) where TGroupKey : IEquatable<TGroupKey> {
|
||||||
|
var groupTables = ImmutableDictionary<TGroupKey, GroupTableBuilder>.Empty;
|
||||||
|
var inputToOutputIds = ImmutableDictionary<int, int>.Empty;
|
||||||
|
var outputToInputIds = ImmutableDictionary<int, int>.Empty;
|
||||||
|
int idCounter = 0;
|
||||||
|
|
||||||
|
return Observable.Create<TableUpdate<IBlueRow, int>>(Subscribe);
|
||||||
|
|
||||||
|
int GetNextId() =>
|
||||||
|
Interlocked.Increment(ref idCounter);
|
||||||
|
|
||||||
|
IDisposable Subscribe(IObserver<TableUpdate<IBlueRow, int>> observer) {
|
||||||
|
int hashCounter = 0;
|
||||||
|
var hashIndex = new Dictionary<TGroupKey, int>();
|
||||||
|
var tableOut = TableUpdate<IBlueRow, int>.mk_empty(r => r.Id());
|
||||||
|
return inputStream.Subscribe(OnNext, OnError, OnCompleted);
|
||||||
|
|
||||||
|
void OnNext(TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey> upd) {
|
||||||
|
var bd = tableOut.to_builder();
|
||||||
|
foreach (var group in upd.removed) {
|
||||||
|
var hash = hashIndex[group.group_key];
|
||||||
|
bd.remove(hash);
|
||||||
|
}
|
||||||
|
var groupTablesBd = groupTables.ToBuilder();
|
||||||
|
foreach (var group in upd.added.Concat(upd.changed)) {
|
||||||
|
Log.assert(group.table.state.Count > 0);
|
||||||
|
var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++);
|
||||||
|
var groupRowCollection = new TableUpdateGroupingRowCollection<TGroupKey>(group, group.table.state.Values.First().schema);
|
||||||
|
var applyArgs = new (string?, int?, object?)[] { ("rows", 0, groupRowCollection) };
|
||||||
|
var resRows = apply(group.group_key, groupRowCollection);
|
||||||
|
if (resRows is null) {
|
||||||
|
bd.remove(hash);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
UpdateGroupTable(group.group_key, resRows);
|
||||||
|
}
|
||||||
|
groupTables = groupTablesBd.ToImmutable();
|
||||||
|
tableOut = bd.to_table();
|
||||||
|
observer.OnNext(tableOut);
|
||||||
|
|
||||||
|
void UpdateGroupTable(TGroupKey groupKey, IBlueRowCollection groupRows) {
|
||||||
|
var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows);
|
||||||
|
foreach (var r in indexedRows) {
|
||||||
|
bd.add_or_update(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void OnError(Exception ex) {
|
||||||
|
Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }""");
|
||||||
|
}
|
||||||
|
|
||||||
|
void OnCompleted() {
|
||||||
|
Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": "{{inputStream.GetType()}}" }""");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<IBlueRow, int>> CreateApplyTable<TGroupKey>(IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey>> inputStream, IFunctionEvaluator apply) where TGroupKey : IEquatable<TGroupKey> {
|
||||||
|
var groupTables = ImmutableDictionary<TGroupKey, GroupTableBuilder>.Empty;
|
||||||
|
var inputToOutputIds = ImmutableDictionary<int, int>.Empty;
|
||||||
|
var outputToInputIds = ImmutableDictionary<int, int>.Empty;
|
||||||
|
int idCounter = 0;
|
||||||
|
|
||||||
|
return Observable.Create<TableUpdate<IBlueRow, int>>(Subscribe);
|
||||||
|
|
||||||
|
int GetNextId() =>
|
||||||
|
Interlocked.Increment(ref idCounter);
|
||||||
|
|
||||||
|
IDisposable Subscribe(IObserver<TableUpdate<IBlueRow, int>> observer) {
|
||||||
|
int hashCounter = 0;
|
||||||
|
var hashIndex = new Dictionary<TGroupKey, int>();
|
||||||
|
var tableOut = TableUpdate<IBlueRow, int>.mk_empty(r => r.Id());
|
||||||
|
return inputStream.Subscribe(OnNext, OnError, OnCompleted);
|
||||||
|
|
||||||
|
void OnNext(TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey> upd) {
|
||||||
|
var bd = tableOut.to_builder();
|
||||||
|
foreach (var group in upd.removed) {
|
||||||
|
var hash = hashIndex[group.group_key];
|
||||||
|
bd.remove(hash);
|
||||||
|
}
|
||||||
|
var groupTablesBd = groupTables.ToBuilder();
|
||||||
|
foreach (var group in upd.added.Concat(upd.changed)) {
|
||||||
|
Log.assert(group.table.state.Count > 0);
|
||||||
|
var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++);
|
||||||
|
var groupRowCollection = new TableUpdateGroupingRowCollection<TGroupKey>(group, group.table.state.Values.First().schema);
|
||||||
|
var applyArgs = new StreamArgument[] { ("rows", 0, groupRowCollection) };
|
||||||
|
var res = apply.Evaluate(string.Empty, applyArgs);
|
||||||
|
if (res is IBlueRow resSingleRow) {
|
||||||
|
UpdateGroupTable(group.group_key, new SingleRowTable(resSingleRow));
|
||||||
|
} else if (res is IBlueRowCollection resRows) {
|
||||||
|
UpdateGroupTable(group.group_key, resRows);
|
||||||
|
} else {
|
||||||
|
throw new InvalidOperationException("Invalid result returned by 'Apply' fn");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
groupTables = groupTablesBd.ToImmutable();
|
||||||
|
tableOut = bd.to_table();
|
||||||
|
observer.OnNext(tableOut);
|
||||||
|
|
||||||
|
void UpdateGroupTable(TGroupKey groupKey, IBlueRowCollection groupRows) {
|
||||||
|
var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows);
|
||||||
|
foreach (var r in indexedRows) {
|
||||||
|
bd.add_or_update(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void OnError(Exception ex) {
|
||||||
|
Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }""");
|
||||||
|
}
|
||||||
|
|
||||||
|
void OnCompleted() {
|
||||||
|
Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": "{{inputStream.GetType()}}" }""");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>> CreateGroupByTable(ViewStream inputStream, IFunctionEvaluator groupBy) {
|
||||||
|
return Observable.Create<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>>(Subscribe);
|
||||||
|
|
||||||
|
IDisposable Subscribe(IObserver<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>> observer) {
|
||||||
|
TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper> tableOut = TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>.mk_empty(r => r.group_key);
|
||||||
|
return inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), GroupKey).Subscribe(observer);
|
||||||
|
|
||||||
|
EquatableKeyWrapper GroupKey(IBlueRow r) {
|
||||||
|
var groupByArgs = new StreamArgument[] { ("row", 0, r) };
|
||||||
|
var groupKey = new EquatableKeyWrapper(groupBy.Evaluate(string.Empty, groupByArgs) ?? DBNull.Value);
|
||||||
|
return groupKey;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyArrayWrapper>, EquatableKeyArrayWrapper>> CreateGroupByTable(ViewStream inputStream, string[] columns, bool useColumnIndices = true) {
|
||||||
|
Func<IBlueRow, EquatableKeyArrayWrapper> groupBy;
|
||||||
|
if (!useColumnIndices) {
|
||||||
|
groupBy = GroupKeyByName;
|
||||||
|
|
||||||
|
EquatableKeyArrayWrapper GroupKeyByName(IBlueRow r) =>
|
||||||
|
new(Array.ConvertAll(columns, c => r[c]));
|
||||||
|
} else {
|
||||||
|
int[]? columnIndices = null;
|
||||||
|
groupBy = GroupKeyByIndex;
|
||||||
|
|
||||||
|
EquatableKeyArrayWrapper GroupKeyByIndex(IBlueRow r) {
|
||||||
|
columnIndices ??= r.schema.ResolveIndices(columns);
|
||||||
|
return new(Array.ConvertAll(columnIndices, c => r[c]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return CreateGroupByTable(inputStream, groupBy);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>> CreateGroupByTable<TKey>(ViewStream inputStream, Func<IBlueRow, TKey> groupBy) where TKey : IEquatable<TKey> {
|
||||||
|
return Observable.Create<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>>(Subscribe);
|
||||||
|
|
||||||
|
IDisposable Subscribe(IObserver<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>> observer) =>
|
||||||
|
inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), groupBy).Subscribe(observer);
|
||||||
|
}
|
||||||
|
|
||||||
|
class GroupTableBuilder {
|
||||||
|
readonly Func<int> getUniqueId;
|
||||||
|
ImmutableDictionary<int, int> inputToOutputIds = ImmutableDictionary<int, int>.Empty;
|
||||||
|
ImmutableDictionary<int, int> outputToInputIds = ImmutableDictionary<int, int>.Empty;
|
||||||
|
IndexTable? current;
|
||||||
|
|
||||||
|
public GroupTableBuilder(Func<int> getUniqueId) =>
|
||||||
|
this.getUniqueId = getUniqueId;
|
||||||
|
|
||||||
|
public IndexTable? Current => current;
|
||||||
|
|
||||||
|
public IndexTable Reset(IBlueRowCollection rows) {
|
||||||
|
var inputToOutputIdsBd = inputToOutputIds.ToBuilder();
|
||||||
|
var outputToInputIdsBd = outputToInputIds.ToBuilder();
|
||||||
|
foreach (var r in rows) {
|
||||||
|
int inputId = r.Id();
|
||||||
|
if (!inputToOutputIdsBd.TryGetValue(inputId, out var outputId)) {
|
||||||
|
outputId = getUniqueId();
|
||||||
|
inputToOutputIdsBd.Add(inputId, outputId);
|
||||||
|
outputToInputIdsBd.Add(outputId, inputId);
|
||||||
|
}
|
||||||
|
Log.assert(outputToInputIdsBd.TryGetValue(outputId, out var oinputId) && oinputId == inputId);
|
||||||
|
Log.assert(inputToOutputIdsBd.TryGetValue(inputId, out var ioutputId) && ioutputId == outputId);
|
||||||
|
}
|
||||||
|
inputToOutputIds = inputToOutputIdsBd.ToImmutable();
|
||||||
|
outputToInputIds = outputToInputIdsBd.ToImmutable();
|
||||||
|
current = new(rows, outputToInputIds, inputToOutputIds);
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
//using bts.SilverPine;
|
||||||
|
using BCG;
|
||||||
|
using mbts.utils;
|
||||||
|
|
||||||
|
namespace bts.BlueOrb {
|
||||||
|
[DataStreamUserRespresentation("GroupByStream")]
|
||||||
|
public class GroupByStream : ViewStream {
|
||||||
|
|
||||||
|
public interface INonIncGroupByEvaluator {
|
||||||
|
// grouped_rows
|
||||||
|
IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
||||||
|
string[]? get_group_by_cols();
|
||||||
|
}
|
||||||
|
|
||||||
|
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
||||||
|
Func<string[]?> _groupby_cols_func;
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
||||||
|
public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
||||||
|
_groupby_cols_func = groupby_cols_func;
|
||||||
|
_evaluate_func = evaluate_func;
|
||||||
|
}
|
||||||
|
public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||||
|
BlueTable? rv = null;
|
||||||
|
foreach (var kv in grouped_rows) {
|
||||||
|
List<IBlueRow> g_rows = kv.Value;
|
||||||
|
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
||||||
|
if (out_rows is null || out_rows.Count == 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rv ??= new BlueTable(out_rows.ElementAt(0).schema);
|
||||||
|
rv.addRows(out_rows);
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
public string[]? get_group_by_cols() {
|
||||||
|
return _groupby_cols_func();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
readonly ViewStream? _main_stream;
|
||||||
|
readonly INonIncGroupByEvaluator? _groupby_func;
|
||||||
|
readonly string[]? _groupby_cols;
|
||||||
|
readonly Stopwatch _timer = new();
|
||||||
|
|
||||||
|
BlueRowHasher? _row_hasher;
|
||||||
|
int _n_total_fires = 0;
|
||||||
|
int _n_expensive_fires = 0;
|
||||||
|
|
||||||
|
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
||||||
|
_main_stream = main_stream;
|
||||||
|
_groupby_func = groupby_func;
|
||||||
|
_groupby_cols = _groupby_func.get_group_by_cols();
|
||||||
|
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
||||||
|
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
||||||
|
|
||||||
|
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
|
||||||
|
: this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
|
||||||
|
|
||||||
|
// this is the main constructor that does everything
|
||||||
|
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
|
||||||
|
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
|
||||||
|
Func<string[]?> groupby_cols_func = () => {
|
||||||
|
if (String.IsNullOrEmpty(groupby_cols))
|
||||||
|
return null;
|
||||||
|
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
|
||||||
|
};
|
||||||
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
||||||
|
if (!String.IsNullOrEmpty(optional_sort_col))
|
||||||
|
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
|
||||||
|
// this si where the group by aggregation happens
|
||||||
|
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
|
||||||
|
if (rv == null)
|
||||||
|
return new List<IBlueRow>();
|
||||||
|
return new IBlueRow[] { rv }.ToList();
|
||||||
|
};
|
||||||
|
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
|
||||||
|
this(parent_str, String.Empty, standard_group_by_text) {
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() { Log.notnull(_main_stream).onChange += _main_stream_onchange; }
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() { Log.notnull(_main_stream).onChange -= _main_stream_onchange; }
|
||||||
|
|
||||||
|
private void _main_stream_onchange(object nv) {
|
||||||
|
var dp = ExceptionAssert.Argument.Is<IDataPacket>(nv);
|
||||||
|
///grouping together
|
||||||
|
_timer.Restart();
|
||||||
|
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
||||||
|
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
|
||||||
|
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
|
||||||
|
_timer.Stop();
|
||||||
|
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
|
if (groupElapsedMs >= 50) {
|
||||||
|
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
||||||
|
}
|
||||||
|
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||||
|
/// reducing
|
||||||
|
_timer.Restart();
|
||||||
|
var total_state = Log.notnull(_groupby_func).evaluate(grouped_rows);
|
||||||
|
if (total_state == null)
|
||||||
|
// mayeb returna n empty table ?
|
||||||
|
return;
|
||||||
|
_timer.Stop();
|
||||||
|
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
|
_n_total_fires++;
|
||||||
|
if (applyElapsedMs >= 50) {
|
||||||
|
_n_expensive_fires++;
|
||||||
|
Log.DataFlow.info(
|
||||||
|
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||||
|
}
|
||||||
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class GroupByTable<TKey> : IBlueRowCollection {
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
readonly IBlueRowCollection inputRows;
|
||||||
|
readonly List<IBlueRow> rows;
|
||||||
|
readonly Dictionary<int, int> idToIndex;
|
||||||
|
|
||||||
|
public GroupByTable(IBlueRowCollection inputRows, Func<IBlueRow, TKey> getKey, IEqualityComparer<TKey> keyEqualityComparer, IComparer<TKey> keyComparer, string? groupKeyCol = "group") {
|
||||||
|
this.inputRows = inputRows;
|
||||||
|
var cols = inputRows.schema.columnInfo.ToList();
|
||||||
|
rows = new(inputRows.Count);
|
||||||
|
int groupKeyColIdx = -1;
|
||||||
|
if (!string.IsNullOrEmpty(groupKeyCol)) {
|
||||||
|
groupKeyColIdx = cols.Count;
|
||||||
|
cols.Add(new(groupKeyCol, groupKeyColIdx, typeof(TKey)));
|
||||||
|
}
|
||||||
|
for (int i = 0; i < cols.Count; i++) {
|
||||||
|
Log.assert(cols[i].colNo == i);
|
||||||
|
}
|
||||||
|
outputSchema = new BlueTableSchema(cols.ToArray(), "GroupBy");
|
||||||
|
rows.AddRange(inputRows
|
||||||
|
.GroupBy(getKey, keyEqualityComparer)
|
||||||
|
.OrderBy(g => g.Key, keyComparer)
|
||||||
|
.SelectMany(rg => rg.Select(r => new Row(rg.Key, r, outputSchema, groupKeyColIdx))));
|
||||||
|
idToIndex = rows.Select((r, i) => (r, i)).ToDictionary(t => t.r.Id(), t => t.i);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
rows[idToIndex[id]];
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => outputSchema;
|
||||||
|
|
||||||
|
public int Count => rows.Count;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
idToIndex.ContainsKey(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() =>
|
||||||
|
rows.GetEnumerator();
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
rows[index];
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) =>
|
||||||
|
idToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(item, rows[idx])
|
||||||
|
? idx
|
||||||
|
: -1;
|
||||||
|
|
||||||
|
public class Row : IBlueRow {
|
||||||
|
readonly IBlueRow inputRow;
|
||||||
|
readonly IBlueTableSchema outputSchema;
|
||||||
|
readonly int keyColIdx;
|
||||||
|
|
||||||
|
public Row(TKey key, IBlueRow inputRow, IBlueTableSchema outputSchema, int keyColIdx = -1) {
|
||||||
|
Key = key;
|
||||||
|
this.inputRow = inputRow;
|
||||||
|
this.outputSchema = outputSchema;
|
||||||
|
this.keyColIdx = keyColIdx;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object? this[int index] {
|
||||||
|
get {
|
||||||
|
if (index == keyColIdx) {
|
||||||
|
return Key;
|
||||||
|
}
|
||||||
|
return inputRow[index];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public TKey Key { get; }
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => outputSchema;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,229 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.Immutable;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
public class ImmutableOrderedTable : IBlueRowCollection {
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
readonly ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows;
|
||||||
|
readonly IReadOnlyDictionary<long, IBlueRow> ordinalToRow;
|
||||||
|
readonly ImmutableDictionary<int, long> idToOrdinal;
|
||||||
|
long newRowOrdinal = 0;
|
||||||
|
|
||||||
|
public ImmutableOrderedTable(IBlueTableSchema schema)
|
||||||
|
: this(
|
||||||
|
schema,
|
||||||
|
ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<long, IBlueRow>()),
|
||||||
|
ImmutableDictionary<int, long>.Empty, 0) { }
|
||||||
|
|
||||||
|
ImmutableOrderedTable(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows, ImmutableDictionary<int, long> idToOrdinal, long newRowOrdinal) {
|
||||||
|
this.schema = schema;
|
||||||
|
this.rows = rows;
|
||||||
|
this.idToOrdinal = idToOrdinal;
|
||||||
|
this.newRowOrdinal = newRowOrdinal;
|
||||||
|
ordinalToRow = ImmutableSortedSetDictionary.Create(rows);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static OrderedTable FromRowCollection(IBlueRowCollection rowCollection) {
|
||||||
|
var rowList = new List<IBlueRow>(rowCollection.Count);
|
||||||
|
var idToIndex = new Dictionary<int, int>();
|
||||||
|
foreach (var r in rowCollection) {
|
||||||
|
idToIndex[r.Id()] = rowList.Count;
|
||||||
|
rowList.Add(r);
|
||||||
|
}
|
||||||
|
return new(rowCollection.schema, rowList, idToIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema Schema => schema;
|
||||||
|
|
||||||
|
public int Count => rows.Count;
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
idToOrdinal.TryGetValue(id, out var ord) ? ordinalToRow[ord] : throw new KeyNotFoundException();
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRowCollection.schema => schema;
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
idToOrdinal.ContainsKey(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
foreach (var (_, row) in rows) {
|
||||||
|
yield return row;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
rows[index].Value;
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) =>
|
||||||
|
idToOrdinal.TryGetValue(item.Id(), out var ord) ? rows.IndexOf(new(ord, null!)) : -1;
|
||||||
|
|
||||||
|
public Builder ToBuilder() =>
|
||||||
|
new(schema, rows, idToOrdinal, newRowOrdinal);
|
||||||
|
|
||||||
|
public class Builder : IBlueRowCollection {
|
||||||
|
readonly IBlueTableSchema schema;
|
||||||
|
|
||||||
|
ImmutableSortedSet<KeyValuePair<long, IBlueRow>> originalRows;
|
||||||
|
ImmutableSortedSet<KeyValuePair<long, IBlueRow>>.Builder rows;
|
||||||
|
ImmutableDictionary<int, long> originalIdToOrdinal;
|
||||||
|
ImmutableDictionary<int, long>.Builder idToOrdinal;
|
||||||
|
IReadOnlyDictionary<long, IBlueRow> originalOrdinalToRow;
|
||||||
|
IReadOnlyDictionary<long, IBlueRow> ordinalToRow;
|
||||||
|
|
||||||
|
readonly HashSet<int> removes = new();
|
||||||
|
readonly HashSet<int> adds = new();
|
||||||
|
long newRowOrdinal;
|
||||||
|
|
||||||
|
public Builder(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows, ImmutableDictionary<int, long> idToOrdinal, long newRowOrdinal) {
|
||||||
|
this.schema = schema;
|
||||||
|
originalRows = rows;
|
||||||
|
this.rows = rows.ToBuilder();
|
||||||
|
originalIdToOrdinal = idToOrdinal;
|
||||||
|
this.idToOrdinal = idToOrdinal.ToBuilder();
|
||||||
|
originalOrdinalToRow = ImmutableSortedSetDictionary.Create(originalRows);
|
||||||
|
ordinalToRow = ImmutableSortedSetDictionary.CreateBuilder(this.rows);
|
||||||
|
this.newRowOrdinal = newRowOrdinal;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow this[int id] {
|
||||||
|
get {
|
||||||
|
if (idToOrdinal.TryGetValue(id, out var ord) && !removes.Contains(id)) {
|
||||||
|
return ordinalToRow[ord];
|
||||||
|
}
|
||||||
|
throw new KeyNotFoundException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Count => rows.Count + adds.Count - removes.Count;
|
||||||
|
|
||||||
|
IBlueTableSchema IBlueRowCollection.schema => schema;
|
||||||
|
|
||||||
|
public void Add(IBlueRow row) =>
|
||||||
|
addOrUpdate(row, false);
|
||||||
|
|
||||||
|
public void AddOrUpdate(IBlueRow row) =>
|
||||||
|
addOrUpdate(row, true);
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
(idToOrdinal.ContainsKey(id) && !removes.Contains(id)) || adds.Contains(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
foreach (var row in rows) {
|
||||||
|
if (!removes.Contains(row.Value.Id())) {
|
||||||
|
yield return row.Value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) =>
|
||||||
|
rows[index].Value;
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
if (idToOrdinal.TryGetValue(item.Id(), out var ord)) {
|
||||||
|
return rows.IndexOf(new(ord, item));
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool Remove(int id) {
|
||||||
|
if (!idToOrdinal.TryGetValue(id, out var ord)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
idToOrdinal.Remove(id);
|
||||||
|
Log.assert(rows.Remove(new(ord, null!)));
|
||||||
|
Log.assert(ordinalToRow.Count == rows.Count);
|
||||||
|
adds.Remove(id);
|
||||||
|
if (originalIdToOrdinal.ContainsKey(id)) {
|
||||||
|
removes.Add(id);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ICollection<IBlueRow>.Remove(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
if (!idToOrdinal.TryGetValue(id, out var ord) || !BlueRowEqualityComparer.Equals(item, ordinalToRow[ord])) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
rows.Remove(new(ord, null!));
|
||||||
|
adds.Remove(id);
|
||||||
|
if (originalIdToOrdinal.ContainsKey(id)) {
|
||||||
|
removes.Add(id);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ImmutableOrderedTable ToImmutable(out IDataPacket dp) {
|
||||||
|
var newRows = new ImmutableOrderedTable(schema, rows.ToImmutable(), idToOrdinal.ToImmutable(), newRowOrdinal);
|
||||||
|
dp = new DataPacket(
|
||||||
|
newRows,
|
||||||
|
new BlueTableListWrapper(schema, adds.Select(id => ordinalToRow[idToOrdinal[id]]).ToList()),
|
||||||
|
new BlueTableListWrapper(schema, removes.Select(id => originalOrdinalToRow[originalIdToOrdinal[id]]).ToList()));
|
||||||
|
return newRows;
|
||||||
|
}
|
||||||
|
|
||||||
|
void addOrUpdate(IBlueRow row, bool canUpdate) {
|
||||||
|
if (newRowOrdinal == long.MaxValue) {
|
||||||
|
resetOrdinal();
|
||||||
|
}
|
||||||
|
int id = row.Id();
|
||||||
|
if (idToOrdinal.TryGetValue(id, out var ord)) {
|
||||||
|
if (!canUpdate) {
|
||||||
|
throw new ArgumentException("Row id already present", nameof(row));
|
||||||
|
}
|
||||||
|
rows.Remove(new(ord, row));
|
||||||
|
}
|
||||||
|
ord = newRowOrdinal++;
|
||||||
|
idToOrdinal[id] = ord;
|
||||||
|
if (originalIdToOrdinal.ContainsKey(id)) {
|
||||||
|
removes.Add(id);
|
||||||
|
}
|
||||||
|
rows.Add(new(ord, row));
|
||||||
|
adds.Add(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
void resetOrdinal() {
|
||||||
|
Log.DataFlow.warn($$"""[immutable_ordered_table_reset_ordinal.json]{ "tableName": "{{schema.tableName}}", "origNumRows": {{originalRows.Count}}, "numRows": {{rows.Count}}, "ts": {{DateTime.Now}} }""");
|
||||||
|
var originalRowsBd = originalRows.Clear().ToBuilder();
|
||||||
|
var originalIdToOrdinalBd = originalIdToOrdinal.Clear().ToBuilder();
|
||||||
|
var newRowsBd = ImmutableSortedSet.CreateBuilder(rows.KeyComparer);
|
||||||
|
var newIdToOrdinalBd = ImmutableDictionary.CreateBuilder<int, long>(idToOrdinal.KeyComparer);
|
||||||
|
long ord = 0;
|
||||||
|
long? prevRowOrd = null;
|
||||||
|
foreach (var kv in originalRows) {
|
||||||
|
if (prevRowOrd.HasValue) {
|
||||||
|
Log.assert(prevRowOrd.Value < kv.Key);
|
||||||
|
}
|
||||||
|
prevRowOrd = kv.Key;
|
||||||
|
originalRowsBd.Add(new(ord, kv.Value));
|
||||||
|
originalIdToOrdinalBd.Add(new(kv.Value.Id(), ord));
|
||||||
|
ord++;
|
||||||
|
}
|
||||||
|
prevRowOrd = null;
|
||||||
|
foreach (var kv in rows) {
|
||||||
|
if (prevRowOrd.HasValue) {
|
||||||
|
Log.assert(prevRowOrd.Value < kv.Key);
|
||||||
|
}
|
||||||
|
prevRowOrd = kv.Key;
|
||||||
|
newRowsBd.Add(new(ord, kv.Value));
|
||||||
|
newIdToOrdinalBd.Add(new(kv.Value.Id(), ord));
|
||||||
|
ord++;
|
||||||
|
}
|
||||||
|
rows = newRowsBd;
|
||||||
|
originalRows = originalRowsBd.ToImmutable();
|
||||||
|
originalIdToOrdinal = originalIdToOrdinalBd.ToImmutable();
|
||||||
|
idToOrdinal = newIdToOrdinalBd;
|
||||||
|
originalOrdinalToRow = ImmutableSortedSetDictionary.Create(originalRows);
|
||||||
|
ordinalToRow = ImmutableSortedSetDictionary.CreateBuilder(rows);
|
||||||
|
newRowOrdinal = ord;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
public class IndexRow : IBlueRow {
|
||||||
|
readonly IBlueRow inputRow;
|
||||||
|
readonly int id;
|
||||||
|
readonly int idIndex;
|
||||||
|
|
||||||
|
public IndexRow(IBlueRow inputRow, int id, int idIndex, IBlueTableSchema schema) {
|
||||||
|
this.inputRow = inputRow;
|
||||||
|
this.id = id;
|
||||||
|
this.idIndex = idIndex;
|
||||||
|
this.schema = schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow InputRow => inputRow;
|
||||||
|
|
||||||
|
public IBlueTableSchema schema { get; }
|
||||||
|
|
||||||
|
public object? this[int index] =>
|
||||||
|
index == idIndex ? id : inputRow[index];
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Disposables;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Immutable;
|
||||||
|
|
||||||
|
public class IndexStream : ViewStream {
|
||||||
|
readonly ViewStream inputStream;
|
||||||
|
|
||||||
|
ImmutableDictionary<int, int> inputToOutputIds = ImmutableDictionary<int, int>.Empty;
|
||||||
|
ImmutableDictionary<int, int> outputToInputIds = ImmutableDictionary<int, int>.Empty;
|
||||||
|
int idCounter = 0;
|
||||||
|
IDisposable? sub;
|
||||||
|
|
||||||
|
public IndexStream(ViewStream inputStream) =>
|
||||||
|
this.inputStream = inputStream;
|
||||||
|
|
||||||
|
~IndexStream() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
base.myOnFirstSubscriber();
|
||||||
|
sub = inputStream.Subscribe(OnInputChanged);
|
||||||
|
|
||||||
|
void OnInputChanged(IDataPacket dp) {
|
||||||
|
var inputToOutputIdsBd = inputToOutputIds.ToBuilder();
|
||||||
|
var outputToInputIdsBd = outputToInputIds.ToBuilder();
|
||||||
|
foreach (var r in dp.addedRows) {
|
||||||
|
int inputId = r.Id();
|
||||||
|
if (!inputToOutputIds.TryGetValue(inputId, out int outputId)) {
|
||||||
|
outputId = ++idCounter;
|
||||||
|
inputToOutputIdsBd[inputId] = outputId;
|
||||||
|
outputToInputIdsBd[outputId] = inputId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
inputToOutputIds = inputToOutputIdsBd.ToImmutable();
|
||||||
|
outputToInputIds = outputToInputIdsBd.ToImmutable();
|
||||||
|
myFireEvent(new DataPacket(
|
||||||
|
new IndexTable(dp.newRows, outputToInputIds, inputToOutputIds),
|
||||||
|
new IndexTable(dp.addedRows, outputToInputIds, inputToOutputIds),
|
||||||
|
new IndexTable(dp.deletedRows, outputToInputIds, inputToOutputIds)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() =>
|
||||||
|
Disposable.dispose(ref sub);
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
namespace bts.BlueOrb;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using Spark.Core.Collections;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.Immutable;
|
||||||
|
|
||||||
|
public class IndexTable : IBlueRowCollection {
|
||||||
|
readonly IBlueRowCollection inputRows;
|
||||||
|
readonly IDictionary<int, int> outputToInputIdMap;
|
||||||
|
readonly IDictionary<int, int> inputToOutputIdMap;
|
||||||
|
readonly int idColIndex;
|
||||||
|
|
||||||
|
public IndexTable(IBlueRowCollection inputRows, IDictionary<int, int> outputToInputIdMap, IDictionary<int, int> inputToOutputIdMap) {
|
||||||
|
this.inputRows = inputRows;
|
||||||
|
this.outputToInputIdMap = outputToInputIdMap;
|
||||||
|
this.inputToOutputIdMap = inputToOutputIdMap;
|
||||||
|
if (!inputRows.schema.columnInfo.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
|
||||||
|
throw new ArgumentException("No 'id' column", nameof(inputRows));
|
||||||
|
}
|
||||||
|
idColIndex = idColInfo.colNo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => inputRows.schema;
|
||||||
|
|
||||||
|
public int Count => inputRows.Count;
|
||||||
|
|
||||||
|
public IBlueRow this[int id] =>
|
||||||
|
outputToInputIdMap.TryGetValue(id, out var inputId)
|
||||||
|
? new IndexRow(inputRows[inputId], id, idColIndex, schema)
|
||||||
|
: throw new KeyNotFoundException();
|
||||||
|
|
||||||
|
IBlueRow IList<IBlueRow>.this[int index] {
|
||||||
|
get => GetRow(index);
|
||||||
|
set => throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IndexTable Create(IBlueRowCollection inputRows) {
|
||||||
|
var outputToInputIdMap = new Dictionary<int, int>();
|
||||||
|
var inputToOutputIdMap = new Dictionary<int, int>();
|
||||||
|
foreach (var r in inputRows) {
|
||||||
|
int inputId = r.Id();
|
||||||
|
int outputId = outputToInputIdMap.Count + 1;
|
||||||
|
outputToInputIdMap[outputId] = inputId;
|
||||||
|
inputToOutputIdMap[inputId] = outputId;
|
||||||
|
}
|
||||||
|
return new(inputRows, outputToInputIdMap, inputToOutputIdMap);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static BlueTableImmutable CreateImmutable(IBlueTableSchema schema, IEnumerable<IBlueRow> rows) {
|
||||||
|
int idIndex = schema.ResolveIndices("id");
|
||||||
|
var rowSetBd = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, IBlueRow>()).ToBuilder();
|
||||||
|
foreach (var r in rows) {
|
||||||
|
int idx = rowSetBd.Count + 1;
|
||||||
|
var inputRow = (r as IndexRow)?.InputRow ?? r;
|
||||||
|
rowSetBd.Add(new(idx, new IndexRow(inputRow, idx, idIndex, schema)));
|
||||||
|
}
|
||||||
|
return new(schema, rowSetBd.ToImmutable());
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool containsID(int id) =>
|
||||||
|
inputRows.containsID(id);
|
||||||
|
|
||||||
|
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||||
|
foreach (var r in inputRows) {
|
||||||
|
yield return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueRow GetRow(int index) {
|
||||||
|
var r = inputRows.GetRow(index);
|
||||||
|
return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int IndexOf(IBlueRow item) {
|
||||||
|
int id = item.Id();
|
||||||
|
if (outputToInputIdMap.TryGetValue(id, out var inputId)) {
|
||||||
|
var inputRow = inputRows[inputId];
|
||||||
|
var r = new IndexRow(inputRow, id, idColIndex, schema);
|
||||||
|
if (BlueRowEqualityComparer.Equals(r, item)) {
|
||||||
|
var idx = inputRows.IndexOf(inputRow);
|
||||||
|
Log.assert(idx >= 0);
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
namespace bts.BlueOrb {
|
||||||
|
using BCG;
|
||||||
|
using RowHash = System.Int32;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Data.Entity.Core.Mapping;
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("JoinStream")]
|
||||||
|
public class SimpleJoinStream : ViewStream {
|
||||||
|
public class JoinBookkeepingData {
|
||||||
|
// initialized in cons
|
||||||
|
private readonly List<string> _join_col_names;
|
||||||
|
private readonly bool _is_left;
|
||||||
|
private readonly BlueRowHasher _hasher;
|
||||||
|
// initialized on first udpate
|
||||||
|
private IBlueTableSchema _schema;
|
||||||
|
private int[] _join_cols;
|
||||||
|
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
|
||||||
|
|
||||||
|
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
|
||||||
|
_join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
|
||||||
|
_is_left = is_left;
|
||||||
|
_hasher = row_hasher;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IBlueTableSchema schema => _schema;
|
||||||
|
|
||||||
|
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
|
||||||
|
|
||||||
|
public List<IBlueRow> get_matching_rows(int hash) {
|
||||||
|
List<IBlueRow> rv = null;
|
||||||
|
_hash_row_map.TryGetValue(hash, out rv);
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
private RowHash _get_hash(IBlueRow r) {
|
||||||
|
if (_join_cols == null || _join_cols.Length == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return _hasher.get_hash(r, _join_cols);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void update_data(DataPacket dp) {
|
||||||
|
if (!ready) {
|
||||||
|
_schema = dp.newRows.schema;
|
||||||
|
_join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
|
||||||
|
}
|
||||||
|
_hash_row_map.Clear();
|
||||||
|
foreach (IBlueRow r in dp.newRows) {
|
||||||
|
int hash = _get_hash(r);
|
||||||
|
List<IBlueRow> hash_rows = null;
|
||||||
|
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
|
||||||
|
hash_rows = new List<IBlueRow>();
|
||||||
|
_hash_row_map.Add(hash, hash_rows);
|
||||||
|
}
|
||||||
|
hash_rows.Add(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool ready {
|
||||||
|
get {
|
||||||
|
return _schema != null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private ViewStream _ls, _rs;
|
||||||
|
private JoinBookkeepingData _ldata, _rdata;
|
||||||
|
private ViewStreamDatabase _vsd;
|
||||||
|
private IBlueTableSchema _schema = null;
|
||||||
|
|
||||||
|
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
|
||||||
|
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
|
||||||
|
|
||||||
|
public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
|
||||||
|
_ls = ls;
|
||||||
|
_rs = rs;
|
||||||
|
List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
||||||
|
List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
||||||
|
Log.assert(lj_col_list.Count == rj_col_list.Count);
|
||||||
|
int n_cols = lj_col_list.Count;
|
||||||
|
BlueRowHasher hasher = new BlueRowHasher(n_cols);
|
||||||
|
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
|
||||||
|
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_ls.onChange += _ls_onchange;
|
||||||
|
_rs.onChange += _rs_onchange;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _ls_onchange(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
_ldata.update_data(dp);
|
||||||
|
_form_and_fire();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _rs_onchange(object dp_obj) {
|
||||||
|
DataPacket dp = dp_obj as DataPacket;
|
||||||
|
_rdata.update_data(dp);
|
||||||
|
_form_and_fire();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_ls.onChange -= _ls_onchange;
|
||||||
|
_rs.onChange -= _rs_onchange;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _build_schema() {
|
||||||
|
if (_schema != null)
|
||||||
|
return;
|
||||||
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
|
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
|
||||||
|
foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
|
||||||
|
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
|
||||||
|
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
||||||
|
}
|
||||||
|
foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
|
||||||
|
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
|
||||||
|
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
||||||
|
}
|
||||||
|
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _form_and_fire() {
|
||||||
|
if (!_ldata.ready || !_rdata.ready)
|
||||||
|
return;
|
||||||
|
_build_schema();
|
||||||
|
List<object[]> res_rows = new List<object[]>();
|
||||||
|
|
||||||
|
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
|
||||||
|
int offset = 1;
|
||||||
|
JoinBookkeepingData sdata = _ldata;
|
||||||
|
if (is_left) {
|
||||||
|
offset = 1;
|
||||||
|
sdata = _ldata;
|
||||||
|
} else {
|
||||||
|
sdata = _rdata;
|
||||||
|
offset = 1 + _ldata.schema.columnInfo.Length;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
|
||||||
|
rval[offset + i] = r[i];
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
|
||||||
|
object[] rv = new object[_schema.columnNames.Length];
|
||||||
|
for (int i = 0; i < _schema.columnNames.Length; i++) {
|
||||||
|
rv[i] = DBNull.Value;
|
||||||
|
}
|
||||||
|
rv[0] = id;
|
||||||
|
copy_row(rv, l_row, true);
|
||||||
|
return rv;
|
||||||
|
};
|
||||||
|
// this is where the join happens
|
||||||
|
foreach (var kv in _ldata.rows) {
|
||||||
|
RowHash hash = kv.Key;
|
||||||
|
List<IBlueRow> l_rows = kv.Value;
|
||||||
|
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
|
||||||
|
|
||||||
|
foreach (IBlueRow lr in l_rows) {
|
||||||
|
if (r_rows != null) {
|
||||||
|
foreach (IBlueRow r in r_rows) {
|
||||||
|
object[] rowvals = build_row_array(res_rows.Count, lr);
|
||||||
|
copy_row(rowvals, r, false);
|
||||||
|
res_rows.Add(rowvals);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
object[] rowvals = build_row_array(res_rows.Count, lr);
|
||||||
|
res_rows.Add(rowvals);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BlueTable rv = new BlueTable(_schema);
|
||||||
|
foreach (object[] rval in res_rows) {
|
||||||
|
rv.addRow(new BlueRow(_schema, rval));
|
||||||
|
}
|
||||||
|
var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("FullJoinStream")]
|
||||||
|
public class FullJoinStream : ViewStream {
|
||||||
|
public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
|
||||||
|
ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
|
||||||
|
ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
|
||||||
|
mySetRealSource(rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
//namespace bts.Blueorb {
|
||||||
|
// using BCG;
|
||||||
|
|
||||||
|
// public class JsonSaverStream : ViewStream {
|
||||||
|
// private ViewStream _stream;
|
||||||
|
// private string _file_name;
|
||||||
|
// public JsonSaverStream(ViewStream vs, string file_name) {
|
||||||
|
// _stream = vs;
|
||||||
|
// _file_name = file_name;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnFirstSubscriber() {
|
||||||
|
// _stream.onChange += _on_data;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void _on_data(object dp_obj) {
|
||||||
|
// if (dp_obj == null) {
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
// DataPacket dp = dp_obj as DataPacket;
|
||||||
|
// dp.newRows.write_to_json_file(dp.newRows, _file_name);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnLastSubscriber() {
|
||||||
|
// _stream.onChange -= _on_data;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using Amazon.Runtime.Internal.Util;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
using BCG.Utilities;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
namespace bts.BlueOrb{
|
||||||
|
public class LiteralStream : ViewStream {
|
||||||
|
|
||||||
|
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
||||||
|
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
|
||||||
|
string[] pairs = key_value_pairs.Split(' ');
|
||||||
|
foreach (string p in pairs) {
|
||||||
|
string[] kvp = p.Split(':');
|
||||||
|
string col_name = kvp[0];
|
||||||
|
string val = kvp[1];
|
||||||
|
var tup = BlueDataUtility.guess_type(val);
|
||||||
|
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
|
||||||
|
rv.Add(Tuple.Create(ci, tup.Item2));
|
||||||
|
}
|
||||||
|
return rv;
|
||||||
|
}
|
||||||
|
|
||||||
|
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
|
||||||
|
// col_name: col_value ... ; col_name: col_value
|
||||||
|
public LiteralStream(string table_name, string rows) {
|
||||||
|
Func<string, int, IBlueRow> get_row = (json_row, id) => {
|
||||||
|
Dictionary<string, object> map = new Dictionary<string, object>();
|
||||||
|
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
|
||||||
|
//Logger.assert(col_vals.Length == 2);
|
||||||
|
foreach (string t in toks) {
|
||||||
|
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
|
||||||
|
string col = col_vals[0];
|
||||||
|
string val = col_vals[1];
|
||||||
|
map.Add(col, val);
|
||||||
|
}
|
||||||
|
map["id"] = id;
|
||||||
|
return new BlueRow(map);
|
||||||
|
};
|
||||||
|
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
||||||
|
BlueTable table = null;
|
||||||
|
foreach (string json_string in row_arr) {
|
||||||
|
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
||||||
|
get_row(json_string, table == null ? 0 : table.Count);
|
||||||
|
table = table ?? new BlueTable(r.schema);
|
||||||
|
table.addRow(r);
|
||||||
|
}
|
||||||
|
ViewStream str = new BlueTableStream(table);
|
||||||
|
str = new SchemaGuesserStream(str);
|
||||||
|
mySetRealSource(str);
|
||||||
|
}
|
||||||
|
|
||||||
|
public LiteralStream(string tableName, string schema, string value) {
|
||||||
|
string[] columns = schema.Split(' ');
|
||||||
|
|
||||||
|
|
||||||
|
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
|
||||||
|
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
||||||
|
int n = 1;
|
||||||
|
foreach (string c in columns) {
|
||||||
|
string[] columnAndType = c.Split('/');
|
||||||
|
if (columnAndType.Length == 1)
|
||||||
|
ci[n] = new ColumnInfo(c, n, typeof(double));
|
||||||
|
else
|
||||||
|
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
||||||
|
n++;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
|
||||||
|
string[] rows = value.Split(';');
|
||||||
|
n = 0;
|
||||||
|
foreach (string r in rows)
|
||||||
|
try {
|
||||||
|
if (r == string.Empty)
|
||||||
|
continue;
|
||||||
|
t.addRow(
|
||||||
|
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
||||||
|
}
|
||||||
|
|
||||||
|
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loaded 100 of 2573 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user