Compare commits
315
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
39e6e2139d | ||
|
|
9b0c3ae4f3 | ||
|
|
99f390a38c | ||
|
|
6f45e4dd2f | ||
|
|
47de5c60c8 | ||
|
|
871954bcc3 | ||
|
|
f41d85c224 | ||
|
|
18e7658445 | ||
|
|
ca522b7a23 | ||
|
|
32fccbfb69 | ||
|
|
27720a3127 | ||
|
|
6e3d1c7b8e | ||
|
|
625738b7b5 | ||
|
|
ef34d7efd8 | ||
|
|
40917782b0 | ||
|
|
e906bfec64 | ||
|
|
6b2845aa90 | ||
|
|
65fbea141d | ||
|
|
c6635005a1 | ||
|
|
72c21912a1 | ||
|
|
3142e62a10 | ||
|
|
1a82f5152e | ||
|
|
722ab02d2b | ||
|
|
6f352a5c95 | ||
|
|
ef3f7cc430 | ||
|
|
17b98adf74 | ||
|
|
809734eca8 | ||
|
|
ec87a2311d | ||
|
|
58bdb4e00d | ||
|
|
ee629278cf | ||
|
|
09ca8f504f | ||
|
|
fd6926eee8 | ||
|
|
5ae46fef77 | ||
|
|
07d12cbf94 | ||
|
|
e75dd34213 | ||
|
|
73294f1b76 | ||
|
|
1ca34db259 | ||
|
|
c35865a620 | ||
|
|
7a3804f1cd | ||
|
|
dd10feb286 | ||
|
|
f03b571f05 | ||
|
|
07aa53dffa | ||
|
|
68b42bdb01 | ||
|
|
2d7e6b1ec2 | ||
|
|
db0df50b53 | ||
|
|
bb6662a2c1 | ||
|
|
e1f98a3209 | ||
|
|
1f3b2c5cab | ||
|
|
cd1f4631dd | ||
|
|
4a1a7308dd | ||
|
|
7da668c136 | ||
|
|
304e669de2 | ||
|
|
9d86c85686 | ||
|
|
410680cadf | ||
|
|
64bd5eef4f | ||
|
|
3619d4b22c | ||
|
|
baa9dbfe76 | ||
|
|
11241b9b37 | ||
|
|
eb3fad64b0 | ||
|
|
cb97f26c89 | ||
|
|
5739f948b8 | ||
|
|
bdfb00aa22 | ||
|
|
50611fe4a3 | ||
|
|
4a193bd0e2 | ||
|
|
75907e2bd5 | ||
|
|
73d670c0dc | ||
|
|
688a6c8512 | ||
|
|
551430e256 | ||
|
|
c57312fb4d | ||
|
|
418201de6b | ||
|
|
5e1a83db8d | ||
|
|
e0ad6f72c0 | ||
|
|
8a22d7e318 | ||
|
|
856ba177aa | ||
|
|
54819d7d9b | ||
|
|
aaf9229f7e | ||
|
|
009a00c041 | ||
|
|
104742c9c0 | ||
|
|
1ea17a2a4e | ||
|
|
e68f6d5eb1 | ||
|
|
6bc397b497 | ||
|
|
343af5d8a3 | ||
|
|
ff8727b92d | ||
|
|
5d85dfa197 | ||
|
|
842740ceb3 | ||
|
|
79894864a0 | ||
|
|
f3021651eb | ||
|
|
459b808f4a | ||
|
|
49497c9be5 | ||
|
|
585524a8b2 | ||
|
|
c46771e8b6 | ||
|
|
244fc647fb | ||
|
|
57d7b4ba50 | ||
|
|
97e4276e20 | ||
|
|
d947882b0e | ||
|
|
61479c1198 | ||
|
|
1871358bde | ||
|
|
8bdf899da8 | ||
|
|
63c400543f | ||
|
|
6a339c273e | ||
|
|
16fc526b92 | ||
|
|
8f8e0a8b8e | ||
|
|
36fa0a8237 | ||
|
|
9abd1e68f2 | ||
|
|
de56bd3237 | ||
|
|
47712b704a | ||
|
|
6cf2b5fed2 | ||
|
|
e2fe3a04ce | ||
|
|
7aafc1b35e | ||
|
|
0165952b69 | ||
|
|
bc9e07a287 | ||
|
|
67a5f6fea2 | ||
|
|
5e374fcb56 | ||
|
|
639be9c210 | ||
|
|
62e82dbc42 | ||
|
|
6fa1aa036f | ||
|
|
92e825a73c | ||
|
|
f12fa9c524 | ||
|
|
a412f96701 | ||
|
|
ee4308cb73 | ||
|
|
5af3b094fa | ||
|
|
f18d00fd80 | ||
|
|
acb2bb7c1b | ||
|
|
56ca93c9d9 | ||
|
|
2c711a1224 | ||
|
|
12bffe1cd4 | ||
|
|
b6d7111f2d | ||
|
|
4113b02fb4 | ||
|
|
c050f764a7 | ||
|
|
065414cb2c | ||
|
|
c320fadffe | ||
|
|
6899d4ea28 | ||
|
|
4ce6c7930a | ||
|
|
9964d36689 | ||
|
|
d4da7a4e8f | ||
|
|
0c7ed9c384 | ||
|
|
e402b8ad83 | ||
|
|
b523ab22f0 | ||
|
|
2aa6ebac49 | ||
|
|
42f2727268 | ||
|
|
d3d79b5f2a | ||
|
|
44189a8371 | ||
|
|
a5bbb3b08e | ||
|
|
bc96653e77 | ||
|
|
5657fc81e3 | ||
|
|
4751372f9a | ||
|
|
0aaa6e18fd | ||
|
|
d68fe93422 | ||
|
|
9e294e7573 | ||
|
|
37ef7e6369 | ||
|
|
2d27abe4c3 | ||
|
|
2bb3a2b6b5 | ||
|
|
1443007f93 | ||
|
|
81b94a3857 | ||
|
|
2ee020d505 | ||
|
|
6b008ffd08 | ||
|
|
c76adbe257 | ||
|
|
09e3d00025 | ||
|
|
063ce5be73 | ||
|
|
eff9bb399f | ||
|
|
59afa2e2e1 | ||
|
|
38f6c71551 | ||
|
|
b303dfb80c | ||
|
|
de44f120af | ||
|
|
eb2c620ad2 | ||
|
|
276414f38f | ||
|
|
0248b73bde | ||
|
|
5722272435 | ||
|
|
d95ffbcedc | ||
|
|
d267cbbb73 | ||
|
|
1d2825c0ff | ||
|
|
b363a45374 | ||
|
|
f5a638bf8f | ||
|
|
be67f67abb | ||
|
|
1b9ce5f9b7 | ||
|
|
7fb411adce | ||
|
|
cac881ea1b | ||
|
|
07a72f6943 | ||
|
|
b6f64b4804 | ||
|
|
fa5cc6ab38 | ||
|
|
664424c91c | ||
|
|
0b463f4e65 | ||
|
|
ea0f1153e4 | ||
|
|
ff4bab71f3 | ||
|
|
c1fdc2e8be | ||
|
|
2ddb12212f | ||
|
|
06c694ea8a | ||
|
|
2cdea43eb5 | ||
|
|
10f105d48d | ||
|
|
fb69e75019 | ||
|
|
d3736df90f | ||
|
|
337828ff62 | ||
|
|
6ba4354e4c | ||
|
|
c5830493fe | ||
|
|
87c72d43dc | ||
|
|
47e70272d2 | ||
|
|
d04d711706 | ||
|
|
d77effe8e1 | ||
|
|
5606e690e6 | ||
|
|
8f50979f90 | ||
|
|
61e61e8bb8 | ||
|
|
b4baefb802 | ||
|
|
40c09084bd | ||
|
|
c0d622a02b | ||
|
|
99a17a81ff | ||
|
|
86c40fd487 | ||
|
|
e13dcaab58 | ||
|
|
4532c8644f | ||
|
|
cb97978fb0 | ||
|
|
3a13bb389a | ||
|
|
ddf7c89b61 | ||
|
|
3078dfda73 | ||
|
|
331d906a12 | ||
|
|
ef0ffdc243 | ||
|
|
97229a8f1a | ||
|
|
8c77499015 | ||
|
|
e1954f9ac9 | ||
|
|
a4a7ee384b | ||
|
|
5dfd2d96b5 | ||
|
|
d6b3f102de | ||
|
|
949b0b5b72 | ||
|
|
37e2d2ee13 | ||
|
|
bfdf6228be | ||
|
|
bc408c0c75 | ||
|
|
1486c1185c | ||
|
|
cb06d07eea | ||
|
|
c7205fcdd0 | ||
|
|
911347e080 | ||
|
|
a65eaf1cc4 | ||
|
|
fe2235bc8d | ||
|
|
2417b430fd | ||
|
|
be81e9af47 | ||
|
|
5e4858e860 | ||
|
|
9b82083a60 | ||
|
|
30f322a60a | ||
|
|
58015a396e | ||
|
|
939bdc5f56 | ||
|
|
2939cb601f | ||
|
|
2454f89ec4 | ||
|
|
df13b0c76f | ||
|
|
b4f87c21e0 | ||
|
|
9cfa28a91d | ||
|
|
6ade868afe | ||
|
|
1230424577 | ||
|
|
9bbada0740 | ||
|
|
44b3709c8f | ||
|
|
cabcd406f7 | ||
|
|
1c0026c108 | ||
|
|
11e862ae06 | ||
|
|
4deeacda4a | ||
|
|
9a0bf98b2c | ||
|
|
c286c3646b | ||
|
|
50596abce3 | ||
|
|
324a6031ba | ||
|
|
94591eee6d | ||
|
|
41390da06a | ||
|
|
9df9622ac2 | ||
|
|
4b965d1c3b | ||
|
|
d485621d21 | ||
|
|
b8678f6043 | ||
|
|
de15b0ea4b | ||
|
|
bd41e6cb52 | ||
|
|
f07b1f2e5f | ||
|
|
a232261df5 | ||
|
|
c3cd8cf543 | ||
|
|
95814d3dae | ||
|
|
c3ef18e649 | ||
|
|
feaf2cc6b4 | ||
|
|
e184c850d1 | ||
|
|
161fdfa463 | ||
|
|
9c184d7203 | ||
|
|
a2344e1494 | ||
|
|
cc153a1ced | ||
|
|
b19f720e42 | ||
|
|
51d934de27 | ||
|
|
23e47f1749 | ||
|
|
a5a905e20f | ||
|
|
1cecbe9276 | ||
|
|
edfc1fc0d7 | ||
|
|
cb43af0dd4 | ||
|
|
70616d4a09 | ||
|
|
6ec15d34a7 | ||
|
|
48b3852e6f | ||
|
|
595656f298 | ||
|
|
951d68e20c | ||
|
|
d64ddb3477 | ||
|
|
d937054f6f | ||
|
|
61affcf749 | ||
|
|
7270967181 | ||
|
|
008e84dcf0 | ||
|
|
9cc0108a01 | ||
|
|
6c1d1a43f5 | ||
|
|
d96b6da139 | ||
|
|
295300e7fa | ||
|
|
b35b9170e4 | ||
|
|
e57e401780 | ||
|
|
f5c333b84c | ||
|
|
60c784f371 | ||
|
|
68fe50f7e2 | ||
|
|
7ebafd8246 | ||
|
|
8f2e37d644 | ||
|
|
426b0daf61 | ||
|
|
5d71d7f01b | ||
|
|
1fa1dfa01e | ||
|
|
ec28e12a6f | ||
|
|
84b7e74bb6 | ||
|
|
64ffd3050f | ||
|
|
1f2f7dda09 | ||
|
|
60de12277b | ||
|
|
963a521a2c | ||
|
|
00ce3cdf37 | ||
|
|
fbdb4f112d | ||
|
|
2f9f20de06 | ||
|
|
c7b5b5df54 | ||
|
|
9342613501 |
No files matched your search
@@ -9,7 +9,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"fable": {
|
"fable": {
|
||||||
"version": "4.24.0",
|
"version": "4.0.6",
|
||||||
"commands": [
|
"commands": [
|
||||||
"fable"
|
"fable"
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ RUN \
|
|||||||
&& apt-get install -y cmake
|
&& apt-get install -y cmake
|
||||||
|
|
||||||
# Dotnet
|
# Dotnet
|
||||||
RUN apt-get update && apt-get install -y dotnet8
|
RUN apt-get update && apt-get install -y dotnet7
|
||||||
|
|
||||||
# Node
|
# Node
|
||||||
ARG node_major=18
|
ARG node_major=18
|
||||||
@@ -129,3 +129,5 @@ RUN \
|
|||||||
-G sudo \
|
-G sudo \
|
||||||
--password "$(openssl passwd -1 "$docker_pass")" \
|
--password "$(openssl passwd -1 "$docker_pass")" \
|
||||||
"$docker_user"
|
"$docker_user"
|
||||||
|
|
||||||
|
RUN usermod -aG docker "$docker_user"
|
||||||
@@ -1,10 +1,15 @@
|
|||||||
{
|
{
|
||||||
"dockerFile": "Dockerfile",
|
"dockerFile": "Dockerfile",
|
||||||
|
// Creates local dir to mount to HOME in docker
|
||||||
|
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
|
||||||
"hostRequirements": {
|
"hostRequirements": {
|
||||||
"memory": "8gb",
|
"memory": "4gb",
|
||||||
"storage": "10gb"
|
"storage": "6gb"
|
||||||
},
|
},
|
||||||
|
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
|
||||||
"containerUser": "bts",
|
"containerUser": "bts",
|
||||||
|
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
|
||||||
|
"workspaceFolder": "/home/bts/spark",
|
||||||
"updateRemoteUserUID": true,
|
"updateRemoteUserUID": true,
|
||||||
"shutdownAction": "none",
|
"shutdownAction": "none",
|
||||||
// Note: If running multiple containers on the same host, each must use a
|
// Note: If running multiple containers on the same host, each must use a
|
||||||
@@ -12,20 +17,26 @@
|
|||||||
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
||||||
"runArgs": [
|
"runArgs": [
|
||||||
"--shm-size",
|
"--shm-size",
|
||||||
"8gb"
|
"2gb",
|
||||||
|
"--network",
|
||||||
|
"host"
|
||||||
],
|
],
|
||||||
"mounts": [
|
"mounts": [
|
||||||
|
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
|
||||||
"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
|
"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
|
||||||
],
|
],
|
||||||
"customizations": {
|
"customizations": {
|
||||||
"vscode": {
|
"vscode": {
|
||||||
|
"settings": {
|
||||||
|
"clangd.arguments": [
|
||||||
|
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
|
||||||
|
]
|
||||||
|
},
|
||||||
"extensions": [
|
"extensions": [
|
||||||
"eamodio.gitlens",
|
"eamodio.gitlens",
|
||||||
"llvm-vs-code-extensions.vscode-clangd",
|
"llvm-vs-code-extensions.vscode-clangd",
|
||||||
"ms-python.python",
|
|
||||||
"ms-vscode.cmake-tools",
|
"ms-vscode.cmake-tools",
|
||||||
"ms-vscode.cpptools",
|
"ms-vscode.cpptools"
|
||||||
"xaver.clang-format"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,10 +7,10 @@
|
|||||||
# We use Docker Compose to build images and launch containers for various services
|
# We use Docker Compose to build images and launch containers for various services
|
||||||
# that make up our software suite:
|
# that make up our software suite:
|
||||||
# - sparkd
|
# - sparkd
|
||||||
# - TODO: pubsub
|
# - pubsub
|
||||||
# - TODO: md-publisher
|
# - md-publisher
|
||||||
# - TODO: webapp (DSQA on the web)
|
# - webapp (DSQA on the web)
|
||||||
# - TODO: api-server (websocket based)
|
# - api-server (websocket based)
|
||||||
# From inside a devcontainer, you can launch all services in separate containers
|
# From inside a devcontainer, you can launch all services in separate containers
|
||||||
# on your machine by running:
|
# on your machine by running:
|
||||||
#
|
#
|
||||||
@@ -31,19 +31,51 @@
|
|||||||
|
|
||||||
### Sparkd ###
|
### Sparkd ###
|
||||||
# The internal image for sparkd to use. For more info, view:
|
# The internal image for sparkd to use. For more info, view:
|
||||||
# src/server/sparkd/Dockerfile
|
# src/app_sparkd/Dockerfile
|
||||||
SPARKD__IMAGE=sparkd
|
SPARKD__IMAGE=sparkd
|
||||||
# The config dir for this service to use
|
# The 'spark.conf' for this service to use
|
||||||
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
|
SPARKD__CONF_FILE=src/app_sparkd/spark.conf
|
||||||
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
|
# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
|
||||||
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
|
# SPARKD__IMAGE=sparkd_md_replay
|
||||||
# SPARKD__DATA_DIR=/home/bts/sparkdata/data
|
# SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
|
||||||
|
# (configure which md to replay in the new SPARKD__CONF_FILE)
|
||||||
|
SPARKD__MD_RECORDING_DIR=.md_recording
|
||||||
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
|
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
|
||||||
SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
|
|
||||||
SPARKD__CONFIG_PORT=8000
|
SPARKD__CONFIG_PORT=8000
|
||||||
SPARKD__QUICKBUS_PORT=8001
|
SPARKD__QUICKBUS_PORT=8001
|
||||||
SPARKD__SIM_FIX_PORT=8002
|
SPARKD__SIM_FIX_PORT=8002
|
||||||
SPARKD__SHM_SIZE=4gb
|
|
||||||
SPARKD__PYTHON_REQUIREMENTS=requirements.txt
|
|
||||||
|
|
||||||
# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
|
### Pubsub ###
|
||||||
|
# The 'spark.conf' for this service to use
|
||||||
|
PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
|
||||||
|
PUBSUB__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||||
|
PUBSUB__CPP_SOURCE_DIR=install/data-stream-suite-linux
|
||||||
|
# The appsettings to load at runtime. Expects a file named:
|
||||||
|
# src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
|
||||||
|
PUBSUB__DOTNET_ENVIRONMENT=Staging
|
||||||
|
|
||||||
|
### Md Publisher ###
|
||||||
|
# The 'spark.conf' for this service to use
|
||||||
|
MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
|
||||||
|
MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||||
|
# Determines which appsettings to load at runtime. Expects a file named:
|
||||||
|
# src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
|
||||||
|
MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
|
||||||
|
|
||||||
|
### Webapp (DSQA on the web) ###
|
||||||
|
# The 'spark.conf' for this service to use
|
||||||
|
WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
|
||||||
|
WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||||
|
WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
|
||||||
|
# Determines which appsettings to load at runtime. Expects a file named:
|
||||||
|
# src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
|
||||||
|
WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
|
||||||
|
|
||||||
|
### Api Server ###
|
||||||
|
# The 'spark.conf' for this service to use
|
||||||
|
API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
|
||||||
|
API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||||
|
API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
|
||||||
|
# Determines which appsettings to load at runtime. Expects a file named:
|
||||||
|
# src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
|
||||||
|
API_SERVER__ASPNETCORE_ENVIRONMENT=Staging
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
*.cmd text=auto eol=crlf
|
*.cmd text=auto eol=crlf
|
||||||
*.cs text=auto eol=crlf
|
*.cs text=auto eol=crlf
|
||||||
*.sh text=auto eol=lf
|
*.sh text=auto eol=lf
|
||||||
|
*.py text=auto eol=lf
|
||||||
+2
-4
@@ -11,7 +11,6 @@ cmake-build-debug/
|
|||||||
**.vscode/*
|
**.vscode/*
|
||||||
!.vscode/launch.json
|
!.vscode/launch.json
|
||||||
!.vscode/tasks.json
|
!.vscode/tasks.json
|
||||||
!.vscode/*.env
|
|
||||||
.vs
|
.vs
|
||||||
obj/
|
obj/
|
||||||
*.csproj.user
|
*.csproj.user
|
||||||
@@ -55,12 +54,11 @@ trace_data/
|
|||||||
.cache/
|
.cache/
|
||||||
/src/Spark.Admin.WebApp/tsconfig.*.g.json
|
/src/Spark.Admin.WebApp/tsconfig.*.g.json
|
||||||
/src/Spark.Admin.WebApp/src/bin/
|
/src/Spark.Admin.WebApp/src/bin/
|
||||||
/src/client/**/*appsettings.Local.json
|
|
||||||
/src/DataStreamSuite/**/*appsettings.Local.json
|
|
||||||
**/CMakeDeps.cmake
|
**/CMakeDeps.cmake
|
||||||
**/CMakeDeps_win.cmake
|
**/CMakeDeps_win.cmake
|
||||||
.devcontainer/
|
.devcontainer/
|
||||||
*.Generated.cs
|
*.Generated.cs
|
||||||
*.lst
|
*.lst
|
||||||
|
src/app_sparkd/spark.local.conf
|
||||||
.env.local
|
.env.local
|
||||||
.secrets
|
.md_recording
|
||||||
Vendored
+73
-157
@@ -1,14 +1,6 @@
|
|||||||
{
|
{
|
||||||
"version": "0.2.0",
|
"version": "0.2.0",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
{
|
|
||||||
"name": "Python Debugger: Current File with Arguments",
|
|
||||||
"type": "debugpy",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "${file}",
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"args": "${command:pickArgs}"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": ".NET Core Attach",
|
"name": ".NET Core Attach",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
@@ -24,10 +16,19 @@
|
|||||||
"program": "spark_api_server.exe",
|
"program": "spark_api_server.exe",
|
||||||
"args": [
|
"args": [
|
||||||
"run",
|
"run",
|
||||||
"--sparkd",
|
|
||||||
"http://127.0.0.1:8000/config"
|
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
},
|
||||||
|
"linux": {
|
||||||
|
"preLaunchTask": "Build Api Server (Debug)",
|
||||||
|
"program": "spark_api_server",
|
||||||
|
"args": [
|
||||||
|
"run",
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
},
|
||||||
|
"env": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -41,10 +42,18 @@
|
|||||||
"program": "spark_api_server.exe",
|
"program": "spark_api_server.exe",
|
||||||
"args": [
|
"args": [
|
||||||
"run",
|
"run",
|
||||||
"--sparkd",
|
|
||||||
"http://127.0.0.1:8000/config"
|
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
},
|
||||||
|
"linux": {
|
||||||
|
"program": "spark_api_server",
|
||||||
|
"args": [
|
||||||
|
"run",
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
},
|
||||||
|
"env": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -53,9 +62,9 @@
|
|||||||
"name": "Bts.DataStreamWebApp.Server",
|
"name": "Bts.DataStreamWebApp.Server",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"preLaunchTask": "Build DataStream WebApp (Debug)",
|
"preLaunchTask": "Build DataStream (Debug)",
|
||||||
"program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
|
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
||||||
"env": {
|
"env": {
|
||||||
"ASPNETCORE_HTTP_PORT": "8080",
|
"ASPNETCORE_HTTP_PORT": "8080",
|
||||||
"ASPNETCORE_HTTPS_PORT": "8443",
|
"ASPNETCORE_HTTPS_PORT": "8443",
|
||||||
@@ -67,8 +76,8 @@
|
|||||||
"name": "Bts.DataStreamWebApp.Server (Skip Build)",
|
"name": "Bts.DataStreamWebApp.Server (Skip Build)",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
|
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
||||||
"env": {
|
"env": {
|
||||||
"ASPNETCORE_HTTP_PORT": "8080",
|
"ASPNETCORE_HTTP_PORT": "8080",
|
||||||
"ASPNETCORE_HTTPS_PORT": "8443",
|
"ASPNETCORE_HTTPS_PORT": "8443",
|
||||||
@@ -88,10 +97,10 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamPublisher.exe",
|
"program": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
||||||
},
|
},
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -106,10 +115,10 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamPublisher.exe",
|
"program": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
||||||
},
|
},
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -118,39 +127,7 @@
|
|||||||
"type": "cppdbg",
|
"type": "cppdbg",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
||||||
"args": [
|
"args": ["run"],
|
||||||
"run"
|
|
||||||
],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "spreader tests2",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"[spreader2]"
|
|
||||||
],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "spreader tests",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"[spreader]"
|
|
||||||
],
|
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
||||||
"environment": [],
|
"environment": [],
|
||||||
@@ -165,25 +142,12 @@
|
|||||||
"program": "",
|
"program": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
"program": "DataStreamQueryAnalyzer.exe",
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0-windows",
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
|
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "DataStreamQueryAnalyzer (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
|
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -202,7 +166,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "PubSubServer.exe",
|
"program": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
},
|
},
|
||||||
@@ -220,7 +184,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "PubSubServer.exe",
|
"program": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||||
},
|
},
|
||||||
@@ -235,25 +199,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
"preLaunchTask": "Build Spark (Debug)",
|
||||||
"program": "Spark.exe",
|
"program": "Spark.exe",
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
"args": [
|
||||||
"env": {
|
"--sparkd",
|
||||||
"DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
|
"http://127.0.0.1:8000/config"
|
||||||
}
|
],
|
||||||
},
|
"cwd": "build/Debug/bin/net7.0-windows",
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Spark (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "Spark.exe",
|
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -270,40 +220,7 @@
|
|||||||
"--sparkd",
|
"--sparkd",
|
||||||
"${input:sparkConfigServer}"
|
"${input:sparkConfigServer}"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
"cwd": "build/Debug/bin/net7.0-windows",
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Spark Test",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
|
||||||
"program": "Spark.Console.exe",
|
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkTest}"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Spark Test (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "Spark.Console.exe",
|
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkTest}"
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -312,6 +229,7 @@
|
|||||||
"name": "Sparkd",
|
"name": "Sparkd",
|
||||||
"type": "cppdbg",
|
"type": "cppdbg",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
|
"preLaunchTask": "Build Sparkd (Debug | Clang)",
|
||||||
"program": "",
|
"program": "",
|
||||||
"linux": {
|
"linux": {
|
||||||
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
||||||
@@ -364,6 +282,34 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "Sparkd Web (Skip Build)",
|
||||||
|
"type": "cppdbg",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"linux": {
|
||||||
|
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"web"
|
||||||
|
],
|
||||||
|
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"externalConsole": false,
|
||||||
|
"MIMode": "gdb",
|
||||||
|
"setupCommands": [
|
||||||
|
{
|
||||||
|
"description": "Enable pretty-printing for gdb",
|
||||||
|
"text": "-enable-pretty-printing",
|
||||||
|
"ignoreFailures": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Set Disassembly Flavor to Intel",
|
||||||
|
"text": "-gdb-set disassembly-flavor intel",
|
||||||
|
"ignoreFailures": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
],
|
],
|
||||||
"inputs": [
|
"inputs": [
|
||||||
{
|
{
|
||||||
@@ -403,42 +349,12 @@
|
|||||||
"Staging",
|
"Staging",
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"id": "dataStreamQueryAnalyzerEnvironment",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "DataStreamQueryAnalyzer Configuration",
|
|
||||||
"options": [
|
|
||||||
"Development",
|
|
||||||
"Local",
|
|
||||||
],
|
|
||||||
"default": "Development",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkEnvironment",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Spark Configuration",
|
|
||||||
"options": [
|
|
||||||
"Development",
|
|
||||||
"Local",
|
|
||||||
],
|
|
||||||
"default": "Development",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "sparkConfigServer",
|
"id": "sparkConfigServer",
|
||||||
"type": "promptString",
|
"type": "promptString",
|
||||||
"description": "Config Server URL",
|
"description": "Config Server URL",
|
||||||
"default": "http://localhost:8000/config",
|
"default": "http://localhost:8000/config",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"id": "sparkTest",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Test to run",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdTest",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Test to run",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "isSparkApiEnabled",
|
"id": "isSparkApiEnabled",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
|
|||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
|
|
||||||
Vendored
+61
-169
@@ -485,12 +485,12 @@
|
|||||||
"command": "",
|
"command": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamQueryAnalyzer.exe",
|
"command": "DataStreamQueryAnalyzer.exe",
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
|
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -510,7 +510,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -530,7 +530,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -549,11 +549,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -571,11 +572,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -593,11 +595,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -698,11 +701,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -720,11 +724,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -741,11 +746,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -763,11 +769,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -784,11 +791,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -803,7 +811,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -819,7 +827,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -835,22 +843,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Spark (Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.exe",
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -875,7 +868,7 @@
|
|||||||
"${input:sparkConfigQuery}",
|
"${input:sparkConfigQuery}",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -901,24 +894,6 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"label": "Run Sparkd (Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run"
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"label": "Run Sparkd Config Query",
|
"label": "Run Sparkd Config Query",
|
||||||
"dependsOn": "Build Sparkd",
|
"dependsOn": "Build Sparkd",
|
||||||
@@ -940,65 +915,6 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"label": "Test Spark (Debug)",
|
|
||||||
"dependsOn": "Build Spark (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"icon": {
|
|
||||||
"id": "coverage",
|
|
||||||
"color": "terminal.ansiYellow",
|
|
||||||
},
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.Console.exe",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkTest}"
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Test Sparkd",
|
|
||||||
"dependsOn": "Build Sparkd",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkdTest}",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "coverage",
|
|
||||||
"color": "terminal.ansiYellow",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Test Sparkd (Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkdTest}",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "coverage",
|
|
||||||
"color": "terminal.ansiYellow",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
@@ -1087,40 +1003,26 @@
|
|||||||
"default": "sparkd-debug-clang",
|
"default": "sparkd-debug-clang",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "dataStreamPublisherEnvironment",
|
"id": "dataStreamPublisherConfig",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
"description": "Publisher Configuration",
|
"description": "Publisher Configuration",
|
||||||
"options": [
|
"options": [
|
||||||
"Development",
|
"MarketDataPublisher",
|
||||||
"MarketData",
|
"SimpleEchoClient",
|
||||||
"Production",
|
"SimpleTestDataPublisher",
|
||||||
"ProductionAws",
|
"TestDataPublisher",
|
||||||
"Staging",
|
|
||||||
],
|
],
|
||||||
"default": "MarketDataPublisher",
|
"default": "MarketDataPublisher",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "dataStreamQueryAnalyzerEnvironment",
|
"id": "pubSubServerConfig",
|
||||||
"type": "pickString",
|
|
||||||
"description": "DataStreamQueryAnalyzer Configuration",
|
|
||||||
"options": [
|
|
||||||
"Development",
|
|
||||||
"Local",
|
|
||||||
],
|
|
||||||
"default": "Development",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "pubSubServerEnvironment",
|
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
"description": "PubSub Configuration",
|
"description": "PubSub Configuration",
|
||||||
"options": [
|
"options": [
|
||||||
"Development",
|
"PubSubServer",
|
||||||
"EchoServer",
|
"SimpleEchoClient",
|
||||||
"Production",
|
|
||||||
"ProductionAws",
|
|
||||||
"Staging",
|
|
||||||
],
|
],
|
||||||
"default": "Development",
|
"default": "PubSubServer",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "sparkConfigQuery",
|
"id": "sparkConfigQuery",
|
||||||
@@ -1131,16 +1033,6 @@
|
|||||||
"id": "sparkdConfigQuery",
|
"id": "sparkdConfigQuery",
|
||||||
"type": "promptString",
|
"type": "promptString",
|
||||||
"description": "Config Type(s)"
|
"description": "Config Type(s)"
|
||||||
},
|
}
|
||||||
{
|
|
||||||
"id": "sparkTest",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Test to run",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdTest",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Test to run",
|
|
||||||
},
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+66
-82
@@ -1,10 +1,7 @@
|
|||||||
cmake_minimum_required(VERSION 3.12)
|
cmake_minimum_required(VERSION 3.12)
|
||||||
project(spark)
|
project(spark)
|
||||||
include(GNUInstallDirs)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
|
|
||||||
set(CMAKE_CXX_STANDARD 20)
|
set(CMAKE_CXX_STANDARD 20)
|
||||||
|
|
||||||
get_version_from_git()
|
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
set(platform "windows")
|
set(platform "windows")
|
||||||
else()
|
else()
|
||||||
@@ -36,98 +33,85 @@ else()
|
|||||||
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
|
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
|
|
||||||
link_directories(${EXTERNALS_LIB})
|
|
||||||
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
||||||
if(DEFINED APP_DATA_DIR)
|
if(DEFINED APP_DATA_DIR)
|
||||||
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
||||||
endif()
|
endif()
|
||||||
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
||||||
if(WIN32)
|
|
||||||
# subdirs
|
|
||||||
add_subdirectory(src/lib_spark)
|
|
||||||
else()
|
|
||||||
if(BUILD_SPARKD)
|
|
||||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
|
||||||
# read by their code). We override some below.
|
|
||||||
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
|
||||||
|
|
||||||
# This allows us to override hardcoded exchange-urls with ones read from
|
if(BUILD_SPARKD)
|
||||||
# config (i.e., resolved at runtime rather than compile-time)
|
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||||
add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
|
# read by their code). We override some below.
|
||||||
|
add_compile_definitions(SPARK_ENABLE_CCAPI)
|
||||||
|
|
||||||
# Enable ccapi services
|
# This allows us to override hardcoded exchange-urls with ones read from
|
||||||
add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
|
# config (i.e., resolved at runtime rather than compile-time)
|
||||||
add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
|
add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
|
||||||
# Enable ccapi exchanges
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
|
|
||||||
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
|
|
||||||
|
|
||||||
set(CCAPI_LOG_LEVEL
|
# Enable ccapi services
|
||||||
"Warn"
|
add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
|
||||||
CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
|
add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
|
||||||
string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
|
# Enable ccapi exchanges
|
||||||
add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
|
||||||
|
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
|
||||||
|
|
||||||
# ** Binance **
|
set(CCAPI_LOG_LEVEL
|
||||||
add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
|
"Warn"
|
||||||
add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
|
||||||
add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
|
||||||
|
add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
|
||||||
|
|
||||||
# ** Binance US **
|
# ** Binance **
|
||||||
add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
|
||||||
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
||||||
|
add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
||||||
|
|
||||||
# ** Binance USDS Futures **
|
# ** Binance US **
|
||||||
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
|
add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
|
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
|
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
|
|
||||||
|
|
||||||
# ** Binance Coin Futures **
|
# ** Binance USDS Futures **
|
||||||
add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
||||||
add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
||||||
add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
||||||
|
|
||||||
# ** Deribit **
|
# ** Deribit **
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
|
add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
|
||||||
add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
|
add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
|
||||||
add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
|
add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
|
||||||
add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
|
add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
|
add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
|
add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
|
||||||
add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
|
add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
|
||||||
add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
|
add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
|
||||||
|
|
||||||
# ** Okx **
|
# ** Okx **
|
||||||
add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
|
add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
|
||||||
add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
|
add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
# subdirs
|
# subdirs
|
||||||
add_subdirectory(src/lib_spark)
|
add_subdirectory(src/lib_system)
|
||||||
if (BUILD_SPARKD)
|
add_subdirectory(src/lib_domain)
|
||||||
add_subdirectory(src/lib_python_interop)
|
add_subdirectory(src/lib_http)
|
||||||
add_subdirectory(src/lib_onixs_fix)
|
if (BUILD_SPARKD)
|
||||||
add_subdirectory(src/lib_binance)
|
add_subdirectory(src/lib_python_md)
|
||||||
add_subdirectory(src/lib_okx_execution)
|
add_subdirectory(src/app_sparkd)
|
||||||
add_subdirectory(src/app_sparkd)
|
add_subdirectory(src/data_files)
|
||||||
add_subdirectory(src/data_files)
|
endif()
|
||||||
endif()
|
add_subdirectory(src/lib_spark)
|
||||||
if (BUILD_ADMIN_WEB_APP)
|
if (BUILD_ADMIN_WEB_APP)
|
||||||
add_subdirectory(src/Spark.Admin.WebApp)
|
add_subdirectory(src/Spark.Admin.WebApp)
|
||||||
endif()
|
endif()
|
||||||
if(BUILD_DOCS)
|
if(BUILD_DOCS)
|
||||||
add_subdirectory(doc/user)
|
add_subdirectory(doc/user)
|
||||||
endif()
|
endif()
|
||||||
if(BUILD_SDK)
|
if(BUILD_SDK)
|
||||||
add_subdirectory(src/sdk)
|
add_subdirectory(src/sdk)
|
||||||
endif()
|
|
||||||
endif()
|
endif()
|
||||||
+12
-15
@@ -24,13 +24,12 @@
|
|||||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||||
"CMAKE_CXX_STANDARD": "20",
|
"CMAKE_CXX_STANDARD": "20",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
|
||||||
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
|
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
|
||||||
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
||||||
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -50,20 +49,11 @@
|
|||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"platform": "",
|
"platform": "",
|
||||||
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "OFF"
|
|
||||||
},
|
|
||||||
"CMAKE_CXX_STANDARD": "20",
|
"CMAKE_CXX_STANDARD": "20",
|
||||||
"CMAKE_INSTALL_RPATH": ".",
|
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
|
"CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
|
||||||
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
||||||
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
"DATABENTO_USE_EXTERNAL_JSON": {
|
|
||||||
"type": "BOOL",
|
|
||||||
"value": "ON"
|
|
||||||
},
|
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
||||||
"BUILD_SDK": {
|
"BUILD_SDK": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
@@ -174,13 +164,12 @@
|
|||||||
"CMAKE_CXX_COMPILER": "clang++",
|
"CMAKE_CXX_COMPILER": "clang++",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
|
||||||
"CMAKE_EXPORT_COMPILE_COMMANDS": {
|
"CMAKE_EXPORT_COMPILE_COMMANDS": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
},
|
},
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -219,7 +208,7 @@
|
|||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -252,7 +241,7 @@
|
|||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -305,6 +294,14 @@
|
|||||||
"inherits": "sparkd"
|
"inherits": "sparkd"
|
||||||
},
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"name": "sparkd-debug-clang-j6",
|
||||||
|
"configurePreset": "sparkd-debug-clang",
|
||||||
|
"displayName": "Sparkd (Debug | Clang | jobs=6)",
|
||||||
|
"inherits": "sparkd",
|
||||||
|
"jobs": 6
|
||||||
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "sparkd-debug-gcc",
|
"name": "sparkd-debug-gcc",
|
||||||
"configurePreset": "sparkd-debug-gcc",
|
"configurePreset": "sparkd-debug-gcc",
|
||||||
|
|||||||
+45
-134
@@ -14,33 +14,15 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
|
|
||||||
EndProject
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
@@ -98,102 +80,46 @@ Global
|
|||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
|
||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
@@ -204,19 +130,4 @@ Global
|
|||||||
GlobalSection(Performance) = preSolution
|
GlobalSection(Performance) = preSolution
|
||||||
HasPerformanceSessions = true
|
HasPerformanceSessions = true
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(NestedProjects) = preSolution
|
|
||||||
{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
EndGlobalSection
|
|
||||||
EndGlobal
|
EndGlobal
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
include_guard(GLOBAL)
|
|
||||||
|
|
||||||
function(get_version_from_git)
|
|
||||||
find_package(Git QUIET)
|
|
||||||
if(NOT Git_FOUND)
|
|
||||||
message(WARNING "Git not found")
|
|
||||||
return()
|
|
||||||
endif()
|
|
||||||
|
|
||||||
execute_process(
|
|
||||||
COMMAND ${GIT_EXECUTABLE} describe --tags --always
|
|
||||||
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
|
||||||
OUTPUT_VARIABLE GIT_TAG
|
|
||||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
|
||||||
RESULT_VARIABLE GIT_RESULT
|
|
||||||
)
|
|
||||||
|
|
||||||
if(NOT GIT_RESULT EQUAL 0)
|
|
||||||
message(WARNING "Failed to get git tag")
|
|
||||||
return()
|
|
||||||
endif()
|
|
||||||
|
|
||||||
execute_process(
|
|
||||||
COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
|
|
||||||
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
|
||||||
OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
|
|
||||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
|
||||||
)
|
|
||||||
set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
|
|
||||||
set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
|
|
||||||
|
|
||||||
string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
|
|
||||||
if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
|
|
||||||
set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
|
|
||||||
set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
|
|
||||||
set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
|
|
||||||
set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
|
|
||||||
set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
|
|
||||||
set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
|
|
||||||
|
|
||||||
set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}")
|
|
||||||
set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
|
|
||||||
set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
|
|
||||||
set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
|
|
||||||
else()
|
|
||||||
message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
|
|
||||||
endif()
|
|
||||||
endfunction()
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
include(GNUInstallDirs)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
|
||||||
|
|
||||||
function(init_python_module)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args
|
|
||||||
NAME
|
|
||||||
)
|
|
||||||
set(_multi_value_args
|
|
||||||
SOURCES
|
|
||||||
)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
|
||||||
set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
|
|
||||||
|
|
||||||
set(module_proj ${_name}_module)
|
|
||||||
set(${module_proj}_SOURCES ${_sources})
|
|
||||||
pybind11_add_module(${module_proj} SHARED ${_sources})
|
|
||||||
target_link_libraries(
|
|
||||||
${module_proj}
|
|
||||||
PUBLIC
|
|
||||||
pybind11::embed
|
|
||||||
spark
|
|
||||||
${_name}
|
|
||||||
)
|
|
||||||
target_include_directories(
|
|
||||||
${module_proj}
|
|
||||||
PRIVATE
|
|
||||||
$<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
|
|
||||||
)
|
|
||||||
target_include_directories(
|
|
||||||
${module_proj}
|
|
||||||
PUBLIC
|
|
||||||
${Python3_INCLUDE_DIRS}
|
|
||||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
|
||||||
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
|
||||||
)
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
|
||||||
DEPENDS ${_name} ${module_proj}
|
|
||||||
COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
|
|
||||||
COMMENT "Generating py stubs"
|
|
||||||
)
|
|
||||||
add_custom_target(
|
|
||||||
"${_name}_py_stubs"
|
|
||||||
ALL
|
|
||||||
DEPENDS
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
|
||||||
)
|
|
||||||
install(
|
|
||||||
TARGETS ${module_proj}
|
|
||||||
EXPORT ${module_proj}Targets
|
|
||||||
RUNTIME
|
|
||||||
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
|
||||||
LIBRARY
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
|
||||||
ARCHIVE
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
|
||||||
)
|
|
||||||
foreach(dir ${pybind11_INCLUDE_DIRS})
|
|
||||||
install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
|
|
||||||
endforeach()
|
|
||||||
install(
|
|
||||||
EXPORT ${module_proj}Targets
|
|
||||||
FILE ${module_proj}Targets.cmake
|
|
||||||
NAMESPACE ${_name}::
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
|
||||||
)
|
|
||||||
endfunction()
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
include(GNUInstallDirs)
|
|
||||||
|
|
||||||
function(init_python_plugin)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args NAME PLUGIN_NAME)
|
|
||||||
set(_multi_value_args SOURCES)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
|
||||||
set(_plugin_name ${_ARGS_PLUGIN_NAME})
|
|
||||||
if (NOT DEFINED _ARGS_PLUGIN_NAME)
|
|
||||||
set(_plugin_name ${_name}_plugins)
|
|
||||||
endif()
|
|
||||||
set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
|
|
||||||
if (NOT DEFINED _ARGS_SOURCES)
|
|
||||||
file(
|
|
||||||
GLOB_RECURSE ${_plugin_name}_SOURCES
|
|
||||||
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
|
||||||
${_plugin_name}/*.py)
|
|
||||||
endif()
|
|
||||||
set(src_outs)
|
|
||||||
foreach(_src ${${_plugin_name}_SOURCES})
|
|
||||||
set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
|
|
||||||
set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT ${_src_out}
|
|
||||||
COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
|
|
||||||
DEPENDS ${_src_in}
|
|
||||||
COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
|
|
||||||
)
|
|
||||||
list(APPEND src_outs ${_src_out})
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
|
|
||||||
|
|
||||||
add_library(${_plugin_name} INTERFACE)
|
|
||||||
add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
|
|
||||||
|
|
||||||
set_target_properties(
|
|
||||||
${_plugin_name}
|
|
||||||
PROPERTIES
|
|
||||||
INTERFACE_PYTHON_SCRIPTS_DIR
|
|
||||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
|
|
||||||
)
|
|
||||||
|
|
||||||
set_target_properties(
|
|
||||||
${_plugin_name}
|
|
||||||
PROPERTIES
|
|
||||||
INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
|
|
||||||
)
|
|
||||||
|
|
||||||
install(
|
|
||||||
TARGETS ${_plugin_name}
|
|
||||||
EXPORT ${_plugin_name}Targets
|
|
||||||
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
|
||||||
)
|
|
||||||
|
|
||||||
install(
|
|
||||||
EXPORT ${_plugin_name}Targets
|
|
||||||
FILE ${_plugin_name}Targets.cmake
|
|
||||||
NAMESPACE ${_name}::
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
|
||||||
)
|
|
||||||
install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
|
|
||||||
endfunction()
|
|
||||||
@@ -1,182 +0,0 @@
|
|||||||
set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
|
|
||||||
set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
|
|
||||||
set(PROTOBUF_STATIC_LIB "libprotobuf")
|
|
||||||
set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
|
|
||||||
set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
|
|
||||||
|
|
||||||
function(Protobuf_Generate_Cpp)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
|
|
||||||
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
set(${_ARGS_CXX_SOURCES})
|
|
||||||
set(${_ARGS_CXX_HEADERS})
|
|
||||||
|
|
||||||
set(_proto_include_cli_args)
|
|
||||||
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
|
||||||
list(APPEND _proto_include_cli_args -I ${_dir})
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
foreach(FIL ${_ARGS_PROTO_SOURCES})
|
|
||||||
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
|
|
||||||
get_filename_component(FIL_WE ${FIL} NAME_WE)
|
|
||||||
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
|
|
||||||
if(FIL_DIR)
|
|
||||||
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
|
|
||||||
endif()
|
|
||||||
|
|
||||||
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
|
|
||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
|
|
||||||
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
|
||||||
COMMAND
|
|
||||||
${Protobuf_PROTOC_EXECUTABLE}
|
|
||||||
ARGS
|
|
||||||
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
|
|
||||||
DEPENDS ${ABS_FIL}
|
|
||||||
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
|
|
||||||
VERBATIM
|
|
||||||
)
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
set_source_files_properties(
|
|
||||||
${${_ARGS_CXX_SOURCES}}
|
|
||||||
${${_ARGS_CXX_HEADERS}}
|
|
||||||
PROPERTIES
|
|
||||||
GENERATED TRUE
|
|
||||||
)
|
|
||||||
|
|
||||||
# remove some warnings from protobuf generated files
|
|
||||||
if(WIN32)
|
|
||||||
set_source_files_properties(
|
|
||||||
${${_ARGS_CXX_SOURCES}}
|
|
||||||
PROPERTIES
|
|
||||||
COMPILE_FLAGS "/wd4309 /wd4838"
|
|
||||||
)
|
|
||||||
else()
|
|
||||||
set_source_files_properties(
|
|
||||||
${${_ARGS_CXX_SOURCES}}
|
|
||||||
PROPERTIES
|
|
||||||
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
|
||||||
)
|
|
||||||
endif()
|
|
||||||
|
|
||||||
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
|
||||||
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
|
||||||
endfunction()
|
|
||||||
|
|
||||||
function(Service_Compiler_Generate_Cpp)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
|
|
||||||
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
set(${_ARGS_CXX_SOURCES})
|
|
||||||
set(${_ARGS_CXX_HEADERS})
|
|
||||||
|
|
||||||
set(_proto_include_cli_args)
|
|
||||||
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
|
||||||
list(APPEND _proto_include_cli_args -I ${_dir})
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
foreach(FIL ${_ARGS_PROTO_SOURCES})
|
|
||||||
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
|
|
||||||
get_filename_component(FIL_WE ${FIL} NAME_WE)
|
|
||||||
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
|
|
||||||
if(FIL_DIR)
|
|
||||||
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
|
|
||||||
endif()
|
|
||||||
|
|
||||||
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
|
|
||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
|
|
||||||
|
|
||||||
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
|
|
||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
|
|
||||||
|
|
||||||
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
|
|
||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
|
|
||||||
|
|
||||||
set(TEMPLATES
|
|
||||||
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
|
|
||||||
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
|
|
||||||
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
|
|
||||||
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
|
|
||||||
)
|
|
||||||
|
|
||||||
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
|
|
||||||
if(WIN32)
|
|
||||||
set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
|
||||||
COMMAND SET PYTHONPATH=${SPARK_PYTHON}
|
|
||||||
COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
|
|
||||||
COMMAND
|
|
||||||
${Protobuf_PROTOC_EXECUTABLE}
|
|
||||||
ARGS
|
|
||||||
--cpp_out=${_ARGS_CXX_OUT_DIR}
|
|
||||||
--plugin=protoc-gen-service=${_plugin_cmd}
|
|
||||||
--service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
|
|
||||||
-I ${PROTOBUF_INCLUDE_DIRS}
|
|
||||||
-I ${_protobuf_src_dir}
|
|
||||||
${ABS_FIL}
|
|
||||||
DEPENDS
|
|
||||||
${ABS_FIL}
|
|
||||||
${TEMPLATES}
|
|
||||||
${_plugin_cmd}
|
|
||||||
${_plugin_py}
|
|
||||||
COMMENT "Running C++ service compiler on ${FIL}"
|
|
||||||
VERBATIM
|
|
||||||
)
|
|
||||||
else()
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
|
||||||
COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
|
|
||||||
ARGS
|
|
||||||
${Protobuf_PROTOC_EXECUTABLE}
|
|
||||||
--cpp_out=${_ARGS_CXX_OUT_DIR}
|
|
||||||
--plugin=protoc-gen-service=${_plugin_py}
|
|
||||||
--service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
|
|
||||||
${_proto_include_cli_args} ${ABS_FIL}
|
|
||||||
DEPENDS
|
|
||||||
${ABS_FIL}
|
|
||||||
${TEMPLATES}
|
|
||||||
${_plugin_py}
|
|
||||||
COMMENT "Running C++ service compiler on ${FIL}"
|
|
||||||
VERBATIM
|
|
||||||
)
|
|
||||||
endif()
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
# remove some warnings from protobuf generated files
|
|
||||||
if(WIN32)
|
|
||||||
set_source_files_properties(
|
|
||||||
${${_ARGS_CXX_SOURCES}}
|
|
||||||
PROPERTIES
|
|
||||||
COMPILE_FLAGS "/wd4309 /wd4838"
|
|
||||||
)
|
|
||||||
else()
|
|
||||||
set_source_files_properties(
|
|
||||||
${${_ARGS_CXX_SOURCES}}
|
|
||||||
PROPERTIES
|
|
||||||
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
|
||||||
)
|
|
||||||
endif()
|
|
||||||
|
|
||||||
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
|
||||||
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
|
||||||
endfunction()
|
|
||||||
|
|
||||||
@@ -39,8 +39,6 @@ function(protobuf_generate_cpp)
|
|||||||
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
|
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
|
||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
||||||
message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
|
||||||
# TODO: Pass proto sources by relative path (so this fn can be called from
|
|
||||||
# arbitrary subdirs).
|
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT
|
OUTPUT
|
||||||
"${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
|
||||||
@@ -48,7 +46,7 @@ function(protobuf_generate_cpp)
|
|||||||
COMMAND
|
COMMAND
|
||||||
${_ARGS_PROTOC}
|
${_ARGS_PROTOC}
|
||||||
ARGS
|
ARGS
|
||||||
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
|
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
|
||||||
DEPENDS ${_abs_fil}
|
DEPENDS ${_abs_fil}
|
||||||
COMMENT "Running C++ protocol buffer compiler on ${_fil}"
|
COMMENT "Running C++ protocol buffer compiler on ${_fil}"
|
||||||
VERBATIM
|
VERBATIM
|
||||||
@@ -80,4 +78,3 @@ function(protobuf_generate_cpp)
|
|||||||
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
||||||
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
@@ -1,5 +1,3 @@
|
|||||||
include_guard(GLOBAL)
|
|
||||||
|
|
||||||
function(protobuf_generate_cpp_config)
|
function(protobuf_generate_cpp_config)
|
||||||
set(_options)
|
set(_options)
|
||||||
set(_one_value_args
|
set(_one_value_args
|
||||||
@@ -58,4 +56,3 @@ function(protobuf_generate_cpp_config)
|
|||||||
file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
|
file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
|
||||||
endif()
|
endif()
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
|
|
||||||
|
|
||||||
function(sym_link_python_module)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
|
|
||||||
set(_multi_value_args)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
|
|
||||||
"${_multi_value_args}" ${ARGN})
|
|
||||||
if(NOT DEFINED _ARGS_LIB_NAME)
|
|
||||||
message(
|
|
||||||
FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
|
|
||||||
endif()
|
|
||||||
set(_lib_name ${_ARGS_LIB_NAME})
|
|
||||||
set(_module_name ${_ARGS_MODULE_NAME})
|
|
||||||
if(NOT DEFINED _ARGS_MODULE_NAME)
|
|
||||||
set(_module_name ${_lib_name}_module)
|
|
||||||
endif()
|
|
||||||
set(_plugin_name ${_ARGS_PLUGIN_NAME})
|
|
||||||
set(_deps ${_lib_name} ${_module_name})
|
|
||||||
if(DEFINED _ARGS_PLUGIN_NAME)
|
|
||||||
list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
|
|
||||||
endif()
|
|
||||||
add_dependencies(${PROJECT_NAME} ${_deps})
|
|
||||||
sym_link_local_lib(${_lib_name})
|
|
||||||
set(_module_suffix cpython-310-x86_64-linux-gnu.so)
|
|
||||||
sym_link_file(
|
|
||||||
SOURCE
|
|
||||||
${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
|
|
||||||
DEST
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
|
|
||||||
INSTALL_DEST
|
|
||||||
${CMAKE_INSTALL_BINDIR}/)
|
|
||||||
if(DEFINED _ARGS_PLUGIN_NAME)
|
|
||||||
get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
|
|
||||||
sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
|
|
||||||
${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
|
|
||||||
endif()
|
|
||||||
endfunction()
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
#ifndef BTS_SPARK_VERSION_H
|
|
||||||
#define BTS_SPARK_VERSION_H
|
|
||||||
|
|
||||||
#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
|
|
||||||
#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
|
|
||||||
#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
|
|
||||||
#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
|
|
||||||
#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
|
|
||||||
#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
cmake_minimum_required(VERSION 3.5)
|
||||||
|
|
||||||
|
# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
|
||||||
|
# setting it explicitly prevents a compile-warning)
|
||||||
|
cmake_policy(SET CMP0076 NEW)
|
||||||
|
|
||||||
|
# add protos to the library to facilitate source_group
|
||||||
|
target_sources(spark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
|
||||||
|
|
||||||
|
function(assign_source_group)
|
||||||
|
foreach(_source IN ITEMS ${ARGN})
|
||||||
|
if(IS_ABSOLUTE "${_source}")
|
||||||
|
file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
|
||||||
|
else()
|
||||||
|
set(_source_rel "${_source}")
|
||||||
|
endif()
|
||||||
|
get_filename_component(_source_path "${_source_rel}" PATH)
|
||||||
|
string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
|
||||||
|
source_group("${_source_path_msvc}" FILES "${_source}")
|
||||||
|
set(SOURCE_FILES "${_source}")
|
||||||
|
endforeach()
|
||||||
|
endfunction(assign_source_group)
|
||||||
|
|
||||||
|
assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
services:
|
||||||
|
sparkd:
|
||||||
|
build:
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
context: ./sparkd
|
||||||
|
command: [ "./sparkd", "run" ]
|
||||||
|
ports: [ "8000:8000", "8001:8001", "8002:8002" ]
|
||||||
|
|
||||||
|
pubsub:
|
||||||
|
build:
|
||||||
|
dockerfile: pubsub.Dockerfile
|
||||||
|
context: ./DataStream
|
||||||
|
command: [ "./PubSubServer" ]
|
||||||
|
environment:
|
||||||
|
- DOTNET_ENVIRONMENT=Production
|
||||||
|
ports:
|
||||||
|
- 22019:22019
|
||||||
|
|
||||||
|
md-publisher:
|
||||||
|
build:
|
||||||
|
dockerfile: md-publisher.Dockerfile
|
||||||
|
context: ./DataStream
|
||||||
|
depends_on:
|
||||||
|
- pubsub
|
||||||
|
- sparkd
|
||||||
|
command:
|
||||||
|
[
|
||||||
|
"dockerize",
|
||||||
|
"-wait",
|
||||||
|
"tcp://pubsub:22019",
|
||||||
|
"-wait",
|
||||||
|
"tcp://sparkd:8000",
|
||||||
|
"-timeout",
|
||||||
|
"10s",
|
||||||
|
"./DataStreamPublisher"
|
||||||
|
]
|
||||||
|
environment:
|
||||||
|
- DOTNET_ENVIRONMENT=MarketData
|
||||||
|
|
||||||
|
webapp:
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
- 8443:8443
|
||||||
|
build:
|
||||||
|
dockerfile: webapp.Dockerfile
|
||||||
|
context: ./DataStream
|
||||||
|
command: [ "./Bts.DataStreamWebApp.Server" ]
|
||||||
|
environment:
|
||||||
|
- ASPNETCORE_HTTP_PORT=8080
|
||||||
|
- ASPNETCORE_HTTPS_PORT=8443
|
||||||
|
- ASPNETCORE_ENVIRONMENT=Staging
|
||||||
+53
-137
@@ -1,100 +1,69 @@
|
|||||||
services:
|
services:
|
||||||
sparkd:
|
sparkd:
|
||||||
volumes:
|
|
||||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
|
||||||
build:
|
build:
|
||||||
dockerfile: src/app_sparkd/Dockerfile
|
dockerfile: src/app_sparkd/Dockerfile
|
||||||
args:
|
args:
|
||||||
- IMAGE=${SPARKD__IMAGE}
|
- IMAGE=${SPARKD__IMAGE}
|
||||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
- CONF_FILE=${SPARKD__CONF_FILE}
|
||||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
- MD_RECORDING_DIR=${SPARKD__MD_RECORDING_DIR}
|
||||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
command: [ "./sparkd", "run" ]
|
||||||
command: >
|
|
||||||
sh -c "
|
|
||||||
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
|
|
||||||
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
|
||||||
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
|
||||||
fi
|
|
||||||
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
|
||||||
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
|
||||||
set -a
|
|
||||||
. /home/bts/sparkdata/conf/.env.local
|
|
||||||
set +a
|
|
||||||
fi
|
|
||||||
if [ -d /home/bts/sparkdata/conf/plugins ]; then
|
|
||||||
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
|
|
||||||
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
|
|
||||||
fi
|
|
||||||
./sparkd run"
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
- ${SPARKD__CONFIG_DIR}/.env
|
|
||||||
ports:
|
ports:
|
||||||
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
- "8000:8000"
|
||||||
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
- "8001:8001"
|
||||||
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
- "8002:8002"
|
||||||
deploy:
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
memory: "${SPARKD__MEMORY:-4gb}"
|
|
||||||
|
|
||||||
sparkd-symbols:
|
pubsub:
|
||||||
volumes:
|
environment:
|
||||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
- DOTNET_ENVIRONMENT=${PUBSUB__DOTNET_ENVIRONMENT}
|
||||||
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
|
|
||||||
build:
|
build:
|
||||||
dockerfile: src/app_sparkd/Dockerfile
|
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
|
||||||
args:
|
args:
|
||||||
- IMAGE=${SPARKD__IMAGE}
|
- CS_SOURCE_DIR=${PUBSUB__CS_SOURCE_DIR}
|
||||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
- CPP_SOURCE_DIR=${PUBSUB__CPP_SOURCE_DIR}
|
||||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
- CONF_FILE=${PUBSUB__CONF_FILE}
|
||||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
|
||||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
|
||||||
command: >
|
|
||||||
sh -c "
|
|
||||||
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
|
|
||||||
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
|
||||||
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
|
||||||
fi
|
|
||||||
ls -alh /home/bts/sparkdata/conf/
|
|
||||||
cat /home/bts/sparkdata/conf/.env.local
|
|
||||||
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
|
||||||
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
|
||||||
set -a
|
|
||||||
. /home/bts/sparkdata/conf/.env.local
|
|
||||||
set +a
|
|
||||||
fi
|
|
||||||
if [ -d /home/bts/sparkdata/conf/plugins ]; then
|
|
||||||
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
|
|
||||||
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
|
|
||||||
fi
|
|
||||||
./sparkd run"
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
- ${SPARKD__CONFIG_DIR}/.env
|
|
||||||
ports:
|
ports:
|
||||||
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
- "22019:22019"
|
||||||
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
command: [ "./PubSubServer" ]
|
||||||
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
|
||||||
deploy:
|
|
||||||
restart_policy:
|
|
||||||
condition: any
|
|
||||||
delay: 3s
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
memory: "${SPARKD__MEMORY:-4gb}"
|
|
||||||
|
|
||||||
sparkd-tail:
|
md-publisher:
|
||||||
image: bluetradingsystems/linux_dotnet7
|
build:
|
||||||
volumes:
|
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
|
||||||
- type: volume
|
args:
|
||||||
read_only: true
|
- SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
|
||||||
source: sparkd_data
|
- CONF_FILE=${MD_PUBLISHER__CONF_FILE}
|
||||||
target: /home/bts/sparkdata/data/logging.sparkd
|
depends_on:
|
||||||
volume:
|
- pubsub
|
||||||
subpath: logging.sparkd
|
- sparkd
|
||||||
command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
|
command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
|
||||||
|
environment:
|
||||||
|
- DOTNET_ENVIRONMENT=${MD_PUBLISHER__DOTNET_ENVIRONMENT}
|
||||||
|
|
||||||
|
webapp:
|
||||||
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
|
- "8443:8443"
|
||||||
|
build:
|
||||||
|
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
|
||||||
|
args:
|
||||||
|
- SOURCE_DIR=${WEBAPP__SOURCE_DIR}
|
||||||
|
- WEB_ROOT=${WEBAPP__WEB_ROOT}
|
||||||
|
- CONF_FILE=${WEBAPP__CONF_FILE}
|
||||||
|
command: [ "./Bts.DataStreamWebApp.Server" ]
|
||||||
|
environment:
|
||||||
|
- ASPNETCORE_ENVIRONMENT=${WEBAPP__ASPNETCORE_ENVIRONMENT}
|
||||||
|
|
||||||
|
api-server:
|
||||||
|
depends_on:
|
||||||
|
- sparkd
|
||||||
|
ports:
|
||||||
|
- 8008:8008
|
||||||
|
- 9443:9443
|
||||||
|
build:
|
||||||
|
dockerfile: src/client/spark_api_server/Dockerfile
|
||||||
|
command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
|
||||||
|
environment:
|
||||||
|
- ASPNETCORE_ENVIRONMENT=${API_SERVER__ASPNETCORE_ENVIRONMENT}
|
||||||
|
|
||||||
sparkd-ping-test:
|
sparkd-ping-test:
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -108,56 +77,3 @@ services:
|
|||||||
USER "root"
|
USER "root"
|
||||||
RUN apt-get update && apt-get -y install iputils-ping
|
RUN apt-get update && apt-get -y install iputils-ping
|
||||||
command: [ "ping", "sparkd" ]
|
command: [ "ping", "sparkd" ]
|
||||||
|
|
||||||
pubsub:
|
|
||||||
environment:
|
|
||||||
- DOTNET_ENVIRONMENT=Staging
|
|
||||||
build:
|
|
||||||
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
|
|
||||||
args:
|
|
||||||
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
|
||||||
- CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
|
|
||||||
ports:
|
|
||||||
- 22019:22019
|
|
||||||
command: [ "./PubSubServer" ]
|
|
||||||
|
|
||||||
md-publisher:
|
|
||||||
build:
|
|
||||||
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
|
|
||||||
args:
|
|
||||||
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
|
||||||
- CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
|
|
||||||
depends_on:
|
|
||||||
- pubsub
|
|
||||||
- sparkd
|
|
||||||
command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
|
|
||||||
environment:
|
|
||||||
- DOTNET_ENVIRONMENT=MarketData
|
|
||||||
|
|
||||||
api-server:
|
|
||||||
depends_on:
|
|
||||||
- sparkd
|
|
||||||
ports:
|
|
||||||
- 8008:8008
|
|
||||||
- 9443:9443
|
|
||||||
build:
|
|
||||||
dockerfile: src/client/spark_api_server/Dockerfile
|
|
||||||
command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
|
|
||||||
|
|
||||||
webapp:
|
|
||||||
ports:
|
|
||||||
- 8080:8080
|
|
||||||
- 8443:8443
|
|
||||||
build:
|
|
||||||
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
|
|
||||||
args:
|
|
||||||
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
|
||||||
- WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
|
|
||||||
- CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
|
|
||||||
command: [ "./Bts.DataStreamWebApp.Server" ]
|
|
||||||
environment:
|
|
||||||
- ASPNETCORE_ENVIRONMENT=Staging
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
sparkd_data:
|
|
||||||
name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
|
|
||||||
@@ -124,8 +124,8 @@ service OrderService {
|
|||||||
```
|
```
|
||||||
### C++ Example
|
### C++ Example
|
||||||
```cpp
|
```cpp
|
||||||
#include "spark/api/public/ServiceContext.h"
|
#include "lib/api/public/ServiceContext.h"
|
||||||
#include "spark/api/services/OrderService.client.pb.h"
|
#include "lib/api/services/OrderService.client.pb.h"
|
||||||
using namespace bts::api;
|
using namespace bts::api;
|
||||||
ServiceContext ctx;
|
ServiceContext ctx;
|
||||||
auto orders = ctx.locate<orderman::OrderService>();
|
auto orders = ctx.locate<orderman::OrderService>();
|
||||||
|
|||||||
@@ -29,9 +29,9 @@ namespace sample {
|
|||||||
```
|
```
|
||||||
|
|
||||||
```cpp
|
```cpp
|
||||||
#include "spark/api/public/ServiceContext.h"
|
#include "lib/api/public/ServiceContext.h"
|
||||||
#include "spark/domain_model/Security.h"
|
#include "spark/lib/domain_model/Security.h"
|
||||||
#include "spark/api/services/MarketDataService.client.pb.h"
|
#include "lib/api/services/MarketDataService.client.pb.h"
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
using namespace bts::api;
|
using namespace bts::api;
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
|
|||||||
// Subscribe to security trading status
|
// Subscribe to security trading status
|
||||||
void IAlgoServices::subscribe_status(SecurityKey);
|
void IAlgoServices::subscribe_status(SecurityKey);
|
||||||
// You implement
|
// You implement
|
||||||
virtual void on_status(SecurityKey, co2::TradingStatus) override;
|
virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
|
||||||
```
|
```
|
||||||
|
|
||||||
# Child Orders
|
# Child Orders
|
||||||
|
|||||||
@@ -1,38 +0,0 @@
|
|||||||
FROM ubuntu:22.04
|
|
||||||
|
|
||||||
ARG DEBIAN_FRONTEND=noninteractive
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y pkg-config
|
|
||||||
RUN apt-get update && apt-get install -y locales
|
|
||||||
RUN apt-get update && apt-get install -y ca-certificates
|
|
||||||
RUN apt-get update && apt-get install -y libssh2-1-dev
|
|
||||||
RUN apt-get update && apt-get install -y software-properties-common
|
|
||||||
RUN apt-get update && apt-get install -y sudo
|
|
||||||
|
|
||||||
# GCC
|
|
||||||
ARG gcc_major=13
|
|
||||||
RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y g++-$gcc_major
|
|
||||||
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y dotnet7
|
|
||||||
|
|
||||||
RUN locale-gen en_US.UTF-8
|
|
||||||
|
|
||||||
ARG docker_user="bts"
|
|
||||||
ARG docker_uid="1000"
|
|
||||||
ARG docker_pass="bts"
|
|
||||||
|
|
||||||
RUN \
|
|
||||||
groupadd -g "$docker_uid" "$docker_user" \
|
|
||||||
&& useradd \
|
|
||||||
-m \
|
|
||||||
-u "$docker_uid" \
|
|
||||||
-g "$docker_uid" \
|
|
||||||
-G sudo \
|
|
||||||
--password "$(openssl passwd -1 "$docker_pass")" \
|
|
||||||
"$docker_user"
|
|
||||||
|
|
||||||
USER bts
|
|
||||||
WORKDIR /home/bts
|
|
||||||
+1
-1
Submodule externals updated: 4f6734703c...4f5e385556.
@@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"sdk": {
|
||||||
|
"version": "6.0.200",
|
||||||
|
"rollForward": "latestMajor"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,42 +1,22 @@
|
|||||||
astroid==2.7.3
|
astroid==2.7.3
|
||||||
autopep8==1.5.7
|
autopep8==1.5.7
|
||||||
blinker==1.9.0
|
|
||||||
boto3==1.37.9
|
|
||||||
botocore==1.37.9
|
|
||||||
certifi==2023.5.7
|
certifi==2023.5.7
|
||||||
charset-normalizer==3.1.0
|
charset-normalizer==3.1.0
|
||||||
click==8.1.8
|
|
||||||
colorama==0.4.4
|
colorama==0.4.4
|
||||||
Flask==3.1.0
|
|
||||||
idna==3.4
|
idna==3.4
|
||||||
isort==5.9.3
|
isort==5.9.3
|
||||||
itsdangerous==2.2.0
|
|
||||||
Jinja2==3.1.6
|
|
||||||
jmespath==1.0.1
|
|
||||||
lazy-object-proxy==1.6.0
|
lazy-object-proxy==1.6.0
|
||||||
MarkupSafe==3.0.2
|
|
||||||
mccabe==0.6.1
|
mccabe==0.6.1
|
||||||
numpy==2.2.6
|
|
||||||
platformdirs==2.3.0
|
platformdirs==2.3.0
|
||||||
protobuf==4.21.12
|
protobuf==4.21.12
|
||||||
pybars3==0.9.7
|
pybars3==0.9.7
|
||||||
pybind11==2.13.6
|
|
||||||
pybind11-stubgen==2.5.3
|
|
||||||
pybit==5.11.0
|
|
||||||
pycodestyle==2.7.0
|
pycodestyle==2.7.0
|
||||||
pycparser==2.20
|
pycparser==2.20
|
||||||
pycryptodome==3.23.0
|
|
||||||
pylint==2.10.2
|
pylint==2.10.2
|
||||||
PyMeta3==0.5.1
|
PyMeta3==0.5.1
|
||||||
python-dateutil==2.9.0.post0
|
|
||||||
python-dotenv==1.0.1
|
|
||||||
requests==2.31.0
|
requests==2.31.0
|
||||||
s3transfer==0.11.4
|
|
||||||
six==1.17.0
|
|
||||||
termcolor==1.1.0
|
termcolor==1.1.0
|
||||||
toml==0.10.2
|
toml==0.10.2
|
||||||
urllib3==2.0.3
|
urllib3==2.0.3
|
||||||
websocket-client==1.8.0
|
|
||||||
Werkzeug==3.1.3
|
|
||||||
wrapt==1.12.1
|
wrapt==1.12.1
|
||||||
yapf==0.31.0
|
yapf==0.31.0
|
||||||
+103
-164
@@ -1,164 +1,103 @@
|
|||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio Version 17
|
# Visual Studio Version 17
|
||||||
VisualStudioVersion = 17.3.32922.545
|
VisualStudioVersion = 17.3.32922.545
|
||||||
MinimumVisualStudioVersion = 15.0.26124.0
|
MinimumVisualStudioVersion = 15.0.26124.0
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
|
Global
|
||||||
EndProject
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
|
Debug|x64 = Debug|x64
|
||||||
EndProject
|
MinSizeRel|x64 = MinSizeRel|x64
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
|
Release|x64 = Release|x64
|
||||||
EndProject
|
RelWithDebInfo|x64 = RelWithDebInfo|x64
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
|
EndGlobalSection
|
||||||
EndProject
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
EndProject
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
EndProject
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
|
||||||
EndProject
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
EndProject
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
Global
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
|
||||||
Debug|x64 = Debug|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
MinSizeRel|x64 = MinSizeRel|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
Release|x64 = Release|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
|
||||||
RelWithDebInfo|x64 = RelWithDebInfo|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
|
||||||
EndGlobalSection
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
|
EndGlobalSection
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
HideSolutionNode = FALSE
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
|
EndGlobalSection
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
|
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
|
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
EndGlobalSection
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
|
GlobalSection(Performance) = preSolution
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
|
HasPerformanceSessions = true
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
|
EndGlobalSection
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
EndGlobal
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
|
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
|
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
|
||||||
HideSolutionNode = FALSE
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
|
||||||
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(Performance) = preSolution
|
|
||||||
HasPerformanceSessions = true
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(NestedProjects) = preSolution
|
|
||||||
{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
|
|
||||||
{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
|
|
||||||
EndGlobalSection
|
|
||||||
EndGlobal
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -6,80 +6,23 @@ using Spark.Core.Collections;
|
|||||||
|
|
||||||
static class ArgumentComparer {
|
static class ArgumentComparer {
|
||||||
public static int Compare(
|
public static int Compare(
|
||||||
IArgument value1, IArgument value2, IDictionary<string, object?> env) {
|
IArgument value1, IArgument value2, IDictionary<string,object?> env) {
|
||||||
if (value1 is null) {
|
if (value1 is null) {
|
||||||
throw new ArgumentNullException(nameof(value1));
|
throw new ArgumentNullException(nameof(value1));
|
||||||
}
|
}
|
||||||
if (value2 is null) {
|
if (value2 is null) {
|
||||||
throw new ArgumentNullException(nameof(value2));
|
throw new ArgumentNullException(nameof(value2));
|
||||||
}
|
}
|
||||||
var obj1 = value1.GetValue(env);
|
var cmp1 = value1.Cast<IComparable>();
|
||||||
var obj2 = value2.GetValue(env);
|
var cmp2 = value2.Cast<IComparable>();
|
||||||
if (obj1 is null) {
|
|
||||||
if (obj2 is null) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
if (obj2 is null) {
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return compareObjs(obj1, obj2);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int compareObjs(object obj1, object obj2) {
|
|
||||||
try {
|
try {
|
||||||
var typ1 = obj1.GetType();
|
var v1 = cmp1.GetValue(env);
|
||||||
var typ2 = obj2.GetType();
|
var v2 = cmp2.GetValue(env);
|
||||||
bool areSameTypes = typ1 == typ2;
|
return v1.GetType().IsNumeric() ?
|
||||||
if (areSameTypes) {
|
new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
|
||||||
return ((IComparable)obj1).CompareTo(obj2);
|
|
||||||
}
|
|
||||||
if (typ1.IsNumeric()) {
|
|
||||||
return tryCompareNumeric(obj1, obj2);
|
|
||||||
} else if (typ2.IsNumeric()) {
|
|
||||||
return -tryCompareNumeric(obj2, obj1);
|
|
||||||
} else if (typ1.IsEnum) {
|
|
||||||
if (typ2 == typeof(string)) {
|
|
||||||
return -((string)obj2).CompareTo(obj1.ToString());
|
|
||||||
}
|
|
||||||
} else if (typ1 == typeof(string)) {
|
|
||||||
return ((string)obj1).CompareTo(obj2.ToString());
|
|
||||||
} else if (typ1 == typeof(DateTime)) {
|
|
||||||
if (typ2 == typeof(TimeSpan)) {
|
|
||||||
return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
|
|
||||||
}
|
|
||||||
} else if (typ1 == typeof(TimeSpan)) {
|
|
||||||
if (typ2 == typeof(DateTime)) {
|
|
||||||
return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
Log.Ui.error($"Comparison failed: {ex}");
|
Log.Ui.error($"Comparison failed: {ex}");
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
|
|
||||||
}
|
|
||||||
|
|
||||||
static int tryCompareNumeric(object numericObj1, object obj2) {
|
|
||||||
var typ2 = obj2.GetType();
|
|
||||||
if (typ2.IsNumeric()) {
|
|
||||||
return new NumericComparer().Compare(numericObj1, obj2);
|
|
||||||
} else if (typ2.IsEnum) {
|
|
||||||
var undTyp2 = typ2.GetEnumUnderlyingType();
|
|
||||||
if (undTyp2 == typeof(byte)) {
|
|
||||||
return new NumericComparer().Compare(numericObj1, (byte)obj2);
|
|
||||||
}
|
|
||||||
if (undTyp2 == typeof(short)) {
|
|
||||||
return new NumericComparer().Compare(numericObj1, (short)obj2);
|
|
||||||
}
|
|
||||||
if (undTyp2 == typeof(int)) {
|
|
||||||
return new NumericComparer().Compare(numericObj1, (int)obj2);
|
|
||||||
}
|
|
||||||
if (undTyp2 == typeof(long)) {
|
|
||||||
return new NumericComparer().Compare(numericObj1, (long)obj2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -41,6 +41,7 @@ Identifier {Letter}({Letter}*[0-9]*_*)*
|
|||||||
OpenParenthesis \(
|
OpenParenthesis \(
|
||||||
CloseParenthesis \)
|
CloseParenthesis \)
|
||||||
Comma ,
|
Comma ,
|
||||||
|
Negate '!'
|
||||||
MapsTo ':'
|
MapsTo ':'
|
||||||
IsOfType '/'
|
IsOfType '/'
|
||||||
|
|
||||||
@@ -93,6 +94,10 @@ IsOfType '/'
|
|||||||
return (int)Token.COMMA;
|
return (int)Token.COMMA;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
{Negate} {
|
||||||
|
return (int)Token.NEGATE;
|
||||||
|
}
|
||||||
|
|
||||||
{MapsTo} {
|
{MapsTo} {
|
||||||
return (int)Token.MAPS_TO;
|
return (int)Token.MAPS_TO;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,14 +1,14 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.3" />
|
<PackageReference Include="YaccLexTools" Version="1.2.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -2,8 +2,6 @@ namespace Bts.BooleanExpressions;
|
|||||||
|
|
||||||
using Bts.BooleanExpressions.Arguments;
|
using Bts.BooleanExpressions.Arguments;
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
class BoundBooleanExpression : IBooleanExpression {
|
class BoundBooleanExpression : IBooleanExpression {
|
||||||
readonly IBooleanExpression expression;
|
readonly IBooleanExpression expression;
|
||||||
readonly IDictionary<string, FunctionDefinition> definitions;
|
readonly IDictionary<string, FunctionDefinition> definitions;
|
||||||
@@ -19,10 +17,10 @@ class BoundBooleanExpression : IBooleanExpression {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public bool Evaluate(IDictionary<string, object?> env) {
|
public bool Evaluate(IDictionary<string, object?> env) {
|
||||||
if (identifiers.Count > 0) {
|
foreach (var kv in env) {
|
||||||
env = AggregateDictionary.Create(env, identifiers);
|
identifiers[kv.Key] = kv.Value;
|
||||||
}
|
}
|
||||||
env = new EvaluationEnvironment(env, definitions);
|
env = new EvaluationEnvironment(identifiers, definitions);
|
||||||
return expression.Evaluate(env);
|
return expression.Evaluate(env);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
|
|||||||
public BufferException(string message) : base(message) { }
|
public BufferException(string message) : base(message) { }
|
||||||
public BufferException(string message, Exception innerException)
|
public BufferException(string message, Exception innerException)
|
||||||
: base(message, innerException) { }
|
: base(message, innerException) { }
|
||||||
|
protected BufferException(SerializationInfo info, StreamingContext context)
|
||||||
|
: base(info, context) { }
|
||||||
}
|
}
|
||||||
|
|
||||||
internal abstract class ScanBuff
|
internal abstract class ScanBuff
|
||||||
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
|
|||||||
return new BuildBuffer(source);
|
return new BuildBuffer(source);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if (!BYTEMODE)
|
||||||
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
|
||||||
{
|
{
|
||||||
return new BuildBuffer(source, fallbackCodePage);
|
return new BuildBuffer(source, fallbackCodePage);
|
||||||
}
|
}
|
||||||
|
#endif // !BYTEMODE
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
|
|||||||
{
|
{
|
||||||
FileStream fStrm = (stream as FileStream);
|
FileStream fStrm = (stream as FileStream);
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
if (fStrm != null) FileName = fStrm.Name;
|
||||||
NextBlk = BlockReaderFactory.Get(stream);
|
NextBlk = BlockReaderFactory.Raw(stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if (!BYTEMODE)
|
||||||
public BuildBuffer(Stream stream, int fallbackCodePage)
|
public BuildBuffer(Stream stream, int fallbackCodePage)
|
||||||
{
|
{
|
||||||
FileStream fStrm = (stream as FileStream);
|
FileStream fStrm = (stream as FileStream);
|
||||||
if (fStrm != null) FileName = fStrm.Name;
|
if (fStrm != null) FileName = fStrm.Name;
|
||||||
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Marks a conservative lower bound for the buffer,
|
/// Marks a conservative lower bound for the buffer,
|
||||||
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
|
|||||||
//
|
//
|
||||||
internal static class BlockReaderFactory
|
internal static class BlockReaderFactory
|
||||||
{
|
{
|
||||||
|
public static BlockReader Raw(Stream stream)
|
||||||
|
{
|
||||||
|
return delegate(char[] block, int index, int number)
|
||||||
|
{
|
||||||
|
byte[] b = new byte[number];
|
||||||
|
int count = stream.Read(b, 0, number);
|
||||||
|
int i = 0;
|
||||||
|
int j = index;
|
||||||
|
for (; i < count; i++, j++)
|
||||||
|
block[j] = (char)b[i];
|
||||||
|
return count;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
|
#if (!BYTEMODE)
|
||||||
|
public static BlockReader Get(Stream stream, int fallbackCodePage)
|
||||||
{
|
{
|
||||||
Encoding encoding;
|
Encoding encoding;
|
||||||
int preamble = Preamble(stream);
|
int preamble = Preamble(stream);
|
||||||
|
|
||||||
if (preamble != 0) // There is a valid BOM here!
|
if (preamble != 0) // There is a valid BOM here!
|
||||||
encoding = Encoding.GetEncoding(preamble);
|
encoding = Encoding.GetEncoding(preamble);
|
||||||
else if (fallbackCodePage >= 0) // Anything but "guess"
|
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
|
||||||
|
return Raw(stream);
|
||||||
|
else if (fallbackCodePage != -2) // Anything but "guess"
|
||||||
encoding = Encoding.GetEncoding(fallbackCodePage);
|
encoding = Encoding.GetEncoding(fallbackCodePage);
|
||||||
else // This is the "guess" option
|
else // This is the "guess" option
|
||||||
encoding = GuessEncoding(stream);
|
{
|
||||||
|
int guess = new Guesser(stream).GuessCodePage();
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
if (guess == -1) // ==> this is a 7-bit file
|
||||||
|
encoding = Encoding.ASCII;
|
||||||
|
else if (guess == 65001)
|
||||||
|
encoding = Encoding.UTF8;
|
||||||
|
else // ==> use the machine default
|
||||||
|
encoding = Encoding.Default;
|
||||||
|
}
|
||||||
StreamReader reader = new StreamReader(stream, encoding);
|
StreamReader reader = new StreamReader(stream, encoding);
|
||||||
return reader.Read;
|
return reader.Read;
|
||||||
}
|
}
|
||||||
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
|
|||||||
int b2 = stream.ReadByte();
|
int b2 = stream.ReadByte();
|
||||||
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
|
||||||
return 65001; // UTF8
|
return 65001; // UTF8
|
||||||
|
|
||||||
int b3 = stream.ReadByte();
|
|
||||||
// Check for UTF-32 BOM (Byte Order Mark)
|
|
||||||
if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
|
|
||||||
return 12001; // UTF-32BE
|
|
||||||
if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
|
|
||||||
return 12000; // UTF-32LE
|
|
||||||
|
|
||||||
//
|
//
|
||||||
// There is no unicode preamble, so we
|
// There is no unicode preamble, so we
|
||||||
// return denoter for the machine default.
|
// return denoter for the machine default.
|
||||||
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
|
|||||||
stream.Seek(0, SeekOrigin.Begin);
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
#endif // !BYTEMODE
|
||||||
private static Encoding GuessEncoding(Stream stream)
|
|
||||||
{
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (IsUtf16(stream, out Encoding encoding))
|
|
||||||
{
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return encoding;
|
|
||||||
}
|
|
||||||
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (IsUtf8(stream))
|
|
||||||
{
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return Encoding.UTF8;
|
|
||||||
}
|
|
||||||
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (IsAscii(stream))
|
|
||||||
{
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return Encoding.ASCII;
|
|
||||||
}
|
|
||||||
|
|
||||||
#if !NET462_OR_GREATER
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
if (IsLatin1(stream))
|
|
||||||
{
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return Encoding.Latin1;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
return Encoding.Default;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsUtf8(Stream stream)
|
|
||||||
{
|
|
||||||
int b;
|
|
||||||
while ((b = stream.ReadByte()) != -1)
|
|
||||||
{
|
|
||||||
if ((b & 0x80) == 0) // 0xxxxxxx
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
|
|
||||||
{
|
|
||||||
if (!IsValidUtf8ContinuationByte(stream)) return false;
|
|
||||||
}
|
|
||||||
else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
|
|
||||||
{
|
|
||||||
if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
|
|
||||||
}
|
|
||||||
else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
|
||||||
{
|
|
||||||
if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsValidUtf8ContinuationByte(Stream stream)
|
|
||||||
{
|
|
||||||
int b = stream.ReadByte();
|
|
||||||
return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsUtf16(Stream stream, out Encoding encoding)
|
|
||||||
{
|
|
||||||
encoding = Encoding.Default;
|
|
||||||
|
|
||||||
if (stream.Length >= 2)
|
|
||||||
{
|
|
||||||
// Read the first two bytes
|
|
||||||
int b0 = stream.ReadByte();
|
|
||||||
int b1 = stream.ReadByte();
|
|
||||||
|
|
||||||
// Check for UTF-16 BOM (Byte Order Mark)
|
|
||||||
if (b0 == 0xFF && b1 == 0xFE)
|
|
||||||
{
|
|
||||||
encoding = Encoding.Unicode;
|
|
||||||
return true; // UTF-16LE
|
|
||||||
}
|
|
||||||
if (b0 == 0xFE && b1 == 0xFF)
|
|
||||||
{
|
|
||||||
encoding = Encoding.BigEndianUnicode;
|
|
||||||
return true; // UTF-16BE
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reset the stream position
|
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
|
||||||
|
|
||||||
// Check for common UTF-16 patterns
|
|
||||||
byte[] buffer = new byte[2];
|
|
||||||
while (stream.Read(buffer, 0, 2) == 2)
|
|
||||||
{
|
|
||||||
if (buffer[0] == 0x00 && buffer[1] != 0x00)
|
|
||||||
{
|
|
||||||
encoding = Encoding.BigEndianUnicode;
|
|
||||||
return true; // Likely UTF-16BE
|
|
||||||
}
|
|
||||||
if (buffer[0] != 0x00 && buffer[1] == 0x00)
|
|
||||||
{
|
|
||||||
encoding = Encoding.Unicode;
|
|
||||||
return true; // Likely UTF-16LE
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsAscii(Stream stream)
|
|
||||||
{
|
|
||||||
int b;
|
|
||||||
while ((b = stream.ReadByte()) != -1)
|
|
||||||
{
|
|
||||||
// Latin1 includes byte values from 0x00 to 0xFF
|
|
||||||
if (b > 0x7F)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsLatin1(Stream stream)
|
|
||||||
{
|
|
||||||
int b;
|
|
||||||
while ((b = stream.ReadByte()) != -1)
|
|
||||||
{
|
|
||||||
// Latin1 includes byte values from 0x00 to 0xFF
|
|
||||||
if (b > 0xFF)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
#endregion Buffer classes
|
#endregion Buffer classes
|
||||||
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#region guesser
|
||||||
|
#if (!BYTEMODE)
|
||||||
|
// ==============================================================
|
||||||
|
// ============ Encoding Guesser =============
|
||||||
|
// ==============================================================
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// This class provides a simple finite state automaton that
|
||||||
|
/// scans the file looking for (1) valid UTF-8 byte patterns,
|
||||||
|
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
|
||||||
|
/// The method then guesses whether it is UTF-8 or maybe some
|
||||||
|
/// local machine default encoding. This works well for the
|
||||||
|
/// various Latin encodings.
|
||||||
|
/// </summary>
|
||||||
|
internal class Guesser
|
||||||
|
{
|
||||||
|
ScanBuff buffer;
|
||||||
|
|
||||||
|
public int GuessCodePage() { return Scan(); }
|
||||||
|
|
||||||
|
const int maxAccept = 10;
|
||||||
|
const int initial = 0;
|
||||||
|
const int eofNum = 0;
|
||||||
|
const int goStart = -1;
|
||||||
|
const int INITIAL = 0;
|
||||||
|
const int EndToken = 0;
|
||||||
|
|
||||||
|
#region user code
|
||||||
|
/*
|
||||||
|
* Reads the bytes of a file to determine if it is
|
||||||
|
* UTF-8 or a single-byte code page file.
|
||||||
|
*/
|
||||||
|
public long utfX;
|
||||||
|
public long uppr;
|
||||||
|
#endregion user code
|
||||||
|
|
||||||
|
int state;
|
||||||
|
int currentStart = startState[0];
|
||||||
|
int code;
|
||||||
|
|
||||||
|
#region ScannerTables
|
||||||
|
static int[] startState = new int[] { 11, 0 };
|
||||||
|
|
||||||
|
#region CharacterMap
|
||||||
|
static sbyte[] map = new sbyte[256] {
|
||||||
|
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||||
|
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||||
|
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||||
|
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||||
|
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||||
|
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||||
|
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||||
|
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
static sbyte[][] nextState = new sbyte[][] {
|
||||||
|
new sbyte[] {0, 0, 0, 0, 0, 0},
|
||||||
|
new sbyte[] {-1, -1, 10, -1, -1, -1},
|
||||||
|
new sbyte[] {-1, -1, -1, -1, -1, -1},
|
||||||
|
new sbyte[] {-1, -1, 8, -1, -1, -1},
|
||||||
|
new sbyte[] {-1, -1, 5, -1, -1, -1},
|
||||||
|
new sbyte[] {-1, -1, 6, -1, -1, -1},
|
||||||
|
new sbyte[] {-1, -1, 7, -1, -1, -1},
|
||||||
|
null,
|
||||||
|
new sbyte[] {-1, -1, 9, -1, -1, -1},
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
new sbyte[] {-1, 1, 2, 3, 4, 2}
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
|
||||||
|
// Reason for suppression: cannot have self-reference in array initializer.
|
||||||
|
static Guesser()
|
||||||
|
{
|
||||||
|
nextState[7] = nextState[2];
|
||||||
|
nextState[9] = nextState[2];
|
||||||
|
nextState[10] = nextState[2];
|
||||||
|
}
|
||||||
|
|
||||||
|
int NextState()
|
||||||
|
{
|
||||||
|
if (code == ScanBuff.EndOfFile)
|
||||||
|
return eofNum;
|
||||||
|
else
|
||||||
|
return nextState[state][map[code]];
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
public Guesser(System.IO.Stream file) { SetSource(file); }
|
||||||
|
|
||||||
|
public void SetSource(System.IO.Stream source)
|
||||||
|
{
|
||||||
|
this.buffer = new BuildBuffer(source);
|
||||||
|
code = buffer.Read();
|
||||||
|
}
|
||||||
|
|
||||||
|
int Scan()
|
||||||
|
{
|
||||||
|
for (; ; )
|
||||||
|
{
|
||||||
|
int next;
|
||||||
|
state = currentStart;
|
||||||
|
while ((next = NextState()) == goStart)
|
||||||
|
code = buffer.Read();
|
||||||
|
|
||||||
|
state = next;
|
||||||
|
code = buffer.Read();
|
||||||
|
|
||||||
|
while ((next = NextState()) > eofNum)
|
||||||
|
{
|
||||||
|
state = next;
|
||||||
|
code = buffer.Read();
|
||||||
|
}
|
||||||
|
if (state <= maxAccept)
|
||||||
|
{
|
||||||
|
#region ActionSwitch
|
||||||
|
#pragma warning disable 162
|
||||||
|
switch (state)
|
||||||
|
{
|
||||||
|
case eofNum:
|
||||||
|
switch (currentStart)
|
||||||
|
{
|
||||||
|
case 11:
|
||||||
|
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
|
||||||
|
else if (uppr * 10 > utfX) return 0; /* default code page */
|
||||||
|
else return 65001; /* UTF-8 encoding */
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return EndToken;
|
||||||
|
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
|
||||||
|
case 2: // Recognized '{Upper128}', Shortest string "\x80"
|
||||||
|
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
|
||||||
|
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
|
||||||
|
uppr++;
|
||||||
|
break;
|
||||||
|
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
|
||||||
|
uppr += 2;
|
||||||
|
break;
|
||||||
|
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
|
||||||
|
uppr += 3;
|
||||||
|
break;
|
||||||
|
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
|
||||||
|
utfX += 3;
|
||||||
|
break;
|
||||||
|
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
|
||||||
|
uppr += 2;
|
||||||
|
break;
|
||||||
|
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
|
||||||
|
utfX += 2;
|
||||||
|
break;
|
||||||
|
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
|
||||||
|
utfX++;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
#pragma warning restore 162
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} // end class Guesser
|
||||||
|
|
||||||
|
#endif // !BYTEMODE
|
||||||
|
#endregion
|
||||||
#endif // !NOFILES
|
#endif // !NOFILES
|
||||||
|
|
||||||
// End of code copied from embedded resource
|
// End of code copied from embedded resource
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,20 +0,0 @@
|
|||||||
module.exports = {
|
|
||||||
root: true,
|
|
||||||
env: { browser: true, es2020: true },
|
|
||||||
extends: [
|
|
||||||
'eslint:recommended',
|
|
||||||
'plugin:react/recommended',
|
|
||||||
'plugin:react/jsx-runtime',
|
|
||||||
'plugin:react-hooks/recommended',
|
|
||||||
],
|
|
||||||
ignorePatterns: ['dist', '.eslintrc.cjs'],
|
|
||||||
parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
|
|
||||||
settings: { react: { version: '18.3.1' } },
|
|
||||||
plugins: ['react-refresh'],
|
|
||||||
rules: {
|
|
||||||
'react-refresh/only-export-components': [
|
|
||||||
'warn',
|
|
||||||
{ allowConstantExport: true },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import importPlugin from "eslint-plugin-import";
|
||||||
|
import reactPkg from "eslint-plugin-react";
|
||||||
|
const { configs: reactConfigs } = reactPkg;
|
||||||
|
import js from "@eslint/js";
|
||||||
|
import defineConfig from "./vite.config.js";
|
||||||
|
|
||||||
|
export default [
|
||||||
|
js.configs.recommended,
|
||||||
|
importPlugin.flatConfigs.recommended,
|
||||||
|
reactConfigs.flat.recommended,
|
||||||
|
reactConfigs.flat["jsx-runtime"],
|
||||||
|
{
|
||||||
|
files: ["**/*.{js,jsx}"],
|
||||||
|
languageOptions: {
|
||||||
|
ecmaVersion: "latest",
|
||||||
|
sourceType: "module",
|
||||||
|
globals: {
|
||||||
|
console: "readonly"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
settings: {
|
||||||
|
"import/resolver": {
|
||||||
|
vite: {
|
||||||
|
viteConfig: defineConfig({ mode: "development" })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
rules: {
|
||||||
|
semi: "error",
|
||||||
|
"prefer-const": "error",
|
||||||
|
"no-unused-vars": "warn",
|
||||||
|
"import/no-dynamic-require": "warn",
|
||||||
|
"import/no-nodejs-modules": "warn",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
];
|
||||||
+1458
-1912
File diff suppressed because it is too large.
Load diff
@@ -10,34 +10,37 @@
|
|||||||
"preview": "vite preview"
|
"preview": "vite preview"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@angular/material": "^18.0.4",
|
"@angular/material": "^18.2.8",
|
||||||
"@emotion/react": "^11.11.4",
|
"@emotion/react": "^11.13.3",
|
||||||
"@emotion/styled": "^11.11.5",
|
"@emotion/styled": "^11.13.0",
|
||||||
"@fontsource/noto-sans": "^5.0.22",
|
"@fontsource/noto-sans": "^5.1.0",
|
||||||
"@material/data-table": "^15.0.0-canary.7f224ddd4.0",
|
"@material/data-table": "^15.0.0-canary.423edc3dc.0",
|
||||||
"@mui/icons-material": "^5.15.20",
|
"@mui/icons-material": "^6.1.4",
|
||||||
"@mui/material": "^5.15.20",
|
"@mui/material": "^6.1.4",
|
||||||
"@mui/x-data-grid": "^7.7.1",
|
"@mui/x-data-grid": "^7.20.0",
|
||||||
"bulma": "^1.0.1",
|
"bulma": "^1.0.2",
|
||||||
"classnames": "^2.5.1",
|
"classnames": "^2.5.1",
|
||||||
"react": "^18.3.1",
|
"react": "^18.3.1",
|
||||||
"react-data-grid": "^7.0.0-beta.44",
|
"react-data-grid": "^7.0.0-beta.47",
|
||||||
"react-data-table-component": "^7.6.2",
|
"react-data-table-component": "^7.6.2",
|
||||||
"react-dom": "^18.3.1",
|
"react-dom": "^18.3.1",
|
||||||
"react-json-view-lite": "^1.4.0",
|
"react-json-view-lite": "^1.5.0",
|
||||||
"react-pro-sidebar": "^1.1.0",
|
"react-pro-sidebar": "^1.1.0",
|
||||||
"react-use-websocket": "^4.8.1",
|
"react-use-websocket": "^4.9.0",
|
||||||
"rpc-websockets": "^7.11.2"
|
"rpc-websockets": "^9.0.4"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/react": "^18.3.3",
|
"@types/react": "^18.3.11",
|
||||||
"@types/react-dom": "^18.3.0",
|
"@types/react-dom": "^18.3.1",
|
||||||
"@vitejs/plugin-react": "^4.3.1",
|
"@vitejs/plugin-react": "^4.3.2",
|
||||||
"eslint": "^8.57.0",
|
"eslint": "^9.12.0",
|
||||||
"eslint-plugin-react": "^7.34.2",
|
"@eslint/js": "^9.12.0",
|
||||||
"eslint-plugin-react-hooks": "^4.6.2",
|
"eslint-plugin-import": "^2.31.0",
|
||||||
"eslint-plugin-react-refresh": "^0.4.7",
|
"eslint-plugin-react": "^7.37.1",
|
||||||
"react-mui-sidebar": "^1.3.5",
|
"eslint-plugin-react-hooks": "^5.0.0",
|
||||||
"vite": "^5.2.13"
|
"eslint-plugin-react-refresh": "^0.4.12",
|
||||||
|
"eslint-import-resolver-vite": "^2.0.1",
|
||||||
|
"react-mui-sidebar": "^1.3.8",
|
||||||
|
"vite": "^5.4.9"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -12,7 +12,7 @@ const QueryStatusView = ({ queryState, details, error }) => {
|
|||||||
|
|
||||||
let header = queryState;
|
let header = queryState;
|
||||||
let headerContent = null;
|
let headerContent = null;
|
||||||
let detailElements = [];
|
const detailElements = [];
|
||||||
let severity = queryState == "Subscribed" ? "success" : "info";
|
let severity = queryState == "Subscribed" ? "success" : "info";
|
||||||
let cursor = details == null ? "default" : "pointer";
|
let cursor = details == null ? "default" : "pointer";
|
||||||
if (error != null) {
|
if (error != null) {
|
||||||
@@ -20,14 +20,14 @@ const QueryStatusView = ({ queryState, details, error }) => {
|
|||||||
cursor = "pointer";
|
cursor = "pointer";
|
||||||
header = `Error Code: ${error.code}`;
|
header = `Error Code: ${error.code}`;
|
||||||
headerContent = error.message;
|
headerContent = error.message;
|
||||||
detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
|
detailElements.push(<Typography key="error-msg" variant="body1">{error.data.error_message}</Typography>);
|
||||||
detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
|
detailElements.push(<Typography key="error-details" variant="subtitle1">Stack Trace:</Typography>);
|
||||||
(error.data.stack_trace ?? []).forEach(line => detailElements.push(
|
(error.data.stack_trace ?? []).forEach((line, idx) => detailElements.push(
|
||||||
<Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
|
<Typography key={`stacktrace-${idx}`} variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
|
||||||
{line}
|
{line}
|
||||||
</Typography>));
|
</Typography>));
|
||||||
} else if (details != null) {
|
} else if (details != null) {
|
||||||
detailElements.push(<Typography variant="body1">{details}</Typography>);
|
detailElements.push(<Typography key="status-msg" variant="body1">{details}</Typography>);
|
||||||
}
|
}
|
||||||
return (
|
return (
|
||||||
<Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
|
<Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
|
||||||
|
|||||||
@@ -29,13 +29,12 @@ if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const target = env.ASPNETCORE_HTTPS_PORT
|
|
||||||
? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
|
|
||||||
: env.ASPNETCORE_URLS
|
|
||||||
? env.ASPNETCORE_URLS.split(';')[0]
|
|
||||||
: 'https://localhost:8443';
|
|
||||||
|
|
||||||
export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
|
export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
|
||||||
|
const target = env.ASPNETCORE_HTTPS_PORT
|
||||||
|
? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
|
||||||
|
: env.ASPNETCORE_URLS
|
||||||
|
? env.ASPNETCORE_URLS.split(';')[0]
|
||||||
|
: 'https://localhost:8443';
|
||||||
return {
|
return {
|
||||||
plugins: [plugin()],
|
plugins: [plugin()],
|
||||||
resolve: {
|
resolve: {
|
||||||
@@ -68,6 +67,10 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
|
|||||||
'^/ws': {
|
'^/ws': {
|
||||||
target,
|
target,
|
||||||
secure: false
|
secure: false
|
||||||
|
},
|
||||||
|
'^/browse': {
|
||||||
|
target,
|
||||||
|
secure: false
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
port: 5173,
|
port: 5173,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<InvariantGlobalization>true</InvariantGlobalization>
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
@@ -12,10 +12,10 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="CommandLineParser" Version="2.9.1" />
|
<PackageReference Include="CommandLineParser" Version="2.9.1" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="8.0.18" />
|
<PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="8.0.18" />
|
<PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
|
||||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.3" />
|
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
+37
-34
@@ -1,34 +1,37 @@
|
|||||||
namespace Bts.DataStreamWebApp.Server.Controllers;
|
namespace Bts.DataStreamWebApp.Server.Controllers;
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
using Bts.DataStream;
|
||||||
|
|
||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
|
||||||
[Route("/ws")]
|
[Route("/ws")]
|
||||||
public class DataStreamSubscriptionController : ControllerBase {
|
public class DataStreamSubscriptionController : ControllerBase {
|
||||||
readonly ILogger logger;
|
readonly ILogger logger;
|
||||||
readonly IDataStreamFactory streamFactory;
|
readonly IDataStreamFactory streamFactory;
|
||||||
|
|
||||||
public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
|
public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
|
||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
logger.LogInformation("Creating {ControllerType}", GetType().Name);
|
logger.LogInformation("Creating {ControllerType}", GetType().Name);
|
||||||
streamFactory = DataStreamFactory.Create(streamFactories);
|
var streamFactoryList = streamFactories.ToList();
|
||||||
}
|
streamFactory = streamFactoryList.Count == 1
|
||||||
|
? streamFactoryList[0]
|
||||||
[Route("/ws")]
|
: new CompositeDataStreamFactory(streamFactoryList);
|
||||||
public async Task Get(CancellationToken cancellationToken) {
|
}
|
||||||
if (!HttpContext.WebSockets.IsWebSocketRequest) {
|
|
||||||
logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
|
[Route("/ws")]
|
||||||
return;
|
public async Task Get(CancellationToken cancellationToken) {
|
||||||
}
|
if (!HttpContext.WebSockets.IsWebSocketRequest) {
|
||||||
logger.LogInformation("Establishing websocket connection...");
|
logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
|
||||||
try {
|
return;
|
||||||
using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
|
}
|
||||||
logger.LogInformation("Websocket connection established.");
|
logger.LogInformation("Establishing websocket connection...");
|
||||||
using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
|
try {
|
||||||
await server.Run(cancellationToken);
|
using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
|
||||||
} catch (Exception ex) {
|
logger.LogInformation("Websocket connection established.");
|
||||||
logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
|
using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
|
||||||
}
|
await server.Run(cancellationToken);
|
||||||
}
|
} catch (Exception ex) {
|
||||||
}
|
logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,7 +2,7 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
|
|||||||
|
|
||||||
using bts.websocket_core.json_rpc;
|
using bts.websocket_core.json_rpc;
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
using Bts.DataStream;
|
||||||
using Bts.DataStreamWebApp.Server.Messages;
|
using Bts.DataStreamWebApp.Server.Messages;
|
||||||
|
|
||||||
using System.Net.WebSockets;
|
using System.Net.WebSockets;
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
FROM bluetradingsystems/linux_dotnet7
|
FROM bluetradingsystems/linux_dotnet7
|
||||||
|
|
||||||
ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
|
ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
|
||||||
|
ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
|
||||||
ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
|
ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
|
||||||
ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
|
ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
|
||||||
|
|
||||||
RUN dotnet dev-certs https --trust
|
RUN dotnet dev-certs https --trust
|
||||||
COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
|
COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
|
||||||
|
COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
|
||||||
COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
|
COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
|
||||||
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
|
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
|
||||||
WORKDIR "/home/bts/bin"
|
WORKDIR "/home/bts/bin"
|
||||||
@@ -4,7 +4,7 @@ using BCG;
|
|||||||
|
|
||||||
using bts.websocket_core.json_rpc;
|
using bts.websocket_core.json_rpc;
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
using Bts.DataStream;
|
||||||
using Bts.DataStreamExpressions;
|
using Bts.DataStreamExpressions;
|
||||||
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
using System.Diagnostics.CodeAnalysis;
|
||||||
@@ -60,7 +60,7 @@ public class SubscribeRequest {
|
|||||||
}
|
}
|
||||||
Exception? exception = null;
|
Exception? exception = null;
|
||||||
try {
|
try {
|
||||||
stream = exp.Evaluate(streamFactory) as ViewStream;
|
stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
exception = ex;
|
exception = ex;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,8 +79,8 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
|
|||||||
.AddSparkApi(config.GetSection("SparkApi"))
|
.AddSparkApi(config.GetSection("SparkApi"))
|
||||||
.AddMasterWorker(config.GetSection("MasterWorker"))
|
.AddMasterWorker(config.GetSection("MasterWorker"))
|
||||||
.AddViewStreamDatabase()
|
.AddViewStreamDatabase()
|
||||||
.AddPrimitiveStreams(config.GetSection("PrimitiveStreams"))
|
.AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams"))
|
||||||
.AddMarketDataStreams()
|
.AddOhlcvStreams(config.GetSection("StreamDatabase"))
|
||||||
.AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
|
.AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
|
||||||
.AddCompositeDataStreamFactory()
|
.AddCompositeDataStreamFactory()
|
||||||
.AddPubSubClient(config.GetSection("PubSub"));
|
.AddPubSubClient(config.GetSection("PubSub"));
|
||||||
|
|||||||
@@ -116,7 +116,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
|
"natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
|
||||||
"dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", exclusive_end: true)"
|
"dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\")"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"natural_language": "Retrieve NVIDIA L1 books from Oct 3",
|
"natural_language": "Retrieve NVIDIA L1 books from Oct 3",
|
||||||
@@ -147,10 +147,10 @@
|
|||||||
"interval (sample frequency of the data), ",
|
"interval (sample frequency of the data), ",
|
||||||
"levels (the number of price levels to examine)",
|
"levels (the number of price levels to examine)",
|
||||||
"begin and end (inclusive beginning and end of the time window to query, respectively), ",
|
"begin and end (inclusive beginning and end of the time window to query, respectively), ",
|
||||||
"inclusive_end (a bool flag that, when true, makes the end time inclusive). ",
|
|
||||||
"Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
|
"Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
|
||||||
"Depending on the wording of the request, some arguments may be omitted. ",
|
"Depending on the wording of the request, some arguments may be omitted. ",
|
||||||
"The begin, end, and interval args should be expressed in iso 8601 format. ",
|
"The begin, end, and interval args should be expressed in iso 8601 format. ",
|
||||||
|
"If the prompt doesn't contain an explicit interval, please choose a reasonable value depending on the duration between begin and end.",
|
||||||
"Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
|
"Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
|
||||||
"In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
|
"In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
|
||||||
"which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
|
"which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.7" />
|
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
@@ -3,12 +3,15 @@
|
|||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
|
||||||
public static class HostBuilderExtensions {
|
public static class HostBuilderExtensions {
|
||||||
public static IHostBuilder CreateBtsBuilder(string[] args) {
|
public static IHostBuilder CreateBtsBuilder(string[] args) {
|
||||||
// The calling assembly's name serves as the "appsettings" json file prefix.
|
// Obtain calling assembly name here, rather than in the lambda where it is actually needed,
|
||||||
|
// because want the name of the assembly that called this function, not the assembly that will eventually invoke
|
||||||
|
// the lambda...that will be the Microsoft.Extensions.Hosting assembly.
|
||||||
|
// Get the name of the assembly that called this function. That will act as the "appsettings"
|
||||||
|
// json file prefix
|
||||||
var exeName = GetCallingAssemblyName();
|
var exeName = GetCallingAssemblyName();
|
||||||
return Host.CreateDefaultBuilder(args)
|
return Host.CreateDefaultBuilder(args)
|
||||||
.ConfigureAppConfiguration((context, configBuilder) =>
|
.ConfigureAppConfiguration((context, configBuilder) =>
|
||||||
@@ -53,21 +56,4 @@ public static class HostBuilderExtensions {
|
|||||||
?.GetName()
|
?.GetName()
|
||||||
?.Name
|
?.Name
|
||||||
?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
|
?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Separates command line args from config args, using '--' as the delimiter. Args
|
|
||||||
/// before the delimiter are considered command line args, while those after it are
|
|
||||||
/// config args.
|
|
||||||
/// </summary>
|
|
||||||
public static (string[] Args, string[] ConfigArgs) SplitConfigArgs(string[] args) {
|
|
||||||
int splitIdx = Array.IndexOf(args, "--");
|
|
||||||
if (splitIdx == -1) {
|
|
||||||
return (args, Array.Empty<string>());
|
|
||||||
}
|
|
||||||
var cfgArgs = new string[args.Length - splitIdx - 1];
|
|
||||||
Array.Copy(args, splitIdx + 1, cfgArgs, 0, cfgArgs.Length);
|
|
||||||
var nonCfgArgs = new string[args.Length - cfgArgs.Length - 1];
|
|
||||||
Array.Copy(args, 0, nonCfgArgs, 0, nonCfgArgs.Length);
|
|
||||||
return (nonCfgArgs, cfgArgs);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
public class AddColumnRow : IBlueRow {
|
|
||||||
readonly IBlueRow inputRow;
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
readonly object? value;
|
|
||||||
readonly int valueIndex;
|
|
||||||
|
|
||||||
public AddColumnRow(IBlueRow inputRow, IBlueTableSchema outputSchema, object? value, int valueIndex) {
|
|
||||||
this.inputRow = inputRow;
|
|
||||||
this.outputSchema = outputSchema;
|
|
||||||
this.value = value;
|
|
||||||
this.valueIndex = valueIndex;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] {
|
|
||||||
get {
|
|
||||||
if (index == valueIndex) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
if (index >= valueIndex) {
|
|
||||||
index--;
|
|
||||||
}
|
|
||||||
return inputRow[index];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRow.schema => outputSchema;
|
|
||||||
}
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public class AddColumnTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection inputRows;
|
|
||||||
readonly Func<IBlueRow, object?> getValue;
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
readonly int valueIndex;
|
|
||||||
|
|
||||||
public AddColumnTable(IBlueRowCollection inputRows, string columnName, Type columnType, Func<IBlueRow, object?> getValue) {
|
|
||||||
this.inputRows = inputRows;
|
|
||||||
this.getValue = getValue;
|
|
||||||
var inputCols = inputRows.schema.columnInfo;
|
|
||||||
var outputCols = new ColumnInfo[inputCols.Length + 1];
|
|
||||||
Array.Copy(inputCols, outputCols, inputCols.Length);
|
|
||||||
outputCols[^1] = new(columnName, inputCols.Length, columnType);
|
|
||||||
outputSchema = new BlueTableSchema(outputCols, "AddColumnTable");
|
|
||||||
valueIndex = inputCols.Length;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] {
|
|
||||||
get {
|
|
||||||
var r = inputRows[id];
|
|
||||||
return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => outputSchema;
|
|
||||||
|
|
||||||
public int Count => inputRows.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
inputRows.containsID(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
foreach (var r in inputRows) {
|
|
||||||
yield return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) {
|
|
||||||
var r = inputRows.GetRow(index);
|
|
||||||
return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
|
|
||||||
}
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
if (!inputRows.containsID(id)) {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
var r = inputRows[id];
|
|
||||||
return BlueRowEqualityComparer.Equals(new AddColumnRow(r, outputSchema, getValue(r), valueIndex), item)
|
|
||||||
? inputRows.IndexOf(r)
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("AddColumnsStream")]
|
|
||||||
public class AddColumnsStream: ViewStream {
|
|
||||||
public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
|
|
||||||
ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
#nullable enable
|
|
||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class ApplyStream : ViewStream {
|
|
||||||
readonly ViewStream source;
|
|
||||||
readonly Func<IBlueRow, object?> apply;
|
|
||||||
IBlueTableSchema? outputSchema;
|
|
||||||
BlueTableImmutable? lastFired;
|
|
||||||
IDisposable? sub;
|
|
||||||
|
|
||||||
public ApplyStream(ViewStream source, IFunctionEvaluator applyEvaluator) {
|
|
||||||
this.source = source;
|
|
||||||
apply = Apply;
|
|
||||||
|
|
||||||
object? Apply(IBlueRow r) {
|
|
||||||
var res = applyEvaluator.Evaluate(string.Empty, new StreamArgument[] { (null, 0, r) });
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public ApplyStream(ViewStream source, Func<IBlueRow, object?> apply) {
|
|
||||||
this.source = source;
|
|
||||||
this.apply = apply;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
sub = source.Subscribe(OnSourceChanged);
|
|
||||||
|
|
||||||
void OnSourceChanged(IDataPacket dp) {
|
|
||||||
Lazy<BlueTableImmutable> lastFired = new(() => this.lastFired ??= new(Log.notnull(outputSchema)));
|
|
||||||
Lazy<BlueTableImmutable.DataPacketBuilder> bd = new(() => lastFired.Value.ToDataPacketBuilder());
|
|
||||||
|
|
||||||
foreach (var r in dp.deletedRows) {
|
|
||||||
var res = apply(r);
|
|
||||||
if (res is IBlueRow blueRow) {
|
|
||||||
bd.Value.Remove(blueRow.Id());
|
|
||||||
} else if (res is IBlueRowCollection blueRows) {
|
|
||||||
foreach (var br in blueRows) {
|
|
||||||
bd.Value.Remove(br.Id());
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
foreach (var r in dp.addedRows) {
|
|
||||||
var res = apply(r);
|
|
||||||
if (res is IBlueRow blueRow) {
|
|
||||||
outputSchema ??= blueRow.schema;
|
|
||||||
bd.Value.AddOrUpdate(blueRow);
|
|
||||||
} else if (res is IBlueRowCollection blueRows) {
|
|
||||||
outputSchema ??= blueRows.schema;
|
|
||||||
foreach (var br in blueRows) {
|
|
||||||
bd.Value.AddOrUpdate(br);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (outputSchema is null) {
|
|
||||||
// We get the outputSchema from the result of the first apply, if it's null, we
|
|
||||||
// cannot determine the output data type.
|
|
||||||
// TODO: maybe support specifying return type in DSL?
|
|
||||||
Log.notnull(res);
|
|
||||||
}
|
|
||||||
var resType = res?.GetType();
|
|
||||||
if (outputSchema is null) {
|
|
||||||
Log.notnull(resType);
|
|
||||||
Log.assert(ColumnInfo.isAllowedType(resType));
|
|
||||||
outputSchema = new BlueTableSchema(new ColumnInfo[] {
|
|
||||||
new("id", 0, typeof(int)),
|
|
||||||
new("result", 1, resType),
|
|
||||||
}, dp.newRows.schema.tableName + "_Apply");
|
|
||||||
}
|
|
||||||
bd.Value.AddOrUpdate(new BlueRow(outputSchema, new object?[] { r.Id(), res }));
|
|
||||||
// Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (bd.IsValueCreated) {
|
|
||||||
myFireEvent(bd.Value.ToDataPacket(out this.lastFired));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public class ApplyTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection source;
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
readonly List<IBlueRow> outputRows;
|
|
||||||
readonly Dictionary<int, int> idToIndex = new();
|
|
||||||
readonly Func<IBlueRow, IBlueRow> applyFn;
|
|
||||||
|
|
||||||
public ApplyTable(IBlueRowCollection source, IBlueTableSchema? outputSchema, Func<IBlueRow, IBlueRow> applyFn) {
|
|
||||||
this.source = source;
|
|
||||||
this.applyFn = applyFn;
|
|
||||||
outputRows = new(source.Count);
|
|
||||||
foreach (var r in source) {
|
|
||||||
outputRows.Add(applyFn(r));
|
|
||||||
}
|
|
||||||
if (outputRows.Count > 0) {
|
|
||||||
outputSchema ??= outputRows[0].schema;
|
|
||||||
}
|
|
||||||
this.outputSchema = outputSchema ?? source.schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] => outputRows[idToIndex[id]];
|
|
||||||
|
|
||||||
IBlueRow IList<IBlueRow>.this[int index] {
|
|
||||||
get => GetRow(index);
|
|
||||||
set => throw new NotSupportedException();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => outputSchema;
|
|
||||||
|
|
||||||
public int Count => source.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
source.containsID(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() =>
|
|
||||||
outputRows.GetEnumerator();
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
outputRows[index];
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
if (!source.containsID(id)) {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
var inputRow = source[id];
|
|
||||||
int idx = source.IndexOf(inputRow);
|
|
||||||
if (BlueRowEqualityComparer.Equals(outputRows[idx], item)) {
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
namespace Bts.DataStream.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
using Bts.PubSub;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
public static class AsyncViewStream {
|
|
||||||
public static ViewStream Create(IWorkScheduler workScheduler, Func<Task<ViewStream?>> factory) {
|
|
||||||
return ViewStreamFactory.Create(Subscribe);
|
|
||||||
|
|
||||||
IDisposable Subscribe(Action<IDataPacket> fire) {
|
|
||||||
var cts = new CancellationTokenSource();
|
|
||||||
var tcs = new TaskCompletionSource<IDisposable>();
|
|
||||||
var syncRoot = new object();
|
|
||||||
IDisposable? streamSub = null;
|
|
||||||
workScheduler.Post(Init);
|
|
||||||
return Disposable.Create(Dispose);
|
|
||||||
|
|
||||||
async void Init() {
|
|
||||||
lock (syncRoot) {
|
|
||||||
if (cts.IsCancellationRequested) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var stream = await factory();
|
|
||||||
if (stream is null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
var tempStreamSub = stream.Subscribe(fire);
|
|
||||||
lock (syncRoot) {
|
|
||||||
if (cts.IsCancellationRequested) {
|
|
||||||
tempStreamSub.Dispose();
|
|
||||||
} else {
|
|
||||||
streamSub = tempStreamSub;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void Dispose() {
|
|
||||||
IDisposable? tempStreamSub;
|
|
||||||
lock (syncRoot) {
|
|
||||||
tempStreamSub = streamSub;
|
|
||||||
if (tempStreamSub is null) {
|
|
||||||
cts.Cancel();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
streamSub = null;
|
|
||||||
}
|
|
||||||
tempStreamSub?.Dispose();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
using bts.utils;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class BlueAddIndexColumnTable : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly IList<object?>[] columnValues;
|
|
||||||
readonly int count;
|
|
||||||
|
|
||||||
BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
|
|
||||||
this.schema = schema;
|
|
||||||
this.columnValues = columnValues;
|
|
||||||
this.count = count ?? columnValues[0].Count;
|
|
||||||
}
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public int Count => count;
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
id > 0 && id <= count
|
|
||||||
? GetRow(id - 1)
|
|
||||||
: throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
|
|
||||||
if (columnInfos.Any(c => c.colName == "id")) {
|
|
||||||
throw new ArgumentException("'id' column already exists", nameof(columnInfos));
|
|
||||||
}
|
|
||||||
if (columnInfos.Length != columnValues.Length) {
|
|
||||||
throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
|
|
||||||
}
|
|
||||||
int count = columnValues.FirstOrDefault()?.Count ?? 0;
|
|
||||||
var origCols = columnInfos;
|
|
||||||
var origColValues = columnValues;
|
|
||||||
columnInfos = new ColumnInfo[origCols.Length + 1];
|
|
||||||
columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
|
|
||||||
columnValues = new IList<object?>[origCols.Length + 1];
|
|
||||||
columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
|
|
||||||
for (int i = 1; i < columnInfos.Length; i++) {
|
|
||||||
columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
|
|
||||||
columnValues[i] = origColValues[i - 1];
|
|
||||||
}
|
|
||||||
var schema = new BlueTableSchema(columnInfos, tableName);
|
|
||||||
return new(schema, columnValues, count);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
|
|
||||||
int count = columnValues.FirstOrDefault()?.Count ?? 0;
|
|
||||||
int? idColIdx = schema.TryResolveIndices("id");
|
|
||||||
var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
|
|
||||||
var columnInfos = schema.columnInfo;
|
|
||||||
if (idColIdx.HasValue) {
|
|
||||||
// Replace existing id column. columnValues may or may not contain the old ids (if its
|
|
||||||
// length is one less column than that of columnInfos, it doesn't contain them).
|
|
||||||
switch (columnValues.Length - schema.columnInfo.Length) {
|
|
||||||
case 0:
|
|
||||||
columnValues[idColIdx.Value] = idColValues;
|
|
||||||
break;
|
|
||||||
case -1:
|
|
||||||
columnValues = ArrayUtils.InsertExact(columnValues, idColIdx.Value, true, idColValues);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (columnValues.Length != columnInfos.Length) {
|
|
||||||
throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
|
|
||||||
}
|
|
||||||
var origColInfos = columnInfos;
|
|
||||||
columnInfos = new ColumnInfo[columnInfos.Length + 1];
|
|
||||||
columnInfos[0] = new("id", 0, typeof(int));
|
|
||||||
for (int i = 1; i < columnInfos.Length; i++) {
|
|
||||||
var c = origColInfos[i - 1];
|
|
||||||
columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
|
|
||||||
}
|
|
||||||
columnValues = ArrayUtils.InsertExact(columnValues, 0, true, idColValues);
|
|
||||||
}
|
|
||||||
return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
id > 0 && id <= count;
|
|
||||||
|
|
||||||
public IList<object?> GetColumn(int index) =>
|
|
||||||
columnValues[index];
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new BlueRowColumnWrapper(schema, index, columnValues);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
|
|
||||||
? id - 1
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
int count = Count;
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
yield return GetRow(i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public class BlueRowColumnWrapper : IBlueRow {
|
|
||||||
readonly int index;
|
|
||||||
readonly IList<object?>[] columns;
|
|
||||||
|
|
||||||
public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
|
|
||||||
this.schema = schema;
|
|
||||||
this.index = index;
|
|
||||||
this.columns = columns;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] =>
|
|
||||||
columns[index][this.index];
|
|
||||||
|
|
||||||
public IBlueTableSchema schema { get; }
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class BlueIndexedColumnTable : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly IList<object?>[] columnValues;
|
|
||||||
readonly Dictionary<int, int> rowIdToIndex;
|
|
||||||
|
|
||||||
public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
|
|
||||||
this.schema = schema;
|
|
||||||
this.columnValues = columnValues;
|
|
||||||
int idColumnIndex = schema.ResolveIndices("id");
|
|
||||||
rowIdToIndex = columnValues[idColumnIndex]
|
|
||||||
.Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
|
|
||||||
.ToDictionary();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
rowIdToIndex.TryGetValue(id, out var idx)
|
|
||||||
? GetRow(idx)
|
|
||||||
: throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public int Count => rowIdToIndex.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
rowIdToIndex.ContainsKey(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
int count = Count;
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
yield return GetRow(i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new BlueRowColumnWrapper(schema, index, columnValues);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) =>
|
|
||||||
rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
|
|
||||||
? idx
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using System;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("CSVReaderStream")]
|
|
||||||
public class CSVReaderStream : ViewStream {
|
|
||||||
public CSVReaderStream(string file_name)
|
|
||||||
: this(file_name, false) { }
|
|
||||||
public CSVReaderStream(string file_name, bool guess_schema) {
|
|
||||||
ViewStream rs = new BCG.CSVReaderStream(file_name);
|
|
||||||
if (guess_schema) {
|
|
||||||
rs = new SchemaGuesserStream(rs);
|
|
||||||
}
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("CSVSaverStream")]
|
|
||||||
public class CSVSaverStream: ViewStream {
|
|
||||||
public CSVSaverStream(ViewStream parent_str, string file_name) {
|
|
||||||
BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class CastStream : ViewStream, IDisposable {
|
|
||||||
readonly ViewStream inputStream;
|
|
||||||
readonly (string, Type)[] casts;
|
|
||||||
IDisposable? sub;
|
|
||||||
|
|
||||||
public CastStream(ViewStream inputStream, params (string, Type)[] casts) {
|
|
||||||
this.inputStream = inputStream;
|
|
||||||
this.casts = casts;
|
|
||||||
}
|
|
||||||
|
|
||||||
~CastStream() =>
|
|
||||||
Dispose();
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
inputStream.Subscribe(OnInputChanged);
|
|
||||||
|
|
||||||
void OnInputChanged(IDataPacket dp) {
|
|
||||||
var dpOut = new DataPacket(
|
|
||||||
new CastTable(dp.newRows, casts),
|
|
||||||
new CastTable(dp.addedRows, casts),
|
|
||||||
new CastTable(dp.deletedRows, casts));
|
|
||||||
myFireEvent(dpOut);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
}
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public class CastTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection rows;
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
readonly (int Index, Func<IBlueRow, object?> GetValue)[] casts;
|
|
||||||
readonly IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer =
|
|
||||||
TupleComparer.Comparer<int, Func<IBlueRow, object?>>(item1Ordinal: 0);
|
|
||||||
|
|
||||||
public CastTable(IBlueRowCollection rows, params (string, Type)[] casts) {
|
|
||||||
this.rows = rows;
|
|
||||||
var inputSchema = rows.schema;
|
|
||||||
var inputCols = inputSchema.columnInfo;
|
|
||||||
var castIndices = inputSchema.ResolveIndices(Array.ConvertAll(casts, c => c.Item1));
|
|
||||||
this.casts = new (int Index, Func<IBlueRow, object?> GetValue)[casts.Length];
|
|
||||||
var outputCols = inputCols.ToArray();
|
|
||||||
for (int i = 0; i < casts.Length; i++) {
|
|
||||||
var colIdx = castIndices[i];
|
|
||||||
this.casts[i] = (colIdx, GetCaster(casts[i].Item2));
|
|
||||||
var inputCol = inputCols[colIdx];
|
|
||||||
Log.assert(inputCol.colNo == colIdx);
|
|
||||||
outputCols[colIdx] = new(inputCol.colName, inputCol.colNo, casts[i].Item2);
|
|
||||||
|
|
||||||
Func<IBlueRow, object?> GetCaster(Type to) {
|
|
||||||
if (to == typeof(short)) {
|
|
||||||
return r => r.GetInt16(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(int)) {
|
|
||||||
return r => r.GetInt32(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(long)) {
|
|
||||||
return r => r.GetInt64(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(ushort)) {
|
|
||||||
return r => r.GetUInt16(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(uint)) {
|
|
||||||
return r => r.GetUInt32(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(ulong)) {
|
|
||||||
return r => r.GetUInt64(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(double)) {
|
|
||||||
return r => r.GetFloat64(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(float)) {
|
|
||||||
return r => r.GetFloat32(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(DateTime)) {
|
|
||||||
return r => r.GetDateTime(colIdx);
|
|
||||||
}
|
|
||||||
if (to == typeof(TimeSpan)) {
|
|
||||||
return r => r.GetTimeSpan(colIdx);
|
|
||||||
}
|
|
||||||
throw new NotImplementedException($"CastTable cast not implemented for (out) type {to}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
outputSchema = new BlueTableSchema(inputSchema.tableName + "_Cast", outputCols);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
new Row(rows[id], outputSchema, casts, castComparer);
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => outputSchema;
|
|
||||||
|
|
||||||
public int Count => rows.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
rows.containsID(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
foreach (var r in rows) {
|
|
||||||
yield return new Row(r, outputSchema, casts, castComparer);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new Row(rows.GetRow(index), outputSchema, casts, castComparer);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
var inputRow = rows[item.Id()];
|
|
||||||
return BlueRowEqualityComparer.Equals(item, new Row(inputRow, outputSchema, casts, castComparer))
|
|
||||||
? rows.IndexOf(inputRow)
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
class Row : IBlueRow {
|
|
||||||
readonly IBlueRow source;
|
|
||||||
readonly (int Index, Func<IBlueRow, object?> GetValue)[] casts;
|
|
||||||
readonly IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer;
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
|
|
||||||
public Row(IBlueRow source, IBlueTableSchema outputSchema, (int Index, Func<IBlueRow, object?> GetValue)[] casts, IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer) {
|
|
||||||
this.source = source;
|
|
||||||
this.casts = casts;
|
|
||||||
this.castComparer = castComparer;
|
|
||||||
this.outputSchema = outputSchema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] {
|
|
||||||
get {
|
|
||||||
var castIdx = casts.BinarySearch((index, null!), castComparer);
|
|
||||||
return castIdx > -1
|
|
||||||
? casts[castIdx].GetValue(source)
|
|
||||||
: source[index];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRow.schema => outputSchema;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
|
|
||||||
using System.Reactive.Linq;
|
|
||||||
|
|
||||||
public class ColumnInjectionStream : ViewStream {
|
|
||||||
public ColumnInjectionStream(ViewStream inputStream, IInjectedColumn[] outputColumns) {
|
|
||||||
IBlueTableSchema? outputSchema = null;
|
|
||||||
mySetRealSource(inputStream.AsObservable().Select(Transform).AsViewStream());
|
|
||||||
|
|
||||||
IDataPacket Transform(IDataPacket dp) {
|
|
||||||
outputSchema ??= ColumnInjectionTable.GetOutputSchema(dp.Schema, outputColumns);
|
|
||||||
var newRows = new ColumnInjectionTable(dp.newRows, outputSchema, outputColumns);
|
|
||||||
var addedRows = new ColumnInjectionTable(dp.addedRows, outputSchema, outputColumns);
|
|
||||||
var deletedRows = new ColumnInjectionTable(dp.deletedRows, outputSchema, outputColumns);
|
|
||||||
return new DataPacket(newRows, addedRows, deletedRows);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,156 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public interface IInjectedColumn {
|
|
||||||
string Name { get; }
|
|
||||||
Type Type { get; }
|
|
||||||
int Ordinal { get; }
|
|
||||||
ColumnInfo ColumnInfo => new(Name, Ordinal, Type);
|
|
||||||
object? GetValue(IBlueRow row);
|
|
||||||
}
|
|
||||||
|
|
||||||
public interface IInjectedColumn<T> : IInjectedColumn {
|
|
||||||
Type IInjectedColumn.Type => typeof(T);
|
|
||||||
|
|
||||||
new T? GetValue(IBlueRow row);
|
|
||||||
|
|
||||||
object? IInjectedColumn.GetValue(IBlueRow row) =>
|
|
||||||
GetValue(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
public class InjectedColumn : IInjectedColumn {
|
|
||||||
readonly Func<IBlueRow, object?> getValue;
|
|
||||||
|
|
||||||
public InjectedColumn(string name, int index, Type type, Func<IBlueRow, object?> getValue) {
|
|
||||||
Name = name;
|
|
||||||
Ordinal = index;
|
|
||||||
Type = type;
|
|
||||||
this.getValue = getValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string Name { get; }
|
|
||||||
public int Ordinal { get; }
|
|
||||||
public Type Type { get; }
|
|
||||||
|
|
||||||
public object? GetValue(IBlueRow row) =>
|
|
||||||
getValue(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
public class InjectedColumn<T> : IInjectedColumn<T> {
|
|
||||||
readonly Func<IBlueRow, T?> getValue;
|
|
||||||
|
|
||||||
public InjectedColumn(string name, int index, Func<IBlueRow, T?> getValue) {
|
|
||||||
Name = name;
|
|
||||||
Ordinal = index;
|
|
||||||
this.getValue = getValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string Name { get; }
|
|
||||||
public int Ordinal { get; }
|
|
||||||
|
|
||||||
public T? GetValue(IBlueRow row) =>
|
|
||||||
getValue(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
public class ColumnInjectionTable : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
readonly IInjectedColumn[] outputColumns;
|
|
||||||
readonly IBlueRowCollection source;
|
|
||||||
|
|
||||||
public ColumnInjectionTable(IBlueRowCollection source, IBlueTableSchema outputSchema, IInjectedColumn[] outputColumns) {
|
|
||||||
this.source = source;
|
|
||||||
this.outputSchema = outputSchema;
|
|
||||||
this.outputColumns = outputColumns;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ColumnInjectionTable Create(IBlueRowCollection source, IEnumerable<IInjectedColumn> injectedColumns) {
|
|
||||||
var inputCols = source.schema.columnInfo;
|
|
||||||
var outputColumns = new List<IInjectedColumn>(inputCols.Length + 1);
|
|
||||||
var outputColInfos = new List<ColumnInfo>(inputCols.Length + 1);
|
|
||||||
int injectionIdx = 0;
|
|
||||||
using var en = injectedColumns.GetEnumerator();
|
|
||||||
bool hasMoreInjections = en.MoveNext();
|
|
||||||
for (int i = 0; i < inputCols.Length;) {
|
|
||||||
if (hasMoreInjections && en.Current.Ordinal <= i) {
|
|
||||||
outputColInfos.Add(new(en.Current.Name, outputColumns.Count, en.Current.Type));
|
|
||||||
outputColumns.Add(en.Current);
|
|
||||||
injectionIdx++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
var ci = inputCols[i];
|
|
||||||
outputColInfos.Add(new(ci.colName, outputColumns.Count, ci.colType));
|
|
||||||
outputColumns.Add(new InjectedColumn(ci.colName, outputColumns.Count, ci.colType, r => r[ci.colNo]));
|
|
||||||
}
|
|
||||||
var outputSchema = new BlueTableSchema(outputColInfos.ToArray(), $"ColumnInjection({source.schema.tableName})");
|
|
||||||
return new(source, outputSchema, outputColumns.ToArray());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, IEnumerable<IInjectedColumn> injectedColumns) {
|
|
||||||
var inputCols = inputSchema.columnInfo;
|
|
||||||
var outputColumns = new List<IInjectedColumn>(inputCols.Length + 1);
|
|
||||||
var outputColInfos = new List<ColumnInfo>(inputCols.Length + 1);
|
|
||||||
using var en = injectedColumns.GetEnumerator();
|
|
||||||
bool hasMoreInjections = en.MoveNext();
|
|
||||||
for (int i = 0; i < inputCols.Length;) {
|
|
||||||
if (hasMoreInjections && en.Current.Ordinal <= i) {
|
|
||||||
outputColInfos.Add(new(en.Current.Name, outputColumns.Count, en.Current.Type));
|
|
||||||
outputColumns.Add(en.Current);
|
|
||||||
hasMoreInjections = en.MoveNext();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
var ci = inputCols[i];
|
|
||||||
outputColInfos.Add(new(ci.colName, outputColumns.Count, ci.colType));
|
|
||||||
outputColumns.Add(new InjectedColumn(ci.colName, outputColumns.Count, ci.colType, r => r[ci.colNo]));
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
return new BlueTableSchema(outputColInfos.ToArray(), $"ColumnInjection({inputSchema.tableName})");
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
new ColumnInjectionRow(source[id], outputSchema, outputColumns);
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => throw new System.NotImplementedException();
|
|
||||||
|
|
||||||
public int Count => source.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
source.containsID(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
foreach (var r in source) {
|
|
||||||
yield return new ColumnInjectionRow(r, outputSchema, outputColumns);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new ColumnInjectionRow(source.GetRow(index), outputSchema, outputColumns);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
return source.containsID(id) && BlueRowEqualityComparer.Equals(item, this[id])
|
|
||||||
? source.IndexOf(source[id])
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
class ColumnInjectionRow : IBlueRow {
|
|
||||||
readonly IBlueRow row;
|
|
||||||
readonly IInjectedColumn[] columns;
|
|
||||||
|
|
||||||
public ColumnInjectionRow(IBlueRow row, IBlueTableSchema schema, IInjectedColumn[] columns) {
|
|
||||||
this.row = row;
|
|
||||||
this.schema = schema;
|
|
||||||
this.columns = columns;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema { get; }
|
|
||||||
|
|
||||||
public object? this[int index] =>
|
|
||||||
columns[index].GetValue(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
public class ColumnMaskRow : IBlueRow {
|
|
||||||
readonly IBlueRow inputRow;
|
|
||||||
readonly (string? Name, int Index)[] columnMap;
|
|
||||||
|
|
||||||
public ColumnMaskRow(IBlueRow inputRow, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
|
|
||||||
this.inputRow = inputRow;
|
|
||||||
this.columnMap = columnMap;
|
|
||||||
this.schema = schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema { get; }
|
|
||||||
|
|
||||||
public object? this[int index] =>
|
|
||||||
inputRow[columnMap[index].Index];
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
|
||||||
public class ColumnMaskStream : ViewStream {
|
|
||||||
readonly ViewStream source;
|
|
||||||
readonly string? columnMaskExpression;
|
|
||||||
|
|
||||||
IDisposable? sub;
|
|
||||||
IBlueTableSchema? outputSchema;
|
|
||||||
// Each item in this array represents an output column as a tuple of
|
|
||||||
// new name, and corresponding index in the input schema.
|
|
||||||
(string? OutputName, int InputIndex)[]? columnMappings;
|
|
||||||
(string From, string To)[]? columnNameMappings;
|
|
||||||
readonly bool includeUnmappedColumns;
|
|
||||||
|
|
||||||
public ColumnMaskStream(ViewStream source, string columnMaskExpression) {
|
|
||||||
this.source = source;
|
|
||||||
this.columnMaskExpression = columnMaskExpression;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ColumnMaskStream(ViewStream source, bool includeUnmappedColumns, params (string From, string To)[] columnNameMappings) {
|
|
||||||
this.source = source;
|
|
||||||
this.columnNameMappings = columnNameMappings;
|
|
||||||
this.includeUnmappedColumns = includeUnmappedColumns;
|
|
||||||
}
|
|
||||||
|
|
||||||
~ColumnMaskStream() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
Log.isnull(sub);
|
|
||||||
sub = source.Subscribe(OnSourceChanged);
|
|
||||||
|
|
||||||
void OnSourceChanged(IDataPacket dp) {
|
|
||||||
ensureOutputSchema(dp.Schema);
|
|
||||||
var addedRows = new ColumnMaskTable(dp.addedRows, columnMappings, outputSchema);
|
|
||||||
var deletedRows = new ColumnMaskTable(dp.deletedRows, columnMappings, outputSchema);
|
|
||||||
var allRows = new ColumnMaskTable(dp.newRows, columnMappings, outputSchema);
|
|
||||||
myFireEvent(new DataPacket(allRows, addedRows, deletedRows));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
|
|
||||||
[MemberNotNull(nameof(outputSchema), nameof(columnMappings))]
|
|
||||||
void ensureOutputSchema(IBlueTableSchema inputSchema) {
|
|
||||||
if (outputSchema is not null) {
|
|
||||||
Log.notnull(columnMappings);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (columnMaskExpression is not null) {
|
|
||||||
outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, columnMaskExpression, out columnMappings);
|
|
||||||
} else if (columnNameMappings is not null) {
|
|
||||||
outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, includeUnmappedColumns, columnNameMappings, out columnMappings);
|
|
||||||
} else {
|
|
||||||
throw new InvalidOperationException("Cannot build output schema");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,120 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
using bts.utils;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class ColumnMaskTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection inputRows;
|
|
||||||
readonly (string? Name, int Index)[] columnMap;
|
|
||||||
|
|
||||||
public ColumnMaskTable(IBlueRowCollection inputRows, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
|
|
||||||
this.inputRows = inputRows;
|
|
||||||
this.columnMap = columnMap;
|
|
||||||
this.schema = schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema { get; }
|
|
||||||
|
|
||||||
public int Count => inputRows.Count;
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
new ColumnMaskRow(inputRows[id], columnMap, schema);
|
|
||||||
|
|
||||||
public static ColumnMaskTable Create(IBlueRowCollection table, string columnMaskExpression) {
|
|
||||||
var outputSchema = GetOutputSchema(table.schema, columnMaskExpression, out var columnMappings);
|
|
||||||
return new ColumnMaskTable(table, columnMappings, outputSchema);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, string columnMaskExpression, out (string? Name, int Index)[] columnMappings) {
|
|
||||||
bool includeOthers;
|
|
||||||
if (columnMaskExpression.StartsWith('!')) {
|
|
||||||
columnMaskExpression = columnMaskExpression[1..].TrimStart();
|
|
||||||
includeOthers = false;
|
|
||||||
} else {
|
|
||||||
includeOthers = true;
|
|
||||||
}
|
|
||||||
var map = columnMaskExpression.Split(' ').Select(MapColumn).ToOrderedDictionary(tup => tup.Index, tup => tup.Name);
|
|
||||||
var inputCols = inputSchema.columnInfo;
|
|
||||||
if (includeOthers) {
|
|
||||||
var implicitCols = inputCols.Where(c => !map.ContainsKey(c.colNo)).ToList();
|
|
||||||
map.AddRange(implicitCols.Select(c => new KeyValuePair<int, string?>(c.colNo, null)));
|
|
||||||
} else if (!map.Any(c => inputCols[c.Key].colName == "id") && inputCols.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
|
|
||||||
// Add 'id' col if missing
|
|
||||||
map.Insert(0, new(idColInfo.colNo, "id"));
|
|
||||||
}
|
|
||||||
var outputCols = new ColumnInfo[map.Count];
|
|
||||||
for (int i = 0; i < outputCols.Length; i++) {
|
|
||||||
var inputColInfo = inputCols[map.Keys[i]];
|
|
||||||
outputCols[i] = new(map.Values[i] ?? inputColInfo.colName, i, inputColInfo.colType);
|
|
||||||
}
|
|
||||||
columnMappings = map.WrapSelector(kv => (kv.Value, kv.Key), null).ToArray();
|
|
||||||
return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
|
|
||||||
|
|
||||||
(string? Name, int Index) MapColumn(string columnStr) {
|
|
||||||
var parts = columnStr.Split(':');
|
|
||||||
return parts.Length switch {
|
|
||||||
1 => ((string?)null, inputSchema[parts[0]].colNo),
|
|
||||||
2 => (parts[1], inputSchema[parts[0]].colNo),
|
|
||||||
_ => throw new InvalidOperationException("Unable to get output schema due to invalid column expression"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, bool includeUnmapped, (string From, string To)[] columnNameMappings, out (string? OutputName, int InputIndex)[] columnMappings) {
|
|
||||||
var inputCols = inputSchema.columnInfo;
|
|
||||||
ColumnInfo[]? outputCols;
|
|
||||||
if (includeUnmapped) {
|
|
||||||
columnMappings = Array.ConvertAll(inputCols, c => ((string?)c.colName, c.colNo));
|
|
||||||
outputCols = ArrayUtils.Clone(inputCols);
|
|
||||||
foreach (var (from, to) in columnNameMappings) {
|
|
||||||
var idx = inputCols.ResolveIndices(from);
|
|
||||||
var c = outputCols[idx];
|
|
||||||
Log.assert(c.colNo == idx);
|
|
||||||
outputCols[idx] = new(to, c.colNo, c.colType);
|
|
||||||
columnMappings[idx] = (to, idx);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
outputCols = new ColumnInfo[columnNameMappings.Length];
|
|
||||||
columnMappings = Array.ConvertAll(columnNameMappings, m => ((string?)m.To, inputCols.ResolveIndices(m.From)));
|
|
||||||
Array.Copy(inputCols, outputCols, inputCols.Length);
|
|
||||||
for (int i = 0; i < columnNameMappings.Length; i++) {
|
|
||||||
var (from, to) = columnNameMappings[i];
|
|
||||||
var idx = inputCols.ResolveIndices(from);
|
|
||||||
outputCols[idx] = new(to, i, inputCols[columnMappings[i].InputIndex].colType);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
inputRows.containsID(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
foreach (var r in inputRows) {
|
|
||||||
yield return new ColumnMaskRow(r, columnMap, schema);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new ColumnMaskRow(inputRows.GetRow(index), columnMap, schema);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
if (inputRows.containsID(id)) {
|
|
||||||
var inputRow = inputRows[id];
|
|
||||||
var outputRow = new ColumnMaskRow(inputRow, columnMap, schema);
|
|
||||||
if (BlueRowEqualityComparer.Equals(item, outputRow)) {
|
|
||||||
return inputRows.IndexOf(inputRow);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
// namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
// using BCG;
|
|
||||||
|
|
||||||
// using System;
|
|
||||||
// using System.Linq;
|
|
||||||
|
|
||||||
// /// <summary>
|
|
||||||
// /// Represents a row transformation table (all columns except 'id' can be changed).
|
|
||||||
// /// </summary>
|
|
||||||
// public class ColumnSelectorRow : IBlueRow {
|
|
||||||
// readonly IBlueRow source;
|
|
||||||
// readonly IBlueTableSchema outputSchema;
|
|
||||||
// readonly Func<object?, object?>[] columnSelectors;
|
|
||||||
|
|
||||||
// public ColumnSelectorRow(IBlueRow source, params (int Index, Type Type, Func<object?, object?> GetValue)[] columnSelectors) {
|
|
||||||
// this.source = source;
|
|
||||||
// this.columnSelectors = Array.ConvertAll(columnSelectors, c => (c.Index, c.GetValue));
|
|
||||||
// outputSchema = getSchema(source.schema, columnSelectors);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public ColumnSelectorRow(IBlueRow source, IBlueTableSchema outputSchema, params (int Index, Func<object?, object?> GetValue)[] columnSelectors) {
|
|
||||||
// this.source = source;
|
|
||||||
// this.outputSchema = outputSchema;
|
|
||||||
// this.columnSelectors = columnSelectors;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public object? this[int index] {
|
|
||||||
// get {
|
|
||||||
// for (int i = 0; i < columnSelectors.Length; i++) {
|
|
||||||
// if (columnSelectors[i].Index == index) {
|
|
||||||
// return columnSelectors[i].GetValue(source[index]);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// return source[index];
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public IBlueTableSchema schema => outputSchema;
|
|
||||||
|
|
||||||
// public static Func<object?, object?>[] BuildColumnSelectors(IBlueTableSchema schema, params (int Index, Func<object?, object?> GetValue)[] columnSelectors) {
|
|
||||||
// var cols = schema.columnInfo;
|
|
||||||
// var rv = Array.ConvertAll(schema.columnInfo, c => new Func<object?, object?>(_ => _));
|
|
||||||
// for (int i = 0; i < columnSelectors.Length; i++) {
|
|
||||||
// var cs = columnSelectors[i];
|
|
||||||
// rv[cs.Index] = cs.GetValue;
|
|
||||||
// }
|
|
||||||
// return rv;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// static IBlueTableSchema getSchema(IBlueTableSchema inputSchema, (int Index, Type Type, Func<object?, object?> GetValue)[] columnSelectors) {
|
|
||||||
// var inputCols = inputSchema.columnInfo;
|
|
||||||
// var outputCols = inputCols.ToArray();
|
|
||||||
// for (int i = 0; i < columnSelectors.Length; i++) {
|
|
||||||
// var (idx, typ, _) = columnSelectors[i];
|
|
||||||
// var col = inputCols[idx];
|
|
||||||
// outputCols[idx] = new(col.colName, col.colNo, typ);
|
|
||||||
// }
|
|
||||||
// return new BlueTableSchema(outputCols, "ColumnSelector");
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@@ -1,107 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class ColumnSelectorRowFactory {
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
readonly Func<object?, object?>[] columnSelectors;
|
|
||||||
|
|
||||||
public ColumnSelectorRowFactory(IBlueTableSchema outputSchema, Func<object?, object?>[] columnSelectors) {
|
|
||||||
this.outputSchema = outputSchema;
|
|
||||||
this.columnSelectors = columnSelectors;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema OutputSchema => outputSchema;
|
|
||||||
|
|
||||||
public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, params (string Name, Type Type, Func<object?, object?> GetValue)[] columnOverrides) =>
|
|
||||||
Create(inputSchema, columnOverrides.AsEnumerable());
|
|
||||||
|
|
||||||
public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, params (int Index, Type Type, Func<object?, object?> GetValue)[] columnOverrides) =>
|
|
||||||
Create(inputSchema, columnOverrides.AsEnumerable());
|
|
||||||
|
|
||||||
public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, IEnumerable<(string Name, Type Type, Func<object?, object?> GetValue)> columnOverrides) =>
|
|
||||||
Create(inputSchema, columnOverrides.Select(c => (inputSchema.ResolveIndices(c.Name), c.Type, c.GetValue)));
|
|
||||||
|
|
||||||
public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, IEnumerable<(int Index, Type Type, Func<object?, object?> GetValue)> columnOverrides) {
|
|
||||||
var bd = new Builder(inputSchema);
|
|
||||||
foreach (var c in columnOverrides) {
|
|
||||||
bd.AddOrUpdate(c);
|
|
||||||
}
|
|
||||||
return bd.ToImmutable();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow Create(IBlueRow row) =>
|
|
||||||
new Row(row, outputSchema, columnSelectors);
|
|
||||||
|
|
||||||
public Builder ToBuilder() =>
|
|
||||||
new(outputSchema);
|
|
||||||
|
|
||||||
class Row : IBlueRow {
|
|
||||||
readonly IBlueRow source;
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly Func<object?, object?>[] columnSelectors;
|
|
||||||
|
|
||||||
public Row(IBlueRow source, IBlueTableSchema schema, Func<object?, object?>[] columnSelectors) {
|
|
||||||
this.source = source;
|
|
||||||
this.schema = schema;
|
|
||||||
this.columnSelectors = columnSelectors;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] =>
|
|
||||||
columnSelectors[index](source[index]);
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRow.schema => schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public class Builder {
|
|
||||||
readonly IBlueTableSchema inputSchema;
|
|
||||||
readonly List<(int Index, Type Type, Func<object?, object?> GetValue)> columnOverrides = new();
|
|
||||||
|
|
||||||
public Builder(IBlueTableSchema inputSchema) {
|
|
||||||
this.inputSchema = inputSchema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema ToSchema() =>
|
|
||||||
ToSchema(inputSchema.tableName + "_ColumnSelector");
|
|
||||||
|
|
||||||
public IBlueTableSchema ToSchema(string tableName) {
|
|
||||||
var inputCols = inputSchema.columnInfo;
|
|
||||||
var outputCols = inputCols.ToArray();
|
|
||||||
foreach (var c in columnOverrides) {
|
|
||||||
var inputCol = inputCols[c.Index];
|
|
||||||
outputCols[c.Index] = new(inputCol.colName, c.Index, c.Type);
|
|
||||||
}
|
|
||||||
return new BlueTableSchema(outputCols, tableName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public ColumnSelectorRowFactory ToImmutable() {
|
|
||||||
var outputSchema = ToSchema();
|
|
||||||
var columnSelectors = Array.ConvertAll(outputSchema.columnInfo, c => new Func<object?, object?>(_ => _));
|
|
||||||
foreach (var cs in columnOverrides) {
|
|
||||||
columnSelectors[cs.Index] = cs.GetValue;
|
|
||||||
}
|
|
||||||
return new(outputSchema, columnSelectors);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void AddOrUpdate((int Index, Type Type, Func<object?, object?> GetValue) item) =>
|
|
||||||
columnOverrides.Add(item);
|
|
||||||
|
|
||||||
public void AddOrUpdate(int index, Type type, Func<object?, object?> getValue) =>
|
|
||||||
columnOverrides.Add((index, type, getValue));
|
|
||||||
|
|
||||||
public void AddOrUpdate<T>(int index, Func<object?, T> getValue) {
|
|
||||||
columnOverrides.Add((index, typeof(T), GetValueWrapper));
|
|
||||||
|
|
||||||
object? GetValueWrapper(object? inputVal) =>
|
|
||||||
getValue(inputVal);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
using System;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
[DataStreamUserRespresentation("CountStream")]
|
|
||||||
public class CountStream : ViewStream {
|
|
||||||
private DataPacket myLastFiredDataPacket;
|
|
||||||
private IBlueTableSchema mySchema = new BlueTableSchema("count",
|
|
||||||
new ColumnInfo("id", 0, typeof(int)),
|
|
||||||
new ColumnInfo("count", 1, typeof(int)));
|
|
||||||
private int myId = 0;
|
|
||||||
private ViewStream myVS;
|
|
||||||
|
|
||||||
public CountStream(ViewStream vs) {
|
|
||||||
myVS = vs;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
myVS.onChange += myHandleChanges;
|
|
||||||
}
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
myVS.onChange -= myHandleChanges;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void myHandleChanges(object newValue) {
|
|
||||||
myFireEvent(
|
|
||||||
myLastFiredDataPacket =
|
|
||||||
myLastFiredDataPacket == null ?
|
|
||||||
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
|
||||||
new BlueTable(mySchema))
|
|
||||||
:
|
|
||||||
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
|
||||||
myLastFiredDataPacket.newRows)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private IBlueRowCollection myGetRows(object dp, int id) {
|
|
||||||
BlueTable retVal = new BlueTable(mySchema);
|
|
||||||
retVal.addRow
|
|
||||||
(new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
|
|
||||||
return retVal;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,117 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using MathNet.Numerics.Statistics;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class CovarianceTable : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly Lazy<IBlueRow> row;
|
|
||||||
|
|
||||||
public CovarianceTable(IBlueRowCollection rows1, IBlueRowCollection rows2) {
|
|
||||||
var inputSchema = rows1.schema;
|
|
||||||
var inputCols = inputSchema.columnInfo;
|
|
||||||
var outputCols = Array.ConvertAll(rows1.schema.columnInfo, ColumnFactory);
|
|
||||||
schema = new BlueTableSchema(outputCols, inputSchema.tableName + "_" + rows2.schema.tableName + "_Covariance");
|
|
||||||
row = new(RowFactory);
|
|
||||||
|
|
||||||
static ColumnInfo ColumnFactory(ColumnInfo inputCol) {
|
|
||||||
var t = inputCol.colType;
|
|
||||||
if (inputCol.colName != "id") {
|
|
||||||
if (t == typeof(short) || t == typeof(int) || t == typeof(long)
|
|
||||||
|| t == typeof(ushort) || t == typeof(uint) || t == typeof(ulong)
|
|
||||||
|| t == typeof(double) || t == typeof(float)) {
|
|
||||||
t = typeof(double);
|
|
||||||
} else if (t == typeof(DateTime)) {
|
|
||||||
t = typeof(TimeSpan);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new(inputCol.colName, inputCol.colNo, t);
|
|
||||||
}
|
|
||||||
|
|
||||||
IBlueRow RowFactory() {
|
|
||||||
var rowVals = new object[inputCols.Length];
|
|
||||||
for (int i = 0; i < inputCols.Length; i++) {
|
|
||||||
var col = inputCols[i];
|
|
||||||
var colType = col.colType;
|
|
||||||
if (col.colName == "id") {
|
|
||||||
rowVals[i] = 1;
|
|
||||||
} else if (colType == typeof(short)) {
|
|
||||||
rowVals[i] = GetValue(v => v as short?);
|
|
||||||
} else if (colType == typeof(int)) {
|
|
||||||
rowVals[i] = GetValue(v => v as int?);
|
|
||||||
} else if (colType == typeof(long)) {
|
|
||||||
rowVals[i] = GetValue(v => v as long?);
|
|
||||||
} else if (colType == typeof(ushort)) {
|
|
||||||
rowVals[i] = GetValue(v => v as ushort?);
|
|
||||||
} else if (colType == typeof(uint)) {
|
|
||||||
rowVals[i] = GetValue(v => v as uint?);
|
|
||||||
} else if (colType == typeof(ulong)) {
|
|
||||||
rowVals[i] = GetValue(v => v as ulong?);
|
|
||||||
} else if (colType == typeof(double)) {
|
|
||||||
rowVals[i] = GetValue(v => v as double?);
|
|
||||||
} else if (colType == typeof(float)) {
|
|
||||||
rowVals[i] = GetValue(v => v as float?);
|
|
||||||
} else if (colType == typeof(DateTime)) {
|
|
||||||
rowVals[i] = TimeSpan.FromSeconds(GetValue(v => (v as DateTime?)?.Ticks / 10_000_000));
|
|
||||||
}
|
|
||||||
|
|
||||||
double GetValue(Func<object?, double?> convert) {
|
|
||||||
var vals1 = rows1.GetColumn(i)
|
|
||||||
.Select(convert)
|
|
||||||
.SelectWhereValues();
|
|
||||||
var vals2 = rows2.GetColumn(i)
|
|
||||||
.Select(convert)
|
|
||||||
.SelectWhereValues();
|
|
||||||
return vals1.Covariance(vals2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new Row(schema, rowVals);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
id == 1 ? row.Value : throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public int Count => 1;
|
|
||||||
|
|
||||||
IBlueRow IList<IBlueRow>.this[int index] {
|
|
||||||
get => GetRow(index);
|
|
||||||
set => throw new NotSupportedException();
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
id == 1;
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
yield return row.Value;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
index == 0 ? row.Value : throw new IndexOutOfRangeException();
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) =>
|
|
||||||
BlueRowEqualityComparer.Equals(item, row.Value) ? 0 : -1;
|
|
||||||
|
|
||||||
class Row : IBlueRow {
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly object?[] values;
|
|
||||||
|
|
||||||
public Row(IBlueTableSchema schema, object?[] values) {
|
|
||||||
this.schema = schema;
|
|
||||||
this.values = values;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] => values[index];
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRow.schema => schema;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class CummulativeSumStream : ViewStream, IDisposable {
|
|
||||||
readonly ViewStream inputStream;
|
|
||||||
readonly CummulativeSumTable.StringBehavior stringBehavior;
|
|
||||||
IBlueTableSchema? schema;
|
|
||||||
IDisposable? sub;
|
|
||||||
SingleRowTable? lastFired;
|
|
||||||
object?[]? lastFiredValues;
|
|
||||||
Func<object?, object?, object?>[]? columnOperators;
|
|
||||||
int idColumnIdx = -1;
|
|
||||||
|
|
||||||
public CummulativeSumStream(ViewStream inputStream, CummulativeSumTable.StringBehavior stringBehavior = CummulativeSumTable.StringBehavior.LastValue) {
|
|
||||||
this.inputStream = inputStream;
|
|
||||||
this.stringBehavior = stringBehavior;
|
|
||||||
}
|
|
||||||
|
|
||||||
~CummulativeSumStream() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
sub = inputStream.Subscribe(OnInputStreamChanged);
|
|
||||||
|
|
||||||
void OnInputStreamChanged(IDataPacket dp) {
|
|
||||||
ensureInit(dp);
|
|
||||||
var values = lastFiredValues?.ToArray() ?? new object?[schema.columnInfo.Length];
|
|
||||||
foreach (var r in dp.addedRows) {
|
|
||||||
for (int i = 0; i < values.Length; i++) {
|
|
||||||
values[i] = columnOperators[i](values[i], r[i]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var table = new SingleRowTable(new BlueRow(schema, values));
|
|
||||||
DataPacket dpOut = new(table, table, lastFired as IBlueRowCollection ?? new BlueTable(schema));
|
|
||||||
lastFired = table;
|
|
||||||
lastFiredValues = values;
|
|
||||||
myFireEvent(dpOut);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
|
|
||||||
[MemberNotNull(nameof(schema), nameof(columnOperators))]
|
|
||||||
void ensureInit(IDataPacket dp) {
|
|
||||||
if (schema is not null) {
|
|
||||||
Log.notnull(columnOperators);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
schema = dp.Schema;
|
|
||||||
idColumnIdx = schema.ResolveIndices("id");
|
|
||||||
var colInfos = schema.columnInfo;
|
|
||||||
columnOperators = new Func<object?, object?, object?>[schema.columnInfo.Length];
|
|
||||||
for (int i = 0; i < colInfos.Length; i++) {
|
|
||||||
if (i == idColumnIdx) {
|
|
||||||
columnOperators[i] = (x, y) => 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
columnOperators[i] = schema.columnInfo[i].type_code() switch {
|
|
||||||
TypeCode.Int32 => (x, y) => (x as int? ?? 0) + (BlueDataUtility.GetInt32(y) ?? 0),
|
|
||||||
TypeCode.Int64 => (x, y) => (x as long? ?? 0) + (BlueDataUtility.GetInt64(y) ?? 0),
|
|
||||||
TypeCode.UInt32 => (x, y) => (x as uint? ?? 0) + (BlueDataUtility.GetUInt32(y) ?? 0),
|
|
||||||
TypeCode.UInt64 => (x, y) => (x as ulong? ?? 0) + (BlueDataUtility.GetUInt64(y) ?? 0),
|
|
||||||
TypeCode.Single => (x, y) => (x as float? ?? 0) + (BlueDataUtility.GetFloat32(y) ?? 0),
|
|
||||||
TypeCode.Double => (x, y) => (x as double? ?? 0) + (BlueDataUtility.GetFloat64(y) ?? 0),
|
|
||||||
TypeCode.String => stringBehavior switch {
|
|
||||||
CummulativeSumTable.StringBehavior.Concatenate => (x, y) => (x as string ?? string.Empty) + (y as string ?? string.Empty),
|
|
||||||
_ => (accum, next) => next,
|
|
||||||
},
|
|
||||||
TypeCode.DateTime => (x, y) => y,
|
|
||||||
_ => throw new InvalidOperationException($"Failed to add row values for col '{schema.columnInfo[i]}'"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,119 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class CummulativeSumTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection input;
|
|
||||||
readonly IList<object?>[] columns;
|
|
||||||
readonly StringBehavior stringBehavior;
|
|
||||||
|
|
||||||
public CummulativeSumTable(IBlueRowCollection input, StringBehavior stringBehavior = StringBehavior.LastValue) {
|
|
||||||
this.input = input;
|
|
||||||
var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
|
|
||||||
if (colInfos.Length > 0) {
|
|
||||||
Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
|
|
||||||
}
|
|
||||||
columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// By default, we take the last value for strings intstead of adding (concatenating) them.
|
|
||||||
/// Otherwise it would be too easy to create massive concatenated strings by mistake.
|
|
||||||
/// </summary>
|
|
||||||
public enum StringBehavior {
|
|
||||||
LastValue,
|
|
||||||
Concatenate
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
id == 1 ? GetRow(0) : throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => input.schema;
|
|
||||||
|
|
||||||
public int Count => 1;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
id == 1;
|
|
||||||
|
|
||||||
public IList<object?> GetColumn(int colIndex) {
|
|
||||||
var count = input.Count;
|
|
||||||
var colInfo = input.schema.columnInfo[colIndex];
|
|
||||||
if (colInfo.colName == "id") {
|
|
||||||
return new object?[] { 1 };
|
|
||||||
}
|
|
||||||
object? res = null;
|
|
||||||
var colVals = input.GetColumn(colIndex);
|
|
||||||
if (colInfo.colType == typeof(TimeSpan)) {
|
|
||||||
// TimeSpan is a special case because it doesn't have a TypeCode.
|
|
||||||
res = TimeSpanImpl(colVals);
|
|
||||||
} else {
|
|
||||||
var typeCode = colInfo.type_code();
|
|
||||||
const int MAX_STRING_LENGTH = 1024;
|
|
||||||
res = typeCode switch {
|
|
||||||
TypeCode.Int32 => colVals.Cast<int>().Sum(),
|
|
||||||
TypeCode.Int64 => colVals.Cast<long>().Sum(),
|
|
||||||
TypeCode.UInt32 => colVals.Cast<uint>().Sum(),
|
|
||||||
TypeCode.UInt64 => colVals.Cast<ulong>().Aggregate(0UL, (s, v) => s + v),
|
|
||||||
TypeCode.Single => colVals.Cast<float>().Sum(),
|
|
||||||
TypeCode.Double => colVals.Cast<double>().Sum(),
|
|
||||||
TypeCode.DateTime => colVals.Cast<DateTime>().Aggregate(DateTime.MinValue, (s, v) => new DateTime(s.Ticks + v.Ticks)),
|
|
||||||
TypeCode.String => stringBehavior switch {
|
|
||||||
StringBehavior.Concatenate => colVals.Cast<string>().Aggregate(string.Empty, (s, v) => {
|
|
||||||
if (s.Length + v.Length > MAX_STRING_LENGTH) {
|
|
||||||
throw new InvalidOperationException("Cannot sum strings because the result is too long");
|
|
||||||
}
|
|
||||||
return s + v;
|
|
||||||
}),
|
|
||||||
_ => count == 0 ? null : colVals[^1] as string,
|
|
||||||
},
|
|
||||||
_ => null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return new object?[] { res };
|
|
||||||
|
|
||||||
static object? TimeSpanImpl(IList<object?> colVals) {
|
|
||||||
var ticks = colVals.Cast<TimeSpan>().Select(ts => ts.Ticks).Sum();
|
|
||||||
return TimeSpan.FromTicks(ticks);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
yield return GetRow(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new BlueRowColumnWrapper(input.schema, index, columns);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
var inputRow = input[id];
|
|
||||||
int idx = input.IndexOf(inputRow);
|
|
||||||
if (idx != -1) {
|
|
||||||
var row = GetRow(idx);
|
|
||||||
if (BlueRowEqualityComparer.Equals(item, row)) {
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public class CummulativeSumTableEvaluator : IFunctionEvaluator {
|
|
||||||
public object? Evaluate(string functionId, IEnumerable<StreamArgument> args) {
|
|
||||||
if (args.TryGetFirst(out var a)) {
|
|
||||||
var table = a.Value as IBlueRowCollection;
|
|
||||||
if (table is not null) {
|
|
||||||
return new CummulativeSumTable(table);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,163 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq.Expressions;
|
|
||||||
|
|
||||||
public class DiffStream : ViewStream, IDisposable {
|
|
||||||
readonly ViewStream source;
|
|
||||||
readonly int period;
|
|
||||||
IBlueTableSchema? outputSchema;
|
|
||||||
IDisposable? sub;
|
|
||||||
BlueTableImmutable? lastFired;
|
|
||||||
|
|
||||||
public DiffStream(ViewStream source, int period) {
|
|
||||||
this.source = source;
|
|
||||||
this.period = period;
|
|
||||||
}
|
|
||||||
|
|
||||||
~DiffStream() =>
|
|
||||||
Dispose();
|
|
||||||
|
|
||||||
public void Dispose() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
Log.isnull(sub);
|
|
||||||
sub = source.Subscribe(OnSourceChange);
|
|
||||||
|
|
||||||
void OnSourceChange(IDataPacket dp) {
|
|
||||||
if (outputSchema is null) {
|
|
||||||
var inputSchema = dp.Schema;
|
|
||||||
var outputCols = new ColumnInfo[inputSchema.columnInfo.Length];
|
|
||||||
for (int i = 0; i < inputSchema.columnInfo.Length; i++) {
|
|
||||||
outputCols[i] = GetOutputColumn(inputSchema.columnInfo[i]);
|
|
||||||
}
|
|
||||||
outputSchema = new BlueTableSchema($"{inputSchema.tableName}_Diff{period}", outputCols);
|
|
||||||
}
|
|
||||||
lastFired ??= new(outputSchema);
|
|
||||||
var newTable = new DiffTable(dp.newRows, period);
|
|
||||||
var allRows = new List<object?[]>();
|
|
||||||
foreach (IBlueRow r in newTable) {
|
|
||||||
var rvals = new object?[r.schema.columnInfo.Length];
|
|
||||||
int x = 0;
|
|
||||||
foreach (var col in r.schema.columnInfo) {
|
|
||||||
var val = r[col.colNo];
|
|
||||||
rvals[x++] = val;
|
|
||||||
}
|
|
||||||
allRows.Add(rvals);
|
|
||||||
}
|
|
||||||
var bd = lastFired.ToDataPacketBuilder();
|
|
||||||
var removedIndices = new List<int>();
|
|
||||||
var addedIndices = new List<int>();
|
|
||||||
var ops = new List<(int Index, bool IsAdd)>();
|
|
||||||
foreach (var r in dp.deletedRows) {
|
|
||||||
int id = r.Id();
|
|
||||||
var rOut = lastFired[id];
|
|
||||||
var idx = lastFired.IndexOf(rOut);
|
|
||||||
Log.assert(idx >= 0 && idx < lastFired.Count);
|
|
||||||
removedIndices.Add(idx);
|
|
||||||
ops.Add((idx, false));
|
|
||||||
}
|
|
||||||
foreach (var r in dp.addedRows) {
|
|
||||||
var idx = dp.newRows.IndexOf(r);
|
|
||||||
Log.assert(idx >= 0 && idx < dp.newRows.Count);
|
|
||||||
addedIndices.Add(idx);
|
|
||||||
ops.Add((idx, true));
|
|
||||||
}
|
|
||||||
ops.Sort(Comparer<(int Index, bool IsAdd)>.Create((a, b) => a.Index.CompareTo(b.Index)));
|
|
||||||
int rmCount = 0;
|
|
||||||
var removedRows = new Dictionary<int, IBlueRow>();
|
|
||||||
for (int i = 0; i < ops.Count; i++) {
|
|
||||||
var op = ops[i];
|
|
||||||
if (op.IsAdd) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
rmCount++;
|
|
||||||
int j = op.Index/* - rmCount*/;
|
|
||||||
var stop = Math.Min(j + period, lastFired.Count/* - rmCount*/);
|
|
||||||
// Remove the row that was removed in the incoming DataPacket.
|
|
||||||
bd.Remove(lastFired.GetRow(j).Id());
|
|
||||||
for (j++; j < stop; j++) {
|
|
||||||
// Remove rows that changed as a result. We will add the updated versions
|
|
||||||
// of these rows in the next pass.
|
|
||||||
var r = lastFired.GetRow(j);
|
|
||||||
bd.Remove(r.Id());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
int addCount = ops.Count - rmCount;
|
|
||||||
for (int i = ops.Count - 1; i >= 0; i--) {
|
|
||||||
var op = ops[i];
|
|
||||||
if (op.IsAdd) {
|
|
||||||
addCount--;
|
|
||||||
int j = op.Index;
|
|
||||||
var stop = Math.Min(j + period, newTable.Count);
|
|
||||||
for (; j < stop; j++) {
|
|
||||||
var r = newTable.GetRow(j);
|
|
||||||
bd.AddOrUpdate(r);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
rmCount--;
|
|
||||||
int j = op.Index - rmCount + addCount;
|
|
||||||
var stop = Math.Min(j + period, newTable.Count);
|
|
||||||
for (; j < stop; j++) {
|
|
||||||
// Add rows that changed due to the removed row (the row with which they compute
|
|
||||||
// the diff has changed).
|
|
||||||
var r = newTable.GetRow(j);
|
|
||||||
bd.AddOrUpdate(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var dpOut = bd.ToDataPacket(out lastFired, false);
|
|
||||||
myFireEvent(dpOut);
|
|
||||||
}
|
|
||||||
|
|
||||||
ColumnInfo GetOutputColumn(ColumnInfo inputColumn) {
|
|
||||||
return inputColumn.type_code() switch {
|
|
||||||
TypeCode.Boolean => inputColumn.Clone(),
|
|
||||||
TypeCode.Byte or TypeCode.SByte or TypeCode.Int16 => WithType(TypeCode.Int32),
|
|
||||||
TypeCode.UInt32 or TypeCode.UInt64 => WithType(TypeCode.Int64),
|
|
||||||
_ => inputColumn.Clone(),
|
|
||||||
};
|
|
||||||
|
|
||||||
ColumnInfo WithType(TypeCode typeCode) =>
|
|
||||||
new(inputColumn.colName, inputColumn.colNo, typeCode);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
|
|
||||||
static class ExpressionBuilder {
|
|
||||||
public static Func<IBlueRow, IBlueRow, int?> Int32DiffExpression(int colIndex) {
|
|
||||||
var colIndexParam = Expression.Constant(colIndex, typeof(int));
|
|
||||||
var rowParam = Expression.Parameter(typeof(IBlueRow), "row");
|
|
||||||
var getValMethod = typeof(ExpressionBuilder).GetMethod(nameof(getInt32Value))
|
|
||||||
?? throw new InvalidOperationException("Failed to find int32 value getter via reflection");
|
|
||||||
var getVal = Expression.Lambda<Func<IBlueRow, int?>>(Expression.Call(null, getValMethod, rowParam, colIndexParam), rowParam).Compile();
|
|
||||||
return ComputeDiff;
|
|
||||||
|
|
||||||
int? ComputeDiff(IBlueRow leftRow, IBlueRow rightRow) {
|
|
||||||
var leftVal = getVal(leftRow);
|
|
||||||
var rightVal = getVal(rightRow);
|
|
||||||
if (leftVal.HasValue && rightVal.HasValue) {
|
|
||||||
return leftVal.Value - rightVal.Value;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static int? getInt32Value(IBlueRow row, int colIndex) {
|
|
||||||
var obj = row[colIndex];
|
|
||||||
return obj as int?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,101 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class DiffTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection input;
|
|
||||||
readonly int period;
|
|
||||||
readonly IList<object?>[] columns;
|
|
||||||
|
|
||||||
public DiffTable(IBlueRowCollection input, int period) {
|
|
||||||
this.input = input;
|
|
||||||
this.period = period;
|
|
||||||
var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
|
|
||||||
if (colInfos.Length > 0) {
|
|
||||||
Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
|
|
||||||
}
|
|
||||||
columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
GetRow(input.IndexOf(input[id]));
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => input.schema;
|
|
||||||
|
|
||||||
public int Count => input.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
input.containsID(id);
|
|
||||||
|
|
||||||
public IList<object?> GetColumn(int colIndex) {
|
|
||||||
var count = input.Count;
|
|
||||||
var colInfo = input.schema.columnInfo[colIndex];
|
|
||||||
if (colInfo.colName == "id") {
|
|
||||||
return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetId, null);
|
|
||||||
}
|
|
||||||
var typeCode = colInfo.type_code();
|
|
||||||
return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetValue, null);
|
|
||||||
|
|
||||||
object? GetId(int rowIndex) {
|
|
||||||
var row = input.GetRow(rowIndex);
|
|
||||||
return row.Id();
|
|
||||||
}
|
|
||||||
|
|
||||||
object? GetValue(int rowIndex) {
|
|
||||||
var rightIndex = rowIndex - period;
|
|
||||||
if (rowIndex < 0 || rowIndex >= count || rightIndex < 0 || rightIndex >= count) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
var leftRow = input.GetRow(rowIndex);
|
|
||||||
var rightRow = input.GetRow(rightIndex);
|
|
||||||
return GetDiff(leftRow[colIndex], rightRow[colIndex], typeCode);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
var count = input.Count;
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
yield return GetRow(i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new BlueRowColumnWrapper(input.schema, index, columns);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
var inputRow = input[id];
|
|
||||||
int idx = input.IndexOf(inputRow);
|
|
||||||
if (idx != -1) {
|
|
||||||
var row = GetRow(idx);
|
|
||||||
if (BlueRowEqualityComparer.Equals(item, row)) {
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
static object? GetDiff(object? leftVal, object? rightVal, TypeCode colType) {
|
|
||||||
if (leftVal is null || rightVal is null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return colType switch {
|
|
||||||
TypeCode.Int32 => (int)leftVal - (int)rightVal,
|
|
||||||
TypeCode.Int64 => (long)leftVal - (long)rightVal,
|
|
||||||
TypeCode.Double => (double)leftVal - (double)rightVal,
|
|
||||||
TypeCode.String => ((string)leftVal).Replace((string)rightVal, string.Empty),
|
|
||||||
TypeCode.DateTime => new DateTime(((DateTime)leftVal).Ticks - ((DateTime)rightVal).Ticks),
|
|
||||||
_ => null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,506 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using Apache.Arrow;
|
|
||||||
using Apache.Arrow.Ipc;
|
|
||||||
using Apache.Arrow.Types;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
using Spark.Core.Threading;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Immutable;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("featherReader")]
|
|
||||||
public class FeatherReaderStream : ViewStream {
|
|
||||||
readonly string file;
|
|
||||||
FileStream fileStream;
|
|
||||||
// Let the table leak in case some dangling references linger around after
|
|
||||||
// myOnLastSubscriber is called. This should be fine since it disposes itself
|
|
||||||
// in its finalizer.
|
|
||||||
FeatherTable? table;
|
|
||||||
|
|
||||||
public FeatherReaderStream(string file) =>
|
|
||||||
this.file = file;
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
if (!Path.Exists(file)) {
|
|
||||||
throw new InvalidOperationException("Feather file does not exist");
|
|
||||||
}
|
|
||||||
fileStream = File.OpenRead(file);
|
|
||||||
table = new(new ArrowStreamReader(fileStream));
|
|
||||||
myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
|
|
||||||
}
|
|
||||||
|
|
||||||
interface IFeatherColumn {
|
|
||||||
int Count { get; }
|
|
||||||
|
|
||||||
object? GetValue(int i);
|
|
||||||
}
|
|
||||||
|
|
||||||
interface IFeatherColumn<T> : IFeatherColumn {
|
|
||||||
new T? GetValue(int i);
|
|
||||||
|
|
||||||
object? IFeatherColumn.GetValue(int i) =>
|
|
||||||
GetValue(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherColumn : IFeatherColumn {
|
|
||||||
readonly IArrowArray array;
|
|
||||||
|
|
||||||
public FeatherColumn(IArrowArray array) {
|
|
||||||
this.array = array;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public static IFeatherColumn Create(IArrowArray array) =>
|
|
||||||
array switch {
|
|
||||||
Int32Array a => new FeatherInt32Column(a),
|
|
||||||
Int64Array a => new FeatherInt64Column(a),
|
|
||||||
DoubleArray a => new FeatherFloat64Column(a),
|
|
||||||
StringArray a => new FeatherStringColumn(a),
|
|
||||||
BooleanArray a => new FeatherBooleanColumn(a),
|
|
||||||
Date64Array a => new FeatherDateTimeColumn(a),
|
|
||||||
Time64Array a => new FeatherTimeStampColumn(a),
|
|
||||||
_ => throw new NotSupportedException($"Unsupported type: {array.GetType()}")
|
|
||||||
};
|
|
||||||
|
|
||||||
public object? GetValue(int i) =>
|
|
||||||
array switch {
|
|
||||||
Int32Array a => a.GetValue(i),
|
|
||||||
Int64Array a => a.GetValue(i),
|
|
||||||
DoubleArray a => a.GetValue(i),
|
|
||||||
StringArray a => a.GetString(i),
|
|
||||||
BooleanArray a => a.GetValue(i),
|
|
||||||
Date64Array a => a.GetValue(i),
|
|
||||||
Time64Array a => a.GetValue(i),
|
|
||||||
_ => throw new NotSupportedException($"Unsupported type: {array.GetType()}")
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherInt32Column : IFeatherColumn<int?> {
|
|
||||||
readonly Int32Array array;
|
|
||||||
|
|
||||||
public FeatherInt32Column(Int32Array array) =>
|
|
||||||
this.array = array;
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public int? GetValue(int i) => array.GetValue(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherInt64Column : IFeatherColumn<long?> {
|
|
||||||
readonly Int64Array array;
|
|
||||||
|
|
||||||
public FeatherInt64Column(Int64Array array) =>
|
|
||||||
this.array = array;
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public long? GetValue(int i) => array.GetValue(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherFloat64Column : IFeatherColumn<double?> {
|
|
||||||
readonly DoubleArray array;
|
|
||||||
|
|
||||||
public FeatherFloat64Column(DoubleArray array) =>
|
|
||||||
this.array = array;
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public double? GetValue(int i) => array.GetValue(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherFloat32Column : IFeatherColumn<float?> {
|
|
||||||
readonly FloatArray array;
|
|
||||||
|
|
||||||
public FeatherFloat32Column(FloatArray array) =>
|
|
||||||
this.array = array;
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public float? GetValue(int i) => array.GetValue(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherStringColumn : IFeatherColumn<string?> {
|
|
||||||
readonly StringArray array;
|
|
||||||
|
|
||||||
public FeatherStringColumn(StringArray array) =>
|
|
||||||
this.array = array;
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public string? GetValue(int i) => array.GetString(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherBooleanColumn : IFeatherColumn<bool?> {
|
|
||||||
readonly BooleanArray array;
|
|
||||||
|
|
||||||
public FeatherBooleanColumn(BooleanArray array) =>
|
|
||||||
this.array = array;
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public bool? GetValue(int i) => array.GetValue(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherDateTimeColumn : IFeatherColumn<DateTime?> {
|
|
||||||
readonly Date64Array array;
|
|
||||||
|
|
||||||
public FeatherDateTimeColumn(Date64Array array) =>
|
|
||||||
this.array = array;
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public DateTime? GetValue(int i) {
|
|
||||||
var nanos = array.GetValue(i);
|
|
||||||
if (!nanos.HasValue) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return new(nanos.Value / 100);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherTimeStampColumn : IFeatherColumn<DateTime?> {
|
|
||||||
readonly Time64Array array;
|
|
||||||
|
|
||||||
public FeatherTimeStampColumn(Time64Array array) =>
|
|
||||||
this.array = array;
|
|
||||||
|
|
||||||
public int Count => array.Length;
|
|
||||||
|
|
||||||
public DateTime? GetValue(int i) {
|
|
||||||
var nanos = array.GetValue(i);
|
|
||||||
if (!nanos.HasValue) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return new(nanos.Value / 100);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherBlueRow : IBlueRow {
|
|
||||||
readonly int id;
|
|
||||||
readonly int rowIndex;
|
|
||||||
readonly IFeatherColumn[] data;
|
|
||||||
|
|
||||||
public FeatherBlueRow(int id, int rowIndex, IFeatherColumn[] data, IBlueTableSchema schema) {
|
|
||||||
this.id = id;
|
|
||||||
this.rowIndex = rowIndex;
|
|
||||||
this.data = data;
|
|
||||||
Schema = schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] =>
|
|
||||||
index == 0
|
|
||||||
? id
|
|
||||||
: data[index - 1].GetValue(rowIndex);
|
|
||||||
|
|
||||||
public IBlueTableSchema Schema { get; }
|
|
||||||
IBlueTableSchema IBlueRow.schema => Schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherColumnDataList : IReadOnlyList<object?> {
|
|
||||||
// Represents 1 column split into batches (hence it being an array)
|
|
||||||
readonly IReadOnlyList<IFeatherColumn> columns;
|
|
||||||
int? count;
|
|
||||||
|
|
||||||
public FeatherColumnDataList(IReadOnlyList<IFeatherColumn> columns) {
|
|
||||||
this.columns = columns;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] {
|
|
||||||
get {
|
|
||||||
for (int i = 0; i < columns.Count; i++) {
|
|
||||||
var col = columns[i];
|
|
||||||
if (index < col.Count) {
|
|
||||||
return col.GetValue(index);
|
|
||||||
}
|
|
||||||
index -= col.Count;
|
|
||||||
}
|
|
||||||
throw new IndexOutOfRangeException();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public int Count =>
|
|
||||||
count ??= columns.Sum(c => c.Count);
|
|
||||||
|
|
||||||
public IEnumerator<object?> GetEnumerator() {
|
|
||||||
for (int i = 0; i < columns.Count; i++) {
|
|
||||||
var col = columns[i];
|
|
||||||
for (int j = 0; j < col.Count; j++) {
|
|
||||||
yield return col.GetValue(i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() =>
|
|
||||||
GetEnumerator();
|
|
||||||
}
|
|
||||||
|
|
||||||
class FeatherTable : IImmutableRowCollection, IDisposable {
|
|
||||||
ImmutableList<(RecordBatch Records, IFeatherColumn[] Columns)> recordBatches = ImmutableList<(RecordBatch Records, IFeatherColumn[] Columns)>.Empty;
|
|
||||||
readonly IReadOnlyDictionary<string, Field> fields;
|
|
||||||
readonly AtomicGate gate = new();
|
|
||||||
int count;
|
|
||||||
ArrowStreamReader? reader;
|
|
||||||
|
|
||||||
public FeatherTable(ArrowStreamReader reader) {
|
|
||||||
this.reader = reader;
|
|
||||||
if (reader.Schema is null) {
|
|
||||||
var batch = reader.ReadNextRecordBatch();
|
|
||||||
Log.notnull(batch);
|
|
||||||
Log.notnull(reader.Schema);
|
|
||||||
var fcols = new IFeatherColumn[batch.ColumnCount];
|
|
||||||
for (int i = 0; i < fcols.Length; i++) {
|
|
||||||
fcols[i] = FeatherColumn.Create(batch.Column(i));
|
|
||||||
}
|
|
||||||
recordBatches = recordBatches.Add((batch, fcols));
|
|
||||||
count = batch.Length;
|
|
||||||
}
|
|
||||||
fields = reader.Schema.Fields;
|
|
||||||
var columns = new ColumnInfo[fields.Count + 1];
|
|
||||||
columns[0] = new("id", 0, typeof(int));
|
|
||||||
for (int i = 0; i < fields.Count; i++) {
|
|
||||||
var f = reader.Schema.GetFieldByIndex(i);
|
|
||||||
Log.assert(f.Name != "id");
|
|
||||||
columns[i + 1] = new(f.Name, i + 1, getFieldType(f));
|
|
||||||
}
|
|
||||||
schema = new BlueTableSchema("FeatherReader", columns);
|
|
||||||
}
|
|
||||||
|
|
||||||
~FeatherTable() =>
|
|
||||||
Disposable.DisposeThreadSafe(ref reader);
|
|
||||||
|
|
||||||
public IBlueTableSchema schema { get; }
|
|
||||||
|
|
||||||
public int Count {
|
|
||||||
get {
|
|
||||||
if (reader is not null) {
|
|
||||||
readBatches().Wait();
|
|
||||||
Log.isnull(reader);
|
|
||||||
}
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] => GetRow(id - 1);
|
|
||||||
|
|
||||||
public bool containsID(int id) {
|
|
||||||
while (id > count && reader is not null) {
|
|
||||||
readBatches(1).Wait();
|
|
||||||
}
|
|
||||||
return id <= count;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
while (id > count && reader is not null) {
|
|
||||||
readBatches(1).Wait();
|
|
||||||
}
|
|
||||||
return id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
|
|
||||||
? id - 1
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) {
|
|
||||||
if (index < 0) {
|
|
||||||
throw new IndexOutOfRangeException();
|
|
||||||
}
|
|
||||||
if (count <= index) {
|
|
||||||
readRows(index + 1).Wait();
|
|
||||||
}
|
|
||||||
int id = index + 1;
|
|
||||||
foreach (var (records, columns) in recordBatches) {
|
|
||||||
if (index < records.Length) {
|
|
||||||
return new FeatherBlueRow(id, index, columns, schema);
|
|
||||||
}
|
|
||||||
index -= records.Length;
|
|
||||||
}
|
|
||||||
throw new IndexOutOfRangeException();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IList<object?> GetColumn(int index) =>
|
|
||||||
new FeatherColumnDataList(new MyColList(this, index)).as_mutable();
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
int i = 0;
|
|
||||||
foreach (var (records, columns) in recordBatches) {
|
|
||||||
yield return new FeatherBlueRow(i + 1, i, columns, schema);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
while (reader is not null) {
|
|
||||||
var task = readBatches(1);
|
|
||||||
task.Wait();
|
|
||||||
if (!task.Result) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
yield return new FeatherBlueRow(i + 1, i, recordBatches[i].Columns, schema);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() =>
|
|
||||||
Disposable.DisposeThreadSafe(ref reader);
|
|
||||||
|
|
||||||
static Type getFieldType(Field f) =>
|
|
||||||
f.DataType.TypeId switch {
|
|
||||||
ArrowTypeId.Null => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Boolean => typeof(bool),
|
|
||||||
ArrowTypeId.UInt8 => typeof(byte),
|
|
||||||
ArrowTypeId.Int8 => typeof(sbyte),
|
|
||||||
ArrowTypeId.UInt16 => typeof(ushort),
|
|
||||||
ArrowTypeId.Int16 => typeof(short),
|
|
||||||
ArrowTypeId.UInt32 => typeof(uint),
|
|
||||||
ArrowTypeId.Int32 => typeof(int),
|
|
||||||
ArrowTypeId.UInt64 => typeof(ulong),
|
|
||||||
ArrowTypeId.Int64 => typeof(long),
|
|
||||||
ArrowTypeId.HalfFloat => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Float => typeof(float),
|
|
||||||
ArrowTypeId.Double => typeof(double),
|
|
||||||
ArrowTypeId.String => typeof(string),
|
|
||||||
ArrowTypeId.Binary => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.FixedSizedBinary => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Date32 => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Date64 => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Timestamp => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Time32 => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Time64 => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Interval => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Decimal128 => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Decimal256 => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.List => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Struct => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Union => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Dictionary => throw new NotImplementedException(),
|
|
||||||
ArrowTypeId.Map => throw new NotImplementedException(),
|
|
||||||
_ => throw new ArgumentException("Unsupported field type", nameof(f)),
|
|
||||||
};
|
|
||||||
|
|
||||||
async Task<bool> readRows(int minCount) {
|
|
||||||
if (reader is null) {
|
|
||||||
return count >= minCount;
|
|
||||||
}
|
|
||||||
int n = 0;
|
|
||||||
using (gate.BeginTransaction()) {
|
|
||||||
if (reader is not null) {
|
|
||||||
int origCount = count;
|
|
||||||
while ((count + n) < minCount) {
|
|
||||||
Log.assert(count == origCount);
|
|
||||||
var batch = await reader.ReadNextRecordBatchAsync();
|
|
||||||
if (batch is null) {
|
|
||||||
Disposable.dispose(ref reader);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
var columns = new IFeatherColumn[batch.ColumnCount];
|
|
||||||
for (int i = 0; i < columns.Length; i++) {
|
|
||||||
columns[i] = FeatherColumn.Create(batch.Column(i));
|
|
||||||
}
|
|
||||||
recordBatches.Add((batch, columns));
|
|
||||||
n += batch.Length;
|
|
||||||
}
|
|
||||||
count += n;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return count >= minCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
async Task<bool> readBatches(int? count = null) {
|
|
||||||
if (reader is null) {
|
|
||||||
return !count.HasValue || count.Value == 0;
|
|
||||||
}
|
|
||||||
if (count.HasValue && count.Value <= recordBatches.Count) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
int n = 0;
|
|
||||||
using (gate.BeginTransaction()) {
|
|
||||||
if (reader is not null) {
|
|
||||||
int origCount = this.count;
|
|
||||||
var recordBatchesBd = recordBatches.ToBuilder();
|
|
||||||
while (!count.HasValue || count-- > recordBatchesBd.Count) {
|
|
||||||
Log.assert(count == origCount);
|
|
||||||
var batch = await reader.ReadNextRecordBatchAsync();
|
|
||||||
if (batch is null) {
|
|
||||||
Disposable.dispose(ref reader);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
var columns = new IFeatherColumn[batch.ColumnCount];
|
|
||||||
for (int i = 0; i < columns.Length; i++) {
|
|
||||||
columns[i] = FeatherColumn.Create(batch.Column(i));
|
|
||||||
}
|
|
||||||
recordBatchesBd.Add((batch, columns));
|
|
||||||
n += batch.Length;
|
|
||||||
}
|
|
||||||
recordBatches = recordBatchesBd.ToImmutable();
|
|
||||||
this.count += n;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return !count.HasValue || count.Value <= recordBatches.Count;
|
|
||||||
}
|
|
||||||
|
|
||||||
class MyColList : IReadOnlyList<IFeatherColumn> {
|
|
||||||
readonly FeatherTable owner;
|
|
||||||
readonly int columnIndex;
|
|
||||||
|
|
||||||
public MyColList(FeatherTable owner, int columnIndex) {
|
|
||||||
this.owner = owner;
|
|
||||||
this.columnIndex = columnIndex;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IFeatherColumn this[int index] {
|
|
||||||
get {
|
|
||||||
if (index < 0) {
|
|
||||||
throw new IndexOutOfRangeException();
|
|
||||||
}
|
|
||||||
if (index >= owner.recordBatches.Count) {
|
|
||||||
var task = owner.readBatches(index + 1);
|
|
||||||
task.Wait();
|
|
||||||
if (!task.Result) {
|
|
||||||
throw new IndexOutOfRangeException();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return owner.recordBatches[index].Columns[columnIndex];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public int Count {
|
|
||||||
get {
|
|
||||||
if (owner.reader is not null) {
|
|
||||||
owner.readBatches().Wait();
|
|
||||||
}
|
|
||||||
return owner.recordBatches.Count;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IEnumerator<IFeatherColumn> GetEnumerator() {
|
|
||||||
int i = 0;
|
|
||||||
foreach (var batch in owner.recordBatches) {
|
|
||||||
yield return batch.Columns[columnIndex];
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
while (owner.reader is not null) {
|
|
||||||
var task = owner.readBatches(1);
|
|
||||||
task.Wait();
|
|
||||||
if (!task.Result) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
yield return owner.recordBatches[i].Columns[columnIndex];
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() =>
|
|
||||||
GetEnumerator();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,159 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("FilterStream")]
|
|
||||||
public class FilterStream : ViewStream {
|
|
||||||
public class RegExpFilter : IStreamFilter {
|
|
||||||
private Regex[] myRegExps;
|
|
||||||
private string[] myCols;
|
|
||||||
|
|
||||||
public RegExpFilter(string col, string regExp)
|
|
||||||
: this(new string[] { col }, new string[] { regExp }) { }
|
|
||||||
|
|
||||||
public RegExpFilter(string[] cols, string[] regExps) {
|
|
||||||
myCols = cols;
|
|
||||||
myRegExps = new Regex[regExps.Length];
|
|
||||||
for (int i = 0; i < regExps.Length; i++)
|
|
||||||
myRegExps[i] = new Regex(regExps[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#region IStreamFilter Members
|
|
||||||
public bool filter(IBlueRow dr) {
|
|
||||||
bool rv = true;
|
|
||||||
for (int i = 0; i < myCols.Length; i++) {
|
|
||||||
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]]?.ToString() ?? string.Empty);
|
|
||||||
if (!rv)
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
BlueTableImmutable? myLastFiredRows;
|
|
||||||
IBlueRowCollection? myLastReceivedRows;
|
|
||||||
private IStreamFilter myFilter;
|
|
||||||
private ViewStream myParentStream;
|
|
||||||
private string myFilterString;
|
|
||||||
private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
|
|
||||||
|
|
||||||
#region constructors
|
|
||||||
public delegate bool OnFilter(IBlueRow r);
|
|
||||||
private class OnFilterWrapper : IStreamFilter {
|
|
||||||
public OnFilterWrapper(OnFilter f) { this.f = f; }
|
|
||||||
|
|
||||||
public bool filter(IBlueRow dr) { return f(dr); }
|
|
||||||
|
|
||||||
private OnFilter f;
|
|
||||||
}
|
|
||||||
public FilterStream(ViewStream vs, OnFilter f)
|
|
||||||
: this(vs, new OnFilterWrapper(f)) { }
|
|
||||||
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, IStreamFilter filter) {
|
|
||||||
myParentStream = vs;
|
|
||||||
myFilter = filter;
|
|
||||||
myFilterString = filter.ToString() ?? string.Empty;
|
|
||||||
if (filter is IDynamicStreamFilter dynFilter) {
|
|
||||||
dynFilter.onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, string filterExpression)
|
|
||||||
: this(vs, new BooleanExpressionFilter(filterExpression)) {
|
|
||||||
myFilterString = filterExpression;
|
|
||||||
}
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, string col, string regExp)
|
|
||||||
: this(vs, new RegExpFilter(col, regExp)) {
|
|
||||||
myFilterString = col + " , " + regExp;
|
|
||||||
}
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs)
|
|
||||||
: this(vs, new BinaryFilter(true)) {
|
|
||||||
myFilterString = true.ToString();
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region overridden methods
|
|
||||||
protected override void myOnFirstSubscriber() =>
|
|
||||||
myParentStream.onChange += myHandleCallbacks;
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
myLastFiredRows = null;
|
|
||||||
myParentStream.onChange -= myHandleCallbacks;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myFireEvent(object o) {
|
|
||||||
var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
|
|
||||||
if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
|
|
||||||
base.myFireEvent(o);
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
private void myHandleCallbacks(object newValue) {
|
|
||||||
var arrivedPacket = ExceptionAssert.Argument.Is<IDataPacket>(newValue);
|
|
||||||
myLastReceivedRows = arrivedPacket.newRows;
|
|
||||||
var bd = (myLastFiredRows ?? new BlueTableImmutable(arrivedPacket.Schema)).ToDataPacketBuilder();
|
|
||||||
foreach (IBlueRow r in arrivedPacket.deletedRows) {
|
|
||||||
// Don't bother evaluating the filter, just try to remove the row. The latter is
|
|
||||||
// cheaper than the former (it's a noop if this row wasn't in the filtered table).
|
|
||||||
bd.Remove(r.Id());
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in arrivedPacket.newRows) {
|
|
||||||
if (myFilter.filter(r)) {
|
|
||||||
bd.AddOrUpdate(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var dpOut = bd.ToDataPacket(out myLastFiredRows);
|
|
||||||
myFireEvent(dpOut);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IStreamFilter filter {
|
|
||||||
set {
|
|
||||||
{
|
|
||||||
myFilter = value;
|
|
||||||
if (numberOfSubscribers > 0) {
|
|
||||||
if (myLastReceivedRows != null) {
|
|
||||||
var bd = (myLastFiredRows ?? new BlueTableImmutable(myLastReceivedRows.schema)).ToDataPacketBuilder();
|
|
||||||
foreach (IBlueRow r in myLastReceivedRows) {
|
|
||||||
int id = r.Id();
|
|
||||||
bool includeNow = myFilter.filter(r);
|
|
||||||
bool includeBefore = myLastFiredRows?.containsID(id) ?? false;
|
|
||||||
if (includeNow == includeBefore) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (includeNow) {
|
|
||||||
bd.AddOrUpdate(r);
|
|
||||||
} else {
|
|
||||||
bd.Remove(id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var dpOut = bd.ToDataPacket(out myLastFiredRows);
|
|
||||||
myFireEvent(dpOut);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
get {
|
|
||||||
return myFilter;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//public string filterString { set { filter = new BooleanExpressionFilter(value); } }
|
|
||||||
|
|
||||||
public override string ToString() {
|
|
||||||
return GetType().ToString() + " " + myFilterString;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void myHandleFilterChange() {
|
|
||||||
filter = myFilter;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public class FilterTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection source;
|
|
||||||
readonly HashSet<int> excludedIndices = new();
|
|
||||||
readonly Dictionary<int, (int Index, int NumSkips, bool Included, IBlueRow Row)> rowMap = new();
|
|
||||||
readonly List<int> filteredIndexToId = new();
|
|
||||||
|
|
||||||
public FilterTable(IBlueRowCollection source, Func<IBlueRow, bool> filter) {
|
|
||||||
this.source = source;
|
|
||||||
int idx = 0;
|
|
||||||
int numSkips = 0;
|
|
||||||
foreach (var r in source) {
|
|
||||||
int id = r.Id();
|
|
||||||
bool included = filter(r);
|
|
||||||
rowMap.Add(id, (idx, numSkips, included, r));
|
|
||||||
if (!included) {
|
|
||||||
excludedIndices.Add(idx);
|
|
||||||
numSkips++;
|
|
||||||
} else {
|
|
||||||
filteredIndexToId.Add(id);
|
|
||||||
}
|
|
||||||
idx++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] {
|
|
||||||
get {
|
|
||||||
var r = rowMap[id];
|
|
||||||
return r.Included ? r.Row : throw new KeyNotFoundException();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => source.schema;
|
|
||||||
|
|
||||||
public int Count => filteredIndexToId.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
rowMap.TryGetValue(id, out var r) && r.Included;
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
int idx = 0;
|
|
||||||
foreach (var r in source) {
|
|
||||||
if (!excludedIndices.Contains(idx)) {
|
|
||||||
yield return r;
|
|
||||||
}
|
|
||||||
idx++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) {
|
|
||||||
if (index < 0 || index >= source.Count) {
|
|
||||||
throw new IndexOutOfRangeException();
|
|
||||||
}
|
|
||||||
var id = filteredIndexToId[index];
|
|
||||||
return rowMap[id].Row;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) =>
|
|
||||||
rowMap.TryGetValue(item.Id(), out var r)
|
|
||||||
&& r.Included
|
|
||||||
&& BlueRowEqualityComparer.Equals(r.Row, item)
|
|
||||||
? r.Index - r.NumSkips
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
namespace Bts.SharedStreams.StreamFactories;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.api.core;
|
|
||||||
using bts.BlueOrb;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Immutable;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Reactive.Linq;
|
|
||||||
using System.Threading;
|
|
||||||
|
|
||||||
public class GroupByApplyStream : ViewStream {
|
|
||||||
readonly ViewStream inputStream;
|
|
||||||
readonly IFunctionEvaluator groupBy;
|
|
||||||
readonly IFunctionEvaluator apply;
|
|
||||||
|
|
||||||
public GroupByApplyStream(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) {
|
|
||||||
this.inputStream = inputStream;
|
|
||||||
this.groupBy = groupBy;
|
|
||||||
this.apply = apply;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<IBlueRow, int>> CreateGroupByApplyTable(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) =>
|
|
||||||
CreateApplyTable(CreateGroupByTable(inputStream, groupBy), apply);
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<IBlueRow, int>> CreateApplyTable<TGroupKey>(IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey>> inputStream, Func<TGroupKey, IBlueRowCollection, IBlueRowCollection?> apply) where TGroupKey : IEquatable<TGroupKey> {
|
|
||||||
var groupTables = ImmutableDictionary<TGroupKey, GroupTableBuilder>.Empty;
|
|
||||||
var inputToOutputIds = ImmutableDictionary<int, int>.Empty;
|
|
||||||
var outputToInputIds = ImmutableDictionary<int, int>.Empty;
|
|
||||||
int idCounter = 0;
|
|
||||||
|
|
||||||
return Observable.Create<TableUpdate<IBlueRow, int>>(Subscribe);
|
|
||||||
|
|
||||||
int GetNextId() =>
|
|
||||||
Interlocked.Increment(ref idCounter);
|
|
||||||
|
|
||||||
IDisposable Subscribe(IObserver<TableUpdate<IBlueRow, int>> observer) {
|
|
||||||
int hashCounter = 0;
|
|
||||||
var hashIndex = new Dictionary<TGroupKey, int>();
|
|
||||||
var tableOut = TableUpdate<IBlueRow, int>.mk_empty(r => r.Id());
|
|
||||||
return inputStream.Subscribe(OnNext, OnError, OnCompleted);
|
|
||||||
|
|
||||||
void OnNext(TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey> upd) {
|
|
||||||
var bd = tableOut.to_builder();
|
|
||||||
foreach (var group in upd.removed) {
|
|
||||||
var hash = hashIndex[group.group_key];
|
|
||||||
bd.remove(hash);
|
|
||||||
}
|
|
||||||
var groupTablesBd = groupTables.ToBuilder();
|
|
||||||
foreach (var group in upd.added.Concat(upd.changed)) {
|
|
||||||
Log.assert(group.table.state.Count > 0);
|
|
||||||
var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++);
|
|
||||||
var groupRowCollection = new TableUpdateGroupingRowCollection<TGroupKey>(group, group.table.state.Values.First().schema);
|
|
||||||
var applyArgs = new (string?, int?, object?)[] { ("rows", 0, groupRowCollection) };
|
|
||||||
var resRows = apply(group.group_key, groupRowCollection);
|
|
||||||
if (resRows is null) {
|
|
||||||
bd.remove(hash);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
UpdateGroupTable(group.group_key, resRows);
|
|
||||||
}
|
|
||||||
groupTables = groupTablesBd.ToImmutable();
|
|
||||||
tableOut = bd.to_table();
|
|
||||||
observer.OnNext(tableOut);
|
|
||||||
|
|
||||||
void UpdateGroupTable(TGroupKey groupKey, IBlueRowCollection groupRows) {
|
|
||||||
var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows);
|
|
||||||
foreach (var r in indexedRows) {
|
|
||||||
bd.add_or_update(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void OnError(Exception ex) {
|
|
||||||
Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }""");
|
|
||||||
}
|
|
||||||
|
|
||||||
void OnCompleted() {
|
|
||||||
Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": "{{inputStream.GetType()}}" }""");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<IBlueRow, int>> CreateApplyTable<TGroupKey>(IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey>> inputStream, IFunctionEvaluator apply) where TGroupKey : IEquatable<TGroupKey> {
|
|
||||||
var groupTables = ImmutableDictionary<TGroupKey, GroupTableBuilder>.Empty;
|
|
||||||
var inputToOutputIds = ImmutableDictionary<int, int>.Empty;
|
|
||||||
var outputToInputIds = ImmutableDictionary<int, int>.Empty;
|
|
||||||
int idCounter = 0;
|
|
||||||
|
|
||||||
return Observable.Create<TableUpdate<IBlueRow, int>>(Subscribe);
|
|
||||||
|
|
||||||
int GetNextId() =>
|
|
||||||
Interlocked.Increment(ref idCounter);
|
|
||||||
|
|
||||||
IDisposable Subscribe(IObserver<TableUpdate<IBlueRow, int>> observer) {
|
|
||||||
int hashCounter = 0;
|
|
||||||
var hashIndex = new Dictionary<TGroupKey, int>();
|
|
||||||
var tableOut = TableUpdate<IBlueRow, int>.mk_empty(r => r.Id());
|
|
||||||
return inputStream.Subscribe(OnNext, OnError, OnCompleted);
|
|
||||||
|
|
||||||
void OnNext(TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey> upd) {
|
|
||||||
var bd = tableOut.to_builder();
|
|
||||||
foreach (var group in upd.removed) {
|
|
||||||
var hash = hashIndex[group.group_key];
|
|
||||||
bd.remove(hash);
|
|
||||||
}
|
|
||||||
var groupTablesBd = groupTables.ToBuilder();
|
|
||||||
foreach (var group in upd.added.Concat(upd.changed)) {
|
|
||||||
Log.assert(group.table.state.Count > 0);
|
|
||||||
var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++);
|
|
||||||
var groupRowCollection = new TableUpdateGroupingRowCollection<TGroupKey>(group, group.table.state.Values.First().schema);
|
|
||||||
var applyArgs = new StreamArgument[] { ("rows", 0, groupRowCollection) };
|
|
||||||
var res = apply.Evaluate(string.Empty, applyArgs);
|
|
||||||
if (res is IBlueRow resSingleRow) {
|
|
||||||
UpdateGroupTable(group.group_key, new SingleRowTable(resSingleRow));
|
|
||||||
} else if (res is IBlueRowCollection resRows) {
|
|
||||||
UpdateGroupTable(group.group_key, resRows);
|
|
||||||
} else {
|
|
||||||
throw new InvalidOperationException("Invalid result returned by 'Apply' fn");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
groupTables = groupTablesBd.ToImmutable();
|
|
||||||
tableOut = bd.to_table();
|
|
||||||
observer.OnNext(tableOut);
|
|
||||||
|
|
||||||
void UpdateGroupTable(TGroupKey groupKey, IBlueRowCollection groupRows) {
|
|
||||||
var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows);
|
|
||||||
foreach (var r in indexedRows) {
|
|
||||||
bd.add_or_update(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void OnError(Exception ex) {
|
|
||||||
Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }""");
|
|
||||||
}
|
|
||||||
|
|
||||||
void OnCompleted() {
|
|
||||||
Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": "{{inputStream.GetType()}}" }""");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>> CreateGroupByTable(ViewStream inputStream, IFunctionEvaluator groupBy) {
|
|
||||||
return Observable.Create<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>>(Subscribe);
|
|
||||||
|
|
||||||
IDisposable Subscribe(IObserver<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>> observer) {
|
|
||||||
TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper> tableOut = TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>.mk_empty(r => r.group_key);
|
|
||||||
return inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), GroupKey).Subscribe(observer);
|
|
||||||
|
|
||||||
EquatableKeyWrapper GroupKey(IBlueRow r) {
|
|
||||||
var groupByArgs = new StreamArgument[] { ("row", 0, r) };
|
|
||||||
var groupKey = new EquatableKeyWrapper(groupBy.Evaluate(string.Empty, groupByArgs) ?? DBNull.Value);
|
|
||||||
return groupKey;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyArrayWrapper>, EquatableKeyArrayWrapper>> CreateGroupByTable(ViewStream inputStream, string[] columns, bool useColumnIndices = true) {
|
|
||||||
Func<IBlueRow, EquatableKeyArrayWrapper> groupBy;
|
|
||||||
if (!useColumnIndices) {
|
|
||||||
groupBy = GroupKeyByName;
|
|
||||||
|
|
||||||
EquatableKeyArrayWrapper GroupKeyByName(IBlueRow r) =>
|
|
||||||
new(Array.ConvertAll(columns, c => r[c]));
|
|
||||||
} else {
|
|
||||||
int[]? columnIndices = null;
|
|
||||||
groupBy = GroupKeyByIndex;
|
|
||||||
|
|
||||||
EquatableKeyArrayWrapper GroupKeyByIndex(IBlueRow r) {
|
|
||||||
columnIndices ??= r.schema.ResolveIndices(columns);
|
|
||||||
return new(Array.ConvertAll(columnIndices, c => r[c]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return CreateGroupByTable(inputStream, groupBy);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>> CreateGroupByTable<TKey>(ViewStream inputStream, Func<IBlueRow, TKey> groupBy) where TKey : IEquatable<TKey> {
|
|
||||||
return Observable.Create<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>>(Subscribe);
|
|
||||||
|
|
||||||
IDisposable Subscribe(IObserver<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>> observer) =>
|
|
||||||
inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), groupBy).Subscribe(observer);
|
|
||||||
}
|
|
||||||
|
|
||||||
class GroupTableBuilder {
|
|
||||||
readonly Func<int> getUniqueId;
|
|
||||||
ImmutableDictionary<int, int> inputToOutputIds = ImmutableDictionary<int, int>.Empty;
|
|
||||||
ImmutableDictionary<int, int> outputToInputIds = ImmutableDictionary<int, int>.Empty;
|
|
||||||
IndexTable? current;
|
|
||||||
|
|
||||||
public GroupTableBuilder(Func<int> getUniqueId) =>
|
|
||||||
this.getUniqueId = getUniqueId;
|
|
||||||
|
|
||||||
public IndexTable? Current => current;
|
|
||||||
|
|
||||||
public IndexTable Reset(IBlueRowCollection rows) {
|
|
||||||
var inputToOutputIdsBd = inputToOutputIds.ToBuilder();
|
|
||||||
var outputToInputIdsBd = outputToInputIds.ToBuilder();
|
|
||||||
foreach (var r in rows) {
|
|
||||||
int inputId = r.Id();
|
|
||||||
if (!inputToOutputIdsBd.TryGetValue(inputId, out var outputId)) {
|
|
||||||
outputId = getUniqueId();
|
|
||||||
inputToOutputIdsBd.Add(inputId, outputId);
|
|
||||||
outputToInputIdsBd.Add(outputId, inputId);
|
|
||||||
}
|
|
||||||
Log.assert(outputToInputIdsBd.TryGetValue(outputId, out var oinputId) && oinputId == inputId);
|
|
||||||
Log.assert(inputToOutputIdsBd.TryGetValue(inputId, out var ioutputId) && ioutputId == outputId);
|
|
||||||
}
|
|
||||||
inputToOutputIds = inputToOutputIdsBd.ToImmutable();
|
|
||||||
outputToInputIds = outputToInputIdsBd.ToImmutable();
|
|
||||||
current = new(rows, outputToInputIds, inputToOutputIds);
|
|
||||||
return current;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,127 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text.Json;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
//using bts.SilverPine;
|
|
||||||
using BCG;
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
namespace bts.BlueOrb {
|
|
||||||
[DataStreamUserRespresentation("GroupByStream")]
|
|
||||||
public class GroupByStream : ViewStream {
|
|
||||||
|
|
||||||
public interface INonIncGroupByEvaluator {
|
|
||||||
// grouped_rows
|
|
||||||
IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
|
||||||
string[]? get_group_by_cols();
|
|
||||||
}
|
|
||||||
|
|
||||||
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
|
||||||
Func<string[]?> _groupby_cols_func;
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
|
||||||
public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
|
||||||
_groupby_cols_func = groupby_cols_func;
|
|
||||||
_evaluate_func = evaluate_func;
|
|
||||||
}
|
|
||||||
public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
|
||||||
BlueTable? rv = null;
|
|
||||||
foreach (var kv in grouped_rows) {
|
|
||||||
List<IBlueRow> g_rows = kv.Value;
|
|
||||||
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
|
||||||
if (out_rows is null || out_rows.Count == 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
rv ??= new BlueTable(out_rows.ElementAt(0).schema);
|
|
||||||
rv.addRows(out_rows);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
public string[]? get_group_by_cols() {
|
|
||||||
return _groupby_cols_func();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
readonly ViewStream? _main_stream;
|
|
||||||
readonly INonIncGroupByEvaluator? _groupby_func;
|
|
||||||
readonly string[]? _groupby_cols;
|
|
||||||
readonly Stopwatch _timer = new();
|
|
||||||
|
|
||||||
BlueRowHasher? _row_hasher;
|
|
||||||
int _n_total_fires = 0;
|
|
||||||
int _n_expensive_fires = 0;
|
|
||||||
|
|
||||||
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
|
||||||
_main_stream = main_stream;
|
|
||||||
_groupby_func = groupby_func;
|
|
||||||
_groupby_cols = _groupby_func.get_group_by_cols();
|
|
||||||
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
|
||||||
}
|
|
||||||
|
|
||||||
public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
|
||||||
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
|
||||||
|
|
||||||
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
|
|
||||||
: this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
|
|
||||||
|
|
||||||
// this is the main constructor that does everything
|
|
||||||
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
|
|
||||||
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
|
|
||||||
Func<string[]?> groupby_cols_func = () => {
|
|
||||||
if (String.IsNullOrEmpty(groupby_cols))
|
|
||||||
return null;
|
|
||||||
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
|
|
||||||
};
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
|
||||||
if (!String.IsNullOrEmpty(optional_sort_col))
|
|
||||||
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
|
|
||||||
// this si where the group by aggregation happens
|
|
||||||
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
|
|
||||||
if (rv == null)
|
|
||||||
return new List<IBlueRow>();
|
|
||||||
return new IBlueRow[] { rv }.ToList();
|
|
||||||
};
|
|
||||||
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
|
|
||||||
this(parent_str, String.Empty, standard_group_by_text) {
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { Log.notnull(_main_stream).onChange += _main_stream_onchange; }
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() { Log.notnull(_main_stream).onChange -= _main_stream_onchange; }
|
|
||||||
|
|
||||||
private void _main_stream_onchange(object nv) {
|
|
||||||
var dp = ExceptionAssert.Argument.Is<IDataPacket>(nv);
|
|
||||||
///grouping together
|
|
||||||
_timer.Restart();
|
|
||||||
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
|
||||||
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
|
|
||||||
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
|
|
||||||
_timer.Stop();
|
|
||||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
|
||||||
if (groupElapsedMs >= 50) {
|
|
||||||
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
|
||||||
}
|
|
||||||
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
|
||||||
/// reducing
|
|
||||||
_timer.Restart();
|
|
||||||
var total_state = Log.notnull(_groupby_func).evaluate(grouped_rows);
|
|
||||||
if (total_state == null)
|
|
||||||
// mayeb returna n empty table ?
|
|
||||||
return;
|
|
||||||
_timer.Stop();
|
|
||||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
|
||||||
_n_total_fires++;
|
|
||||||
if (applyElapsedMs >= 50) {
|
|
||||||
_n_expensive_fires++;
|
|
||||||
Log.DataFlow.info(
|
|
||||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
|
||||||
}
|
|
||||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class GroupByTable<TKey> : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
readonly IBlueRowCollection inputRows;
|
|
||||||
readonly List<IBlueRow> rows;
|
|
||||||
readonly Dictionary<int, int> idToIndex;
|
|
||||||
|
|
||||||
public GroupByTable(IBlueRowCollection inputRows, Func<IBlueRow, TKey> getKey, IEqualityComparer<TKey> keyEqualityComparer, IComparer<TKey> keyComparer, string? groupKeyCol = "group") {
|
|
||||||
this.inputRows = inputRows;
|
|
||||||
var cols = inputRows.schema.columnInfo.ToList();
|
|
||||||
rows = new(inputRows.Count);
|
|
||||||
int groupKeyColIdx = -1;
|
|
||||||
if (!string.IsNullOrEmpty(groupKeyCol)) {
|
|
||||||
groupKeyColIdx = cols.Count;
|
|
||||||
cols.Add(new(groupKeyCol, groupKeyColIdx, typeof(TKey)));
|
|
||||||
}
|
|
||||||
for (int i = 0; i < cols.Count; i++) {
|
|
||||||
Log.assert(cols[i].colNo == i);
|
|
||||||
}
|
|
||||||
outputSchema = new BlueTableSchema(cols.ToArray(), "GroupBy");
|
|
||||||
rows.AddRange(inputRows
|
|
||||||
.GroupBy(getKey, keyEqualityComparer)
|
|
||||||
.OrderBy(g => g.Key, keyComparer)
|
|
||||||
.SelectMany(rg => rg.Select(r => new Row(rg.Key, r, outputSchema, groupKeyColIdx))));
|
|
||||||
idToIndex = rows.Select((r, i) => (r, i)).ToDictionary(t => t.r.Id(), t => t.i);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
rows[idToIndex[id]];
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => outputSchema;
|
|
||||||
|
|
||||||
public int Count => rows.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
idToIndex.ContainsKey(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() =>
|
|
||||||
rows.GetEnumerator();
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
rows[index];
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) =>
|
|
||||||
idToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(item, rows[idx])
|
|
||||||
? idx
|
|
||||||
: -1;
|
|
||||||
|
|
||||||
public class Row : IBlueRow {
|
|
||||||
readonly IBlueRow inputRow;
|
|
||||||
readonly IBlueTableSchema outputSchema;
|
|
||||||
readonly int keyColIdx;
|
|
||||||
|
|
||||||
public Row(TKey key, IBlueRow inputRow, IBlueTableSchema outputSchema, int keyColIdx = -1) {
|
|
||||||
Key = key;
|
|
||||||
this.inputRow = inputRow;
|
|
||||||
this.outputSchema = outputSchema;
|
|
||||||
this.keyColIdx = keyColIdx;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] {
|
|
||||||
get {
|
|
||||||
if (index == keyColIdx) {
|
|
||||||
return Key;
|
|
||||||
}
|
|
||||||
return inputRow[index];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public TKey Key { get; }
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => outputSchema;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,229 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Immutable;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class ImmutableOrderedTable : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows;
|
|
||||||
readonly IReadOnlyDictionary<long, IBlueRow> ordinalToRow;
|
|
||||||
readonly ImmutableDictionary<int, long> idToOrdinal;
|
|
||||||
long newRowOrdinal = 0;
|
|
||||||
|
|
||||||
public ImmutableOrderedTable(IBlueTableSchema schema)
|
|
||||||
: this(
|
|
||||||
schema,
|
|
||||||
ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<long, IBlueRow>()),
|
|
||||||
ImmutableDictionary<int, long>.Empty, 0) { }
|
|
||||||
|
|
||||||
ImmutableOrderedTable(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows, ImmutableDictionary<int, long> idToOrdinal, long newRowOrdinal) {
|
|
||||||
this.schema = schema;
|
|
||||||
this.rows = rows;
|
|
||||||
this.idToOrdinal = idToOrdinal;
|
|
||||||
this.newRowOrdinal = newRowOrdinal;
|
|
||||||
ordinalToRow = ImmutableSortedSetDictionary.Create(rows);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static OrderedTable FromRowCollection(IBlueRowCollection rowCollection) {
|
|
||||||
var rowList = new List<IBlueRow>(rowCollection.Count);
|
|
||||||
var idToIndex = new Dictionary<int, int>();
|
|
||||||
foreach (var r in rowCollection) {
|
|
||||||
idToIndex[r.Id()] = rowList.Count;
|
|
||||||
rowList.Add(r);
|
|
||||||
}
|
|
||||||
return new(rowCollection.schema, rowList, idToIndex);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema Schema => schema;
|
|
||||||
|
|
||||||
public int Count => rows.Count;
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
idToOrdinal.TryGetValue(id, out var ord) ? ordinalToRow[ord] : throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
idToOrdinal.ContainsKey(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
foreach (var (_, row) in rows) {
|
|
||||||
yield return row;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
rows[index].Value;
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) =>
|
|
||||||
idToOrdinal.TryGetValue(item.Id(), out var ord) ? rows.IndexOf(new(ord, null!)) : -1;
|
|
||||||
|
|
||||||
public Builder ToBuilder() =>
|
|
||||||
new(schema, rows, idToOrdinal, newRowOrdinal);
|
|
||||||
|
|
||||||
public class Builder : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
|
|
||||||
ImmutableSortedSet<KeyValuePair<long, IBlueRow>> originalRows;
|
|
||||||
ImmutableSortedSet<KeyValuePair<long, IBlueRow>>.Builder rows;
|
|
||||||
ImmutableDictionary<int, long> originalIdToOrdinal;
|
|
||||||
ImmutableDictionary<int, long>.Builder idToOrdinal;
|
|
||||||
IReadOnlyDictionary<long, IBlueRow> originalOrdinalToRow;
|
|
||||||
IReadOnlyDictionary<long, IBlueRow> ordinalToRow;
|
|
||||||
|
|
||||||
readonly HashSet<int> removes = new();
|
|
||||||
readonly HashSet<int> adds = new();
|
|
||||||
long newRowOrdinal;
|
|
||||||
|
|
||||||
public Builder(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows, ImmutableDictionary<int, long> idToOrdinal, long newRowOrdinal) {
|
|
||||||
this.schema = schema;
|
|
||||||
originalRows = rows;
|
|
||||||
this.rows = rows.ToBuilder();
|
|
||||||
originalIdToOrdinal = idToOrdinal;
|
|
||||||
this.idToOrdinal = idToOrdinal.ToBuilder();
|
|
||||||
originalOrdinalToRow = ImmutableSortedSetDictionary.Create(originalRows);
|
|
||||||
ordinalToRow = ImmutableSortedSetDictionary.CreateBuilder(this.rows);
|
|
||||||
this.newRowOrdinal = newRowOrdinal;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] {
|
|
||||||
get {
|
|
||||||
if (idToOrdinal.TryGetValue(id, out var ord) && !removes.Contains(id)) {
|
|
||||||
return ordinalToRow[ord];
|
|
||||||
}
|
|
||||||
throw new KeyNotFoundException();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public int Count => rows.Count + adds.Count - removes.Count;
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public void Add(IBlueRow row) =>
|
|
||||||
addOrUpdate(row, false);
|
|
||||||
|
|
||||||
public void AddOrUpdate(IBlueRow row) =>
|
|
||||||
addOrUpdate(row, true);
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
(idToOrdinal.ContainsKey(id) && !removes.Contains(id)) || adds.Contains(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
foreach (var row in rows) {
|
|
||||||
if (!removes.Contains(row.Value.Id())) {
|
|
||||||
yield return row.Value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
rows[index].Value;
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
if (idToOrdinal.TryGetValue(item.Id(), out var ord)) {
|
|
||||||
return rows.IndexOf(new(ord, item));
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool Remove(int id) {
|
|
||||||
if (!idToOrdinal.TryGetValue(id, out var ord)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
idToOrdinal.Remove(id);
|
|
||||||
Log.assert(rows.Remove(new(ord, null!)));
|
|
||||||
Log.assert(ordinalToRow.Count == rows.Count);
|
|
||||||
adds.Remove(id);
|
|
||||||
if (originalIdToOrdinal.ContainsKey(id)) {
|
|
||||||
removes.Add(id);
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool ICollection<IBlueRow>.Remove(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
if (!idToOrdinal.TryGetValue(id, out var ord) || !BlueRowEqualityComparer.Equals(item, ordinalToRow[ord])) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
rows.Remove(new(ord, null!));
|
|
||||||
adds.Remove(id);
|
|
||||||
if (originalIdToOrdinal.ContainsKey(id)) {
|
|
||||||
removes.Add(id);
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ImmutableOrderedTable ToImmutable(out IDataPacket dp) {
|
|
||||||
var newRows = new ImmutableOrderedTable(schema, rows.ToImmutable(), idToOrdinal.ToImmutable(), newRowOrdinal);
|
|
||||||
dp = new DataPacket(
|
|
||||||
newRows,
|
|
||||||
new BlueTableListWrapper(schema, adds.Select(id => ordinalToRow[idToOrdinal[id]]).ToList()),
|
|
||||||
new BlueTableListWrapper(schema, removes.Select(id => originalOrdinalToRow[originalIdToOrdinal[id]]).ToList()));
|
|
||||||
return newRows;
|
|
||||||
}
|
|
||||||
|
|
||||||
void addOrUpdate(IBlueRow row, bool canUpdate) {
|
|
||||||
if (newRowOrdinal == long.MaxValue) {
|
|
||||||
resetOrdinal();
|
|
||||||
}
|
|
||||||
int id = row.Id();
|
|
||||||
if (idToOrdinal.TryGetValue(id, out var ord)) {
|
|
||||||
if (!canUpdate) {
|
|
||||||
throw new ArgumentException("Row id already present", nameof(row));
|
|
||||||
}
|
|
||||||
rows.Remove(new(ord, row));
|
|
||||||
}
|
|
||||||
ord = newRowOrdinal++;
|
|
||||||
idToOrdinal[id] = ord;
|
|
||||||
if (originalIdToOrdinal.ContainsKey(id)) {
|
|
||||||
removes.Add(id);
|
|
||||||
}
|
|
||||||
rows.Add(new(ord, row));
|
|
||||||
adds.Add(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
void resetOrdinal() {
|
|
||||||
Log.DataFlow.warn($$"""[immutable_ordered_table_reset_ordinal.json]{ "tableName": "{{schema.tableName}}", "origNumRows": {{originalRows.Count}}, "numRows": {{rows.Count}}, "ts": {{DateTime.Now}} }""");
|
|
||||||
var originalRowsBd = originalRows.Clear().ToBuilder();
|
|
||||||
var originalIdToOrdinalBd = originalIdToOrdinal.Clear().ToBuilder();
|
|
||||||
var newRowsBd = ImmutableSortedSet.CreateBuilder(rows.KeyComparer);
|
|
||||||
var newIdToOrdinalBd = ImmutableDictionary.CreateBuilder<int, long>(idToOrdinal.KeyComparer);
|
|
||||||
long ord = 0;
|
|
||||||
long? prevRowOrd = null;
|
|
||||||
foreach (var kv in originalRows) {
|
|
||||||
if (prevRowOrd.HasValue) {
|
|
||||||
Log.assert(prevRowOrd.Value < kv.Key);
|
|
||||||
}
|
|
||||||
prevRowOrd = kv.Key;
|
|
||||||
originalRowsBd.Add(new(ord, kv.Value));
|
|
||||||
originalIdToOrdinalBd.Add(new(kv.Value.Id(), ord));
|
|
||||||
ord++;
|
|
||||||
}
|
|
||||||
prevRowOrd = null;
|
|
||||||
foreach (var kv in rows) {
|
|
||||||
if (prevRowOrd.HasValue) {
|
|
||||||
Log.assert(prevRowOrd.Value < kv.Key);
|
|
||||||
}
|
|
||||||
prevRowOrd = kv.Key;
|
|
||||||
newRowsBd.Add(new(ord, kv.Value));
|
|
||||||
newIdToOrdinalBd.Add(new(kv.Value.Id(), ord));
|
|
||||||
ord++;
|
|
||||||
}
|
|
||||||
rows = newRowsBd;
|
|
||||||
originalRows = originalRowsBd.ToImmutable();
|
|
||||||
originalIdToOrdinal = originalIdToOrdinalBd.ToImmutable();
|
|
||||||
idToOrdinal = newIdToOrdinalBd;
|
|
||||||
originalOrdinalToRow = ImmutableSortedSetDictionary.Create(originalRows);
|
|
||||||
ordinalToRow = ImmutableSortedSetDictionary.CreateBuilder(rows);
|
|
||||||
newRowOrdinal = ord;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
public class IndexRow : IBlueRow {
|
|
||||||
readonly IBlueRow inputRow;
|
|
||||||
readonly int id;
|
|
||||||
readonly int idIndex;
|
|
||||||
|
|
||||||
public IndexRow(IBlueRow inputRow, int id, int idIndex, IBlueTableSchema schema) {
|
|
||||||
this.inputRow = inputRow;
|
|
||||||
this.id = id;
|
|
||||||
this.idIndex = idIndex;
|
|
||||||
this.schema = schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow InputRow => inputRow;
|
|
||||||
|
|
||||||
public IBlueTableSchema schema { get; }
|
|
||||||
|
|
||||||
public object? this[int index] =>
|
|
||||||
index == idIndex ? id : inputRow[index];
|
|
||||||
}
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Immutable;
|
|
||||||
|
|
||||||
public class IndexStream : ViewStream {
|
|
||||||
readonly ViewStream inputStream;
|
|
||||||
|
|
||||||
ImmutableDictionary<int, int> inputToOutputIds = ImmutableDictionary<int, int>.Empty;
|
|
||||||
ImmutableDictionary<int, int> outputToInputIds = ImmutableDictionary<int, int>.Empty;
|
|
||||||
int idCounter = 0;
|
|
||||||
IDisposable? sub;
|
|
||||||
|
|
||||||
public IndexStream(ViewStream inputStream) =>
|
|
||||||
this.inputStream = inputStream;
|
|
||||||
|
|
||||||
~IndexStream() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
base.myOnFirstSubscriber();
|
|
||||||
sub = inputStream.Subscribe(OnInputChanged);
|
|
||||||
|
|
||||||
void OnInputChanged(IDataPacket dp) {
|
|
||||||
var inputToOutputIdsBd = inputToOutputIds.ToBuilder();
|
|
||||||
var outputToInputIdsBd = outputToInputIds.ToBuilder();
|
|
||||||
foreach (var r in dp.addedRows) {
|
|
||||||
int inputId = r.Id();
|
|
||||||
if (!inputToOutputIds.TryGetValue(inputId, out int outputId)) {
|
|
||||||
outputId = ++idCounter;
|
|
||||||
inputToOutputIdsBd[inputId] = outputId;
|
|
||||||
outputToInputIdsBd[outputId] = inputId;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
inputToOutputIds = inputToOutputIdsBd.ToImmutable();
|
|
||||||
outputToInputIds = outputToInputIdsBd.ToImmutable();
|
|
||||||
myFireEvent(new DataPacket(
|
|
||||||
new IndexTable(dp.newRows, outputToInputIds, inputToOutputIds),
|
|
||||||
new IndexTable(dp.addedRows, outputToInputIds, inputToOutputIds),
|
|
||||||
new IndexTable(dp.deletedRows, outputToInputIds, inputToOutputIds)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose(ref sub);
|
|
||||||
}
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Immutable;
|
|
||||||
|
|
||||||
public class IndexTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection inputRows;
|
|
||||||
readonly IDictionary<int, int> outputToInputIdMap;
|
|
||||||
readonly IDictionary<int, int> inputToOutputIdMap;
|
|
||||||
readonly int idColIndex;
|
|
||||||
|
|
||||||
public IndexTable(IBlueRowCollection inputRows, IDictionary<int, int> outputToInputIdMap, IDictionary<int, int> inputToOutputIdMap) {
|
|
||||||
this.inputRows = inputRows;
|
|
||||||
this.outputToInputIdMap = outputToInputIdMap;
|
|
||||||
this.inputToOutputIdMap = inputToOutputIdMap;
|
|
||||||
if (!inputRows.schema.columnInfo.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
|
|
||||||
throw new ArgumentException("No 'id' column", nameof(inputRows));
|
|
||||||
}
|
|
||||||
idColIndex = idColInfo.colNo;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => inputRows.schema;
|
|
||||||
|
|
||||||
public int Count => inputRows.Count;
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
outputToInputIdMap.TryGetValue(id, out var inputId)
|
|
||||||
? new IndexRow(inputRows[inputId], id, idColIndex, schema)
|
|
||||||
: throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
IBlueRow IList<IBlueRow>.this[int index] {
|
|
||||||
get => GetRow(index);
|
|
||||||
set => throw new NotSupportedException();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IndexTable Create(IBlueRowCollection inputRows) {
|
|
||||||
var outputToInputIdMap = new Dictionary<int, int>();
|
|
||||||
var inputToOutputIdMap = new Dictionary<int, int>();
|
|
||||||
foreach (var r in inputRows) {
|
|
||||||
int inputId = r.Id();
|
|
||||||
int outputId = outputToInputIdMap.Count + 1;
|
|
||||||
outputToInputIdMap[outputId] = inputId;
|
|
||||||
inputToOutputIdMap[inputId] = outputId;
|
|
||||||
}
|
|
||||||
return new(inputRows, outputToInputIdMap, inputToOutputIdMap);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static BlueTableImmutable CreateImmutable(IBlueTableSchema schema, IEnumerable<IBlueRow> rows) {
|
|
||||||
int idIndex = schema.ResolveIndices("id");
|
|
||||||
var rowSetBd = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, IBlueRow>()).ToBuilder();
|
|
||||||
foreach (var r in rows) {
|
|
||||||
int idx = rowSetBd.Count + 1;
|
|
||||||
var inputRow = (r as IndexRow)?.InputRow ?? r;
|
|
||||||
rowSetBd.Add(new(idx, new IndexRow(inputRow, idx, idIndex, schema)));
|
|
||||||
}
|
|
||||||
return new(schema, rowSetBd.ToImmutable());
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
inputRows.containsID(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
foreach (var r in inputRows) {
|
|
||||||
yield return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) {
|
|
||||||
var r = inputRows.GetRow(index);
|
|
||||||
return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema);
|
|
||||||
}
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
if (outputToInputIdMap.TryGetValue(id, out var inputId)) {
|
|
||||||
var inputRow = inputRows[inputId];
|
|
||||||
var r = new IndexRow(inputRow, id, idColIndex, schema);
|
|
||||||
if (BlueRowEqualityComparer.Equals(r, item)) {
|
|
||||||
var idx = inputRows.IndexOf(inputRow);
|
|
||||||
Log.assert(idx >= 0);
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,196 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
using RowHash = System.Int32;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Data.Entity.Core.Mapping;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("JoinStream")]
|
|
||||||
public class SimpleJoinStream : ViewStream {
|
|
||||||
public class JoinBookkeepingData {
|
|
||||||
// initialized in cons
|
|
||||||
private readonly List<string> _join_col_names;
|
|
||||||
private readonly bool _is_left;
|
|
||||||
private readonly BlueRowHasher _hasher;
|
|
||||||
// initialized on first udpate
|
|
||||||
private IBlueTableSchema _schema;
|
|
||||||
private int[] _join_cols;
|
|
||||||
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
|
|
||||||
|
|
||||||
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
|
|
||||||
_join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
|
|
||||||
_is_left = is_left;
|
|
||||||
_hasher = row_hasher;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => _schema;
|
|
||||||
|
|
||||||
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
|
|
||||||
|
|
||||||
public List<IBlueRow> get_matching_rows(int hash) {
|
|
||||||
List<IBlueRow> rv = null;
|
|
||||||
_hash_row_map.TryGetValue(hash, out rv);
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
private RowHash _get_hash(IBlueRow r) {
|
|
||||||
if (_join_cols == null || _join_cols.Length == 0) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return _hasher.get_hash(r, _join_cols);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void update_data(DataPacket dp) {
|
|
||||||
if (!ready) {
|
|
||||||
_schema = dp.newRows.schema;
|
|
||||||
_join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
|
|
||||||
}
|
|
||||||
_hash_row_map.Clear();
|
|
||||||
foreach (IBlueRow r in dp.newRows) {
|
|
||||||
int hash = _get_hash(r);
|
|
||||||
List<IBlueRow> hash_rows = null;
|
|
||||||
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
|
|
||||||
hash_rows = new List<IBlueRow>();
|
|
||||||
_hash_row_map.Add(hash, hash_rows);
|
|
||||||
}
|
|
||||||
hash_rows.Add(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool ready {
|
|
||||||
get {
|
|
||||||
return _schema != null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private ViewStream _ls, _rs;
|
|
||||||
private JoinBookkeepingData _ldata, _rdata;
|
|
||||||
private ViewStreamDatabase _vsd;
|
|
||||||
private IBlueTableSchema _schema = null;
|
|
||||||
|
|
||||||
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
|
|
||||||
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
|
|
||||||
|
|
||||||
public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
|
|
||||||
_ls = ls;
|
|
||||||
_rs = rs;
|
|
||||||
List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
|
||||||
List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
|
||||||
Log.assert(lj_col_list.Count == rj_col_list.Count);
|
|
||||||
int n_cols = lj_col_list.Count;
|
|
||||||
BlueRowHasher hasher = new BlueRowHasher(n_cols);
|
|
||||||
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
|
|
||||||
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_ls.onChange += _ls_onchange;
|
|
||||||
_rs.onChange += _rs_onchange;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _ls_onchange(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
_ldata.update_data(dp);
|
|
||||||
_form_and_fire();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _rs_onchange(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
_rdata.update_data(dp);
|
|
||||||
_form_and_fire();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_ls.onChange -= _ls_onchange;
|
|
||||||
_rs.onChange -= _rs_onchange;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _build_schema() {
|
|
||||||
if (_schema != null)
|
|
||||||
return;
|
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
|
||||||
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
|
|
||||||
foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
|
|
||||||
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
|
|
||||||
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
|
||||||
}
|
|
||||||
foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
|
|
||||||
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
|
|
||||||
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
|
||||||
}
|
|
||||||
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _form_and_fire() {
|
|
||||||
if (!_ldata.ready || !_rdata.ready)
|
|
||||||
return;
|
|
||||||
_build_schema();
|
|
||||||
List<object[]> res_rows = new List<object[]>();
|
|
||||||
|
|
||||||
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
|
|
||||||
int offset = 1;
|
|
||||||
JoinBookkeepingData sdata = _ldata;
|
|
||||||
if (is_left) {
|
|
||||||
offset = 1;
|
|
||||||
sdata = _ldata;
|
|
||||||
} else {
|
|
||||||
sdata = _rdata;
|
|
||||||
offset = 1 + _ldata.schema.columnInfo.Length;
|
|
||||||
}
|
|
||||||
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
|
|
||||||
rval[offset + i] = r[i];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
|
|
||||||
object[] rv = new object[_schema.columnNames.Length];
|
|
||||||
for (int i = 0; i < _schema.columnNames.Length; i++) {
|
|
||||||
rv[i] = DBNull.Value;
|
|
||||||
}
|
|
||||||
rv[0] = id;
|
|
||||||
copy_row(rv, l_row, true);
|
|
||||||
return rv;
|
|
||||||
};
|
|
||||||
// this is where the join happens
|
|
||||||
foreach (var kv in _ldata.rows) {
|
|
||||||
RowHash hash = kv.Key;
|
|
||||||
List<IBlueRow> l_rows = kv.Value;
|
|
||||||
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
|
|
||||||
|
|
||||||
foreach (IBlueRow lr in l_rows) {
|
|
||||||
if (r_rows != null) {
|
|
||||||
foreach (IBlueRow r in r_rows) {
|
|
||||||
object[] rowvals = build_row_array(res_rows.Count, lr);
|
|
||||||
copy_row(rowvals, r, false);
|
|
||||||
res_rows.Add(rowvals);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
object[] rowvals = build_row_array(res_rows.Count, lr);
|
|
||||||
res_rows.Add(rowvals);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
BlueTable rv = new BlueTable(_schema);
|
|
||||||
foreach (object[] rval in res_rows) {
|
|
||||||
rv.addRow(new BlueRow(_schema, rval));
|
|
||||||
}
|
|
||||||
var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("FullJoinStream")]
|
|
||||||
public class FullJoinStream : ViewStream {
|
|
||||||
public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
|
|
||||||
ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
|
|
||||||
ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
//namespace bts.Blueorb {
|
|
||||||
// using BCG;
|
|
||||||
|
|
||||||
// public class JsonSaverStream : ViewStream {
|
|
||||||
// private ViewStream _stream;
|
|
||||||
// private string _file_name;
|
|
||||||
// public JsonSaverStream(ViewStream vs, string file_name) {
|
|
||||||
// _stream = vs;
|
|
||||||
// _file_name = file_name;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
|
||||||
// _stream.onChange += _on_data;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _on_data(object dp_obj) {
|
|
||||||
// if (dp_obj == null) {
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
// DataPacket dp = dp_obj as DataPacket;
|
|
||||||
// dp.newRows.write_to_json_file(dp.newRows, _file_name);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
|
||||||
// _stream.onChange -= _on_data;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
using Amazon.Runtime.Internal.Util;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
using BCG.Utilities;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
namespace bts.BlueOrb{
|
|
||||||
public class LiteralStream : ViewStream {
|
|
||||||
|
|
||||||
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
|
||||||
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
|
|
||||||
string[] pairs = key_value_pairs.Split(' ');
|
|
||||||
foreach (string p in pairs) {
|
|
||||||
string[] kvp = p.Split(':');
|
|
||||||
string col_name = kvp[0];
|
|
||||||
string val = kvp[1];
|
|
||||||
var tup = BlueDataUtility.guess_type(val);
|
|
||||||
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
|
|
||||||
rv.Add(Tuple.Create(ci, tup.Item2));
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
|
|
||||||
// col_name: col_value ... ; col_name: col_value
|
|
||||||
public LiteralStream(string table_name, string rows) {
|
|
||||||
Func<string, int, IBlueRow> get_row = (json_row, id) => {
|
|
||||||
Dictionary<string, object> map = new Dictionary<string, object>();
|
|
||||||
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
|
|
||||||
//Logger.assert(col_vals.Length == 2);
|
|
||||||
foreach (string t in toks) {
|
|
||||||
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
|
|
||||||
string col = col_vals[0];
|
|
||||||
string val = col_vals[1];
|
|
||||||
map.Add(col, val);
|
|
||||||
}
|
|
||||||
map["id"] = id;
|
|
||||||
return new BlueRow(map);
|
|
||||||
};
|
|
||||||
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
|
||||||
BlueTable table = null;
|
|
||||||
foreach (string json_string in row_arr) {
|
|
||||||
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
|
||||||
get_row(json_string, table == null ? 0 : table.Count);
|
|
||||||
table = table ?? new BlueTable(r.schema);
|
|
||||||
table.addRow(r);
|
|
||||||
}
|
|
||||||
ViewStream str = new BlueTableStream(table);
|
|
||||||
str = new SchemaGuesserStream(str);
|
|
||||||
mySetRealSource(str);
|
|
||||||
}
|
|
||||||
|
|
||||||
public LiteralStream(string tableName, string schema, string value) {
|
|
||||||
string[] columns = schema.Split(' ');
|
|
||||||
|
|
||||||
|
|
||||||
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
|
|
||||||
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
|
||||||
int n = 1;
|
|
||||||
foreach (string c in columns) {
|
|
||||||
string[] columnAndType = c.Split('/');
|
|
||||||
if (columnAndType.Length == 1)
|
|
||||||
ci[n] = new ColumnInfo(c, n, typeof(double));
|
|
||||||
else
|
|
||||||
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
|
||||||
n++;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
|
|
||||||
string[] rows = value.Split(';');
|
|
||||||
n = 0;
|
|
||||||
foreach (string r in rows)
|
|
||||||
try {
|
|
||||||
if (r == string.Empty)
|
|
||||||
continue;
|
|
||||||
t.addRow(
|
|
||||||
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
|
||||||
}
|
|
||||||
|
|
||||||
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,96 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class MeanTable : IBlueRowCollection {
|
|
||||||
readonly IBlueRowCollection input;
|
|
||||||
readonly IList<object?>[] columns;
|
|
||||||
|
|
||||||
public MeanTable(IBlueRowCollection input) {
|
|
||||||
this.input = input;
|
|
||||||
var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
|
|
||||||
if (colInfos.Length > 0) {
|
|
||||||
Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
|
|
||||||
}
|
|
||||||
columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
|
|
||||||
for (int i = 0; i < colInfos.Length; i++) {
|
|
||||||
if (colInfos[i].colName == "id") {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
var ci = colInfos[i];
|
|
||||||
switch (ci.type_code()) {
|
|
||||||
case TypeCode.Int16:
|
|
||||||
case TypeCode.Int32:
|
|
||||||
case TypeCode.Int64:
|
|
||||||
case TypeCode.UInt16:
|
|
||||||
case TypeCode.UInt32:
|
|
||||||
case TypeCode.UInt64:
|
|
||||||
case TypeCode.Single:
|
|
||||||
colInfos[i] = new(ci.colName, ci.colNo, typeof(double));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
schema = new BlueTableSchema(colInfos, input.schema.tableName + "_Mean");
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
id == 1 ? GetRow(0) : throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
public IBlueTableSchema schema { get; }
|
|
||||||
|
|
||||||
public int Count => 1;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
id == 1;
|
|
||||||
|
|
||||||
public IList<object?> GetColumn(int colIndex) {
|
|
||||||
var count = input.Count;
|
|
||||||
var colInfo = input.schema.columnInfo[colIndex];
|
|
||||||
if (colInfo.colName == "id") {
|
|
||||||
return new object?[] { 1 };
|
|
||||||
}
|
|
||||||
var typeCode = colInfo.type_code();
|
|
||||||
var colVals = input.GetColumn(colIndex);
|
|
||||||
object? res = typeCode switch {
|
|
||||||
TypeCode.Int16 => colVals.Select(x => x is short v ? (double?)v : null).SelectWhereValues().Average(),
|
|
||||||
TypeCode.Int32 => colVals.Select(x => x is int v ? (double?)v : null).SelectWhereValues().Average(),
|
|
||||||
TypeCode.Int64 => colVals.Select(x => x is long v ? (double?)v : null).SelectWhereValues().Average(),
|
|
||||||
TypeCode.UInt16 => colVals.Select(x => x is ushort v ? (double?)v : null).SelectWhereValues().Average(),
|
|
||||||
TypeCode.UInt32 => colVals.Select(x => x is uint v ? (double?)v : null).SelectWhereValues().Average(),
|
|
||||||
TypeCode.UInt64 => colVals.Select(x => x is ulong v ? (double?)v : null).SelectWhereValues().Average(),
|
|
||||||
TypeCode.Single => colVals.Select(x => x is float v ? (double?)v : null).SelectWhereValues().Average(),
|
|
||||||
TypeCode.Double => colVals.Select(x => x as double?).SelectWhereValues().Average(),
|
|
||||||
TypeCode.DateTime => new DateTime((long)colVals.Cast<DateTime>().Select(dt => (double)dt.Ticks).Average()),
|
|
||||||
_ => null,
|
|
||||||
};
|
|
||||||
return new object?[] { res };
|
|
||||||
}
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
yield return GetRow(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new BlueRowColumnWrapper(input.schema, index, columns);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
var inputRow = input[id];
|
|
||||||
int idx = input.IndexOf(inputRow);
|
|
||||||
if (idx != -1) {
|
|
||||||
var row = GetRow(idx);
|
|
||||||
if (BlueRowEqualityComparer.Equals(item, row)) {
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using System.Reactive.Linq;
|
|
||||||
|
|
||||||
public class MergeAsofStream : ViewStream {
|
|
||||||
public MergeAsofStream(ViewStream left, int[] leftJoinIndices, ViewStream right, int[] rightJoinIndices, MergeAsofTable.MergeType mergeType = MergeAsofTable.MergeType.LessOrEqual) {
|
|
||||||
mySetRealSource(
|
|
||||||
left.AsObservable()
|
|
||||||
.CombineLatest(right.AsObservable(), GetMergeTable)
|
|
||||||
.Scan((IDataPacket)null!, Update)
|
|
||||||
.AsViewStream());
|
|
||||||
|
|
||||||
MergeAsofTable GetMergeTable(IDataPacket l, IDataPacket r) =>
|
|
||||||
new(l.newRows, leftJoinIndices, r.newRows, rightJoinIndices, mergeType);
|
|
||||||
|
|
||||||
IDataPacket Update(IDataPacket? lastDp, MergeAsofTable table) {
|
|
||||||
if (lastDp is null) {
|
|
||||||
return new DataPacket(table, table, new BlueTable(table.Schema));
|
|
||||||
}
|
|
||||||
BlueDataUtility.computeDifferences(table, lastDp.newRows, out var removes, out var adds);
|
|
||||||
return new DataPacket(table, adds, removes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public MergeAsofStream(ViewStream left, string[] leftJoinNames, ViewStream right, string[] rightJoinNames, MergeAsofTable.MergeType mergeType = MergeAsofTable.MergeType.LessOrEqual) {
|
|
||||||
int[]? leftJoinIndices = null;
|
|
||||||
int[]? rightJoinIndices = null;
|
|
||||||
mySetRealSource(
|
|
||||||
left.AsObservable()
|
|
||||||
.CombineLatest(right.AsObservable(), GetMergeTable)
|
|
||||||
.Scan((IDataPacket)null!, Update)
|
|
||||||
.AsViewStream());
|
|
||||||
|
|
||||||
MergeAsofTable GetMergeTable(IDataPacket l, IDataPacket r) =>
|
|
||||||
new(l.newRows, leftJoinIndices ??= l.Schema.ResolveIndices(leftJoinNames), r.newRows, rightJoinIndices ??= r.Schema.ResolveIndices(rightJoinNames), mergeType);
|
|
||||||
|
|
||||||
IDataPacket Update(IDataPacket? lastDp, MergeAsofTable table) {
|
|
||||||
if (lastDp is null) {
|
|
||||||
return new DataPacket(table, table, new BlueTable(table.Schema));
|
|
||||||
}
|
|
||||||
BlueDataUtility.computeDifferences(table, lastDp.newRows, out var removes, out var adds);
|
|
||||||
return new DataPacket(table, adds, removes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,236 +0,0 @@
|
|||||||
namespace bts.BlueOrb;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Immutable;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class MergeAsofTable : IBlueRowCollection {
|
|
||||||
const int ID_COLUMN_SENTINEL = int.MinValue;
|
|
||||||
readonly ImmutableDictionary<int, int> rowMap;
|
|
||||||
readonly ImmutableList<MergeRow> rowList;
|
|
||||||
readonly ImmutableList<(IBlueRow Row, bool IsLeft)> mergedRows;
|
|
||||||
readonly IComparer<(IBlueRow Row, bool IsLeft)> mergedComparer;
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly MergeType mergeType;
|
|
||||||
|
|
||||||
public MergeAsofTable(IBlueRowCollection left, int[] leftJoinIndices, IBlueRowCollection right, int[] rightJoinIndices, MergeType mergeType = MergeType.LessOrEqual) {
|
|
||||||
var leftSchema = left.schema;
|
|
||||||
var rightSchema = right.schema;
|
|
||||||
var leftCols = leftSchema.columnInfo;
|
|
||||||
var rightCols = rightSchema.columnInfo;
|
|
||||||
var colInfos = left.schema.columnInfo.OrderBy(c => c.colNo).ToList();
|
|
||||||
this.mergeType = mergeType;
|
|
||||||
if (colInfos.Count > 0) {
|
|
||||||
Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Count - 1);
|
|
||||||
}
|
|
||||||
foreach (var c in rightCols.OrderBy(c => c.colNo)) {
|
|
||||||
if (colInfos.Any(c => c.colName == c.colName)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
colInfos.Add(new(c.colName, colInfos.Count, c.colType));
|
|
||||||
}
|
|
||||||
schema = buildSchema(leftCols, rightCols, out var mergedColIndices);
|
|
||||||
rowList = buildRowList(left, leftJoinIndices, right, rightJoinIndices, schema, mergedColIndices, mergeType, out mergedRows, out mergedComparer);
|
|
||||||
rowMap = Enumerable.Range(0, rowList.Count).ToImmutableDictionary(i => rowList[i].Id());
|
|
||||||
}
|
|
||||||
|
|
||||||
public enum MergeType {
|
|
||||||
Less,
|
|
||||||
LessOrEqual,
|
|
||||||
Greater,
|
|
||||||
GreaterOrEqual
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
rowList[rowMap[id]];
|
|
||||||
|
|
||||||
IBlueRow IList<IBlueRow>.this[int index] {
|
|
||||||
get => GetRow(index);
|
|
||||||
set => throw new NotSupportedException();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema Schema => schema;
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public int Count => rowList.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
rowMap.ContainsKey(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() =>
|
|
||||||
rowList.GetEnumerator();
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
rowList[index];
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
return rowMap.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(item, rowList[idx])
|
|
||||||
? idx
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
static ImmutableList<MergeRow> buildRowList(
|
|
||||||
IBlueRowCollection left,
|
|
||||||
int[] leftJoinIndices,
|
|
||||||
IBlueRowCollection right,
|
|
||||||
int[] rightJoinIndices,
|
|
||||||
IBlueTableSchema schema,
|
|
||||||
int[] mergedColIndices,
|
|
||||||
MergeType mergeType,
|
|
||||||
out ImmutableList<(IBlueRow Row, bool IsLeft)> mergedRows,
|
|
||||||
out IComparer<(IBlueRow Row, bool IsLeft)> mergedComparer
|
|
||||||
) {
|
|
||||||
var leftCols = left.schema.columnInfo;
|
|
||||||
var rightCols = right.schema.columnInfo;
|
|
||||||
var comp = buildComparer(leftCols, leftJoinIndices, rightCols, rightJoinIndices);
|
|
||||||
mergedComparer = comp;
|
|
||||||
var sortedRows = ImmutableList.CreateBuilder<(IBlueRow Row, bool IsLeft)>();
|
|
||||||
foreach (var r in left) {
|
|
||||||
sortedRows.InsertSorted((r, true), comp);
|
|
||||||
}
|
|
||||||
foreach (var r in right) {
|
|
||||||
sortedRows.InsertSorted((r, false), comp);
|
|
||||||
}
|
|
||||||
mergedRows = sortedRows.ToImmutable();
|
|
||||||
return mergeType switch {
|
|
||||||
MergeType.LessOrEqual => LessOrEqualImpl(),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
ImmutableList<MergeRow> LessOrEqualImpl() {
|
|
||||||
var rowList = ImmutableList.CreateBuilder<MergeRow>();
|
|
||||||
IBlueRow? currLeft = null;
|
|
||||||
IBlueRow? currRight = null;
|
|
||||||
foreach (var (row, isLeft) in sortedRows) {
|
|
||||||
object?[] groupKeys;
|
|
||||||
if (isLeft) {
|
|
||||||
currLeft = row;
|
|
||||||
groupKeys = Array.ConvertAll(leftJoinIndices, i => row[i]);
|
|
||||||
} else {
|
|
||||||
currRight = row;
|
|
||||||
groupKeys = Array.ConvertAll(rightJoinIndices, i => row[i]);
|
|
||||||
}
|
|
||||||
rowList.Add(new(groupKeys, currLeft, currRight, schema, mergedColIndices, 0, rowList.Count + 1));
|
|
||||||
}
|
|
||||||
return rowList.ToImmutable();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static IBlueTableSchema buildSchema(ColumnInfo[] leftCols, ColumnInfo[] rightCols, out int[] mergedColIndices) {
|
|
||||||
var colInfos = new List<ColumnInfo>() {
|
|
||||||
new("id", 0, typeof(int))
|
|
||||||
};
|
|
||||||
var mergedColIndexList = new List<int>();
|
|
||||||
int colNo = 0;
|
|
||||||
foreach (var lc in leftCols) {
|
|
||||||
Log.assert(lc.colNo == colNo);
|
|
||||||
colNo++;
|
|
||||||
if (lc.colName == "id") {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
colInfos.Add(new(lc.colName, colInfos.Count, lc.colType));
|
|
||||||
mergedColIndexList.Add(lc.colNo);
|
|
||||||
}
|
|
||||||
colNo = 0;
|
|
||||||
foreach (var rc in rightCols) {
|
|
||||||
Log.assert(rc.colNo == colNo);
|
|
||||||
colNo++;
|
|
||||||
if (colInfos.Any(c => c.colName == rc.colName)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
colInfos.Add(new(rc.colName, colInfos.Count, rc.colType));
|
|
||||||
mergedColIndexList.Add(-rc.colNo);
|
|
||||||
}
|
|
||||||
mergedColIndices = mergedColIndexList.ToArray();
|
|
||||||
return new BlueTableSchema(colInfos.ToArray(), "MergeAsof");
|
|
||||||
}
|
|
||||||
|
|
||||||
static IComparer<(IBlueRow Row, bool IsLeft)> buildComparer(ColumnInfo[] leftCols, int[] leftJoinIndices, ColumnInfo[] rightCols, int[] rightJoinIndices) {
|
|
||||||
var compBd = new ComparerBuilder<(IBlueRow Row, bool IsLeft)>();
|
|
||||||
for (int i = 0; i < leftJoinIndices.Length; i++) {
|
|
||||||
var li = leftJoinIndices[i];
|
|
||||||
var ri = rightJoinIndices[i];
|
|
||||||
var lc = leftCols[li];
|
|
||||||
var rc = rightCols[ri];
|
|
||||||
Log.assert(lc.colNo == li && rc.colNo == ri);
|
|
||||||
var colType = lc.colType;
|
|
||||||
if (!lc.colType.IsAssignableFrom(rc.colType)) {
|
|
||||||
Log.assert(rc.colType.IsAssignableFrom(lc.colType));
|
|
||||||
colType = rc.colType;
|
|
||||||
}
|
|
||||||
if (colType == typeof(DateTime)) {
|
|
||||||
compBd.Add(t => t.Row.GetDateTime(t.IsLeft ? li : ri));
|
|
||||||
} else if (colType == typeof(TimeSpan)) {
|
|
||||||
compBd.Add(t => t.Row.GetTimeSpan(t.IsLeft ? li : ri));
|
|
||||||
} else if (colType == typeof(short)) {
|
|
||||||
compBd.Add(t => t.Row.GetInt16(t.IsLeft ? li : ri));
|
|
||||||
} else if (colType == typeof(int)) {
|
|
||||||
compBd.Add(t => t.Row.GetInt32(t.IsLeft ? li : ri));
|
|
||||||
} else if (colType == typeof(long)) {
|
|
||||||
compBd.Add(t => t.Row.GetInt64(t.IsLeft ? li : ri));
|
|
||||||
} else if (colType == typeof(ushort)) {
|
|
||||||
compBd.Add(t => t.Row.GetUInt16(t.IsLeft ? li : ri));
|
|
||||||
} else if (colType == typeof(uint)) {
|
|
||||||
compBd.Add(t => t.Row.GetUInt32(t.IsLeft ? li : ri));
|
|
||||||
} else if (colType == typeof(ulong)) {
|
|
||||||
compBd.Add(t => t.Row.GetUInt64(t.IsLeft ? li : ri));
|
|
||||||
} else if (colType == typeof(double)) {
|
|
||||||
compBd.Add(t => t.Row.GetFloat64(t.IsLeft ? li : ri));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return compBd.ToDualComparer();
|
|
||||||
}
|
|
||||||
|
|
||||||
// public class Builder {
|
|
||||||
// readonly ImmutableDictionary<int, int> rowMap;
|
|
||||||
// readonly ImmutableList<MergeRow> rowList;
|
|
||||||
// readonly ImmutableList<(IBlueRow Row, bool IsLeft)> mergedRows;
|
|
||||||
// readonly IComparer<(IBlueRow Row, bool IsLeft)> mergedComparer;
|
|
||||||
// readonly IBlueTableSchema schema;
|
|
||||||
// readonly MergeType mergeType;
|
|
||||||
// readonly Dictionary<int, IBlueRow> leftAdds = new();
|
|
||||||
// readonly HashSet<int> leftRemoves = new();
|
|
||||||
// readonly Dictionary<int, IBlueRow> rightAdds = new();
|
|
||||||
// readonly HashSet<int> rightRemoves = new();
|
|
||||||
|
|
||||||
// public Builder(ImmutableDictionary<int, int> rowMap, ImmutableList<MergeRow> rowList, ImmutableList<(IBlueRow Row, bool IsLeft)> mergedRows, IComparer<(IBlueRow Row, bool IsLeft)> mergedComparer, IBlueTableSchema schema, MergeType mergeType) {
|
|
||||||
// this.rowMap = rowMap;
|
|
||||||
// this.rowList = rowList;
|
|
||||||
// this.mergedRows = mergedRows;
|
|
||||||
// this.mergedComparer = mergedComparer;
|
|
||||||
// this.schema = schema;
|
|
||||||
// this.mergeType = mergeType;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public void AddOrUpdateLeft(IBlueRow row) {
|
|
||||||
// leftAdds[row.Id()] = row;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public void AddOrUpdateRight(IBlueRow row) {
|
|
||||||
// rightAdds[row.Id()] = row;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public void RemoveLeft(int id) {
|
|
||||||
// leftRemoves.Add(id);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public void RemoveRight(int id) {
|
|
||||||
// rightRemoves.Add(id);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public MergeAsofTable ToImmutable(out ImmutableList<MergeRow> removes, out ImmutableList<MergeRow> adds) {
|
|
||||||
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
}
|
|
||||||
Loaded 100 of 2576 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user