Compare commits

..
Author SHA1 Message Date
arham 96767cdfa4 rakurai excanges should automatically work on sim 2026-10-21 14:26:19 +00:00
arham a2987f0102 add reconect to binance futures 2026-10-21 04:30:05 +00:00
arham fbbef6acf7 timestamp for bbo works 2026-10-21 04:29:06 +00:00
arham e38177586f minor fixes and deployment updated public websocket 2026-10-21 04:08:14 +00:00
arham 3e886babe3 spreader working in comand bridge 2026-10-21 04:06:16 +00:00
arham 6a8092c934 docs: document ILINK3 order lifecycle 2026-09-30 07:16:43 +00:00
arham 6f01bde0f5 Add public Spark and BlueOrb WebSocket gateway 2026-09-30 07:15:51 +00:00
arham ab7a5c6bfb feat: add Python Spark command client 2026-09-30 06:59:05 +00:00
arham bf404bc691 feat: add Spark command bridge service 2026-09-30 06:57:40 +00:00
arham 8ba8d55bc2 feat: add versioned Spark command protocol 2026-09-30 06:49:31 +00:00
arham 4614b60218 fix: make error reporting better in webocket api 2026-08-28 02:30:25 +00:00
arham 95d05fa20f add: phoenix md for rakurai 2026-08-28 02:30:03 +00:00
arham c0a8dde7db Add automated venue addition for rakurai exchanges 2026-08-28 02:29:38 +00:00
arham b98eee18f5 add: venue addition is now automatic we dont need to add the venue across the db look at VENUES.md for more info 2026-08-28 02:27:47 +00:00
arham 6ccfbda5aa build: libraries to make rakurai v12 build 2026-08-26 13:31:12 +00:00
arham 6b6f34288d build: in new container build would fail because of missing python package this fixes it 2026-08-26 13:30:28 +00:00
arham eb0d0c5244 update rakurai package 2026-08-24 13:36:53 +00:00
John Colvin 19fef8492d Update externals to remove useless nugets with filenames that are too long
Too long for Windows to clone externals (at least for the Windows Server
box running jenkins)
2026-08-12 14:05:07 -04:00
arham 7521b01022 bug fixes for api stream 2026-08-09 22:13:09 +00:00
arham d0faa95d7a add: fee adjusted stream 2026-08-09 22:12:56 +00:00
arham 36aa2caa2a fractional quantities spreader working 2026-07-30 05:12:58 +00:00
arham 97e7343110 fix: add add raydium and orca to crypto in spreader 2026-07-26 19:28:21 +00:00
arham 5dccc2e9c3 fix: synth name 2026-07-26 19:27:30 +00:00
arham 365012102e basic create synthetic working 2026-07-22 10:25:41 +00:00
arham c3d14c8e31 api server works in default docker 2026-07-02 04:50:44 +00:00
arham 5cc80db09d config change 2026-07-02 04:50:01 +00:00
arham 12c2deb833 market orders for hedge order in spreader 2026-07-02 04:49:44 +00:00
arham 2222fb8e3e add okx to conf 2026-06-30 23:37:26 +00:00
arham 266fdf93e5 fix null check 2026-06-30 23:36:59 +00:00
arham fd121bd85b emnative order hadles limit orders now 2026-06-19 03:13:13 +00:00
arham 1ffaf7c05b Add: rakurai dex execution 2026-06-19 03:12:52 +00:00
arham 5a87288aaa quantity fix 2026-06-15 18:33:37 +00:00
arham a1d62e552c Add:orca raydoum to risk ui 2026-06-11 13:40:10 +00:00
arham 80656a0890 Fix: orca and raydium show up in the api server 2026-06-08 04:10:49 +00:00
arham 188daf2b84 fix: error in cpp 2026-05-31 23:39:38 +00:00
arham 5908adcb09 add: cpp to dex handler 2026-05-15 13:36:09 +00:00
arham 36cf021b70 Add: libs for rakurai to work 2026-05-15 13:35:32 +00:00
arham d45e0e08a2 fix: compiler error 2026-05-15 13:00:25 +00:00
arham 888e751243 raydium orca working 2026-04-24 06:35:50 +00:00
arham bbe3350b4e Add: raydium and orca to frond end 2026-04-24 02:33:24 -04:00
arham 80ea6ad16e add orca nad radium to benue options 2026-04-23 23:46:14 +00:00
arham 0bc0e821f3 basic rakurai hookup done 2026-04-23 17:07:51 +00:00
arham 1b82d07a0d fix: fixed api rounding issues 2026-04-23 17:07:51 +00:00
arham a83404f20f Add:: added orca and raydium 2026-04-23 17:07:51 +00:00
arham 4a74713369 ophir multicast config replay and live working 2026-04-02 00:17:46 +00:00
arham 082beef1da Rename 2026-03-31 23:59:04 +00:00
arham f92d789e90 Add: recording config 2026-03-31 14:16:55 +00:00
arham cd3e845dd9 Fix: set log to diag, it logs every book update 2026-03-31 14:09:37 +00:00
arham a5b704eef1 Add: config for ophir wb 2026-03-31 14:09:05 +00:00
arham afee63e061 Add: new docker compose to support mulitcast 2026-03-31 14:08:42 +00:00
John Colvin fc9356d1a0 deploy: Optionally mount sparkd entrypoint in config dir
To mount a custom entrypoint script, put it in the config profile dir
and define the following env vars:

    SPARKD__IMAGE=sparkd_symbols
    SPARKD__MAIN_ENTRYPOINT=/home/bts/.local/share/bts/sparkd/conf/<custom-entrypoint>

To invoke the "normal" sparkd entrypoint script from the mounted one:

    ./sparkd-entrypoint.sh
2026-03-27 18:28:25 +00:00
John Colvin a85f22c6a8 deploy: Fix sudo issues in sparkd-entrypoint.sh 2026-03-27 17:59:20 +00:00
John Colvin 2024eba4d0 deploy: Add ophir_sim_replay config profile for sparkd 2026-03-26 16:20:41 -04:00
John Colvin 1a49a828ad build: Don't activate venv when generating individual protos
We activate the venv for the whole build
2026-03-26 16:20:41 -04:00
John Colvin c0448c7c8d deploy: Resolve profile dirs with regex patterns 2026-03-26 16:20:41 -04:00
John Colvin a6c6e7faf3 improve: Wait until py work requested before loading py modules
So we don't crash due to missing py modules when no py plugins are running.
2026-03-26 16:20:41 -04:00
John Colvin 56445d2d66 deploy: Add sparkd-shell service to compose.yaml for troubleshooting 2026-03-26 16:20:41 -04:00
John Colvin ccf1bd51c2 cleanup: Remove unused code 2026-03-26 16:20:41 -04:00
John Colvin fd21269df7 refactor: Wrap md run logic in class MarketDataSession
Start in the constructor, gracefully exit in the destructor.
2026-03-26 16:20:41 -04:00
John Colvin fe85c3e648 Allow md recording dir to be mounted as volume in docker compose 2026-03-26 16:20:41 -04:00
John Colvin e1d33ecadf deploy: Configure timezone in entrypoint script 2026-03-26 16:20:41 -04:00
John Colvin 83df96c70f deploy: Move compose entrypoint to dedicated script 2026-03-26 16:20:41 -04:00
John Colvin f718f5b15c deploy: Fix typo in sparkd conf dir path in compose.yaml 2026-03-26 16:20:41 -04:00
John Colvin 5d752f3b73 refactor: Replace DataPacket with IDataPacket in more places 2026-03-26 16:20:40 -04:00
John Colvin d308dbf15a build: Check if venv already active before activating 2026-03-26 16:20:40 -04:00
John Colvin 21c43b9240 refactor: Replace TableFormatter with table.h
Also move to "format" dir
2026-03-26 16:20:40 -04:00
John Colvin e1553f1b04 refactor: Replace StringFinder/Joiner.h with find/join.h
Also move to "algorithm" dir
2026-03-26 16:20:40 -04:00
John Colvin a26918efe9 refactor: Rename a few TitleCase .h files snake_case
* EnumMacros.h -> enum_macros.h
* TimeSpan.h ---> time_span.h
* TimeStamp.h --> time_stamp.h
2026-03-26 16:20:40 -04:00
John Colvin f729fc4c59 diag: Use structured logs in MDP3Listener::onOption 2026-03-26 16:20:40 -04:00
John Colvin fa67adbe36 deploy: Install tzdata in linux_dotnet8 Dockerfile
So we can configure timezone in compose services. Specifically, we need this
to support live cme md (so onixs doesn't think messages are "too old").
2026-03-26 16:20:40 -04:00
John Colvin 72f9ec582e cleanup: Remove unused code 2026-03-26 16:20:40 -04:00
John Colvin 8fea31bdf6 refactor: Add new glbx variable ticks to security (shm and proto) 2026-03-26 16:20:39 -04:00
John Colvin ca96e41391 build: Upgrade cs LangVersion 11 -> 12 2026-03-26 16:20:39 -04:00
John Colvin d4e5c95fde refactor: Rename SparkApiCredentials -> SparkCredentials 2026-03-26 16:20:39 -04:00
John Colvin f6e0ec3617 refactor: Rename SparkdLoginService -> SparkLoginService
This makes it consistent with other type names (we don't use "Sparkd"
anywhere else).
2026-03-26 16:20:39 -04:00
John Colvin 3fa5284838 refactor: Rename SparkdCredentials.cs -> SparkApiCredentials.cs
Matches the class name
2026-03-26 16:20:39 -04:00
John Colvin 4dcaa565f6 cleanup: Remove unnecessary subscription wrapper in cs sdk 2026-03-26 16:20:39 -04:00
John Colvin 75ecfa89d6 build: Clean up spark_client.sln
* Remove unused csprojs related to old sdk
* Remove erroneous build type config
2026-03-26 16:20:39 -04:00
John Colvin 1c930439a4 fix build (dsqa): Add using Spark.Api.MarketData
Needed to reference IMarketDataClient
2026-03-26 16:19:30 -04:00
John Colvin 25084b8cd9 fix build (cs): Add "global::" prefix when using Spark.Api ns
Resolves conflicts with the Bts.Spark.Api namespace when the using
statement is nested inside the root "Bts" namespace.
2026-03-26 16:17:16 -04:00
John Colvin 5b8a8a8882 build: Simplify target to include native assets in cs 2026-03-08 16:00:49 +00:00
John Colvin 21052090a2 fix (pre-emptive): Format dotnet flag opts correctly
In pack_spark_sdk.py, the dotnet pack cmd is constructed with a dict of
opts. If the value part is None, the opt should be formatted as a flag,
"--key" instead of "--key None".
2026-03-07 13:32:03 -05:00
John Colvin 9bd32b8225 fix build (pack cs sdk): Fix --(no-)cs toggle behavior
* Always run "dotnet pack", regardless of csharp toggle opt
* Run "dotnet publish" if csharp toggled on
2026-03-07 13:20:56 -05:00
John Colvin e8e234f420 fix: Don't fire empty updates in Books IObservable 2026-03-07 01:19:38 -05:00
John Colvin be23237a45 fix: Update last val in IObservable FlattenSeries ext
The bootstrap delegate that stitches the sequences together wasn't
getting properly invoked because of this.
2026-03-07 01:19:28 -05:00
arham a9d92650f8 Add: Spread name to synthspreadbbo 2026-03-03 04:33:44 +00:00
arham c03fb13126 Fix: rename EnterSpreaderOrderAsync from CreateSynthAsync 2026-03-03 01:16:10 +00:00
arham 01e6e37db6 Add: Config for ophir live data sim exe 2026-03-03 01:10:34 +00:00
arham 786b0849a7 FIX: removed errenous code that stopped mdp3 from working 2026-03-03 01:09:06 +00:00
John Colvin 4826fc9237 build: Remove dependency on Newtonsoft.Json (in DataStreamCore)
Replace with System.Text.Json (official dotnet)
2026-02-23 13:03:49 -05:00
John Colvin c3341ec29f refactor: Replace a few more DataPacket refs with IDataPacket 2026-02-23 12:56:27 -05:00
John Colvin eb88052c7e chore: Add Metadata prop to IDataPacket
To give it parity with DataPacket
2026-02-23 12:55:23 -05:00
John Colvin 74332f9944 fix: Pass IMarketDataClient to PrimitiveManager to enable md streams
In DataStreamQueryAnalyzer and Spark Api Server
2026-02-23 11:24:33 -05:00
John Colvin 54c97b5610 fix: Set buffer Position to 0 when deserializing pubsub echo payload 2026-02-21 16:36:53 -05:00
John Colvin 5c05afd9b3 fix: Only fire stream update if there are actually changes
In other words, don't fire if addedRows and deletedRows are both empty.
2026-02-21 16:36:53 -05:00
John Colvin dcbba39e4a fix: Add null check in BlueDataUtility.build_dp_inc 2026-02-21 16:36:53 -05:00
John Colvin d09f22007e fix: Ensure PubSubTagHandlerFactory uses correct config
* Add this service as part of the PubSubClient services for dep
  injection (it is needed by PublisherCore)

* Get rid of PublisherCore's default constructor, so dep injection
  treats the tag handler factory as a required param (fails if it isn't
  available)
  - Replace the default ctr with a static "Create" fn, so code that
    hasn't been updated to use dependency injection can still use that.
  - Use PubSubConfig.Instance (runtime) instead of compile time Default

* Fix compile time Default config
  - In fallback handler config, change Tag from "" to ".*" and move it
    from the beginning of the list to the end (it should be the last
    resort, not the other way around).
2026-02-21 16:36:53 -05:00
John Colvin a07e7ea6b8 fix: Don't call MasterWorker/vsd doWork on DSQA ui thread
Now that we use dependency injection, this is handled by a dedicated
service (MasterWorkerService). As a side effect, stream updates aren't
necessarily fired on the ui thread, so we refresh the grid on a form
timer callback.
2026-02-21 16:36:47 -05:00
John Colvin 4da52ff724 fix: Replace DataPacket w/ IDataPacket in DSQA 2026-02-21 14:49:39 -05:00
John Colvin 05c7a32fc5 refactor: Add parameterless ctr to PubsubEchoTestStream
So it's quick/easy to check in DSQA (less typing required)
2026-02-21 14:23:51 -05:00
John Colvin 404d3073f9 ui: Unhide PubsubHeartbeatStream (pss_hb) from DSQA 2026-02-21 14:11:46 -05:00
John Colvin 7f1154d70c cleanup: Fix indentation in PubsubHeartbeatStream.cs 2026-02-21 14:10:11 -05:00
John Colvin 5bc2e29096 fix build: Only check "dsqa" flag when building on windows
The --dsqa opt isn't defined when running on linux.
2026-02-21 17:55:12 +00:00
John Colvin f5731a184c fix: Actually disable Spark in default DSQA config 2026-02-20 11:08:33 -05:00
John Colvin 120fe8e5c6 vscode: Add DSQA debug config to skip build 2026-02-20 10:44:19 -05:00
John Colvin 137fdc537d refactor: Move DSQA config to appsettings.json 2026-02-20 10:43:13 -05:00
John Colvin a605e57bee build: Add dsqa toggle to build_data_stream.py
Windows only
2026-02-20 02:33:38 -05:00
John Colvin 3ca4bf784c ui: Remove bts.BlueOrb namespace filter for DSQA streams 2026-02-20 02:13:17 -05:00
John Colvin 6c3b7ebdaf refactor: Use dependency injection in DSQA 2026-02-20 02:13:17 -05:00
John Colvin 2fbb6fe563 Disable Spark in default DSQA config 2026-02-20 02:13:17 -05:00
John Colvin 9ea33ac110 ui: Unhide PubSubServerStateStream (pss_state) from DSQA 2026-02-20 02:13:17 -05:00
John Colvin 0bc021912a fix: Replace more DataPacket references with IDataPacket 2026-02-20 02:13:17 -05:00
John Colvin fa017bd9ff fix build: Remove old ViewStream.rows references from dsqa
The 'rows' prop no longer exists
2026-02-20 02:13:17 -05:00
John Colvin 57b59215e6 fix build: Don't echo parenthesis in .cmd if block
It's not allowed (because windows).
2026-02-18 02:54:25 -05:00
arham 210768e391 fix: handle badrequests in websocket api 2026-02-18 05:40:27 +00:00
arham f01e8a578d Add: enter synthetic spreader orders through 2026-02-18 05:36:02 +00:00
arham 528a00cf98 Add: config for okx_bin_sim 2026-02-18 05:31:18 +00:00
arham 96a04a7c84 Fix: side and status to json in orderrecodtuple 2026-02-18 05:30:28 +00:00
arham abaf48beb4 Add: other fields to tosjon for security.cs 2026-02-18 05:29:03 +00:00
John Colvin 08fcae94c0 Update externals (moved emumba deribit deb to bitbucket cloud) 2026-02-17 18:16:22 +00:00
John Colvin 30814a2ed6 fix build: Fix Bts.Extensions.Hosting projref path 2026-02-15 16:48:02 +00:00
John Colvin 3c61de5f13 devcontainer: Install Rakurai deb
Download from bitbucket cloud.
2026-02-14 18:43:07 +00:00
John Colvin 771c58d4ad devcontainer: Move emumba deb to bitbucket cloud
I just learned that Bitbucket offers per-repo cloud storage for arbitary
files (the "Downloads" section on the web gui). Doing this before adding
another third-party deb to our build (rakurai), which is quite large at
140MB. We don't want to pack that into our repo.
2026-02-14 18:42:03 +00:00
John Colvin 5db5d40a99 devcontainer: Fix CMake NO_PUBLIC_KEY warning 2026-02-14 18:29:28 +00:00
John Colvin 020ec9c6bf build: Replace proj refs w/ internal nugets when packing 2026-02-03 16:57:32 +00:00
John Colvin 4e28835bbd vscode: Add tools/py to shell PYTHONPATH 2026-02-03 16:57:32 +00:00
John Colvin 6985fbb090 refactor: Move CompressionUtilities into separate csproj
It was nested in PubSub. The new csproj is named Bts.IO.Compression.
2026-02-03 16:57:32 +00:00
John Colvin 2f42311220 build: Update a few python packages
* protobuf         4.21.12 -> 6.33.4
* pybind11         2.13.6 --> 3.0.1
* pybind11-stubgen 2.5.3 ---> 2.5.5
2026-02-03 16:57:32 +00:00
John Colvin 3842efcef1 build: Define build py requirements in separate files
No need to install backend build deps when building the front-end, and
vice versa. The build scripts (build_spark.cmd, build_sparkd.sh, etc.)
automatically install only the necessary requirements now.

Note: We'll keep the monolith requirements.txt file around (at least for
now), as it includes more than just build deps (e.g., numpy, pandas, and
jupyter for data analysis; pybit for bybit plugins, etc.).
2026-02-03 16:57:32 +00:00
John Colvin 635107eb68 cleanup: Remove unused py code 2026-02-03 05:20:29 +00:00
John Colvin 1b9a49b924 cleanup: Add missing files and env vars to sparkd config profiles 2026-02-03 05:20:29 +00:00
John Colvin ead35389d5 build: Make PubSub packable 2026-02-03 05:20:29 +00:00
John Colvin 22b44845fb build: Update Microsoft.AspNetCore.Cors to 2.3.9 2026-02-03 05:20:29 +00:00
John Colvin 8ab26f3e1d build: Remove dependency on Newtonsoft.Json
Replace with System.Text.Json (official dotnet)
2026-02-03 05:16:21 +00:00
John Colvin 01c61bb16d fix build: Publish spark_api_server csproj instead of the sln
There's a version conflict with System.Data.Common when publishing the sln.
2026-01-22 05:50:49 +00:00
John Colvin e09e49ce56 fix build: Add Bts.Net to Spark.Api.Server.sln
It seems like the dep is automatically detected on windows, but not
linux (so we need to add it explicitly).
2026-01-21 15:06:00 -05:00
John Colvin e6cc20858b build: Honor "UseProjectRefs" in spark_websocket, spark_api_server 2026-01-21 14:28:48 -05:00
John Colvin 2eb80082d8 refactor: Move Bts.Extensions.Hosting, Bts.Net to cs dir
They aren't exclusively used by DataStream csprojs
2026-01-21 14:20:39 -05:00
John Colvin d55ef854ba fix build: Don't actually copy files when building with --dry-run 2026-01-21 14:03:01 -05:00
John Colvin b1417ece5d fix build: Fix cmake windows preset in publish_api_server.py
Changed from 'spark' -> 'spark-windows'
2026-01-21 14:03:01 -05:00
John Colvin f6ae219a72 build: Add a few toggle opts to publish py scripts
--(no-)cpp           Enable/disable cpp build
--(no-)cs            Enable/disable csharp build
--(no-)csproj-refs   Depend on csproj refs, or locally built nugets
2026-01-21 14:03:01 -05:00
John Colvin 1e4b9cf833 Add new StartWithOnTimeout ext fn for IObservable
Ophir wanted this functionality for data in the sdk
2026-01-19 10:44:51 -05:00
John Colvin 228931dd2a cleanup: Fix indentation in launch.json
Also rename launch task "(gdb) Attach" -> "Sparkd Attach"
2026-01-19 10:44:51 -05:00
John Colvin f5b4b36ea1 Move devcontainer Dockerfile to bluetradingsystems/linux_dotnet8_dev
We now host it on dockerhub under that name
2026-01-19 10:44:51 -05:00
arham f19d2d695b Fix: Updated path so that profiles work 2026-01-16 06:55:09 +00:00
John Colvin d4b9b6bcba fix: Dispose subscription in TopOfBookQuoter demo 2026-01-14 10:35:42 -05:00
John Colvin 84d74a7873 refactor: Actually implement IDisposable in MarketDataClient 2026-01-14 10:35:42 -05:00
John Colvin 8c34280d75 fix: Init PriceTicker Security to avoid null ref in Equals impl 2026-01-14 10:35:42 -05:00
John Colvin 474d96ced4 feat: Add PriceStats/Ticker to ISimpleSparkClient (cs sdk) 2026-01-14 10:35:42 -05:00
John Colvin 1432692a26 refactor: Make PriceStats/Ticker record structs
Also rename props in PascalCase
2026-01-13 13:57:47 -05:00
arham 1dde5c3ac2 add: trades aggressor side works in the recorder 2026-01-13 18:25:56 +00:00
arham 1cc31e0d2d add: databento datahandler reads cme recorded files with time delay 2026-01-13 18:00:43 +00:00
arham edbd0d2a7b add: conf for es weekly 2026-01-13 17:59:25 +00:00
arham 56a4ec94a2 Fix: glbx config to use databento as marketdata src 2026-01-13 17:55:13 +00:00
arham e3e3d42972 fix: removed double call to std::move 2026-01-13 17:52:16 +00:00
arham 1b9dff3a83 Add: low tick high tick to genereal marketdata listener 2026-01-13 17:51:43 +00:00
John Colvin 8521a98445 ui: Add Tag column to OrderBook window 2026-01-12 10:45:00 -05:00
John Colvin 10eb1ecf1e feat: Add order tag param to ISimpleSparkClient (cs sdk) 2026-01-12 10:44:59 -05:00
John Colvin 88beb0f148 fix: Avoid null ref ex in DiffStream 2026-01-11 15:15:13 -05:00
John Colvin 0f70ae5da2 fix: Fix 'future' -> 'spread' typo in LiveFuturesContractStream 2026-01-11 15:15:13 -05:00
John Colvin bcd207f89e cleanup: Fix more build warnings in DataStreamSuite
Alot more
2026-01-11 15:15:13 -05:00
John Colvin 989d103eef fix: Properly store and dispose md subscription in BBOStream 2026-01-11 15:15:12 -05:00
John Colvin aacd22d496 refactor: Move Exception utils to Spark.Core.Diagnostics ns 2026-01-11 15:15:12 -05:00
John Colvin 5d318b6422 refactor: Rename PropertyNotNull -> MemberNotNull
Because it can be used to null check any member (including fields and
method returns), not just properties.
2026-01-11 15:15:11 -05:00
John Colvin d89164c0ff refactor: Move CollectionExtensions to Spark.Core.Collections namespace 2026-01-11 15:12:33 -05:00
John Colvin cbddb3ae28 fix: Avoid potential race bug in c# OrderPool 2026-01-11 11:20:40 -05:00
John Colvin 2aa25d4085 refactor: Rename ExceptionAssert.not_null -> NotNull 2026-01-11 11:20:40 -05:00
John Colvin cd34f9b1cb cleanup: Tidy up SecondStream, move ThrottleStream to own file
There actually already was a file called "ThrottleStream", but since it
didn't have the ".cs" extension it wasn't getting built.
2026-01-11 11:20:40 -05:00
John Colvin 5f689dbd8e cleanup: Tidy whitespace and using statements 2026-01-11 11:20:40 -05:00
John Colvin 03a8f4e9c0 refactor: Replace DataPacket w/ IDataPacket in many places 2026-01-11 11:20:40 -05:00
John Colvin 49a952fb32 refactor: Generate a few row types with new t4 templates
* PubSubHeartbeatRow
* PubSubRouteRow
* PubSubServerStateRow
* PubSubServerTagStatsRow
* TableMetadataRow
2026-01-10 12:59:14 -05:00
John Colvin a1c6e0ccab build: Add t4 templates to generate c# row types
Produces classes/structs that implement IBlueRow with strongly typed
properties for each column.
2026-01-10 12:59:14 -05:00
John Colvin 63a3eb968c diag: Assert DataStream updates are incremental
In debug builds
2026-01-10 12:59:14 -05:00
John Colvin 9de4b27003 refactor: Ensure stream subscriptions only disposed once
In the debugger, I noticed a few streams were unhooking their internal
onChange callbacks unnecessarily (when they were already unhooked).
2026-01-10 12:59:14 -05:00
John Colvin 849c6c28e5 fix: Don't fire empty updates in MutableAppendStream 2026-01-10 12:59:14 -05:00
John Colvin bf3a8f076f fix: Change DSQA config pubsub port to 22019
The port PubSubServer runs on by default
2026-01-10 12:59:14 -05:00
John Colvin 44749739b9 build: Define centralized SparkVersion prop
All csprojs should be on the same version. At the very least, it
is a sensible default.
2026-01-10 12:59:14 -05:00
John Colvin ddf16bcd5c fix: Add missing LiveMarketDataStream attribute to InstrumentsStream
Otherwise PrimitiveManager returns null for that stream type
2026-01-10 12:59:14 -05:00
John Colvin cfe7d43770 fix: Initialize ViewStreamDatabase.instance in DSQA 2026-01-10 12:59:14 -05:00
John Colvin bb5ea19f69 refactor: Get rid of ViewStream.rows prop
It was causing weird side effects in debugger
2026-01-10 12:59:14 -05:00
John Colvin 327665b4c3 fix: Add null check in ProximityJoinStream 2026-01-08 14:24:27 -05:00
John Colvin 2843638386 fix: Only start Stopwatch when actually doing work 2026-01-08 14:24:27 -05:00
John Colvin 915322927a fix: Dispose md subscription in InstrumentsStream 2026-01-08 14:23:09 -05:00
John Colvin 9905216131 fix: Format PubSub and DSQA config json in PascalCase
Matches what's in appsettings.json files
2026-01-08 14:22:57 -05:00
John Colvin f88915a940 fix build: Add --spark-version opt in build_data_stream.py 2026-01-07 02:34:56 -05:00
John Colvin c5ea913e27 fix: Shutdown application when user closes login dialog
Previously, it would hang until you manually killed the process.
2026-01-06 14:02:10 -05:00
John Colvin 7719aaa2b3 fix: Add EnvironmentVariables config provider to HostBuilder 2026-01-06 14:02:10 -05:00
John Colvin ad3a8a7544 fix: Add json config sources before env vars and cmd line sources
So environment variables override json files (if there is a conflict),
and command line args override everything else.
2026-01-06 02:34:36 -05:00
John Colvin 75d3ba1d59 fix: Support cross-type numeric comparisons in ArgumentComparer
For example, so we can compare ints with doubles. Also adds support for
DateTime and TimeSpan cross-type comparisons.
2026-01-05 13:36:07 -05:00
John Colvin 0ebe9a876c fix: Apply MarketDataClientConfig to MDSource
For throttle intervals, etc.
2026-01-05 13:26:29 -05:00
John Colvin ac1cbaa985 deploy: Add ophir_demo config profile for sparkd 2026-01-05 00:04:57 +00:00
John Colvin e861835690 deploy: Change sparkdata dir to ~/.local/share/bts/spark
In docker scripts (Dockerfile, compose.yaml)
2026-01-04 23:35:04 +00:00
John Colvin 10752257cc vscode: Build sparkd as prelaunch task
Must have been removed by accident
2026-01-04 21:19:00 +00:00
John Colvin 8e67bd45d9 improve: Add 'file' and 'start_date' props to md replay config 2026-01-04 21:19:00 +00:00
John Colvin 140e5bb5ee fix build (sparkd): Disable warnings specifically for clang >= 18
The build was broken for clang < 18 (and gcc) because we were blindly disabling
warnings as errors specific to clang 18, without checking the compiler version.
Namely, "-Weffc++" and "-Wenum-constexpr-conversion".
2026-01-04 21:19:00 +00:00
John Colvin 9ebc282ba5 fix: Use AtomicGate to make TableUpdate Select ext thread-safe 2026-01-04 21:19:00 +00:00
John Colvin 549529aa60 tool: Add script to add BTS github nuget source
Where we host the BTS sdk nugets
2026-01-04 21:19:00 +00:00
John Colvin 5ed4a5c1c5 feat: Add tag property to OrderDefinition 2026-01-04 21:19:00 +00:00
John Colvin 8c458d48bb feat: Add server side csharp sdk
For Ophir
2026-01-04 21:19:00 +00:00
John Colvin bb1e7c30c8 build: Add msbuild flag to reference projs or local nugets
So we can use project refs for dev builds, and package refs for releases.
2026-01-04 21:19:00 +00:00
John Colvin 6568c29ad4 cleanup: Remove unnecessary ProjectReference 2026-01-04 21:19:00 +00:00
John Colvin e5e67e1a01 build: Optionally launch console with Spark exe 2026-01-04 21:19:00 +00:00
John Colvin 4f34e57194 refactor: Move Bus code into Bts.Spark.Core 2026-01-04 21:19:00 +00:00
John Colvin 59b1aead62 refactor: Use HostBuilder to launch spark w/ streams 2026-01-04 21:19:00 +00:00
John Colvin 49a4c65f3b build: Define custom "main" (entry point) for Spark exe
This is a prelude to integrating streams into spark. The plan is to use
an ms HostBuilder to instantiate stream services/dependencies alongside
the spark Application object.
2026-01-04 21:19:00 +00:00
John Colvin cf4158b34c refactor: Consolidate ShellBootstrapper and base class into one 2026-01-04 21:19:00 +00:00
John Colvin db0e0d4311 cleanup: Get rid of stunnel code
This looks like it hasn't been hooked up for years...it would only start
if a stunnel.conf file was detected, which hasn't been around for a very
long time.
2026-01-04 21:19:00 +00:00
John Colvin 50788f8828 fix: Only create mutex if spark app is NOT running in vs designer
I accidentally droped the '!' in a cleanup commit 4 years ago
2026-01-04 21:19:00 +00:00
John Colvin a515468e37 refactor: Rename TableUpdate props to PascalCase
Makes the sdk look more polished, professional
2026-01-04 21:19:00 +00:00
John Colvin 1f68f47b42 build: Propagate py exit codes back to the shell
* publish_spark.cmd
2026-01-04 21:19:00 +00:00
John Colvin 84f6d3d9fe cleanup: Fix build warnings pre-emptively before compiler upgrade
For clang>=18
  * non-constant-expression cannot be narrowed from type 'uint32_t' (aka 'unsigned int')
    to 'float' in initializer list [-Wc++11-narrowing-const-reference]
  * -Wenum-constexpr-conversion
2026-01-04 21:19:00 +00:00
John Colvin 4bc1a4c833 build (qol): Add opts to only build cpp or cs
Usually, we're only working in one language or the other, and don't need
to rebuild both.
2026-01-04 21:19:00 +00:00
John Colvin c50a006d7d deploy: Update dotnet7 -> dotnet8 in docker images 2026-01-04 21:19:00 +00:00
John Colvin 2031f536d6 build: Pack Bts.Spark.Core with libspark 2026-01-04 21:19:00 +00:00
John Colvin 69e4ed84c1 build: Generate version info via git cli in build scripts
Instead of GitVersion.MsBuild nuget
2026-01-04 21:18:58 +00:00
John Colvin 2a284b1e9c cleanup: Remove unused Protobuf.cmake script 2026-01-04 21:08:13 +00:00
John Colvin 4d5d0f8573 build: Consolidate cmake presets 2026-01-04 21:08:13 +00:00
John Colvin c4a6b4591d cleanup: Remove last vestiges of spark proto config from DataStreamCore 2026-01-04 21:08:13 +00:00
John Colvin 8f3fa18bca refactor: Rename spark_api csproj Bts.Spark.Api
This follows the conventional naming practices, where the project shares its
name with the namespace. Doing this in anticipation of the csharp sdk we are
about to release (for Ophir).
2026-01-04 21:08:13 +00:00
John Colvin f60782915c refactor: Rename spark_core csproj Bts.Spark.Core
This follows the conventional naming practices, where the project shares its
name with the namespace. Doing this in anticipation of the csharp sdk we are
about to release (for Ophir).
2026-01-04 21:08:13 +00:00
John Colvin 3565f78be8 cleanup (build): Remove redundant csproj refs
For example, projects that reference spark_api don't need an explicit reference
to spark_core, since spark_api already references it. The same goes for projects
that reference DataStream and DataStreamCore.
2026-01-04 21:08:13 +00:00
John Colvin 41ad7b96f4 build: Only expose spark_core internals to spark and spark_test csprojs 2026-01-04 21:08:13 +00:00
John Colvin c338d26b82 build: Define project specific metadata props for nuget pack 2026-01-04 21:08:13 +00:00
John Colvin 319f054a0c build: Define global company info build props for csprojs
* Authors - Blue Trading Systems
* Company - Blue Trading Systems
2026-01-04 21:08:13 +00:00
John Colvin 389bf6e130 refactor: Rename DictionarySetWrapper -> DictionaryKeySet 2026-01-04 21:08:12 +00:00
John Colvin ded28c23d7 diag: Preserve file and line number in "log_at" ext fn 2026-01-04 21:08:12 +00:00
John Colvin 04acba409f dev qol: Add local appsettings, configurations to gitignore 2026-01-04 21:08:12 +00:00
John Colvin 67f7124ed5 build: Remove DSQA from DataStreamSuite.sln
Because it doesn't build on linux
2026-01-04 21:08:12 +00:00
John Colvin 322794434a fix build: Copy libspark to "net8.0" dir (was net7.0) 2026-01-04 21:08:12 +00:00
John Colvin fbcba118ac fix build (api server): Fix typo DOTNET_BINARY_DIR cmake var (path)
This is the dir into which libspark (c++) is copied for dotnet (c#) exes.
2026-01-04 21:08:12 +00:00
John Colvin 1c579ee063 refactor: Change namespace to Bts.DataStreamCore for a few types
Since they're in the DataStreamCore csproj
2026-01-04 21:08:12 +00:00
John Colvin 7508fa0b7f dev qol: Mark ExceptionAssert 'name' param w/ CallerArgumentExpression 2026-01-04 21:08:12 +00:00
John Colvin 1f18ec4198 refactor: Replace if/else with switch cases 2026-01-04 21:08:12 +00:00
John Colvin 81d7820cca fix: Ensure IDGenerator instance is created before deref 2026-01-04 21:08:12 +00:00
John Colvin e125be1f85 fix: Emplace MDConsumerHandle instead of copy assign 2026-01-04 21:08:12 +00:00
John Colvin 040c8cab59 cleanup: Remove unnecessary code
Trim includes as well
2026-01-04 21:08:12 +00:00
John Colvin 32a12cf1f7 cleanup: Remove redundant macro def 2026-01-04 21:08:12 +00:00
John Colvin db604fbed0 improve: Abort md rec loop if nothing configured to record 2026-01-04 21:08:12 +00:00
John Colvin cc0a3ea15f cleanup: Trivial whitespace cleanup in Protobuf.cmake 2026-01-04 21:08:12 +00:00
John Colvin 4066cd50ce build: Clean up data_files cmake proj (support --target clean) 2026-01-04 21:08:12 +00:00
John Colvin f1ccff9134 build: Resolve app data dir and shared mem prefix at runtime
We can't define them at compile time anymore since we build multiple exes that
depend on libspark in the same dir (e.g., api-server, pubsub, sparkd, etc.),
and each one needs its own shared memory and app data filespaces.

For shared memory:
  * Prefix all shm files with the proc name and an environment variable,
    BTS_SHARED_MEMORY_PREFIX (if defined). The env var is only necessary
    when 2 exe's with the same name run simultaneously.

For app data:
  * On linux, the default is:
    /home/<user_name>/.local/share/bts/<proc_name>
  * On windows, the default is:
    /User/<user_name>/AppData/Roaming/bts/<proc_name>
  * This path can be overridden in the environment variable:
    <proc_name>__APPDATA
    - For example, SPARKD__APPDATA, PUBSUB__APPDATA, etc.
2026-01-04 21:08:12 +00:00
John Colvin 51ab182b2f Don't log "initializing spark config" if it has already been initialized 2026-01-04 21:08:12 +00:00
John Colvin 856cd552eb refactor: Add Api Server service modules for Dependency Injection
* Part of a broader paradigm shift towards Dependency Injection and
  Factory Pattern.
* Adds "api-server" service to docker compose
* Replaces MarketDataClient w/ IMarketDataClient in a few streams
* Replaces ViewStreamDatabase w/ IWorkScheduler in a few streams
2026-01-04 21:08:12 +00:00
John Colvin 997647e917 refactor: Add handy WebsocketExtensions
* IsSocketStateOpen
* IsClosedProperly
2026-01-04 21:08:12 +00:00
John Colvin 687a9741db fix: Don't assert on exception in BooleanExpression.evaluate
Bugs (fixed):
  * DataStream WebApp Server sometimes crashes when the user enters a
    bad query.
2026-01-04 21:08:12 +00:00
John Colvin 7faa0ade24 diag: Assert that local ip address is successfully obtained 2026-01-04 21:08:12 +00:00
John Colvin 4d6d68aff6 improve: Resolve hostnames when comparing ip addrs
So we can to configure pubsub routes by hostname or ipv4 (ipv6 hasn't
been tested yet...so we should assume it doesn't work).
2026-01-04 21:08:12 +00:00
John Colvin dcf291d6b2 diag: Assert TcpConnection endpoint matches expected 2026-01-04 21:08:12 +00:00
John Colvin a6b83988e7 diag: Catch exceptions on AsyncTCPConnector_AsyncWrapper bg thread
Since it spawns a new thread on each attempt, uncaught exceptions were
previously undetected, and caused silent failures down the pipe. This
should make it easier to track such bugs down.
2026-01-04 21:08:12 +00:00
John Colvin 2c097500f4 fix: Format PubSubRoute as json in log msgs
It was previously using the default ToString, which simply returns the type name
2026-01-04 21:08:12 +00:00
John Colvin 81a212e7a3 build: Upgrade packages (deps) in DataStreamTests 2026-01-04 21:08:12 +00:00
John Colvin 449453625c vscode: Add PubSubServer debug cfg to skip build 2026-01-04 21:08:12 +00:00
John Colvin 5058187d5b cleanup: Remove reference to spark protobuf Config in ViewStream
In general, we should keep BlueOrb dependencies on spark/sparkd to a minimum.
The dotnet dependency injection architecture provides a robust config framework
we can use instead, based on json files, environment variables, and command-line
arguments.
2026-01-04 21:08:12 +00:00
John Colvin 0253a0f60f refactor: Run RemoteQueryManager as an IHostedService
This paradigm more accurately represents the RemoteQueryManager as a
Service that can be started/stopped.
2026-01-04 21:08:12 +00:00
John Colvin 6986cb5e8e fix: Ensure spark config init and login are only performed once
As a static function that could be called from anywhere in the code
(directly or indirectly), 'Configuration.Initialize' was vulnerable to
race bugs. The "Configuration.get<T>()" method, whose calls are
ubiquitous in our code, would initialize config if it hadn't been done
so already. Without a reference to the configServer param, this param
would effectively be ignored/overridden with null.

It's important that login only be performed once, for obvious reasons
(our exes that connect to sparkd are generally designed to process data
for a single user). Furthermore, not every routine that requires a
successful login has a reference to the credentials needed to do it. To
address this, we now pass the ISparkdLoginService singleton to these
routines so they can wait until the singular login task has completed
before they begin.
2026-01-04 21:08:12 +00:00
John Colvin 35a45c35ba refactor: Move position model to spark_api
So we can use it in the new sdk (c# server-side api) for Ophil
2026-01-04 21:08:12 +00:00
John Colvin 970555d2a1 diag: Assert ViewStreamDatabase.instance is only created once
The instance must be initialized via ViewStreamDatabase.SetGlobalInstance
2026-01-04 21:08:12 +00:00
John Colvin 1d46ea061a cleanup: Remove unused code from ViewStreamDatabase 2026-01-04 21:08:12 +00:00
John Colvin 4bd12dc465 fix: Define BoooleanConstant before StringConstant
Otherwise the former masks the latter and prevents it from matching
2026-01-04 21:08:12 +00:00
John Colvin ad1d404314 cleanup: Remove unnecessary deps from csprojs 2026-01-04 21:08:12 +00:00
John Colvin a7b6da27ae fix: Catch exception when creating TcpClient 2026-01-04 21:08:12 +00:00
John Colvin de4a55b34c fix: Make SystemSubscriptionMapPublisher threadsafe
The PubSubState.onTagMapChanged event may not be invoked on
the same thread as doWork.
2026-01-04 21:08:12 +00:00
John Colvin 2e14598667 fix: Add a few null checks
* TimeBasedFilter
* TableAccumulator2
* NonIncGroupByStream
2026-01-04 21:08:12 +00:00
John Colvin 3038d9154f improve: Reduce task wait/timeout code in AsyncTCPConnector 2026-01-04 21:08:12 +00:00
John Colvin 5805758187 diag: Assert System publishers 'doWork' is always called on same thread 2026-01-04 21:08:12 +00:00
John Colvin 2c1b7b95a4 improve: Propagate DataStream exceptions via ErrorChanged event 2026-01-04 21:08:12 +00:00
John Colvin 60af47c4eb cleanup: Remove unused PubSubConfig props 2026-01-04 21:08:12 +00:00
John Colvin b0d0594e3d fix: Always call 'handle_internal_pubs' in PubSubMessageReactor main loop 2026-01-04 21:08:12 +00:00
John Colvin b29b2a0adb fix: Fix a few column types in in PubSubServerTagStatsStream
* total_bytes      (int      -> long)
* last_update_time (DateTime -> long)
2026-01-04 21:08:11 +00:00
John Colvin 76fa35382e refactor: Add DataStream service modules for Dependency Injection
Namely, in DataStreamPublisher and PubSubServer. This is part of a broader
paradigm shift towards Dependency Injection and Factory Pattern.

https://learn.microsoft.com/en-us/dotnet/core/extensions/dependency-injection-guidelines

* Use interfaces to decouple services:
  - Decouple ManagedSubscriber from ISubscriberCore impl
  - Decouple PubTableBroker from IPublisherCore impl
  - Decouple RemoteQueryManager from IManagedSubscriber, IPubTableBroker impl

* Rename Publishers -> PublisherNodes in DataStreamPublisherConfig
* Model config profiles after dotnet appsettings
  - Rename Configurations/<name>.json -> appsettings.<name>.json
  - Make json fields match corresponding csharp props verbatim
    * The Dependency Injection framework doesn't support JsonPropertyName
      attributes. Config class props must match json fields verbatim.

* Move reusable startup code into new Bts.Extensions.Hosting proj
* Add pubsub and md-publisher services to compose.yaml
2026-01-04 21:08:11 +00:00
John Colvin 716bc92394 refactor: Only support 1 tag type in IPubSubServerTagHandler 2025-12-30 11:42:23 -05:00
John Colvin 3a6141f3ae refactor: Change Configuration.initialize to be async
Tweak SparkCoreServices to synchronize Configuration.init globally
2025-12-30 11:42:20 -05:00
John Colvin be2368f95a refactor: Add Spark service modules for Dependency Injection 2025-12-30 11:38:21 -05:00
John Colvin 7e32ac9e06 cleanup: Remove unused pubsub tag handler code
* Removed 'IPubSubServerTagHandler.register'
  - All implementations of this function were empty
* Removed 'IPubSubServerTagHandler.supportedTagTypes'
  - This property wasn't referenced anywhere
* Remove IWorker inheritance from IPubSubServerTagHandler
  - No reason these interfaces have to be tied together
2025-12-30 11:38:21 -05:00
John Colvin 8fc4df1427 refactor: Rename CloudStreamManager.cs -> RemoteQueryManager.cs
This matches the file name
2025-12-30 11:38:21 -05:00
John Colvin 2020ebed2d cleanup: Change pub sub schema prefix 'bluto://' -> 'bts-cloud://'
In config json files (not actual code change)
2025-12-30 11:38:20 -05:00
John Colvin cc483ca4b5 diag: Fix typos in log msgs
* Bad string interopolation syntax
2025-12-30 11:38:20 -05:00
John Colvin aae7356dc7 vscode: Add DataStreamPublisher debug cfg to skip build 2025-12-30 11:38:20 -05:00
John Colvin 8d6e94384c cleanup: Fix benign typos in build scripts
Change BUILD_SPARK_PY var name to BUILD_PY (these scripts build other
projects than 'spark').
2025-12-30 11:38:20 -05:00
John Colvin 7d6e4efa4f cleanup: Fix numerous build warnings in DataStreamSuite
Nearly all warnings were due to nullability issues, since the concept of
nullable reference types did not exist when the DataStream code was
originally written.

* Compression related files
  - BZip2InputStream  (disable nullability warnings)
  - BZip2OutputStream (disable nullability warnings)
  - DeflaterHuffman   (disable nullability warnings)
  - Inflater          (disable nullability warnings)
  - InflaterDynHeader (disable nullability warnings)
  - ParallelJoinTest  (disable nullability warnings)
  - TarHeader
  - TarInputStream    (disable nullability warnings)
  - ZipEntry
  - ZipFile
  - ZipInputStream

* AsyncTCPConnection_APM
* AsyncTCPConnection_APM
* InflaterHuffmanTree
* ManagedSubscriber
* ManagedTCPMessageHub
* MapStream
* PubSubClientMessage
  - Normalize indentation/whitespace
  - Fix some typos
  - Address nullability issues to fix build warnings
  - Modernize code
* PubSubMessageReactor
* PubSubServerManagedTCPConnection
* PubSubSingleTagStateEvent
* PubSubState
* PublisherCore
* PubsubDeserializationController
* RemoteMessage
  - make 'message' nullable to resolve warnings elsewhere (in calling code)
* SharedStreamConfig
* SingleTagData
* SubscriberCore
* SuperMapStream
* SystemHeartbeatPublisher
* SystemStatsPublisher
* SystemSubscriptionMapPublisher
* ThreadSafeQueue
* TradeBarsStream

Only use Registry on windows.
2025-12-30 11:38:20 -05:00
John Colvin 02f566a970 cleanup: Modernize PubSubRoute code, json parsing 2025-12-30 11:38:20 -05:00
John Colvin d0d05a5c18 cleanup: Remove unused DateTime grammar from exp parser 2025-12-30 11:38:20 -05:00
John Colvin 3fafeca9ad cleanup: Remove empty file (clearly added by accident) 2025-12-30 11:38:20 -05:00
John Colvin f5a4e7c410 fix: Remove query from literal error stream
The parser can't handle colons in the expression (it throws/crashes)
2025-12-30 11:38:17 -05:00
John Colvin a0b2a8b186 chore: Add async SQLiteEnumerator to read table as IBlueRowCollection 2025-12-30 11:36:35 -05:00
John Colvin 35a207e234 chore: Add various collection classes
These will be useful in stream operators, row indexing, etc:
  * AggregateList
  * CircularBuffer
  * DeltaList
  * DictionarySetWrapper
  * ImmutableOrderedDictionary
  * ImmutableOrderedSet
  * IMutableCollectionFacade
    - inherits IReadOnlyCollection<T> and ICollection<T>
    - throws NotSupportedException in default impls for mutation ops
  * IMutableListFacade
    - inherits IReadOnlyList<T> and IList<T>
    - throws NotSupportedException in default impls for mutation ops
  * IndexDependentSelectorList
  * MergedSortedList
  * OffsetList
  * OrderedDictionary
  * ReverseArrayList
  * SliceList
  * StepList

More notes:
  * Renames ExceptionAssert.is_valid ExceptionAssert.Index.IsValid.
    - Like the 'ExceptionAssert.Argument' class, which provides methods to throw
      ArgumentException if a condition isn't satisfied, this class throws
      IndexOutOfRange exceptions.
2025-12-30 11:36:35 -05:00
John Colvin 2069c64864 chore: Support mutations in AggregateDictionary (optionally) 2025-12-30 11:36:35 -05:00
John Colvin 9a29606fc1 style: Use standard c# capitalization rules
* Use 'PascalCase' for public members (instead of 'snake_case')
* Use 'camelCase' for private members (instead of '_snake_case')
2025-12-30 11:36:34 -05:00
arham e69012a5ad fix: sanitized dsl in websocket 2025-12-03 13:35:08 +00:00
arham 4237d58f76 Add: metadata to dsl callback 2025-12-03 13:34:37 +00:00
arham 1c75f4a53f Add: get last n lines of log for websocketserver 2025-12-03 12:24:16 +00:00
arham aabf1490eb Fix: Trycatch on base program.cs to prevent crash 2025-12-03 12:21:03 +00:00
arham 14a79b6352 Add:: cross exchange arb stream 2025-12-02 15:21:06 -05:00
arham f69b517f83 add: f10 to stop dsqa 2025-12-02 13:05:39 -05:00
arham 2d0d33028a fix: bug fix to rowbasedappendstream 2025-12-02 12:58:22 -05:00
arham f953b3bb8a change: change UnionStream name to AppendStream 2025-12-02 12:47:36 -05:00
arham 73672276f3 Add: add year to datetimexpander stream 2025-12-02 12:46:49 -05:00
1488 changed files with 31558 additions and 22830 deletions

No files matched your search

+7 -1
View File
@@ -1,5 +1,11 @@
{ {
"dockerFile": "Dockerfile", "build": {
"dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
"options": [
"--secret",
"id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
]
},
"hostRequirements": { "hostRequirements": {
"memory": "8gb", "memory": "8gb",
"storage": "10gb" "storage": "10gb"
@@ -1,4 +0,0 @@
#!/usr/bin/env bash
this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
+8
View File
@@ -29,6 +29,9 @@
# tools/deploy.sh --help # tools/deploy.sh --help
# #
### Host/Runtime ###
TZ=Etc/UTC
### Sparkd ### ### Sparkd ###
# The internal image for sparkd to use. For more info, view: # The internal image for sparkd to use. For more info, view:
# src/server/sparkd/Dockerfile # src/server/sparkd/Dockerfile
@@ -38,6 +41,9 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker. # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000). # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
# SPARKD__DATA_DIR=/home/bts/sparkdata/data # SPARKD__DATA_DIR=/home/bts/sparkdata/data
# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
SPARKD__SOURCE_DIR=install/sparkd-release-gcc SPARKD__SOURCE_DIR=install/sparkd-release-gcc
SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/ SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
SPARKD__CONFIG_PORT=8000 SPARKD__CONFIG_PORT=8000
@@ -45,6 +51,8 @@ SPARKD__QUICKBUS_PORT=8001
SPARKD__SIM_FIX_PORT=8002 SPARKD__SIM_FIX_PORT=8002
SPARKD__SHM_SIZE=4gb SPARKD__SHM_SIZE=4gb
SPARKD__PYTHON_REQUIREMENTS=requirements.txt SPARKD__PYTHON_REQUIREMENTS=requirements.txt
SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
### PubSubServer ### ### PubSubServer ###
PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
+76 -22
View File
@@ -142,14 +142,14 @@
"name": "DataStreamQueryAnalyzer", "name": "DataStreamQueryAnalyzer",
"type": "coreclr", "type": "coreclr",
"request": "launch", "request": "launch",
"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)", "preLaunchTask": "Build DataStream (Debug)",
"program": "", "program": "",
"windows": { "windows": {
"program": "DataStreamQueryAnalyzer.exe", "program": "DataStreamQueryAnalyzer.exe",
"cwd": "build/Debug/bin/net8.0-windows", "cwd": "build/Debug/bin/net8.0-windows",
"env": { "env": {
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}", "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
}, },
}, },
"stopAtEntry": false, "stopAtEntry": false,
@@ -164,8 +164,8 @@
"program": "DataStreamQueryAnalyzer.exe", "program": "DataStreamQueryAnalyzer.exe",
"cwd": "build/Debug/bin/net8.0-windows", "cwd": "build/Debug/bin/net8.0-windows",
"env": { "env": {
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}", "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
}, },
}, },
"stopAtEntry": false, "stopAtEntry": false,
@@ -188,9 +188,6 @@
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json" "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
], ],
"cwd": "build/Debug/bin/net8.0", "cwd": "build/Debug/bin/net8.0",
"env": {
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
},
"stopAtEntry": false, "stopAtEntry": false,
"console": "integratedTerminal" "console": "integratedTerminal"
}, },
@@ -212,6 +209,43 @@
"stopAtEntry": false, "stopAtEntry": false,
"console": "integratedTerminal" "console": "integratedTerminal"
}, },
{
"name": "Sdk Demo",
"type": "coreclr",
"request": "launch",
"program": "",
"preLaunchTask": "Build Api Server (Debug)",
"windows": {
"program": "Bts.Spark.Sdk.Demo.exe",
},
"linux": {
"program": "Bts.Spark.Sdk.Demo",
},
"cwd": "build/Debug/bin/net8.0",
"env": {
"DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Sdk Demo (Skip Build)",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"program": "Bts.Spark.Sdk.Demo.exe",
},
"linux": {
"program": "Bts.Spark.Sdk.Demo.exe",
},
"cwd": "build/Debug/bin/net8.0",
"env": {
"DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{ {
"name": "Spark", "name": "Spark",
"type": "coreclr", "type": "coreclr",
@@ -300,6 +334,7 @@
"program": "", "program": "",
"linux": { "linux": {
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd", "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
"preLaunchTask": "Build Sparkd (Debug | Clang)",
}, },
"args": [ "args": [
"run" "run"
@@ -349,6 +384,25 @@
} }
] ]
}, },
{
"name": "Sparkd Attach",
"type": "cppdbg",
"request": "attach",
"program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
},
{
"description": "Set Disassembly Flavor to Intel",
"text": "-gdb-set disassembly-flavor intel",
"ignoreFailures": true
}
]
},
], ],
"inputs": [ "inputs": [
{ {
@@ -374,6 +428,16 @@
], ],
"default": "MarketDataPublisher", "default": "MarketDataPublisher",
}, },
{
"id": "dataStreamQueryAnalyzerEnvironment",
"type": "pickString",
"description": "DataStreamQueryAnalyzer Configuration",
"options": [
"Development",
"Local",
],
"default": "Development",
},
{ {
"id": "dataStreamPublisherEnvironment", "id": "dataStreamPublisherEnvironment",
"type": "pickString", "type": "pickString",
@@ -395,9 +459,9 @@
"default": "PubSubServer", "default": "PubSubServer",
}, },
{ {
"id": "pubSubServerEnvironment", "id": "sdkDemoEnvironment",
"type": "pickString", "type": "pickString",
"description": "Api Server Configuration", "description": "Sdk Demo Configuration",
"options": [ "options": [
"Development", "Development",
"Local", "Local",
@@ -406,14 +470,10 @@
"default": "Development", "default": "Development",
}, },
{ {
"id": "dataStreamQueryAnalyzerEnvironment", "id": "sparkConfigServer",
"type": "pickString", "type": "promptString",
"description": "DataStreamQueryAnalyzer Configuration", "description": "Config Server URL",
"options": [ "default": "http://localhost:8000/config",
"Development",
"Local",
],
"default": "Development",
}, },
{ {
"id": "sparkEnvironment", "id": "sparkEnvironment",
@@ -425,12 +485,6 @@
], ],
"default": "Development", "default": "Development",
}, },
{
"id": "sparkConfigServer",
"type": "promptString",
"description": "Config Server URL",
"default": "http://localhost:8000/config",
},
{ {
"id": "sparkTest", "id": "sparkTest",
"type": "promptString", "type": "promptString",
+85 -70
View File
@@ -121,57 +121,6 @@
"color": "terminal.ansiCyan" "color": "terminal.ansiCyan"
}, },
}, },
{
"label": "Build DataStreamQueryAnalyzer",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_data_stream_query_analyzer.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStreamQueryAnalyzer (Debug)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_data_stream_query_analyzer.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStreamQueryAnalyzer (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_data_stream_query_analyzer.cmd",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{ {
"label": "Build Spark", "label": "Build Spark",
"group": "build", "group": "build",
@@ -223,6 +172,66 @@
"color": "terminal.ansiCyan" "color": "terminal.ansiCyan"
}, },
}, },
{
"label": "Build Spark Sdk",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_spark_sdk.sh",
},
"windows": {
"command": "tools/build_spark_sdk.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark Sdk (Debug)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_spark_sdk.sh",
},
"windows": {
"command": "tools/build_spark_sdk.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark Sdk (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark_sdk.cmd",
},
"linux": {
"command": "tools/build_spark_sdk.sh",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{ {
"label": "Build Sparkd", "label": "Build Sparkd",
"group": "build", "group": "build",
@@ -527,17 +536,16 @@
}, },
{ {
"label": "Run DataStreamQueryAnalyzer", "label": "Run DataStreamQueryAnalyzer",
"dependsOn": "Build DataStreamQueryAnalyzer", "dependsOn": "Build DataStream",
"type": "process", "type": "process",
"command": "", "command": "",
"windows": { "windows": {
"command": "DataStreamQueryAnalyzer.exe", "command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": { "options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows" "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
"env": {
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
},
}, },
}, },
"icon": { "icon": {
@@ -547,17 +555,16 @@
}, },
{ {
"label": "Run DataStreamQueryAnalyzer (Debug)", "label": "Run DataStreamQueryAnalyzer (Debug)",
"dependsOn": "Build DataStreamQueryAnalyzer (Debug)", "dependsOn": "Build DataStream (Debug)",
"type": "process", "type": "process",
"command": "", "command": "",
"windows": { "windows": {
"command": "DataStreamQueryAnalyzer.exe", "command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": { "options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows" "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
"env": {
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
},
}, },
}, },
"icon": { "icon": {
@@ -567,17 +574,16 @@
}, },
{ {
"label": "Run DataStreamQueryAnalyzer (Release)", "label": "Run DataStreamQueryAnalyzer (Release)",
"dependsOn": "Build DataStreamQueryAnalyzer (Release)", "dependsOn": "Build DataStream (Release)",
"type": "process", "type": "process",
"command": "", "command": "",
"windows": { "windows": {
"command": "DataStreamQueryAnalyzer.exe", "command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": { "options": {
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows" "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
"env": {
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
},
}, },
}, },
"icon": { "icon": {
@@ -1039,6 +1045,16 @@
], ],
"default": "Debug", "default": "Debug",
}, },
{
"id": "dataStreamQueryAnalyzerEnvironment",
"type": "pickString",
"description": "DataStreamQueryAnalyzer Configuration",
"options": [
"Development",
"Local",
],
"default": "Development",
},
{ {
"id": "sparkdCmakePreset", "id": "sparkdCmakePreset",
"type": "pickString", "type": "pickString",
@@ -1081,7 +1097,6 @@
"options": [ "options": [
"Development", "Development",
"EchoServer", "EchoServer",
"Local",
"Production", "Production",
"ProductionAws", "ProductionAws",
"Staging", "Staging",
+102
View File
@@ -0,0 +1,102 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.0.31903.59
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.ActiveCfg = Debug|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.Build.0 = Debug|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.ActiveCfg = Debug|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.Build.0 = Debug|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.ActiveCfg = Debug|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.Build.0 = Debug|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.ActiveCfg = Release|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.Build.0 = Release|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.ActiveCfg = Release|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.Build.0 = Release|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.ActiveCfg = Release|x64
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.Build.0 = Release|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.ActiveCfg = Debug|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.Build.0 = Debug|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.ActiveCfg = Debug|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.Build.0 = Debug|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.ActiveCfg = Debug|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.Build.0 = Debug|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.ActiveCfg = Release|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.Build.0 = Release|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.ActiveCfg = Release|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.Build.0 = Release|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.ActiveCfg = Release|x64
{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.Build.0 = Release|x64
{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.ActiveCfg = Debug|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.Build.0 = Debug|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.ActiveCfg = Debug|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.Build.0 = Debug|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.Build.0 = Release|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.ActiveCfg = Release|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.Build.0 = Release|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.ActiveCfg = Release|Any CPU
{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.Build.0 = Release|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.ActiveCfg = Debug|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.Build.0 = Debug|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.ActiveCfg = Debug|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.Build.0 = Debug|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.Build.0 = Release|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.ActiveCfg = Release|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.Build.0 = Release|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.ActiveCfg = Release|Any CPU
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.Build.0 = Release|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.ActiveCfg = Debug|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.Build.0 = Debug|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.ActiveCfg = Debug|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.Build.0 = Debug|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.Build.0 = Release|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.ActiveCfg = Release|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
EndGlobalSection
EndGlobal
+54 -149
View File
@@ -8,14 +8,31 @@
"configurePresets": [ "configurePresets": [
{ {
"name": "spark", "name": "spark-base",
"hidden": true,
"binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": {
"CMAKE_CXX_STANDARD": "20",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
}
}
},
{
"name": "spark-windows",
"inherits": "spark-base",
"displayName": "Spark",
"condition": { "condition": {
"type": "equals", "type": "equals",
"lhs": "${hostSystemName}", "lhs": "${hostSystemName}",
"rhs": "Windows" "rhs": "Windows"
}, },
"generator": "Visual Studio 17 2022", "generator": "Visual Studio 17 2022",
"binaryDir": "${sourceDir}/build/${presetName}",
"environment": { "environment": {
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600" "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
}, },
@@ -23,18 +40,9 @@
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}", "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd", "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT", "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
"CMAKE_CXX_STANDARD": "20",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG", "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG", "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake", "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
}
} }
}, },
@@ -151,17 +159,17 @@
}, },
{ {
"name": "spark-api-server-windows", "name": "spark-linux",
"inherits": "spark", "inherits": "spark-base",
"displayName": "Spark Api Server (Windows)" "displayName": "Spark",
}, "condition": {
"type": "equals",
{ "lhs": "${hostSystemName}",
"name": "spark-api-server-linux", "rhs": "Linux"
"inherits": "sparkd", },
"displayName": "Spark Api Server (Linux)",
"generator": "Ninja Multi-Config", "generator": "Ninja Multi-Config",
"cacheVariables": { "cacheVariables": {
"platform": "linux",
"BUILD_ADMIN_WEB_APP": { "BUILD_ADMIN_WEB_APP": {
"type": "BOOL", "type": "BOOL",
"value": "OFF" "value": "OFF"
@@ -170,21 +178,26 @@
"type": "BOOL", "type": "BOOL",
"value": "OFF" "value": "OFF"
}, },
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
"type": "BOOL",
"value": "OFF"
},
"CMAKE_C_COMPILER": "clang", "CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++", "CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG", "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}", "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
"CMAKE_EXPORT_COMPILE_COMMANDS": { "CMAKE_EXPORT_COMPILE_COMMANDS": {
"type": "BOOL", "type": "BOOL",
"value": "ON" "value": "ON"
}, },
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf", "CMAKE_INSTALL_RPATH": ".",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0", "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
"EXPORT_MD_CONSUMER": { "DATABENTO_USE_EXTERNAL_JSON": {
"type": "BOOL", "type": "BOOL",
"value": "ON" "value": "ON"
} },
"EXTERNALS_ROOT": "${sourceDir}/externals/linux"
} }
}, },
@@ -204,87 +217,32 @@
"cacheVariables": { "cacheVariables": {
"CCAPI_LOG_LEVEL": "Trace" "CCAPI_LOG_LEVEL": "Trace"
} }
},
{
"name": "data-stream-suite-windows",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Windows"
},
"inherits": "spark",
"displayName": "Data Stream Suite (Windows)",
"description": "Data Stream utils ported over from Edge",
"binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": {
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
}
},
{
"name": "data-stream-suite-linux",
"displayName": "Data Stream Suite (Linux)",
"description": "Data Stream utils ported over from Edge",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Linux"
},
"inherits": "sparkd",
"generator": "Ninja Multi-Config",
"cacheVariables": {
"BUILD_ADMIN_WEB_APP": {
"type": "BOOL",
"value": "OFF"
},
"BUILD_SDK": {
"type": "BOOL",
"value": "OFF"
},
"BUILD_SPARKD": {
"type": "BOOL",
"value": "OFF"
},
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
}
}
} }
], ],
"buildPresets": [ "buildPresets": [
{ {
"name": "spark", "name": "spark-windows",
"condition": { "condition": {
"type": "equals", "type": "equals",
"lhs": "${hostSystemName}", "lhs": "${hostSystemName}",
"rhs": "Windows" "rhs": "Windows"
}, },
"hidden": true "hidden": true,
"configurePreset": "spark-windows"
}, },
{ {
"name": "spark-debug", "name": "spark-windows-debug",
"displayName": "Spark (Debug)", "displayName": "Spark (Debug)",
"inherits": "spark", "inherits": "spark-windows",
"configurePreset": "spark",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "spark-release", "name": "spark-windows-release",
"displayName": "Spark (Release)", "displayName": "Spark (Release)",
"inherits": "spark", "inherits": "spark-windows",
"configurePreset": "spark",
"configuration": "Release" "configuration": "Release"
}, },
@@ -327,52 +285,27 @@
}, },
{ {
"name": "spark-api-server-windows", "name": "spark-linux",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Windows"
},
"hidden": true,
"configurePreset": "spark-api-server-windows"
},
{
"name": "spark-api-server-windows-debug",
"displayName": "Spark Api Server (Windows | Debug)",
"inherits": "spark-api-server-windows",
"configuration": "Debug"
},
{
"name": "spark-api-server-windows-release",
"displayName": "Spark Api Server (Windows | Release)",
"inherits": "spark-api-server-windows",
"configuration": "Release"
},
{
"name": "spark-api-server-linux",
"condition": { "condition": {
"type": "equals", "type": "equals",
"lhs": "${hostSystemName}", "lhs": "${hostSystemName}",
"rhs": "Linux" "rhs": "Linux"
}, },
"hidden": true, "hidden": true,
"configurePreset": "spark-api-server-linux" "configurePreset": "spark-linux"
}, },
{ {
"name": "spark-api-server-linux-debug", "name": "spark-linux-debug",
"inherits": "spark-api-server-linux", "inherits": "spark-linux",
"displayName": "Spark Api Server (Linux | Debug)", "displayName": "Spark (Linux | Debug)",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "spark-api-server-linux-release", "name": "spark-linux-release",
"inherits": "spark-api-server-linux", "inherits": "spark-linux",
"displayName": "Spark Api Server (Linux | Release)", "displayName": "Spark (Linux | Release)",
"configuration": "Release" "configuration": "Release"
}, },
@@ -389,34 +322,6 @@
"configurePreset": "sparkd-ccapi-diag", "configurePreset": "sparkd-ccapi-diag",
"displayName": "Debug (ccapi diag)", "displayName": "Debug (ccapi diag)",
"inherits": "sparkd" "inherits": "sparkd"
},
{
"name": "data-stream-suite-windows-debug",
"displayName": "Data Stream Suite (Windows | Debug)",
"configurePreset": "data-stream-suite-windows",
"configuration": "Debug"
},
{
"name": "data-stream-suite-windows-release",
"displayName": "Data Stream Suite (Windows | Release)",
"configurePreset": "data-stream-suite-windows",
"configuration": "Release"
},
{
"name": "data-stream-suite-linux-debug",
"displayName": "Data Stream Suite (Linux | Debug)",
"configurePreset": "data-stream-suite-linux",
"configuration": "Debug"
},
{
"name": "data-stream-suite-linux-release",
"displayName": "Data Stream Suite (Linux | Release)",
"configurePreset": "data-stream-suite-linux",
"configuration": "Release"
} }
] ]
} }
+426 -30
View File
@@ -1,10 +1,11 @@
Microsoft Visual Studio Solution File, Format Version 12.00
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17 # Visual Studio Version 17
VisualStudioVersion = 17.3.32922.545 VisualStudioVersion = 17.3.32922.545
MinimumVisualStudioVersion = 15.0.26124.0 MinimumVisualStudioVersion = 15.0.26124.0
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
EndProject EndProject
@@ -42,180 +43,566 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sdk", "sdk", "{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\sdk\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{70E3DCF5-A3C3-45A0-8F18-B01B2D446532}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_command_bridge", "src\client\spark_command_bridge\spark_command_bridge.csproj", "{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64 Debug|x64 = Debug|x64
Debug|Any CPU = Debug|Any CPU
Debug|x86 = Debug|x86
MinSizeRel|x64 = MinSizeRel|x64 MinSizeRel|x64 = MinSizeRel|x64
MinSizeRel|Any CPU = MinSizeRel|Any CPU
MinSizeRel|x86 = MinSizeRel|x86
Release|x64 = Release|x64 Release|x64 = Release|x64
Release|Any CPU = Release|Any CPU
Release|x86 = Release|x86
RelWithDebInfo|x64 = RelWithDebInfo|x64 RelWithDebInfo|x64 = RelWithDebInfo|x64
RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
RelWithDebInfo|x86 = RelWithDebInfo|x86
EndGlobalSection EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution GlobalSection(ProjectConfigurationPlatforms) = postSolution
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64 {DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64 {DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64 {DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64 {DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64 {DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64 {DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64 {DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64 {6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64 {6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64 {6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64 {6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64 {6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64 {6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64 {6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64 {3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64 {3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64 {3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64 {3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64 {3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64 {3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64 {3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64 {2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64 {2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64 {2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64 {2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64 {2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64 {2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64 {2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64 {EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64 {EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.ActiveCfg = Debug|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.Build.0 = Debug|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64 {EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64 {EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64 {EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64 {EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.Build.0 = Release|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.ActiveCfg = Release|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.Build.0 = Release|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64 {EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64 {EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.ActiveCfg = Debug|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.Build.0 = Debug|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.Build.0 = Release|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.ActiveCfg = Release|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.Build.0 = Release|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU {EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU {EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU {EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU {EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU {EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.Build.0 = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.ActiveCfg = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.Build.0 = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU {911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU {911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU {911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU {911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU {911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.Build.0 = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.ActiveCfg = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.Build.0 = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU {14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU {14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU {14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU {14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU {14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.Build.0 = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|x86.ActiveCfg = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|x86.Build.0 = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU {43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU {43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU {43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU {43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU {43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.Build.0 = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.ActiveCfg = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.Build.0 = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU {6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU {6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU {6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU {6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU {6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.Build.0 = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.ActiveCfg = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.Build.0 = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU {A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU {A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU {A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU {A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU {A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.Build.0 = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.ActiveCfg = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.Build.0 = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.Build.0 = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.ActiveCfg = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.Build.0 = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.Build.0 = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.ActiveCfg = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.Build.0 = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.Build.0 = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.ActiveCfg = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.Build.0 = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU {B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU {9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU {9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU {9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU {9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU {9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.Build.0 = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.ActiveCfg = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.Build.0 = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.Build.0 = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.ActiveCfg = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.Build.0 = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.ActiveCfg = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.Build.0 = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.Build.0 = Debug|Any CPU {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.ActiveCfg = Release|Any CPU {39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.Build.0 = Release|Any CPU {39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.ActiveCfg = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.Build.0 = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.ActiveCfg = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.Build.0 = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.Build.0 = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.Build.0 = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.Build.0 = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.ActiveCfg = Release|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.Build.0 = Release|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.Build.0 = Release|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.ActiveCfg = Release|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.Build.0 = Release|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x64.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x64.Build.0 = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|Any CPU.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|Any CPU.Build.0 = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x86.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x86.Build.0 = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x64.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x64.Build.0 = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|Any CPU.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|Any CPU.Build.0 = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x86.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x86.Build.0 = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x64.ActiveCfg = Release|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x64.Build.0 = Release|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|Any CPU.ActiveCfg = Release|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|Any CPU.Build.0 = Release|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x86.ActiveCfg = Release|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x86.Build.0 = Release|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x64.Build.0 = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|Any CPU.Build.0 = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x86.ActiveCfg = Debug|x64
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x86.Build.0 = Debug|x64
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
EndGlobalSection EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
EndGlobalSection
GlobalSection(Performance) = preSolution
HasPerformanceSessions = true
EndGlobalSection
GlobalSection(NestedProjects) = preSolution GlobalSection(NestedProjects) = preSolution
{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C} {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} {EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
@@ -227,10 +614,19 @@ Global
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} {2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} {FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} {B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} {9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} {6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C} {39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184} = {56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1} {385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
{70E3DCF5-A3C3-45A0-8F18-B01B2D446532} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED} = {70E3DCF5-A3C3-45A0-8F18-B01B2D446532}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
EndGlobalSection
GlobalSection(Performance) = preSolution
HasPerformanceSessions = true
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal
+7 -1
View File
@@ -40,7 +40,13 @@ function(init_python_module)
add_custom_command( add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
DEPENDS ${_name} ${module_proj} DEPENDS ${_name} ${module_proj}
COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module COMMAND
${CMAKE_COMMAND} -E env
"PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}"
${Python3_EXECUTABLE} -m pybind11_stubgen
--numpy-array-use-type-var
--ignore-invalid-expressions '.*'
${_name}_module
COMMENT "Generating py stubs" COMMENT "Generating py stubs"
) )
add_custom_target( add_custom_target(
-182
View File
@@ -1,182 +0,0 @@
set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
set(PROTOBUF_STATIC_LIB "libprotobuf")
set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
function(Protobuf_Generate_Cpp)
set(_options)
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
set(${_ARGS_CXX_SOURCES})
set(${_ARGS_CXX_HEADERS})
set(_proto_include_cli_args)
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
list(APPEND _proto_include_cli_args -I ${_dir})
endforeach()
foreach(FIL ${_ARGS_PROTO_SOURCES})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
if(FIL_DIR)
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
endif()
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
add_custom_command(
OUTPUT
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
COMMAND
${Protobuf_PROTOC_EXECUTABLE}
ARGS
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
DEPENDS ${ABS_FIL}
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
VERBATIM
)
endforeach()
set_source_files_properties(
${${_ARGS_CXX_SOURCES}}
${${_ARGS_CXX_HEADERS}}
PROPERTIES
GENERATED TRUE
)
# remove some warnings from protobuf generated files
if(WIN32)
set_source_files_properties(
${${_ARGS_CXX_SOURCES}}
PROPERTIES
COMPILE_FLAGS "/wd4309 /wd4838"
)
else()
set_source_files_properties(
${${_ARGS_CXX_SOURCES}}
PROPERTIES
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
)
endif()
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
endfunction()
function(Service_Compiler_Generate_Cpp)
set(_options)
set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
set(${_ARGS_CXX_SOURCES})
set(${_ARGS_CXX_HEADERS})
set(_proto_include_cli_args)
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
list(APPEND _proto_include_cli_args -I ${_dir})
endforeach()
foreach(FIL ${_ARGS_PROTO_SOURCES})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
if(FIL_DIR)
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
endif()
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
set(TEMPLATES
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
)
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
if(WIN32)
set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
add_custom_command(
OUTPUT
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
COMMAND SET PYTHONPATH=${SPARK_PYTHON}
COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
COMMAND
${Protobuf_PROTOC_EXECUTABLE}
ARGS
--cpp_out=${_ARGS_CXX_OUT_DIR}
--plugin=protoc-gen-service=${_plugin_cmd}
--service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
-I ${PROTOBUF_INCLUDE_DIRS}
-I ${_protobuf_src_dir}
${ABS_FIL}
DEPENDS
${ABS_FIL}
${TEMPLATES}
${_plugin_cmd}
${_plugin_py}
COMMENT "Running C++ service compiler on ${FIL}"
VERBATIM
)
else()
add_custom_command(
OUTPUT
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
ARGS
${Protobuf_PROTOC_EXECUTABLE}
--cpp_out=${_ARGS_CXX_OUT_DIR}
--plugin=protoc-gen-service=${_plugin_py}
--service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
${_proto_include_cli_args} ${ABS_FIL}
DEPENDS
${ABS_FIL}
${TEMPLATES}
${_plugin_py}
COMMENT "Running C++ service compiler on ${FIL}"
VERBATIM
)
endif()
endforeach()
# remove some warnings from protobuf generated files
if(WIN32)
set_source_files_properties(
${${_ARGS_CXX_SOURCES}}
PROPERTIES
COMPILE_FLAGS "/wd4309 /wd4838"
)
else()
set_source_files_properties(
${${_ARGS_CXX_SOURCES}}
PROPERTIES
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
)
endif()
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
endfunction()
+91 -61
View File
@@ -2,6 +2,7 @@ services:
sparkd: sparkd:
volumes: volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
build: build:
dockerfile: src/app_sparkd/Dockerfile dockerfile: src/app_sparkd/Dockerfile
args: args:
@@ -9,24 +10,38 @@ services:
- SOURCE_DIR=${SPARKD__SOURCE_DIR} - SOURCE_DIR=${SPARKD__SOURCE_DIR}
- CONFIG_DIR=${SPARKD__CONFIG_DIR} - CONFIG_DIR=${SPARKD__CONFIG_DIR}
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS} - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
shm_size: "${SPARKD__SHM_SIZE:-512mb}" shm_size: "${SPARKD__SHM_SIZE:-512mb}"
command: > env_file:
sh -c " - .env
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data - ${SPARKD__CONFIG_DIR}/.env
if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then ports:
mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')" - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
fi - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local' - 8008:8008
set -a - 8080:8080
. /home/bts/.local/share/bts/sparkd/conf/.env.local - 9443:9443
set +a entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
fi deploy:
if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then resources:
echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)' limits:
cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/ memory: "${SPARKD__MEMORY:-4gb}"
fi
./sparkd run" sparkd-shell:
volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
build:
dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
args:
- IMAGE=${SPARKD__IMAGE}
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
command: "do sleep 1; done"
env_file: env_file:
- .env - .env
- ${SPARKD__CONFIG_DIR}/.env - ${SPARKD__CONFIG_DIR}/.env
@@ -42,6 +57,7 @@ services:
sparkd-symbols: sparkd-symbols:
volumes: volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
build: build:
dockerfile: src/app_sparkd/Dockerfile dockerfile: src/app_sparkd/Dockerfile
@@ -50,26 +66,8 @@ services:
- SOURCE_DIR=${SPARKD__SOURCE_DIR} - SOURCE_DIR=${SPARKD__SOURCE_DIR}
- CONFIG_DIR=${SPARKD__CONFIG_DIR} - CONFIG_DIR=${SPARKD__CONFIG_DIR}
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS} - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
shm_size: "${SPARKD__SHM_SIZE:-512mb}" shm_size: "${SPARKD__SHM_SIZE:-512mb}"
command: >
sh -c "
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
fi
ls -alh /home/bts/.local/share/bts/sparkd/conf/
cat /home/bts/.local/share/bts/sparkd/conf/.env.local
if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
set -a
. /home/bts/.local/share/bts/sparkd/conf/.env.local
set +a
fi
if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
fi
./sparkd run"
env_file: env_file:
- .env - .env
- ${SPARKD__CONFIG_DIR}/.env - ${SPARKD__CONFIG_DIR}/.env
@@ -77,6 +75,36 @@ services:
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}" - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}" - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}" - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
- 8008:8008
- 8080:8080
- 9443:9443
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
deploy:
restart_policy:
condition: any
delay: 3s
resources:
limits:
memory: "${SPARKD__MEMORY:-4gb}"
sparkd-multicast:
volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
build:
dockerfile: src/app_sparkd/Dockerfile
args:
- IMAGE=${SPARKD__IMAGE}
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
shm_size: "${SPARKD__SHM_SIZE:-2048mb}"
env_file:
- .env
- ${SPARKD__CONFIG_DIR}/.env
network_mode: "host"
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
deploy: deploy:
restart_policy: restart_policy:
condition: any condition: any
@@ -84,16 +112,15 @@ services:
resources: resources:
limits: limits:
memory: "${SPARKD__MEMORY:-4gb}" memory: "${SPARKD__MEMORY:-4gb}"
sparkd-tail: sparkd-tail:
image: bluetradingsystems/linux_dotnet7 image: bluetradingsystems/linux_dotnet8
volumes: volumes:
- type: volume - type: volume
read_only: true read_only: true
source: sparkd_data source: sparkd_data
target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
volume: volume:
subpath: logging subpath: logging.sparkd
command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"] command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
sparkd-ping-test: sparkd-ping-test:
@@ -104,14 +131,14 @@ services:
build: build:
context: . context: .
dockerfile_inline: | dockerfile_inline: |
FROM bluetradingsystems/linux_dotnet7 FROM bluetradingsystems/linux_dotnet8
USER "root" USER "root"
RUN apt-get update && apt-get -y install iputils-ping RUN apt-get update && apt-get -y install iputils-ping
command: [ "ping", "sparkd" ] command: [ "ping", "sparkd" ]
pubsub: pubsub:
volumes: volumes:
- ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/PubSubServer/data - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
build: build:
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
args: args:
@@ -121,20 +148,20 @@ services:
- 22019:22019 - 22019:22019
command: > command: >
sh -c " sh -c "
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/PubSubServer echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
if [ -e "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" ]; then if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
mv "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')" mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
fi fi
if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env" ]; then if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env' echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
set -a set -a
. /home/bts/.local/share/bts/PubSubServer/conf/.env . /home/bts/.local/share/bts/pubsub/conf/.env
set +a set +a
fi fi
if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env.local" ]; then if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env.local' echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
set -a set -a
. /home/bts/.local/share/bts/PubSubServer/conf/.env.local . /home/bts/.local/share/bts/pubsub/conf/.env.local
set +a set +a
fi fi
./PubSubServer" ./PubSubServer"
@@ -144,8 +171,8 @@ services:
md-publisher: md-publisher:
volumes: volumes:
- ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/DataStreamPublisher/data - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
- historic_md:/home/bts/.local/share/bts/databento - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
build: build:
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
args: args:
@@ -153,20 +180,20 @@ services:
- CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR} - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
command: > command: >
sh -c " sh -c "
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/DataStreamPublisher echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" ]; then if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
mv "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')" mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
fi fi
if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env" ]; then if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env' echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
set -a set -a
. /home/bts/.local/share/bts/DataStreamPublisher/conf/.env . /home/bts/.local/share/bts/md_publisher/conf/.env
set +a set +a
fi fi
if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local" ]; then if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local' echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
set -a set -a
. /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local . /home/bts/.local/share/bts/md_publisher/conf/.env.local
set +a set +a
fi fi
dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher" dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
@@ -195,3 +222,6 @@ volumes:
name: "md_publisher_22019" name: "md_publisher_22019"
historic_md: historic_md:
name: "historic_md" name: "historic_md"
md_recording:
name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
external: true
@@ -0,0 +1,46 @@
FROM ubuntu:22.04
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y pkg-config
RUN apt-get update && apt-get install -y locales
RUN apt-get update && apt-get install -y ca-certificates
RUN apt-get update && apt-get install -y libssh2-1-dev
RUN apt-get update && apt-get install -y software-properties-common
RUN apt-get update && apt-get install -y sudo
RUN apt-get update && apt-get install -y tzdata
# GCC
ARG gcc_major=13
RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-get update \
&& apt-get install -y g++-$gcc_major
RUN apt-get update \
&& apt-get install -y dotnet8
RUN apt-get update && apt-get install -y libpq-dev
RUN apt-get update && apt-get install -y libsodium-dev
RUN apt-get update && apt-get install -y fish
RUN apt-get update && apt-get install -y less
RUN apt-get update && apt-get install -y vim
RUN locale-gen en_US.UTF-8
ARG docker_user="bts"
ARG docker_uid="1000"
ARG docker_pass="bts"
RUN \
groupadd -g "$docker_uid" "$docker_user" \
&& useradd \
-m \
-u "$docker_uid" \
-g "$docker_uid" \
-G sudo \
--password "$(openssl passwd -1 "$docker_pass")" \
"$docker_user"
USER bts
WORKDIR /home/bts
@@ -55,14 +55,16 @@ RUN \
clangd-$clang_major clangd-$clang_major
# CMake # CMake
RUN \ RUN set -eux \
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \ && apt-get update \
| gpg --dearmor - \ && apt-get install -y --no-install-recommends ca-certificates curl gpg \
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \ && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \ | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
&& apt-get update \ > /etc/apt/sources.list.d/kitware.list \
&& apt-get install -y cmake && apt-get update \
&& apt-get install -y --no-install-recommends cmake \
&& rm -rf /var/lib/apt/lists/*
# Dotnet # Dotnet
RUN apt-get update && apt-get install -y dotnet8 RUN apt-get update && apt-get install -y dotnet8
@@ -80,9 +82,14 @@ RUN mkdir -p /etc/apt/keyrings \
# Emumba (deps) # Emumba (deps)
RUN apt-get update && apt-get -y install rapidjson-dev RUN apt-get update && apt-get -y install rapidjson-dev
# Emumba # Emumba
COPY emumba /emumba ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \ RUN --mount=type=secret,id=bitbucket_access_token \
&& rm -r /emumba set -eux \
&& apt-get update \
&& TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
&& curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
&& apt-get install -y /tmp/emumba-deribit-connector.deb \
&& rm -f /tmp/emumba-deribit-connector.deb
# Pcap # Pcap
RUN apt-get update && apt-get install -y libpcap0.8-dev RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -90,6 +97,18 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
# Databento (deps) # Databento (deps)
RUN apt-get update && apt-get -y install libzstd-dev RUN apt-get update && apt-get -y install libzstd-dev
# Rakurai
ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.12_amd64.deb"
RUN --mount=type=secret,id=bitbucket_access_token \
set -eux \
&& apt-get update \
&& TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
&& curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
# Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
# spdlog headers.
&& apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
&& rm -f /tmp/rakuirai.deb
# Docker in Docker # Docker in Docker
RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \
docker.io \ docker.io \
@@ -0,0 +1,11 @@
FROM ubuntu:24.04
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \
fish \
less \
vim
USER bts
WORKDIR /home/bts
+6
View File
@@ -0,0 +1,6 @@
{
"sdk": {
"version": "6.0.200",
"rollForward": "latestMajor"
}
}
+3 -3
View File
@@ -18,10 +18,10 @@ MarkupSafe==3.0.2
mccabe==0.6.1 mccabe==0.6.1
numpy==2.2.6 numpy==2.2.6
platformdirs==2.3.0 platformdirs==2.3.0
protobuf==4.21.12 protobuf==6.33.4
pybars3==0.9.7 pybars3==0.9.7
pybind11==2.13.6 pybind11==3.0.1
pybind11-stubgen==2.5.3 pybind11-stubgen==2.5.5
pybit==5.11.0 pybit==5.11.0
pycodestyle==2.7.0 pycodestyle==2.7.0
pycparser==2.20 pycparser==2.20
+306 -164
View File
@@ -1,164 +1,306 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17 Microsoft Visual Studio Solution File, Format Version 12.00
VisualStudioVersion = 17.3.32922.545 # Visual Studio Version 17
MinimumVisualStudioVersion = 15.0.26124.0 VisualStudioVersion = 17.0.31903.59
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}" MinimumVisualStudioVersion = 10.0.40219.1
EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}" EndProject
EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}" EndProject
EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}" EndProject
EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
Global EndProject
GlobalSection(SolutionConfigurationPlatforms) = preSolution Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
Debug|x64 = Debug|x64 EndProject
MinSizeRel|x64 = MinSizeRel|x64 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
Release|x64 = Release|x64 EndProject
RelWithDebInfo|x64 = RelWithDebInfo|x64 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
EndGlobalSection EndProject
GlobalSection(ProjectConfigurationPlatforms) = postSolution Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64 EndProject
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64 EndProject
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64 EndProject
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64 EndProject
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64 Global
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64 GlobalSection(SolutionConfigurationPlatforms) = preSolution
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64 Debug|Any CPU = Debug|Any CPU
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64 Debug|x64 = Debug|x64
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64 Debug|x86 = Debug|x86
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64 Release|Any CPU = Release|Any CPU
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64 Release|x64 = Release|x64
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64 Release|x86 = Release|x86
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64 EndGlobalSection
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64 GlobalSection(ProjectConfigurationPlatforms) = postSolution
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64 {684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64 {45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64 {AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64 {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU {28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU {7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU {C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU {F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
EndGlobalSection {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
GlobalSection(SolutionProperties) = preSolution {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
HideSolutionNode = FALSE {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
EndGlobalSection {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
GlobalSection(ExtensibilityGlobals) = postSolution {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58} {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
EndGlobalSection {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
GlobalSection(Performance) = preSolution {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
HasPerformanceSessions = true {E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection {ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
GlobalSection(NestedProjects) = preSolution {ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B} {ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B} {ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD} {ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
EndGlobalSection {ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
EndGlobal {ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
EndGlobalSection
EndGlobal
@@ -11,10 +11,20 @@
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.2" />
</ItemGroup> </ItemGroup>
<ItemGroup> <Choose>
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" /> <When Condition="'$(UseProjectRefs)' == 'true'">
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> <ItemGroup>
</ItemGroup> <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
<ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
<PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
</ItemGroup>
</Otherwise>
</Choose>
<ItemGroup Label="ArithmeticExpression"> <ItemGroup Label="ArithmeticExpression">
<Scanner Include="ArithmeticExpressionTokens.lex" /> <Scanner Include="ArithmeticExpressionTokens.lex" />
@@ -5,9 +5,5 @@
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
</ItemGroup>
</Project> </Project>
@@ -6,23 +6,83 @@ using Spark.Core.Collections;
static class ArgumentComparer { static class ArgumentComparer {
public static int Compare( public static int Compare(
IArgument value1, IArgument value2, IDictionary<string,object?> env) { IArgument value1, IArgument value2, IDictionary<string, object?> env) {
if (value1 is null) { if (value1 is null) {
throw new ArgumentNullException(nameof(value1)); throw new ArgumentNullException(nameof(value1));
} }
if (value2 is null) { if (value2 is null) {
throw new ArgumentNullException(nameof(value2)); throw new ArgumentNullException(nameof(value2));
} }
var cmp1 = value1.Cast<IComparable>(); var obj1 = value1.GetValue(env);
var cmp2 = value2.Cast<IComparable>(); var obj2 = value2.GetValue(env);
if (isNullLike(obj1)) {
if (isNullLike(obj2)) {
return 0;
}
return -1;
}
if (isNullLike(obj2)) {
return 1;
}
return compareObjs(obj1!, obj2!);
}
static bool isNullLike(object? value) =>
value is null || DBNull.Value.Equals(value);
static int compareObjs(object obj1, object obj2) {
try { try {
var v1 = cmp1.GetValue(env); var typ1 = obj1.GetType();
var v2 = cmp2.GetValue(env); var typ2 = obj2.GetType();
return v1.GetType().IsNumeric() ? bool areSameTypes = typ1 == typ2;
new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2); if (areSameTypes) {
return ((IComparable)obj1).CompareTo(obj2);
}
if (typ1.IsNumeric()) {
return tryCompareNumeric(obj1, obj2);
} else if (typ2.IsNumeric()) {
return -tryCompareNumeric(obj2, obj1);
} else if (typ1.IsEnum) {
if (typ2 == typeof(string)) {
return -((string)obj2).CompareTo(obj1.ToString());
}
} else if (typ1 == typeof(string)) {
return ((string)obj1).CompareTo(obj2.ToString());
} else if (typ1 == typeof(DateTime)) {
if (typ2 == typeof(TimeSpan)) {
return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
}
} else if (typ1 == typeof(TimeSpan)) {
if (typ2 == typeof(DateTime)) {
return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
}
}
} catch (Exception ex) { } catch (Exception ex) {
Log.Ui.error($"Comparison failed: {ex}"); Log.Ui.error($"Comparison failed: {ex}");
throw; throw;
} }
throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
}
static int tryCompareNumeric(object numericObj1, object obj2) {
var typ2 = obj2.GetType();
if (typ2.IsNumeric()) {
return new NumericComparer().Compare(numericObj1, obj2);
} else if (typ2.IsEnum) {
var undTyp2 = typ2.GetEnumUnderlyingType();
if (undTyp2 == typeof(byte)) {
return new NumericComparer().Compare(numericObj1, (byte)obj2);
}
if (undTyp2 == typeof(short)) {
return new NumericComparer().Compare(numericObj1, (short)obj2);
}
if (undTyp2 == typeof(int)) {
return new NumericComparer().Compare(numericObj1, (int)obj2);
}
if (undTyp2 == typeof(long)) {
return new NumericComparer().Compare(numericObj1, (long)obj2);
}
}
throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
} }
} }
@@ -11,10 +11,20 @@
<PackageReference Include="YaccLexTools" Version="1.2.3" /> <PackageReference Include="YaccLexTools" Version="1.2.3" />
</ItemGroup> </ItemGroup>
<ItemGroup> <Choose>
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" /> <When Condition="'$(UseProjectRefs)' == 'true'">
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> <ItemGroup>
</ItemGroup> <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
<ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
<PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
</ItemGroup>
</Otherwise>
</Choose>
<ItemGroup> <ItemGroup>
<Scanner Include="BooleanExpressionTokens.lex" /> <Scanner Include="BooleanExpressionTokens.lex" />
@@ -10,11 +10,20 @@
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.2" />
</ItemGroup> </ItemGroup>
<ItemGroup> <Choose>
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" /> <When Condition="'$(UseProjectRefs)' == 'true'">
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" /> <ItemGroup>
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
</ItemGroup> <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
<PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
</ItemGroup>
</Otherwise>
</Choose>
<ItemGroup Label="ColumnExpression"> <ItemGroup Label="ColumnExpression">
<Scanner Include="ColumnExpressionTokens.lex" /> <Scanner Include="ColumnExpressionTokens.lex" />
@@ -10,11 +10,20 @@
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.2" />
</ItemGroup> </ItemGroup>
<ItemGroup> <Choose>
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" /> <When Condition="'$(UseProjectRefs)' == 'true'">
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" /> <ItemGroup>
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
</ItemGroup> <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
<PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
</ItemGroup>
</Otherwise>
</Choose>
<ItemGroup Label="ColumnSortExpression"> <ItemGroup Label="ColumnSortExpression">
<Scanner Include="ColumnSortExpressionTokens.lex" /> <Scanner Include="ColumnSortExpressionTokens.lex" />
@@ -10,9 +10,18 @@
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.2" />
</ItemGroup> </ItemGroup>
<ItemGroup> <Choose>
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" /> <When Condition="'$(UseProjectRefs)' == 'true'">
</ItemGroup> <ItemGroup>
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
</ItemGroup>
</Otherwise>
</Choose>
<ItemGroup Label="ColumnMapExpression"> <ItemGroup Label="ColumnMapExpression">
<Scanner Include="ColumnMapExpressionTokens.lex" /> <Scanner Include="ColumnMapExpressionTokens.lex" />
@@ -1,118 +0,0 @@
namespace Bts.DataStream;
using BCG;
using bts.collections;
using bts.utils;
using Bts.DataStreamCore;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
public class BlueAddIndexColumnTable : IBlueRowCollection {
readonly IBlueTableSchema schema;
readonly IList<object?>[] columnValues;
readonly int count;
BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
this.schema = schema;
this.columnValues = columnValues;
this.count = count ?? columnValues[0].Count;
}
IBlueTableSchema IBlueRowCollection.schema => schema;
public int Count => count;
public IBlueRow this[int id] =>
id > 0 && id <= count
? GetRow(id - 1)
: throw new KeyNotFoundException();
public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
if (columnInfos.Any(c => c.colName == "id")) {
throw new ArgumentException("'id' column already exists", nameof(columnInfos));
}
if (columnInfos.Length != columnValues.Length) {
throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
}
int count = columnValues.FirstOrDefault()?.Count ?? 0;
var origCols = columnInfos;
var origColValues = columnValues;
columnInfos = new ColumnInfo[origCols.Length + 1];
columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
columnValues = new IList<object?>[origCols.Length + 1];
columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
for (int i = 1; i < columnInfos.Length; i++) {
columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
columnValues[i] = origColValues[i - 1];
}
var schema = new BlueTableSchema(columnInfos, tableName);
return new(schema, columnValues, count);
}
public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
int count = columnValues.FirstOrDefault()?.Count ?? 0;
int? idColIdx = schema.TryResolveIndices("id");
var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
var columnInfos = schema.columnInfo;
if (idColIdx.HasValue) {
// Replace existing id column. columnValues may or may not contain the old ids (if its
// length is one less column than that of columnInfos, it doesn't contain them).
switch (columnValues.Length - schema.columnInfo.Length) {
case 0:
columnValues[idColIdx.Value] = idColValues;
break;
case -1:
columnValues = ArrayHelper.InsertExact(columnValues, idColIdx.Value, true, idColValues);
break;
default:
throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
}
} else {
if (columnValues.Length != columnInfos.Length) {
throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
}
var origColInfos = columnInfos;
columnInfos = new ColumnInfo[columnInfos.Length + 1];
columnInfos[0] = new("id", 0, typeof(int));
for (int i = 1; i < columnInfos.Length; i++) {
var c = origColInfos[i - 1];
columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
}
columnValues = ArrayHelper.InsertExact(columnValues, 0, true, idColValues);
}
return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
}
public bool containsID(int id) =>
id > 0 && id <= count;
public IList<object?> GetColumn(int index) =>
columnValues[index];
public IBlueRow GetRow(int index) =>
new BlueRowColumnWrapper(schema, index, columnValues);
public int IndexOf(IBlueRow item) {
int id = item.Id();
return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
? id - 1
: -1;
}
public IEnumerator<IBlueRow> GetEnumerator() {
int count = Count;
for (int i = 0; i < count; i++) {
yield return GetRow(i);
}
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
@@ -1,56 +0,0 @@
namespace Bts.DataStream;
using BCG;
using Bts.DataStreamCore;
using mbts.utils;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
public class BlueIndexedColumnTable : IBlueRowCollection {
readonly IBlueTableSchema schema;
readonly IList<object?>[] columnValues;
readonly Dictionary<int, int> rowIdToIndex;
public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
this.schema = schema;
this.columnValues = columnValues;
int idColumnIndex = schema.ResolveIndices("id");
rowIdToIndex = columnValues[idColumnIndex]
.Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
.ToDictionary();
}
public IBlueRow this[int id] =>
rowIdToIndex.TryGetValue(id, out var idx)
? GetRow(idx)
: throw new KeyNotFoundException();
IBlueTableSchema IBlueRowCollection.schema => schema;
public int Count => rowIdToIndex.Count;
public bool containsID(int id) =>
rowIdToIndex.ContainsKey(id);
public IEnumerator<IBlueRow> GetEnumerator() {
int count = Count;
for (int i = 0; i < count; i++) {
yield return GetRow(i);
}
}
public IBlueRow GetRow(int index) =>
new BlueRowColumnWrapper(schema, index, columnValues);
public int IndexOf(IBlueRow item) =>
rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
? idx
: -1;
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
@@ -1,21 +0,0 @@
namespace Bts.DataStream;
using BCG;
using System.Collections.Generic;
public class BlueRowColumnWrapper : IBlueRow {
readonly int index;
readonly IList<object?>[] columns;
public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
this.schema = schema;
this.index = index;
this.columns = columns;
}
public object? this[int index] =>
columns[index][this.index];
public IBlueTableSchema schema { get; }
}
@@ -1,9 +1,7 @@
using System;
namespace BCG { namespace BCG {
[DataStreamUserRespresentation("CountStream")] [DataStreamUserRespresentation("CountStream")]
public class CountStream : ViewStream { public class CountStream : ViewStream {
private DataPacket myLastFiredDataPacket; IDataPacket? myLastFiredDataPacket;
private IBlueTableSchema mySchema = new BlueTableSchema("count", private IBlueTableSchema mySchema = new BlueTableSchema("count",
new ColumnInfo("id", 0, typeof(int)), new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("count", 1, typeof(int))); new ColumnInfo("count", 1, typeof(int)));
@@ -14,6 +12,9 @@ namespace BCG {
myVS = vs; myVS = vs;
} }
public CountStream(string tableName)
: this(new TableStream(tableName)) { }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
myVS.onChange += new ChangeDelegate(myHandleChanges); myVS.onChange += new ChangeDelegate(myHandleChanges);
} }
@@ -21,7 +22,6 @@ namespace BCG {
myVS.onChange -= new ChangeDelegate(myHandleChanges); myVS.onChange -= new ChangeDelegate(myHandleChanges);
} }
private void myHandleChanges(object newValue) { private void myHandleChanges(object newValue) {
myFireEvent( myFireEvent(
myLastFiredDataPacket = myLastFiredDataPacket =
@@ -36,11 +36,8 @@ namespace BCG {
private IBlueRowCollection myGetRows(object dp, int id) { private IBlueRowCollection myGetRows(object dp, int id) {
BlueTable retVal = new BlueTable(mySchema); BlueTable retVal = new BlueTable(mySchema);
retVal.addRow retVal.addRow(new BlueRow(mySchema, id, ((IDataPacket)dp).newRows.Count));
(new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
return retVal; return retVal;
} }
} }
} }
@@ -15,6 +15,7 @@ namespace bts.BlueOrb {
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour"); rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute"); rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day"); rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_year");
return rv; return rv;
}; };
ViewStream rs = new MapStream(parent_stream, ViewStream rs = new MapStream(parent_stream,
@@ -30,6 +31,7 @@ namespace bts.BlueOrb {
rv.setValue($"{inp_col}_hour", dt.Hour); rv.setValue($"{inp_col}_hour", dt.Hour);
rv.setValue($"{inp_col}_minute", dt.Minute); rv.setValue($"{inp_col}_minute", dt.Minute);
rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds); rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
rv.setValue($"{inp_col}_year", dt.Date.Year);
return rv; return rv;
}); });
mySetRealSource(rs); mySetRealSource(rs);
@@ -1,5 +1,8 @@
namespace bts.BlueOrb { namespace bts.BlueOrb {
using BCG; using BCG;
using Bts.DataStreamCore;
using System; using System;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
@@ -21,22 +24,24 @@ namespace bts.BlueOrb {
#region IStreamFilter Members #region IStreamFilter Members
public bool filter(IBlueRow dr) { public bool filter(IBlueRow dr) {
bool rv = true;
for (int i = 0; i < myCols.Length; i++) { for (int i = 0; i < myCols.Length; i++) {
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString()); var val = dr[myCols[i]];
if (!rv) var str = val is DBNull
return rv; ? string.Empty
: (val?.ToString() ?? string.Empty);
if (!myRegExps[i].IsMatch(str)) {
return false;
}
} }
return rv; return true;
} }
#endregion #endregion
} }
private BlueTable myLastFiredRows; BlueTable? myLastFiredRows;
private IBlueRowCollection myLastReceivedRows; IBlueRowCollection? myLastReceivedRows;
private IStreamFilter myFilter; private IStreamFilter myFilter;
private ViewStream myParentStream; private ViewStream myParentStream;
private string myFilterString; private string myFilterString;
private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
#region constructors #region constructors
public delegate bool OnFilter(IBlueRow r); public delegate bool OnFilter(IBlueRow r);
@@ -52,12 +57,11 @@ namespace bts.BlueOrb {
public FilterStream(ViewStream vs, IStreamFilter filter) { public FilterStream(ViewStream vs, IStreamFilter filter) {
{ myParentStream = vs;
myParentStream = vs; myFilter = filter;
myFilter = filter; myFilterString = filter.ToString() ?? string.Empty;
myFilterString = filter.ToString(); if (filter is IDynamicStreamFilter dynFilter) {
if (filter is IDynamicStreamFilter) dynFilter.onFilterChange += myHandleFilterChange;
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
} }
} }
@@ -86,7 +90,7 @@ namespace bts.BlueOrb {
} }
protected override void myFireEvent(object o) { protected override void myFireEvent(object o) {
DataPacket dp = o as DataPacket; var dp = (IDataPacket)o;
if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0) if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
base.myFireEvent(o); base.myFireEvent(o);
} }
@@ -94,23 +98,27 @@ namespace bts.BlueOrb {
private void myHandleCallbacks(object newValue) { private void myHandleCallbacks(object newValue) {
DataPacket retVal; DataPacket retVal;
DataPacket arrivedPacket = newValue as DataPacket; IDataPacket arrivedPacket = (IDataPacket)newValue;
myLastReceivedRows = arrivedPacket.newRows; myLastReceivedRows = arrivedPacket.newRows;
if (myLastFiredRows != null) { if (myLastFiredRows != null) {
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema), retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
new BlueTable(myLastFiredRows.schema)); new BlueTable(myLastFiredRows.schema));
foreach (IBlueRow r in arrivedPacket.deletedRows) var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
if (myLastFiredRows.containsID((int)r["id"])) { foreach (IBlueRow r in arrivedPacket.deletedRows) {
if (myLastFiredRows.containsID(r.Id())) {
myLastFiredRows.deleteRow(r); myLastFiredRows.deleteRow(r);
(retVal.deletedRows as BlueTable).addRow(r); deletedRows.Value.addRow(r);
} }
foreach (IBlueRow r in arrivedPacket.addedRows) }
var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
foreach (IBlueRow r in arrivedPacket.addedRows) {
if (myFilter.filter(r)) { if (myFilter.filter(r)) {
myLastFiredRows.addRow(r); myLastFiredRows.addRow(r);
(retVal.addedRows as BlueTable).addRow(r); addedRows.Value.addRow(r);
} }
}
} else { } else {
myLastFiredRows = new BlueTable(arrivedPacket.schema); myLastFiredRows = new BlueTable(arrivedPacket.Schema);
retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema)); retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
foreach (IBlueRow r in arrivedPacket.newRows) foreach (IBlueRow r in arrivedPacket.newRows)
if (myFilter.filter(r)) if (myFilter.filter(r))
@@ -1,12 +1,13 @@
using System; using BCG;
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Text.Json;
using bts;
//using bts.SilverPine;
using BCG;
namespace bts.BlueOrb { namespace bts.BlueOrb {
[DataStreamUserRespresentation("GroupByStream")] [DataStreamUserRespresentation("GroupByStream")]
@@ -14,20 +15,21 @@ namespace bts.BlueOrb {
public interface INonIncGroupByEvaluator { public interface INonIncGroupByEvaluator {
// grouped_rows // grouped_rows
IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows); IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
string[] get_group_by_cols(); string[]? get_group_by_cols();
} }
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator { private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
Func<string[]> _groupby_cols_func; Func<string[]?> _groupby_cols_func;
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func; Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func, public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) { Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
_groupby_cols_func = groupby_cols_func; _groupby_cols_func = groupby_cols_func;
_evaluate_func = evaluate_func; _evaluate_func = evaluate_func;
} }
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable rv = null; public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable? rv = null;
foreach (var kv in grouped_rows) { foreach (var kv in grouped_rows) {
List<IBlueRow> g_rows = kv.Value; List<IBlueRow> g_rows = kv.Value;
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows); List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
@@ -39,17 +41,19 @@ namespace bts.BlueOrb {
} }
return rv; return rv;
} }
public string[] get_group_by_cols() {
public string[]? get_group_by_cols() {
return _groupby_cols_func(); return _groupby_cols_func();
} }
} }
private ViewStream _main_stream; ViewStream? _main_stream;
private INonIncGroupByEvaluator _groupby_func; INonIncGroupByEvaluator? _groupby_func;
private BlueRowHasher _row_hasher; BlueRowHasher? _row_hasher;
private string[] _groupby_cols; string[]? _groupby_cols;
private Stopwatch _timer = new(); private Stopwatch _timer = new();
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
IDisposable? subscription;
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) { public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
_main_stream = main_stream; _main_stream = main_stream;
@@ -58,7 +62,7 @@ namespace bts.BlueOrb {
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length); _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
} }
public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func, public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { } : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
@@ -68,34 +72,42 @@ namespace bts.BlueOrb {
// this is the main constructor that does everything // this is the main constructor that does everything
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) { public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops); MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
Func<string[]> groupby_cols_func = () => {
if (String.IsNullOrEmpty(groupby_cols))
return null;
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
};
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => { Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
if (!String.IsNullOrEmpty(optional_sort_col)) if (!String.IsNullOrEmpty(optional_sort_col))
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col)); group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
// this si where the group by aggregation happens // this si where the group by aggregation happens
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by); IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
if (rv == null) if (rv == null)
return new List<IBlueRow>(); return new List<IBlueRow>();
return new IBlueRow[] { rv }.ToList(); return new IBlueRow[] { rv }.ToList();
}; };
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func); ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
mySetRealSource(rs); mySetRealSource(rs);
string[]? groupby_cols_func() {
if (string.IsNullOrEmpty(groupby_cols)) {
return null;
}
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
}
} }
public GroupByStream(ViewStream parent_str, string standard_group_by_text) : public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
this(parent_str, String.Empty, standard_group_by_text) { this(parent_str, String.Empty, standard_group_by_text) {
} }
protected override void myOnFirstSubscriber() {
Log.isnull(subscription);
subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
}
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } protected override void myOnLastSubscriber() {
subscription?.Dispose();
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } subscription = null;
}
private void _main_stream_onchange(object nv) { private void _main_stream_onchange(object nv) {
DataPacket dp = nv as DataPacket; Log.notnull(_groupby_func);
var dp = (IDataPacket)nv;
///grouping together ///grouping together
_timer.Restart(); _timer.Restart();
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ? IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -109,7 +121,7 @@ namespace bts.BlueOrb {
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime)); // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing /// reducing
_timer.Restart(); _timer.Restart();
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
if (total_state == null) if (total_state == null)
// mayeb returna n empty table ? // mayeb returna n empty table ?
return; return;
@@ -121,7 +133,10 @@ namespace bts.BlueOrb {
Log.DataFlow.info( Log.DataFlow.info(
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}"); $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
} }
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); var deletedRows = myLastValue is null
? new BlueTable(total_state.schema)
: ((IDataPacket)myLastValue).newRows;
myFireEvent(new DataPacket(total_state, total_state, deletedRows));
} }
} }
} }
@@ -1,41 +1,44 @@
namespace bts.BlueOrb { namespace bts.BlueOrb {
using BCG; using BCG;
using RowHash = System.Int32;
using Bts.DataStream;
using Spark.Core.Diagnostics;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq; using System.Linq;
using System.Text;
using System.Threading.Tasks; using RowHash = System.Int32;
using System.Collections;
using System.Data.Entity.Core.Mapping;
[DataStreamUserRespresentation("JoinStream")] [DataStreamUserRespresentation("JoinStream")]
public class SimpleJoinStream : ViewStream { public class SimpleJoinStream : ViewStream {
public class JoinBookkeepingData { public class JoinBookkeepingData {
// initialized in cons // initialized in cons
private readonly List<string> _join_col_names; readonly string[] _join_col_names;
private readonly bool _is_left;
private readonly BlueRowHasher _hasher; private readonly BlueRowHasher _hasher;
// initialized on first udpate // initialized on first udpate
private IBlueTableSchema _schema; IBlueTableSchema? _schema;
private int[] _join_cols; int[]? _join_cols;
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>(); private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) { public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
_join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList(); _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
_is_left = is_left;
_hasher = row_hasher; _hasher = row_hasher;
} }
public IBlueTableSchema schema => _schema; [NotNullIfNotNull(nameof(_schema))]
public IBlueTableSchema? schema => _schema;
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map; public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
public List<IBlueRow> get_matching_rows(int hash) { [MemberNotNull(nameof(_schema))]
List<IBlueRow> rv = null; public IBlueTableSchema RequireSchema() =>
_hash_row_map.TryGetValue(hash, out rv); ExceptionAssert.MemberNotNull(_schema);
return rv;
} public List<IBlueRow>? get_matching_rows(int hash) =>
_hash_row_map.GetValueOrDefault(hash);
private RowHash _get_hash(IBlueRow r) { private RowHash _get_hash(IBlueRow r) {
if (_join_cols == null || _join_cols.Length == 0) { if (_join_cols == null || _join_cols.Length == 0) {
@@ -44,16 +47,15 @@
return _hasher.get_hash(r, _join_cols); return _hasher.get_hash(r, _join_cols);
} }
public void update_data(DataPacket dp) { public void update_data(IDataPacket dp) {
if (!ready) { if (!ready) {
_schema = dp.newRows.schema; _schema = dp.newRows.schema;
_join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray(); _join_cols = _schema.ResolveIndices(_join_col_names);
} }
_hash_row_map.Clear(); _hash_row_map.Clear();
foreach (IBlueRow r in dp.newRows) { foreach (IBlueRow r in dp.newRows) {
int hash = _get_hash(r); int hash = _get_hash(r);
List<IBlueRow> hash_rows = null; if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
hash_rows = new List<IBlueRow>(); hash_rows = new List<IBlueRow>();
_hash_row_map.Add(hash, hash_rows); _hash_row_map.Add(hash, hash_rows);
} }
@@ -61,6 +63,7 @@
} }
} }
[MemberNotNullWhen(true, nameof(_schema))]
public bool ready { public bool ready {
get { get {
return _schema != null; return _schema != null;
@@ -71,8 +74,7 @@
private ViewStream _ls, _rs; private ViewStream _ls, _rs;
private JoinBookkeepingData _ldata, _rdata; private JoinBookkeepingData _ldata, _rdata;
private ViewStreamDatabase _vsd; IBlueTableSchema? _schema;
private IBlueTableSchema _schema = null;
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) : public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { } this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
@@ -85,8 +87,8 @@
Log.assert(lj_col_list.Count == rj_col_list.Count); Log.assert(lj_col_list.Count == rj_col_list.Count);
int n_cols = lj_col_list.Count; int n_cols = lj_col_list.Count;
BlueRowHasher hasher = new BlueRowHasher(n_cols); BlueRowHasher hasher = new BlueRowHasher(n_cols);
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher); _ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher); _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
@@ -95,13 +97,13 @@
} }
private void _ls_onchange(object dp_obj) { private void _ls_onchange(object dp_obj) {
DataPacket dp = dp_obj as DataPacket; var dp = (IDataPacket)dp_obj;
_ldata.update_data(dp); _ldata.update_data(dp);
_form_and_fire(); _form_and_fire();
} }
private void _rs_onchange(object dp_obj) { private void _rs_onchange(object dp_obj) {
DataPacket dp = dp_obj as DataPacket; var dp = (IDataPacket)dp_obj;
_rdata.update_data(dp); _rdata.update_data(dp);
_form_and_fire(); _form_and_fire();
} }
@@ -111,20 +113,24 @@
_rs.onChange -= _rs_onchange; _rs.onChange -= _rs_onchange;
} }
[MemberNotNull(nameof(_schema))]
private void _build_schema() { private void _build_schema() {
if (_schema != null) if (_schema != null)
return; return;
List<ColumnInfo> cis = new List<ColumnInfo>(); var lschema = Log.notnull(_ldata.schema);
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32)); var rschema = Log.notnull(_rdata.schema);
foreach (ColumnInfo ci in _ldata.schema.columnInfo) { var cis = new List<ColumnInfo> {
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName; new("id", 0, TypeCode.Int32)
};
foreach (ColumnInfo ci in lschema.columnInfo) {
string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType)); cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
} }
foreach (ColumnInfo ci in _rdata.schema.columnInfo) { foreach (ColumnInfo ci in rschema.columnInfo) {
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName; string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType)); cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
} }
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray()); _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
} }
private void _form_and_fire() { private void _form_and_fire() {
@@ -133,34 +139,11 @@
_build_schema(); _build_schema();
List<object[]> res_rows = new List<object[]>(); List<object[]> res_rows = new List<object[]>();
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
int offset = 1;
JoinBookkeepingData sdata = _ldata;
if (is_left) {
offset = 1;
sdata = _ldata;
} else {
sdata = _rdata;
offset = 1 + _ldata.schema.columnInfo.Length;
}
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
rval[offset + i] = r[i];
}
};
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
object[] rv = new object[_schema.columnNames.Length];
for (int i = 0; i < _schema.columnNames.Length; i++) {
rv[i] = DBNull.Value;
}
rv[0] = id;
copy_row(rv, l_row, true);
return rv;
};
// this is where the join happens // this is where the join happens
foreach (var kv in _ldata.rows) { foreach (var kv in _ldata.rows) {
RowHash hash = kv.Key; RowHash hash = kv.Key;
List<IBlueRow> l_rows = kv.Value; List<IBlueRow> l_rows = kv.Value;
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash); List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
foreach (IBlueRow lr in l_rows) { foreach (IBlueRow lr in l_rows) {
if (r_rows != null) { if (r_rows != null) {
@@ -179,8 +162,33 @@
foreach (object[] rval in res_rows) { foreach (object[] rval in res_rows) {
rv.addRow(new BlueRow(_schema, rval)); rv.addRow(new BlueRow(_schema, rval));
} }
var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket); var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
myFireEvent(dp); myFireEvent(dp);
object[] build_row_array(int id, IBlueRow l_row) {
object[] rv = new object[_schema.columnNames.Length];
for (int i = 0; i < _schema.columnNames.Length; i++) {
rv[i] = DBNull.Value;
}
rv[0] = id;
copy_row(rv, l_row, true);
return rv;
}
void copy_row(object?[] rval, IBlueRow r, bool is_left) {
int offset = 1;
JoinBookkeepingData sdata = _ldata;
if (is_left) {
offset = 1;
sdata = _ldata;
} else {
sdata = _rdata;
offset = 1 + _ldata.RequireSchema().columnInfo.Length;
}
for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
rval[offset + i] = r[i];
}
}
} }
} }
@@ -43,7 +43,7 @@ namespace bts.BlueOrb {
_last_fire_time = DateTime.Now; _last_fire_time = DateTime.Now;
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time, IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff")); _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
_table = _table.Add(nr); _table.add_row(nr);
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp); myFireEvent(dp);
} }
@@ -45,8 +45,8 @@ namespace bts.BlueOrb {
} }
// aggregates multiple rows into one - ie does group_by compute // aggregates multiple rows into one - ie does group_by compute
public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) { public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
if (rows.Count() == 0) { if (rows.Count == 0) {
return null; return null;
} }
@@ -73,8 +73,8 @@ namespace bts.BlueOrb {
} }
// aggregates multiple rows into one, used by MultiRowRollingMathStream // aggregates multiple rows into one, used by MultiRowRollingMathStream
public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) { public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
if (rows.Count() == 0) { if (rows.Count == 0) {
return null; return null;
} }
IBlueRow ir = rows.ElementAt(start_idx); IBlueRow ir = rows.ElementAt(start_idx);
@@ -104,8 +104,8 @@ namespace bts.BlueOrb {
// rolling_rows are the rolling window of rows. the 0th row is the one thats gettin // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
// calculated. rows 1 - n-1 are its window rows. the schema is based on the operations. // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) { public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
if (rolling_rows.Count() == 0) { if (rolling_rows.Count == 0) {
return null; return null;
} }
IBlueRow ir = rolling_rows.ElementAt(0); IBlueRow ir = rolling_rows.ElementAt(0);
@@ -156,7 +156,6 @@ namespace bts.BlueOrb {
double first = col_vals.First(); double first = col_vals.First();
double last = col_vals.Last(); double last = col_vals.Last();
return ((first - last) / last) * 100.0; return ((first - last) / last) * 100.0;
return col_vals.First() - col_vals.Last();
} }
default: default:
// TODO // TODO
@@ -1,8 +1,7 @@
namespace bts.BlueOrb { namespace bts.BlueOrb {
using BCG; using BCG;
using System.Collections.Generic; using System;
using System.Net;
[DataStreamUserRespresentation("MultiRowRollingMathStream")] [DataStreamUserRespresentation("MultiRowRollingMathStream")]
public class MultiRowRollingMathStream : ViewStream { public class MultiRowRollingMathStream : ViewStream {
@@ -24,11 +23,12 @@
} }
private void _parent_str_onchange(object dp_obj) { private void _parent_str_onchange(object dp_obj) {
DataPacket dp = dp_obj as DataPacket; var dp = (IDataPacket)dp_obj;
var rows = dp.newRows.to_list(); var rows = dp.newRows.to_list();
BlueTable out_table = null; BlueTable? out_table = null;
for (int i = 0; i < rows.Count; i++) { for (int i = 0; i < rows.Count; i++) {
var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op); var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
?? throw new InvalidOperationException("group_by_compute returned null row");
if (out_table == null) if (out_table == null)
out_table = new BlueTable(out_r.schema); out_table = new BlueTable(out_r.schema);
out_table.addRow(out_r); out_table.addRow(out_r);
@@ -1,29 +0,0 @@
namespace Bts.DataStream.Parquet;
using bts.collections;
using System;
using System.Collections.Generic;
public interface IParquetColumnReader : IDisposable {
IList<object?> Read();
IList<object?> Read(int? start, int? stop);
}
public interface IParquetColumnReader<TLogical, TPhysical> : IParquetColumnReader {
new IList<TLogical?> Read();
new IList<TLogical?> Read(int? start, int? stop);
IList<object?> IParquetColumnReader.Read() =>
Read().WrapSelector(v => (object?)v, null);
IList<object?> IParquetColumnReader.Read(int? start, int? stop) =>
Read(start, stop).WrapSelector(v => (object?)v, null);
}
public interface IParquetColumnReader<T> : IParquetColumnReader<T, T> {
new IList<T?> Read();
IList<object?> IParquetColumnReader.Read() =>
Read().WrapSelector(v => (object?)v, null);
}
@@ -1,507 +0,0 @@
namespace Bts.DataStream.Parquet;
using bts;
using bts.collections;
using mbts.utils;
using ParquetSharp;
using Spark.Core.Collections;
using System;
using System.Collections.Generic;
public static class ParquetAdapter {
public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) {
int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
var colReader = rowGroupReader.Column(index);
return getColumnReader(colReader, rowCount, colType: colType);
}
public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader) {
var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader");
var colInfos = schema.columnInfo;
var colTypes = Array.ConvertAll(colInfos, c => c.colType);
var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
int rowCount = checked((int)fileReader.FileMetaData.NumRows);
for (int i = 0; i < colValues.Length; i++) {
colValues[i] = new List<object?>(rowCount);
}
int rowGroupCount = fileReader.FileMetaData.NumRowGroups;
for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) {
using var rowGroupReader = fileReader.RowGroup(rowGroup);
rowGroupReader.ReadColumnsTo(ref colValues, colTypes);
}
return colValues;
}
public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) {
if (!start.HasValue && !stop.HasValue) {
return ReadColumns(fileReader);
}
var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
var colInfos = schema.columnInfo;
var colTypes = Array.ConvertAll(colInfos, c => c.colType);
var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
int rowCount = checked((int)fileReader.FileMetaData.NumRows);
start ??= 0;
stop ??= rowCount;
var sliceLength = stop.Value - start.Value;
for (int i = 0; i < colValues.Length; i++) {
colValues[i] = new List<object?>(sliceLength);
}
int numRowGroups = fileReader.FileMetaData.NumRowGroups;
int? numRowsPerGroup = null;
int rowIdx = 0;
for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) {
if (stop.Value <= rowIdx) {
break;
}
using var rowGroupReader = fileReader.RowGroup(rowGroup);
int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows);
numRowsPerGroup ??= numRowsInGroup;
if (rowGroup < numRowGroups - 1) {
Log.assert(numRowsInGroup == numRowsPerGroup.Value);
} else {
Log.assert(numRowsInGroup <= numRowsPerGroup.Value);
}
int startIdx = start.Value - rowIdx;
int stopIdx = stop.Value - rowIdx;
rowIdx += numRowsInGroup;
if (startIdx >= numRowsInGroup) {
continue;
}
startIdx = Math.Max(0, startIdx);
stopIdx = Math.Min(stopIdx, numRowsInGroup);
int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx);
Log.assert(n == (stopIdx - startIdx));
}
return colValues;
}
public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes) {
int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
columnValues[colIdx].AddRange(colReader.Read());
}
return rowCount;
}
public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes, int? start, int? stop) {
if (!start.HasValue && !stop.HasValue) {
return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes);
}
int numRows = checked((int)rowGroupReader.MetaData.NumRows);
if (start.HasValue && start.Value >= numRows) {
return 0;
}
int? readRowCount = null;
for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
var colVals = colReader.Read(start, stop);
readRowCount ??= colVals.Count;
Log.assert(readRowCount == colVals.Count);
columnValues[colIdx].AddRange(colVals);
}
return readRowCount ?? throw new ArgumentException("Row group has no columns", nameof(rowGroupReader));
}
static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) {
var desc = reader.ColumnDescriptor;
var logicalType = desc.LogicalType;
return logicalType.Type switch {
LogicalTypeEnum.Int => IntReader(),
LogicalTypeEnum.String => new ParquetColumnReader<string?>(reader, bufferSize),
LogicalTypeEnum.Date => new ParquetColumnReader<DateTime?>(reader, bufferSize),
LogicalTypeEnum.Decimal => DecimalReader(),
LogicalTypeEnum.Timestamp => TimestampReader(),
LogicalTypeEnum.None => DefaultReader(),
_ => throw new NotImplementedException(),
};
IParquetColumnReader DefaultReader() {
return desc.PhysicalType switch {
PhysicalType.Boolean => new ParquetColumnReader<bool?>(reader, bufferSize),
PhysicalType.Int32 => new ParquetColumnReader<int?>(reader, bufferSize),
PhysicalType.Int64 => new ParquetColumnReader<long?>(reader, bufferSize),
PhysicalType.Float => new ParquetColumnReader<float?>(reader, bufferSize),
PhysicalType.Double => new ParquetColumnReader<double?>(reader, bufferSize),
_ => throw new NotImplementedException(),
};
}
IParquetColumnReader TimestampReader() {
var tlt = ExceptionAssert.of_type<TimestampLogicalType>(logicalType);
return new ParquetColumnReader<DateTimeNanos?, DateTime?>(reader, dtn => dtn?.DateTime, bufferSize);
}
IParquetColumnReader IntReader() {
var ilt = ExceptionAssert.of_type<IntLogicalType>(logicalType);
if (!ilt.IsSigned) {
if (colType is null) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<byte?>(reader, bufferSize),
16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
32 => new ParquetColumnReader<uint?>(reader, bufferSize),
64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
_ => throw new NotImplementedException(),
};
} else if (colType == typeof(byte)) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<byte?>(reader, bufferSize),
16 => new ParquetColumnReader<ushort?, byte?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
_ => throw new NotImplementedException(),
};
} else if (colType == typeof(ushort)) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<byte?, ushort?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
_ => throw new NotImplementedException(),
};
} else if (colType == typeof(uint)) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<byte?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
16 => new ParquetColumnReader<ushort?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
32 => new ParquetColumnReader<uint?>(reader, bufferSize),
64 => new ParquetColumnReader<ulong?, uint?>(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize),
_ => throw new NotImplementedException(),
};
} else if (colType == typeof(ulong)) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<byte?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
16 => new ParquetColumnReader<ushort?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
32 => new ParquetColumnReader<uint?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
_ => throw new NotImplementedException(),
};
}
throw new NotImplementedException($"Expected signed int column type, but got {colType}");
} else {
if (colType is null) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
16 => new ParquetColumnReader<short?>(reader, bufferSize),
32 => new ParquetColumnReader<int?>(reader, bufferSize),
64 => new ParquetColumnReader<long?>(reader, bufferSize),
_ => throw new NotImplementedException(),
};
} else if (colType == typeof(sbyte)) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
16 => new ParquetColumnReader<short?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
32 => new ParquetColumnReader<int?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
64 => new ParquetColumnReader<long?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
_ => throw new NotImplementedException(),
};
} else if (colType == typeof(short)) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<char?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
16 => new ParquetColumnReader<short?>(reader, bufferSize),
32 => new ParquetColumnReader<int?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
64 => new ParquetColumnReader<long?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
_ => throw new NotImplementedException(),
};
} else if (colType == typeof(int)) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<char?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
16 => new ParquetColumnReader<short?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
32 => new ParquetColumnReader<int?>(reader, bufferSize),
64 => new ParquetColumnReader<long?, int?>(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize),
_ => throw new NotImplementedException(),
};
} else if (colType == typeof(long)) {
return ilt.BitWidth switch {
8 => new ParquetColumnReader<char?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
16 => new ParquetColumnReader<short?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
32 => new ParquetColumnReader<int?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
64 => new ParquetColumnReader<long?>(reader, bufferSize),
_ => throw new NotImplementedException(),
};
}
throw new NotImplementedException($"Expected unsigned int column type, but got {colType}");
}
}
IParquetColumnReader DecimalReader() {
var dlt = ExceptionAssert.of_type<DecimalLogicalType>(logicalType);
return new ParquetColumnReader<double?>(reader, bufferSize);
}
}
public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) {
return typeCode switch {
TypeCode.Boolean => new ColumnWriter<bool?>(writer, bufferSize),
TypeCode.SByte => new ColumnWriter<sbyte?>(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
TypeCode.Int16 => new ColumnWriter<short?>(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
TypeCode.Int32 => new ColumnWriter<int?>(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
TypeCode.Int64 => new ColumnWriter<long?>(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
TypeCode.Byte => new ColumnWriter<byte?>(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
TypeCode.UInt16 => new ColumnWriter<ushort?>(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
TypeCode.UInt32 => new ColumnWriter<uint?>(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
TypeCode.UInt64 => new ColumnWriter<ulong?>(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
TypeCode.Single => new ColumnWriter<float?>(writer, bufferSize),
TypeCode.Double => new ColumnWriter<double?>(writer, bufferSize),
TypeCode.DateTime => new ColumnWriter<DateTimeNanos?>(writer, bufferSize, ConvertDateTimeObj),
TypeCode.String => new ColumnWriter<string?>(writer, bufferSize),
_ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
};
static DateTimeNanos? ConvertDateTimeObj(object? obj) {
if (obj is DateTime dt) {
return new(dt);
}
if (obj is DateTimeNanos dtn) {
return dtn;
}
throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos");
}
}
public static Func<object?, sbyte?> GetInt8Converter(IntLogicalType logicalType) =>
GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
public static Func<object?, sbyte?> GetInt8Converter(bool isSigned, int bitWidth) =>
isSigned
? bitWidth switch {
8 => obj => obj as sbyte?,
16 => obj => obj is short v ? checked((sbyte)v) : null,
32 => obj => obj is int v ? checked((sbyte)v) : null,
64 => obj => obj is long v ? checked((sbyte)v) : null,
_ => throw new NotImplementedException(),
}
: bitWidth switch {
8 => obj => obj is byte v ? checked((sbyte)v) : null,
16 => obj => obj is ushort v ? checked((sbyte)v) : null,
32 => obj => obj is uint v ? checked((sbyte)v) : null,
64 => obj => obj is ulong v ? checked((sbyte)v) : null,
_ => throw new NotImplementedException(),
};
public static Func<object?, short?> GetInt16Converter(IntLogicalType logicalType) =>
GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
public static Func<object?, short?> GetInt16Converter(bool isSigned, int bitWidth) =>
isSigned
? bitWidth switch {
8 => obj => obj is sbyte v ? v : null,
16 => obj => obj as short?,
32 => obj => obj is int v ? checked((short)v) : null,
64 => obj => obj is long v ? checked((short)v) : null,
_ => throw new NotImplementedException(),
}
: bitWidth switch {
8 => obj => obj is byte v ? v : null,
16 => obj => obj is ushort v ? checked((short)v) : null,
32 => obj => obj is uint v ? checked((short)v) : null,
64 => obj => obj is ulong v ? checked((short)v) : null,
_ => throw new NotImplementedException(),
};
public static Func<object?, int?> GetInt32Converter(IntLogicalType logicalType) =>
GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
public static Func<object?, int?> GetInt32Converter(bool isSigned, int bitWidth) =>
isSigned
? bitWidth switch {
8 => obj => obj is sbyte v ? v : null,
16 => obj => obj is short v ? v : null,
32 => obj => obj as int?,
64 => obj => obj is long v ? checked((int)v) : null,
_ => throw new NotImplementedException(),
}
: bitWidth switch {
8 => obj => obj is byte v ? v : null,
16 => obj => obj is ushort v ? v : null,
32 => obj => obj is uint v ? checked((int)v) : null,
64 => obj => obj is ulong v ? checked((int)v) : null,
_ => throw new NotImplementedException(),
};
public static Func<object?, long?> GetInt64Converter(IntLogicalType logicalType) =>
GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
public static Func<object?, long?> GetInt64Converter(bool isSigned, int bitWidth) =>
isSigned
? bitWidth switch {
8 => obj => obj is sbyte v ? v : null,
16 => obj => obj is short v ? v : null,
32 => obj => obj is int v ? v : null,
64 => obj => obj as long?,
_ => throw new NotImplementedException(),
}
: bitWidth switch {
8 => obj => obj is byte v ? v : null,
16 => obj => obj is ushort v ? v : null,
32 => obj => obj is uint v ? v : null,
64 => obj => obj is ulong v ? checked((long)v) : null,
_ => throw new NotImplementedException(),
};
public static Func<object?, byte?> GetUInt8Converter(IntLogicalType logicalType) =>
GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
public static Func<object?, byte?> GetUInt8Converter(bool isSigned, int bitWidth) =>
isSigned
? bitWidth switch {
8 => obj => obj is sbyte v ? checked((byte)v) : null,
16 => obj => obj is short v ? checked((byte)v) : null,
32 => obj => obj is int v ? checked((byte)v) : null,
64 => obj => obj is long v ? checked((byte)v) : null,
_ => throw new NotImplementedException(),
}
: bitWidth switch {
8 => obj => obj as byte?,
16 => obj => obj is ushort v ? checked((byte)v) : null,
32 => obj => obj is uint v ? checked((byte)v) : null,
64 => obj => obj is ulong v ? checked((byte)v) : null,
_ => throw new NotImplementedException(),
};
public static Func<object?, ushort?> GetUInt16Converter(IntLogicalType logicalType) =>
GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
public static Func<object?, ushort?> GetUInt16Converter(bool isSigned, int bitWidth) =>
isSigned
? bitWidth switch {
8 => obj => obj is sbyte v ? checked((ushort)v) : null,
16 => obj => obj is short v ? checked((ushort)v) : null,
32 => obj => obj is int v ? checked((ushort)v) : null,
64 => obj => obj is long v ? checked((ushort)v) : null,
_ => throw new NotImplementedException(),
}
: bitWidth switch {
8 => obj => obj is byte v ? v : null,
16 => obj => obj is ushort v ? v : null,
32 => obj => obj is uint v ? checked((ushort)v) : null,
64 => obj => obj is ulong v ? checked((ushort)v) : null,
_ => throw new NotImplementedException(),
};
public static Func<object?, uint?> GetUInt32Converter(IntLogicalType logicalType) =>
GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
public static Func<object?, uint?> GetUInt32Converter(bool isSigned, int bitWidth) =>
isSigned
? bitWidth switch {
8 => obj => obj is sbyte v ? checked((uint)v) : null,
16 => obj => obj is short v ? checked((uint)v) : null,
32 => obj => obj is int v ? checked((uint)v) : null,
64 => obj => obj is long v ? checked((uint)v) : null,
_ => throw new NotImplementedException(),
}
: bitWidth switch {
8 => obj => obj is byte v ? v : null,
16 => obj => obj is ushort v ? v : null,
32 => obj => obj is uint v ? v : null,
64 => obj => obj is ulong v ? checked((uint)v) : null,
_ => throw new NotImplementedException(),
};
public static Func<object?, ulong?> GetUInt64Converter(IntLogicalType logicalType) =>
GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
public static Func<object?, ulong?> GetUInt64Converter(bool isSigned, int bitWidth) =>
isSigned
? bitWidth switch {
8 => obj => obj is sbyte v ? checked((ulong)v) : null,
16 => obj => obj is short v ? checked((ulong)v) : null,
32 => obj => obj is int v ? checked((ulong)v) : null,
64 => obj => obj is long v ? checked((ulong)v) : null,
_ => throw new NotImplementedException(),
}
: bitWidth switch {
8 => obj => obj is byte v ? v : null,
16 => obj => obj is ushort v ? v : null,
32 => obj => obj is uint v ? v : null,
64 => obj => obj is ulong v ? v : null,
_ => throw new NotImplementedException(),
};
public static string ColumnEnumTypeKey(string colName) =>
$"{colName}.EnumType";
public static KeyValuePair<string, string> ColumnEnumTypeEntry(string colName, Type enumType) {
if (enumType.IsNullable()) {
enumType = enumType.GetGenericArguments()[0];
}
if (!enumType.IsEnum) {
throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType));
}
return new(ColumnEnumTypeKey(colName), enumType.ToString());
}
public static Type? GetColumnEnumType(IReadOnlyDictionary<string, string>? keyValueMetadata, string columnName) {
if (keyValueMetadata is not null) {
var key = ColumnEnumTypeKey(columnName);
if (keyValueMetadata.TryGetValue(key, out var typeName)) {
return Type.GetType(typeName);
}
}
return null;
}
public interface IColumnWriter : IDisposable {
void Write(IReadOnlyCollection<object?> values);
}
public interface IColumnWriter<T> : IColumnWriter {
void Write(IReadOnlyCollection<T?> values);
}
class ColumnWriter<T> : IColumnWriter<T> {
readonly T?[] buffer;
readonly LogicalColumnWriter<T> writer;
readonly Func<object?, T?> convert;
public ColumnWriter(ColumnWriter writer, int bufferSize)
: this(writer, new T?[bufferSize]) { }
public ColumnWriter(ColumnWriter writer, int bufferSize, Func<object?, T?> convert)
: this(writer, new T?[bufferSize], convert) { }
public ColumnWriter(ColumnWriter writer, T?[] buffer)
: this(writer, buffer, obj => (T?)obj) { }
public ColumnWriter(ColumnWriter writer, T?[] buffer, Func<object?, T?> convert) {
this.buffer = buffer;
this.writer = writer.LogicalWriter<T>();
this.convert = convert;
}
public void Write(IReadOnlyCollection<T?> values) {
if (buffer.Length == 0) {
throw new InvalidOperationException("Cannot write because buffer length is 0");
}
var count = values.Count;
var nPages = count / buffer.Length;
using var en = values.GetEnumerator();
for (int pg = 0; pg < nPages; pg++) {
for (int i = 0; i < buffer.Length; i++) {
Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
buffer[i] = en.Current;
}
writer.WriteBatch(buffer!);
}
var remainder = count - nPages * buffer.Length;
if (remainder != 0) {
for (int i = 0; i < remainder; i++) {
Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
buffer[i] = en.Current;
}
writer.WriteBatch(buffer!, 0, remainder);
}
}
public void Dispose() =>
writer.Dispose();
void IColumnWriter.Write(IReadOnlyCollection<object?> values) =>
Write(ReadOnlySelectorCollection.Create(values, convert));
}
}
@@ -1,111 +0,0 @@
namespace Bts.DataStream.Parquet;
using bts;
using bts.collections;
using ParquetSharp;
using System;
using System.Collections.Generic;
class ParquetColumnReader<T> : IParquetColumnReader<T> {
protected readonly LogicalColumnReader<T> reader;
protected readonly T?[] buffer;
public ParquetColumnReader(ColumnReader reader, int bufferSize)
: this(reader, new T?[bufferSize]) { }
protected ParquetColumnReader(ColumnReader reader, T?[] buffer) {
this.buffer = buffer;
this.reader = reader.LogicalReader<T>();
}
~ParquetColumnReader() =>
Dispose();
public IList<T?> Read() {
if (buffer.Length == 0) {
throw new InvalidOperationException("Cannot write because buffer length is 0");
}
int n = reader.ReadBatch(buffer!);
return buffer.WrapSubRange(0, n);
}
public IList<T?> Read(int? start, int? stop) {
if (buffer.Length == 0) {
throw new InvalidOperationException("Cannot write because buffer length is 0");
}
start ??= 0;
stop ??= buffer.Length;
int length = stop.Value - start.Value;
if (length <= 0) {
return new List<T?>(0);
}
if (start.Value > 0) {
var nSkipped = checked((int)reader.Skip(start.Value));
Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
if (nSkipped < start.Value) {
return new List<T?>(0);
}
}
int n = reader.ReadBatch(buffer!, 0, length);
return buffer.WrapSubRange(0, n);
}
public void Dispose() {
GC.SuppressFinalize(this);
reader.Dispose();
}
}
class ParquetColumnReader<TIn, TOut> : IParquetColumnReader<TOut> {
readonly Func<TIn, TOut?> convert;
readonly LogicalColumnReader<TIn> reader;
readonly TIn[] buffer;
public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, int bufferSize)
: this(reader, convert, new TIn[bufferSize]) { }
public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, TIn[] buffer) {
this.convert = convert;
this.reader = reader.LogicalReader<TIn>();
this.buffer = buffer;
}
~ParquetColumnReader() =>
Dispose();
public IList<TOut?> Read() {
if (buffer.Length == 0) {
throw new InvalidOperationException("Cannot write because buffer length is 0");
}
int n = reader.ReadBatch(buffer!);
return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
}
public IList<TOut?> Read(int? start, int? stop) {
if (buffer.Length == 0) {
throw new InvalidOperationException("Cannot write because buffer length is 0");
}
start ??= 0;
stop ??= buffer.Length;
int length = stop.Value - start.Value;
if (length <= 0) {
return new List<TOut?>(0);
}
if (start.Value > 0) {
var nSkipped = checked((int)reader.Skip(start.Value));
Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
if (nSkipped < start.Value) {
return new List<TOut?>(0);
}
}
int n = reader.ReadBatch(buffer!, 0, length);
return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
}
public void Dispose() {
GC.SuppressFinalize(this);
reader.Dispose();
}
}
@@ -1,159 +0,0 @@
namespace Bts.DataStream.Parquet;
using BCG;
using bts;
using bts.collections;
using Microsoft.Scripting.Utils;
using ParquetSharp;
using System;
using System.Collections.Generic;
using System.Linq;
public static class ParquetExtensions {
public static int RowCount(this FileMetaData fileMetaData) =>
checked((int)fileMetaData.NumRows);
public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
var colInfos = new ColumnInfo[schemaDescriptor.NumColumns];
for (int i = 0; i < colInfos.Length; i++) {
colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata);
}
return new BlueTableSchema(tableName, colInfos);
}
public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") =>
ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata);
public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
var c = schemaDescriptor.Column(index);
var lt = c.LogicalType;
if (lt.Type == LogicalTypeEnum.Enum) {
var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name);
if (enumType is not null) {
return new(c.Name, index, enumType);
}
}
return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType));
}
public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) {
var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
var colInfos = schema.columnInfo;
IBlueRowCollection? rv;
if (!schema.columnInfo.Any(c => c.colName == "id")) {
rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop));
} else {
rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop));
}
Log.assert(!schema.columnInfo.Any(c => c is null));
return rv;
}
public static (Column[] Column, IReadOnlyDictionary<string, string> Meta) ToParquetSchema(this IBlueTableSchema schema) {
var blueColumns = schema.columnInfo;
var parquetColumns = new Column[blueColumns.Length];
var parquetMeta = new Dictionary<string, string>();
KeyValuePair<string, string>[] parquetColMeta;
for (int i = 0; i < blueColumns.Length; i++) {
(parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]);
parquetMeta.AddRange(parquetColMeta);
}
return (parquetColumns, parquetMeta);
}
public static (Column Column, KeyValuePair<string, string>[] Meta) ToParquetColumn(this ColumnInfo columnInfo) {
var colType = columnInfo.colType;
bool isNullable = colType.IsNullable();
if (isNullable) {
colType = colType.GetGenericArguments()[0];
}
KeyValuePair<string, string>[] meta;
TypeCode typeCode;
var colName = columnInfo.colName;
if (colType.IsEnum) {
typeCode = colType.GetEnumUnderlyingType().GetTypeCode();
meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) };
} else {
typeCode = colType.GetTypeCode();
meta = Array.Empty<KeyValuePair<string, string>>();
}
return (ToParquetColumn(colName, typeCode), meta);
static Column ToParquetColumn(string colName, TypeCode typeCode) =>
typeCode switch {
TypeCode.Boolean => new Column<bool?>(colName),
TypeCode.Byte => new Column<byte?>(colName),
TypeCode.Int16 => new Column<short?>(colName),
TypeCode.Int32 => new Column<int?>(colName),
TypeCode.Int64 => new Column<long?>(colName),
TypeCode.UInt16 => new Column<ushort?>(colName),
TypeCode.UInt32 => new Column<uint?>(colName),
TypeCode.UInt64 => new Column<ulong?>(colName),
TypeCode.Single => new Column<float?>(colName),
TypeCode.Double => new Column<double?>(colName),
TypeCode.DateTime => new Column<DateTimeNanos?>(colName),
TypeCode.String => new Column<string?>(colName),
_ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
};
}
static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) {
return logicalType.Type switch {
// LogicalTypeEnum.Int => TypeCode.Int32,
// LogicalTypeEnum.Decimal => TypeCode.Double,
LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime,
LogicalTypeEnum.String => TypeCode.String,
LogicalTypeEnum.Int => FromInt(),
LogicalTypeEnum.Decimal => FromDecimal(),
_ => FromPhysicalType(),
};
TypeCode FromPhysicalType() =>
physicalType switch {
PhysicalType.Boolean => TypeCode.Boolean,
PhysicalType.Int32 => TypeCode.Int32,
PhysicalType.Int64 => TypeCode.Int64,
PhysicalType.Float => TypeCode.Single,
PhysicalType.Double => TypeCode.Double,
_ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"),
};
TypeCode FromInt() {
var ilt = (IntLogicalType)logicalType;
var bitWidth = ilt.BitWidth;
return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt();
TypeCode FromSignedInt() =>
bitWidth switch {
8 => TypeCode.SByte,
16 => TypeCode.Int16,
32 => TypeCode.Int32,
64 => TypeCode.Int64,
_ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"),
};
TypeCode FromUnsignedInt() =>
bitWidth switch {
8 => TypeCode.Byte,
16 => TypeCode.UInt16,
32 => TypeCode.UInt32,
64 => TypeCode.UInt64,
_ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"),
};
}
TypeCode FromDecimal() {
var dlt = (DecimalLogicalType)logicalType;
var scale = dlt.Scale;
var precision = dlt.Precision;
return scale switch {
0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"),
_ => TypeCode.Double,
};
}
}
}
@@ -1,234 +0,0 @@
namespace Bts.DataStream.Parquet;
using BCG;
using bts;
using bts.collections;
using Bts.DataStreamCore;
using ParquetSharp;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
public class ParquetGroupTable : IBlueRowCollection, IDisposable {
readonly RowGroupReader reader;
readonly IBlueTableSchema schema;
readonly bool leaveOpen;
readonly int fullCount;
readonly object syncRoot = new();
readonly int? start;
readonly int? stop;
readonly int startIndex;
readonly int stopIndex;
readonly int idColumnIndex;
readonly int idColumnIndexInFile;
readonly Func<int, int>? rowIdFactory;
DataCache? dataCacheBacking;
public ParquetGroupTable(
RowGroupReader reader,
IBlueTableSchema schema,
int? start = null,
int? stop = null,
int idColumnIndex = -1,
int idColumnIndexInFile = -1,
Func<int, int>? rowIdFactory = null,
bool leaveOpen = false
) {
this.reader = reader;
this.schema = schema;
this.start = start;
this.stop = stop;
this.idColumnIndex = idColumnIndex;
this.idColumnIndexInFile = idColumnIndexInFile;
this.rowIdFactory = rowIdFactory;
this.leaveOpen = leaveOpen;
fullCount = checked((int)reader.MetaData.NumRows);
Log.assert(fullCount >= 0);
startIndex = SliceList.GetStartIndex(start, fullCount, true);
stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
stopIndex = Math.Max(stopIndex, startIndex);
Log.assert(
startIndex >= 0
&& startIndex <= fullCount
&& stopIndex >= 0
&& stopIndex <= fullCount
&& stopIndex >= startIndex);
}
~ParquetGroupTable() =>
Dispose();
public int Count => stopIndex - startIndex;
IBlueTableSchema IBlueRowCollection.schema => schema;
DataCache dataCache =>
EnsureInit();
public IBlueRow this[int id] => dataCache[id];
public bool containsID(int id) =>
dataCache.RowIdToIndex.ContainsKey(id);
public IBlueRow GetRow(int index) =>
dataCache.Rows[index];
public int IndexOf(IBlueRow item) {
int id = item.Id();
return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item)
? idx
: -1;
}
public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) {
if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) {
row = dataCache.Rows[idx];
return true;
}
row = null;
return false;
}
public IEnumerator<IBlueRow> GetEnumerator() {
for (int i = 0; i < dataCache.Rows.Length; i++) {
yield return dataCache.Rows[i];
}
}
public void Dispose() {
GC.SuppressFinalize(this);
if (!leaveOpen) {
reader.Dispose();
}
}
[MemberNotNull(nameof(dataCacheBacking))]
internal DataCache EnsureInit() {
if (dataCacheBacking.HasValue) {
return dataCacheBacking.Value;
}
lock (syncRoot) {
if (dataCacheBacking.HasValue) {
return dataCacheBacking.Value;
}
var rows = new IBlueRow[Count];
int numColumns = reader.MetaData.NumColumns;
var blueColumns = schema.columnInfo;
Log.assert((blueColumns.Length - numColumns) switch {
0 or 1 => true,
_ => false,
});
var columnValues = allocateColumnValues(
reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory);
var rowIdToIndex = new Dictionary<int, int>();
for (int i = 0; i < rows.Length; i++) {
rows[i] = new BlueRowColumnWrapper(schema, i, columnValues);
}
dataCacheBacking = new(rows, rowIdToIndex, columnValues);
}
return dataCacheBacking.Value;
}
static IList<object?>[] allocateColumnValues(
RowGroupReader reader,
int numColumns,
int numRows,
int idColIdx,
int idColIdxInFile,
int startIndex,
int stopIndex,
Func<int, int>? idFactory
) {
if (idColIdx < 0 || idColIdx >= numColumns) {
throw new ArgumentException("Id column index out of bounds", nameof(idColIdx));
}
var columnValues = new IList<object?>[numColumns];
if (idColIdxInFile == -1) {
// Generate ids (there is no id column in the file)
ArgumentNullException.ThrowIfNull(idFactory);
for (int i = 0; i < numColumns; i++) {
if (i < idColIdx) {
ReadColumn(i, i);
} else if (i == idColIdx) {
columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
} else {
ReadColumn(i - 1, i);
}
}
} else if (idFactory is not null) {
// Generate ids (skip id column in the file)
for (int i = 0; i < numColumns; i++) {
if (i == idColIdx) {
columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
} else {
if (i > idColIdx && i < idColIdxInFile) {
ReadColumn(i - 1, i);
} else if (i > idColIdxInFile && i < idColIdx) {
ReadColumn(i + 1, i);
} else {
ReadColumn(i, i);
}
}
}
} else {
// Get ids from the file (output table id column index may differ from file)
// Case 1) File id col index > output
// src cols: a, b, c, d, (id), e, f, ...
// dst cols: a, b, (id), c, d, e, f, ...
//
// Case 2) File id col index == output
// src cols: a, b, (id), c, d, e, f, ...
// dst cols: a, b, (id), c, d, e, f, ...
//
// Case 3) File id col index < output
// src cols: a, b, (id), c, d, e, f, ...
// dst cols: a, b, c, d, (id), e, f, ...
for (int i = 0; i < numColumns; i++) {
if (i == idColIdx) {
ReadColumn(idColIdxInFile, idColIdx);
} else if (i > idColIdx && i < idColIdxInFile) {
ReadColumn(i - 1, i);
} else if (i > idColIdxInFile && i < idColIdx) {
ReadColumn(i, i - 1);
} else {
ReadColumn(i, i);
}
}
}
return columnValues;
void ReadColumn(int srcIdx, int dstIdx) {
using var colReader = reader.GetColumnReader(srcIdx);
columnValues[dstIdx] = colReader.Read(startIndex, stopIndex);
}
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
internal readonly struct DataCache {
public readonly IBlueRow[] Rows;
public readonly Dictionary<int, int> RowIdToIndex;
public readonly IList<object?>[] ColumnValues;
public DataCache(IBlueRow[] rows, Dictionary<int, int> rowIdToIndex, IList<object?>[] columnValues) {
Rows = rows;
RowIdToIndex = rowIdToIndex;
ColumnValues = columnValues;
}
public IBlueRow this[int id] =>
Rows[RowIdToIndex[id]];
public IBlueRow GetRow(int index) =>
Rows[index];
}
}
@@ -1,65 +0,0 @@
namespace Bts.DataStream.Parquet;
using BCG;
using bts;
using bts.collections;
using Bts.DataStreamCore;
using Bts.PubSub;
using Spark.Core.Disposables;
using System;
using System.Threading;
[DataStreamUserRespresentation(false)]
public class ParquetReaderStream : ViewStream {
readonly string file;
readonly int? start;
readonly int? stop;
readonly IWorkScheduler workScheduler;
IBlueRowCollection? table;
IDisposable? workerSub;
public ParquetReaderStream(string file, IWorkScheduler workScheduler) {
this.file = file;
this.workScheduler = workScheduler;
}
public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) {
this.file = file;
this.start = start;
this.stop = stop;
this.workScheduler = workScheduler;
}
protected override void myOnFirstSubscriber() {
Start();
var worker = new AnonymousWorker(DoWork);
workScheduler.Add(worker);
workerSub = Disposable.Create(() => workScheduler.Remove(worker));
async void Start() {
try {
Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop));
} catch (Exception ex) {
Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}");
}
}
void DoWork() {
var table = Volatile.Read(ref this.table);
if (table is null) {
return;
}
Disposable.dispose_thread_safe(ref workerSub);
if (table.schema.columnInfo.Length > 1) {
myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
}
}
}
protected override void myOnLastSubscriber() =>
Disposable.dispose_thread_safe(ref workerSub);
}
@@ -1,26 +0,0 @@
// namespace Bts.DataStream.Parquet;
// using BCG;
// using global::Parquet.Data;
// public class ParquetRow : IBlueRow {
// readonly int id;
// readonly int rowIndex;
// readonly DataColumn[] data;
// public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) {
// this.id = id;
// this.rowIndex = rowIndex;
// this.data = data;
// Schema = schema;
// }
// public object? this[int index] =>
// index == 0
// ? id
// : data[index - 1].Data.GetValue(rowIndex);
// public IBlueTableSchema Schema { get; }
// IBlueTableSchema IBlueRow.schema => Schema;
// }
@@ -1,145 +0,0 @@
namespace Bts.DataStream.Parquet;
using BCG;
using bts;
using bts.collections;
using Bts.DataStreamCore;
using ParquetSharp;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
public class ParquetTable : IBlueRowCollection, IDisposable {
readonly SortedList<int, ParquetGroupTable> groups = new();
readonly int fullCount;
readonly IBlueTableSchema schema;
readonly ParquetFileReader? reader;
readonly int startIndex;
readonly int stopIndex;
public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) {
schema = reader.FileMetaData.ToBlueSchema(tableName);
int idColIdx;
if (schema.TryResolveIndices("id", out int idColIdxInFile)) {
idColIdx = idColIdxInFile;
} else {
var origCols = schema.columnInfo;
var cols = new ColumnInfo[origCols.Length + 1];
idColIdx = 0;
cols[0] = new("id", 0, typeof(int));
for (int i = 0; i < origCols.Length; i++) {
var c = origCols[i];
cols[i + 1] = new(c.colName, i + 1, c.colType);
}
schema = new BlueTableSchema(cols, schema.tableName);
}
fullCount = checked((int)reader.FileMetaData.NumRows);
Log.assert(fullCount >= 0);
startIndex = SliceList.GetStartIndex(start, fullCount, true);
stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
stopIndex = Math.Max(stopIndex, startIndex);
Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex);
int numRowGroups = reader.FileMetaData.NumRowGroups;
int rowGroupLastIdx = 0;
object syncRoot = new();
for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) {
int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null;
int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null;
int offset = rowGroupLastIdx;
var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId);
if (grp.Count == 0) {
continue;
}
rowGroupLastIdx += grp.Count;
groups.Add(rowGroupLastIdx, grp);
int GetAutoId(int index) =>
index + offset;
}
Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit());
if (!leaveOpen) {
this.reader = reader;
}
}
~ParquetTable() =>
Dispose();
public IBlueTableSchema Schema => schema;
public int Count => stopIndex - startIndex;
IBlueTableSchema IBlueRowCollection.schema => schema;
public IBlueRow this[int index] {
get {
foreach (var grp in groups.Values) {
if (grp.TryGetValue(index, out var r)) {
return r;
}
}
throw new KeyNotFoundException();
}
}
public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") =>
new(new(path), tableName, start, stop, false);
public IBlueRow GetRow(int index) {
if (index < 0 || index >= Count) {
throw new IndexOutOfRangeException();
}
int rowGroupIdx = groups.Keys.BinarySearch(index);
if (rowGroupIdx < 0) {
rowGroupIdx = ~rowGroupIdx;
}
Log.assert(rowGroupIdx <= groups.Count);
var grp = groups[rowGroupIdx];
int idxWithinGrp = index;
if (rowGroupIdx > 0) {
idxWithinGrp -= groups.Keys[rowGroupIdx - 1];
}
return grp[idxWithinGrp];
}
public bool containsID(int id) =>
groups.Values.Any(rgl => rgl.containsID(id));
public int IndexOf(IBlueRow item) {
int idx;
int grpFirstIdx = 0;
foreach (var (grpLastIdx, grp) in groups) {
idx = grp.IndexOf(item);
if (idx >= 0) {
return idx + grpFirstIdx;
}
grpFirstIdx = grpLastIdx;
}
return -1;
}
public IEnumerator<IBlueRow> GetEnumerator() {
foreach (var rgl in groups.Values) {
foreach (var r in rgl) {
yield return r;
}
}
}
public void Dispose() {
GC.SuppressFinalize(this);
groups.Values.DisposeAll();
reader?.Dispose();
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
@@ -1,32 +0,0 @@
#nullable enable
namespace Bts.DataStream.Parquet;
using BCG;
using Bts.DataStreamCore;
using ParquetSharp;
using System.IO;
using System.Threading.Tasks;
public class ParquetTableReader {
public static Task<IBlueRowCollection> ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) =>
Task.Run(() => Read(stream, leaveOpen, start, stop));
public static Task<IBlueRowCollection> ReadAsync(string path, int? start = null, int? stop = null) =>
Task.Run(() => Read(path, start, stop));
public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) {
using var fileReader = new ParquetFileReader(stream, leaveOpen);
var rv = fileReader.ReadBlueTable(start, stop);
rv.schema.Validate();
return rv;
}
public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) {
IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop);
rv.schema.Validate();
return rv;
}
}
@@ -1,38 +0,0 @@
namespace Bts.DataStream.Parquet;
using BCG;
using Bts.DataStreamCore;
using bts;
using ParquetSharp;
using System.IO;
using System.Threading.Tasks;
public static class ParquetTableWriter {
public static Task WriteAsync(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) =>
Task.Run(() => Write(blueRows, outStream, leaveOpen));
public static void Write(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) {
var (parquetColumns, parquetMeta) = blueRows.schema.ToParquetSchema();
using var schema = Column.CreateSchemaNode(parquetColumns);
using var writerProps = createWriterProperties();
using var writer = new ParquetFileWriter(outStream, schema, writerProps, keyValueMetadata: parquetMeta, leaveOpen: leaveOpen);
using var rowGroup = writer.AppendRowGroup();
for (int i = 0; i < blueRows.schema.columnInfo.Length; i++) {
var col = blueRows.schema.columnInfo[i];
Log.assert(col.colNo == i);
using var colWriter = ParquetAdapter.GetColumnWriter(col.type_code(), rowGroup.NextColumn());
colWriter.Write(blueRows.GetColumn(i));
}
writer.Close();
}
static WriterProperties createWriterProperties(Compression compression = Compression.Snappy) {
using var builder = new WriterPropertiesBuilder();
builder.Compression(compression);
return builder.Build();
}
}
@@ -1,87 +0,0 @@
namespace Bts.DataStream.Parquet;
using BCG;
using bts.collections;
using Bts.DataStreamCore;
using Bts.PubSub;
using mbts.utils;
using Spark.Core.Disposables;
using Spark.Core.Threading;
using System;
using System.IO;
[DataStreamUserRespresentation(false)]
public class ParquetWriterStream : ViewStream {
readonly string file;
readonly ViewStream inputStream;
readonly IWorkScheduler workScheduler;
readonly AtomicGate gate = new();
IDataPacket? pendingOutput;
IDisposable? dispose;
public ParquetWriterStream(ViewStream inputStream, string file, IWorkScheduler workScheduler) {
this.inputStream = inputStream;
this.file = file;
this.workScheduler = workScheduler;
}
protected override void myOnFirstSubscriber() {
BlueTableImmutable.DataPacketBuilder? outputBuilder = null;
bool isFirstUpdate = false;
var worker = new AnonymousWorker(DoWork);
inputStream.onChange += OnInputStreamChange;
workScheduler.Add(worker);
dispose = Disposable.Create(Dispose);
void Dispose() {
workScheduler.Remove(worker);
inputStream.onChange -= OnInputStreamChange;
}
void OnInputStreamChange(object? dpObj) {
var dp = ExceptionAssert.Argument.Is<IDataPacket>(dpObj);
using (gate.BeginTransaction()) {
outputBuilder ??= new BlueTableImmutable(dp.Schema).ToDataPacketBuilder();
foreach (IBlueRow r in dp.addedRows) {
outputBuilder.AddOrUpdate(r);
}
foreach (IBlueRow r in dp.deletedRows) {
outputBuilder.Remove(r.Id());
}
}
}
async void DoWork() {
using (gate.BeginTransaction()) {
if (outputBuilder is null) {
return;
}
if (pendingOutput is not null) {
myFireEvent(pendingOutput);
pendingOutput = null;
}
var dp = outputBuilder.ToDataPacket();
if (isFirstUpdate) {
isFirstUpdate = false;
} else if (!dp.HasChanges()) {
return;
}
{
using var outStream = File.OpenWrite(file);
await ParquetTableWriter.WriteAsync(dp.newRows, outStream);
}
// Fire the update at the beginning of the next doWork call so it happens on the
// 'work' thread.
pendingOutput = dp;
}
}
}
protected override void myOnLastSubscriber() =>
Disposable.dispose_thread_safe(ref dispose);
}
@@ -1,5 +1,8 @@
namespace bts.BlueOrb { namespace bts.BlueOrb {
using BCG; using BCG;
using Spark.Core.Diagnostics;
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
@@ -54,7 +57,7 @@ namespace bts.BlueOrb {
public bool IsReadOnly { get { return true; } } public bool IsReadOnly { get { return true; } }
public object this[object key] { public object? this[object key] {
get { get {
if (key is string) if (key is string)
return this[(string)key]; return this[(string)key];
@@ -63,16 +66,15 @@ namespace bts.BlueOrb {
set { throw new NotSupportedException("The IBlueRow is readonly"); } set { throw new NotSupportedException("The IBlueRow is readonly"); }
} }
public object this[int index] { public object? this[int index] {
get { get {
return this[schema.columnInfo[index].colName]; return this[schema.columnInfo[index].colName];
} }
} }
public object this[string col_name] { public object? this[string col_name] {
get { get {
string mapped_to_col = null; if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
return _row[mapped_to_col]; return _row[mapped_to_col];
} }
return _row[col_name]; return _row[col_name];
@@ -89,7 +91,8 @@ namespace bts.BlueOrb {
} }
} }
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } public void Add(object key, object? value) =>
throw new NotSupportedException();
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
@@ -114,7 +117,7 @@ namespace bts.BlueOrb {
private ViewStream _stream; private ViewStream _stream;
private string _project_expression; private string _project_expression;
private ProjectData _project_data = null; ProjectData? _project_data;
//public ProjectData //public ProjectData
public NewProjectStream(ViewStream vs, string project_expression) { public NewProjectStream(ViewStream vs, string project_expression) {
_stream = vs; _stream = vs;
@@ -128,9 +131,8 @@ namespace bts.BlueOrb {
return _project_data.out_schema; return _project_data.out_schema;
} }
public IBlueRow map(IBlueRow row) { public IBlueRow map(IBlueRow row) =>
return new NewProjectRow(_project_data, row); new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
}
// this is where most of the logic is, in terms of translating cols // this is where most of the logic is, in terms of translating cols
private static ProjectData _create_project_data(IBlueTableSchema input_schema, private static ProjectData _create_project_data(IBlueTableSchema input_schema,
@@ -1,18 +1,7 @@
using BCG;
using bts.utils;
using Bts.DataStreamCore;
using mbts.utils;
using Spark.Core.Collections;
using Spark.Core.Threading;
using System; using System;
using System.Collections.Generic; using System.Data.Entity.Core.Mapping;
using System.Collections.Immutable;
using System.Linq; using BCG;
namespace bts.BlueOrb { namespace bts.BlueOrb {
// Keeps adding changing rows to the stream, hence preserving the history // Keeps adding changing rows to the stream, hence preserving the history
@@ -20,20 +9,17 @@ namespace bts.BlueOrb {
// into a time series of prices/qty // into a time series of prices/qty
[DataStreamUserRespresentation("TimeSeriesStream")] [DataStreamUserRespresentation("TimeSeriesStream")]
public class TimeSeriesStream : ViewStream { public class TimeSeriesStream : ViewStream {
readonly ViewStream _stream; private CircularBufferTable _table;
readonly string timeColumn; private ViewStream _stream;
readonly AtomicGate gate = new();
CircularBufferTable? _table;
private int _id = 0; private int _id = 0;
private int _size = 0; private int _size = 0;
public TimeSeriesStream(ViewStream stream) public TimeSeriesStream(ViewStream stream)
: this(stream, Int32.MaxValue) { } : this(stream, 1000) { }
public TimeSeriesStream(ViewStream stream, int size, string timeColumn = "ts_event") { public TimeSeriesStream(ViewStream stream, int size) {
_stream = stream; _stream = stream;
_size = size; _size = size;
this.timeColumn = timeColumn;
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
@@ -45,37 +31,27 @@ namespace bts.BlueOrb {
private void _onstream_change(object o) { private void _onstream_change(object o) {
var dp = ExceptionAssert.Argument.Is<IDataPacket>(o); DataPacket dp = o as DataPacket;
DateTime now = DateTime.Now; DateTime now = DateTime.Now;
using var _ = gate.BeginTransaction();
if (_table == null) { if (_table == null) {
var colInfos = dp.newRows.schema.columnInfo; IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
if (colInfos.TryGetFirst(c => c.colName == timeColumn, out var currTimeCol, out var currTimeColIdx)) { schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
colInfos = colInfos.ToArray(); _table = new CircularBufferTable(schema, _size);
colInfos[currTimeColIdx] = new(timeColumn, currTimeCol.colNo, typeof(DateTime));
} else {
colInfos = ArrayHelper.add(colInfos, new(timeColumn, colInfos.Length, typeof(DateTime)));
}
_table = new CircularBufferTable(new BlueTableSchema(colInfos, dp.newRows.schema.tableName), _size);
} }
var addedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
var deletedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
foreach (IBlueRow r in dp.addedRows) { foreach (IBlueRow r in dp.addedRows) {
MutableBlueRow ar = new MutableBlueRow(_table.schema, r); MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
ar.setValue(timeColumn, now); ar.setValue("timestamp", now);
int id = _id++; ar.setValue("id", _id++);
ar.setValue("id", id); _table.add_row(ar);
_table = _table.Add(ar, out var oldRow);
addedRowsBd.AddOrUpdate(ar);
if (oldRow is not null) {
deletedRowsBd.AddOrUpdate(oldRow);
}
} }
var addedRows = addedRowsBd.ToImmutable(); var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
var deletedRows = deletedRowsBd.ToImmutable();
var dp_out = new DataPacket(_table, addedRows, deletedRows);
myFireEvent(dp_out); myFireEvent(dp_out);
} }
} }
} }
@@ -1,197 +0,0 @@
namespace Bts.DataStream;
using bts;
using Spark.Core.Collections;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
/// <summary>
/// Maps a timeseries of keys to unique int32 ids.
/// </summary>
public class TimeseriesIndex<T> where T : notnull {
static readonly IComparer<KeyValuePair<DateTime, int>> idTimeseriesComp =
KeyValuePairComparer.KeyComparer<DateTime, int>();
readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId;
readonly ImmutableDictionary<int, TimeseriesKey<T>> idToName;
readonly int idCounter;
public TimeseriesIndex()
: this(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Empty, ImmutableDictionary<int, TimeseriesKey<T>>.Empty, 0) { }
TimeseriesIndex(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
this.nameToId = nameToId;
this.idToName = idToName;
this.idCounter = idCounter;
}
public int Count => idToName.Count;
public ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> NameToId => nameToId;
public ImmutableDictionary<int, TimeseriesKey<T>> IdToName => idToName;
public bool ContainsId(int id) =>
idToName.ContainsKey(id);
public int GetNearest(TimeseriesKey<T> key) {
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
throw new KeyNotFoundException();
}
int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
if (idx < 0) {
idx = ~idx;
if (idx == 0) {
throw new KeyNotFoundException();
}
idx--;
}
return ids[idx].Value;
}
public bool TryGetNearest(TimeseriesKey<T> key, out int result) {
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
result = -1;
return false;
}
int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
if (idx < 0) {
idx = ~idx;
if (idx == 0) {
result = -1;
return false;
}
idx--;
}
result = ids[idx].Value;
return true;
}
public TimeseriesIndex<T> GetNearestOrAdd(TimeseriesKey<T> key, out int result) {
int idx;
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
idx = 0;
} else {
idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
if (idx < 0) {
idx = ~idx;
}
}
if (idx == 0) {
result = idCounter + 1;
ids = ids.Insert(0, new(key.Timestamp, result));
return new(
nameToId.SetItem(key.Name, ids),
idToName.SetItem(result, key),
result);
}
result = ids[idx].Value;
return this;
}
public TimeseriesIndex<T> GetOrAdd(TimeseriesKey<T> key, out int result) {
int idx;
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
idx = 0;
} else {
idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
if (idx < 0) {
idx = ~idx;
} else {
result = ids[idx].Value;
return this;
}
}
result = idCounter + 1;
ids = ids.Insert(idx, new(key.Timestamp, result));
return new(
nameToId.SetItem(key.Name, ids),
idToName.SetItem(result, key),
result);
}
public IList<KeyValuePair<DateTime, int>> GetIds(T name) =>
nameToId[name];
public Builder ToBuilder() =>
new(this);
public class Builder {
readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Builder nameToId;
readonly ImmutableDictionary<int, TimeseriesKey<T>>.Builder idToName;
int idCounter;
public Builder(TimeseriesIndex<T> source)
: this(source.nameToId, source.idToName, source.idCounter) { }
public Builder(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
this.nameToId = nameToId.ToBuilder();
this.idToName = idToName.ToBuilder();
this.idCounter = idCounter;
}
public int GetNearest(TimeseriesKey<T> key) =>
getOrAdd(key, false, false);
public int GetNearestOrAdd(TimeseriesKey<T> key) =>
getOrAdd(key, false, true);
public int GetOrAdd(TimeseriesKey<T> key) =>
getOrAdd(key, true, true);
public int GetLatest(T key) =>
nameToId.TryGetValue(key, out var ids)
? ids[^1].Value
: throw new KeyNotFoundException();
public bool TryGetLatest(T key, out int result) {
if (nameToId.TryGetValue(key, out var ids)) {
result = ids[^1].Value;
return true;
}
result = 0;
return false;
}
public ImmutableList<KeyValuePair<DateTime, int>> GetIds(T name) =>
nameToId.TryGetValue(name, out var ids)
? ids
: ImmutableList<KeyValuePair<DateTime, int>>.Empty;
public TimeseriesIndex<T> ToImmutable() =>
new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter);
int getOrAdd(TimeseriesKey<T> key, bool requireExactTimestamp, bool canAdd) {
var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList<KeyValuePair<DateTime, int>>.Empty);
int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp);
int id;
if (idx < 0) {
Log.Gen.info($"Adding key to timeseries index ({key.Timestamp}, {key.Name})");
// We didn't get an exact match. If requireExactTimestamp is false and a record with an
// earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp.
idx = ~idx;
if (requireExactTimestamp || idx == 0) {
if (!canAdd) {
throw new KeyNotFoundException();
}
id = ++idCounter;
ids = ids.Insert(idx, new(key.Timestamp, id));
nameToId[key.Name] = ids;
idToName.Add(id, key);
} else {
if (idx == ids.Count) {
--idx;
}
id = ids[idx].Value;
}
} else {
id = ids[idx].Value;
}
return id;
}
}
}
@@ -1,22 +0,0 @@
namespace Bts.DataStream;
using Spark.Core.Collections;
using System;
using System.Collections.Generic;
public readonly record struct TimeseriesKey<T>(T Name, DateTime Timestamp) : IDualComparable<TimeseriesKey<T>> {
public int CompareTo(TimeseriesKey<T> other) {
var rv = Timestamp.CompareTo(other.Timestamp);
if (rv == 0) {
rv = Comparer<T>.Default.Compare(Name, other.Name);
}
return rv;
}
public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey<T> key) =>
(key.Name, key.Timestamp);
public static implicit operator TimeseriesKey<T>((T Name, DateTime Timestamp) tup) =>
new(tup.Name, tup.Timestamp);
}
@@ -5,7 +5,8 @@ namespace bts.BlueOrb {
using mbts; using mbts;
using BCG; using BCG;
[DataStreamUserRespresentation("UnionStream")] // [DataStreamUserRespresentation("UnionStream")]
[DataStreamUserRespresentation("AppendStream")]
public class UnionStream : ViewStream { public class UnionStream : ViewStream {
public UnionStream(ViewStream left_stream, ViewStream right_stream) { public UnionStream(ViewStream left_stream, ViewStream right_stream) {
ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream); ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
@@ -6,24 +6,37 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" /> <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
<ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
<ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
<ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
<ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="IronPython" Version="3.4.2" /> <PackageReference Include="IronPython" Version="3.4.2" />
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" /> <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
<PackageReference Include="MongoDB.Bson" Version="2.23.1" /> <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
<PackageReference Include="MongoDB.Driver" Version="2.23.1" /> <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
<PackageReference Include="ParquetSharp" Version="20.0.0" /> <PackageReference Include="Parquet.Net" Version="5.0.1" />
<PackageReference Include="System.Data.SQLite" Version="1.0.118" /> <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
<PackageReference Include="MathNet.Numerics" Version="5.0.0" /> <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
</ItemGroup> </ItemGroup>
<Choose>
<When Condition="'$(UseProjectRefs)' == 'true'">
<ItemGroup>
<ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
<ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
<ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
<ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
<ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
<ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<PackageReference Include="ArithmeticExpressions" Version="$(SparkVersion)" />
<PackageReference Include="BaseUtilities" Version="$(SparkVersion)" />
<PackageReference Include="BooleanExpressions" Version="$(SparkVersion)" />
<PackageReference Include="Bts.ColumnExpressions" Version="$(SparkVersion)" />
<PackageReference Include="Bts.ColumnSortExpressions" Version="$(SparkVersion)" />
<PackageReference Include="DataStreamExpressions" Version="$(SparkVersion)" />
</ItemGroup>
</Otherwise>
</Choose>
</Project> </Project>
@@ -2,7 +2,6 @@ namespace Bts.DataStream;
using BCG; using BCG;
using bts;
using bts.api.core; using bts.api.core;
using Bts.DataStreamCore; using Bts.DataStreamCore;
@@ -16,13 +15,27 @@ using Spark.Core.Threading;
using System; using System;
using System.Linq; using System.Linq;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
public static class DataStreamExtensions { public static class DataStreamExtensions {
public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
source as ViewStream ?? new ObservableViewStream(source);
public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow => public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger); source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
IDisposable Subscribe(IObserver<IDataPacket> observer) {
source.onChange += OnChange;
return Disposable.Create(() => source.onChange -= OnChange);
void OnChange(object? newValue) =>
observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
$"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
}
}
// public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow => // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
// source.Select(dp => { // source.Select(dp => {
// dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); // dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
@@ -46,16 +59,16 @@ public static class DataStreamExtensions {
public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>( public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> { this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
rowKeySelector ??= r => keySelector(valueSelector(r)); rowKeySelector ??= r => keySelector(valueSelector(r));
return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => { return source.Scan(TableUpdate<T, TKey>.Empty(keySelector), (res, dp) => {
var bd = res.to_builder(); var bd = res.ToBuilder();
dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
foreach (IBlueRow r in addedRows) { foreach (IBlueRow r in addedRows) {
bd.add_or_update(valueSelector(r)); bd.AddOrUpdate(valueSelector(r));
} }
foreach (IBlueRow r in deletedRows) { foreach (IBlueRow r in deletedRows) {
bd.remove(rowKeySelector(r)); bd.Remove(rowKeySelector(r));
} }
return bd.to_table(); return bd.ToImmutable();
}); });
} }
@@ -65,16 +78,16 @@ public static class DataStreamExtensions {
(res, table) => { (res, table) => {
var resTable = res.Table; var resTable = res.Table;
if (resTable is null) { if (resTable is null) {
if (table.state.Count == 0) { if (table.Index.Count == 0) {
return (null, null); return (null, null);
} }
resTable = new(table.state.Values.First().schema); resTable = new(table.Index.Values.First().schema);
} }
var bd = resTable.ToDataPacketBuilder(); var bd = resTable.ToDataPacketBuilder();
foreach (var r in table.removed) { foreach (var r in table.RemovedItems) {
bd.Remove(r.Id()); bd.Remove(r.Id());
} }
foreach (var r in table.added) { foreach (var r in table.AddedItems) {
bd.AddOrUpdate(r); bd.AddOrUpdate(r);
} }
var resDp = bd.ToDataPacket(out var newRows); var resDp = bd.ToDataPacket(out var newRows);
@@ -88,10 +101,10 @@ public static class DataStreamExtensions {
(Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)), (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
(res, table) => { (res, table) => {
var bd = res.Table.ToDataPacketBuilder(); var bd = res.Table.ToDataPacketBuilder();
foreach (var r in table.removed) { foreach (var r in table.RemovedItems) {
bd.Remove(r.Id()); bd.Remove(r.Id());
} }
foreach (var r in table.added) { foreach (var r in table.AddedItems) {
bd.AddOrUpdate(r); bd.AddOrUpdate(r);
} }
var resDp = bd.ToDataPacket(out var newRows); var resDp = bd.ToDataPacket(out var newRows);
@@ -121,6 +134,46 @@ public static class DataStreamExtensions {
public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow => public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
source.AsDataPackets(schema).AsViewStream(); source.AsDataPackets(schema).AsViewStream();
public static int ResolveIndices(this IBlueTableSchema schema, string name) {
var indices = ResolveIndices(schema, new[] { name });
return indices[0];
}
public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
var indices = ResolveIndices(schema, new[] { name1, name2 });
return (indices[0], indices[1]);
}
public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
return (indices[0], indices[1], indices[2]);
}
public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
return (indices[0], indices[1], indices[2], indices[3]);
}
public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
var columns = schema.columnInfo;
var indices = new int[columnNames.Length];
int nResolved = 0;
for (int i = 0; i < columnNames.Length; i++) {
int origNResolved = nResolved;
for (int j = 0; j < columns.Length; j++) {
if (columns[j].colName == columnNames[i]) {
indices[i] = j;
nResolved++;
break;
}
}
if (nResolved == origNResolved) {
throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
}
}
return indices;
}
public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) => public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
Observable.Create<IDataPacket>(o => { Observable.Create<IDataPacket>(o => {
var inputGate = new AtomicGate(); var inputGate = new AtomicGate();
@@ -145,7 +198,7 @@ public static class DataStreamExtensions {
foreach (IBlueRow r in value.addedRows) { foreach (IBlueRow r in value.addedRows) {
bd.AddOrUpdate(r); bd.AddOrUpdate(r);
} }
pendingDpOut = bd.ToDataPacket(out lastTable, false); pendingDpOut = bd.ToDataPacket(out lastTable);
} }
} }
@@ -161,118 +214,5 @@ public static class DataStreamExtensions {
} }
}); });
public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
const int STATE_IDLE = 0;
const int STATE_BUSY = 1;
const int STATE_DISPOSED = 2;
return Observable.Create<IDataPacket>(Subscribe);
IDisposable Subscribe(IObserver<IDataPacket> o) {
var syncRoot = new object();
IBlueRowCollection? lastTableIn = null;
BlueTableImmutable? lastTableOut = null;
int nConcurrentDoWorks = 0;
int state = STATE_IDLE;
IDisposable? dispose = null;
dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
return Disposable.Create(Dispose);
void Dispose() {
lock (syncRoot) {
switch (state) {
case STATE_DISPOSED:
return;
case STATE_BUSY:
state = STATE_DISPOSED;
return;
}
state = STATE_DISPOSED;
}
Disposable.dispose(ref dispose);
}
void OnNext(IDataPacket value) =>
Volatile.Write(ref lastTableIn, value.newRows);
async void DoWork() {
lock (syncRoot) {
nConcurrentDoWorks++;
switch (state) {
case STATE_DISPOSED:
case STATE_BUSY:
return;
case STATE_IDLE:
break;
default:
Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
return;
}
state = STATE_BUSY;
}
var dp = await PublishTable();
bool isDisposed = false;
lock (syncRoot) {
switch (state) {
case STATE_DISPOSED:
isDisposed = true;
break;
case STATE_BUSY:
break;
default:
Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
return;
}
}
if (isDisposed) {
Disposable.dispose(ref dispose);
return;
}
if (dp is not null) {
o.OnNext(dp);
}
lock (syncRoot) {
nConcurrentDoWorks = 0;
switch (state) {
case STATE_DISPOSED:
isDisposed = true;
break;
case STATE_BUSY:
state = STATE_IDLE;
break;
default:
Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
return;
}
}
if (isDisposed) {
Disposable.dispose(ref dispose);
}
async Task<DataPacket?> PublishTable() {
var tableIn = Interlocked.Exchange(ref lastTableIn, null);
if (tableIn is null) {
return null;
}
var tableOut = await transformAsync(tableIn);
if (tableOut is null) {
return null;
}
lastTableOut ??= new(tableOut.schema);
var bd = lastTableOut.ToDataPacketBuilder();
bd.Clear();
foreach (IBlueRow r in tableOut) {
bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
}
return bd.ToDataPacket(out lastTableOut);
}
}
static string StateName(int state) =>
state switch {
STATE_IDLE => "IDLE",
STATE_BUSY => "BUSY",
STATE_DISPOSED => "DISPOSED",
_ => $"STATE_{state}",
};
}
}
} }
@@ -3,27 +3,30 @@ namespace bts.BlueOrb {
using BCG; using BCG;
using System.Collections.Generic; using System.Collections.Generic;
using Bts.DataStreamExpressions; using Bts.DataStreamExpressions;
using Bts.DataStreamCore; using System.Xml;
// All this does is take in a dsl expression, parse it and fire the data that come out of running it. // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
// So for example, if an expresison is passed // So for example, if an expresison is passed
[DataStreamUserRespresentation("dsl")] [DataStreamUserRespresentation("dsl")]
public class DSLStream : ViewStream { public class DSLStream : ViewStream {
private readonly string _dsl; private readonly string _dsl;
public DSLStream(string dsl, IDataStreamFactory streamFactory) { public DSLStream(string dsl) {
Log.Gen.info($"DSLStream.cons dsl: {dsl}"); Log.Gen.info($"DSLStream.cons dsl: {dsl}");
// Remove all colons
string sanitizedDsl = dsl.Replace(":", "COLON");
_dsl = dsl; _dsl = dsl;
var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl"); ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
mySetRealSource(rs); mySetRealSource(rs);
} }
static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) { static ViewStream? build_stream(string stream_expression) {
try { try {
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory); var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv; return rv;
} catch (Exception ex) { } catch (Exception ex) {
//send to blueorb
Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}"); Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
return null; return null;
} }
@@ -1,14 +1,13 @@
namespace Bts.DataStream; namespace Bts.DataStream;
using BCG; using BCG;
using BCG.Extensions;
using BCG.Utilities.General; using BCG.Utilities.General;
using bts; using bts;
using bts.collections;
using Bts.DataStreamCore; using Bts.DataStreamCore;
using Spark.Core.Collections;
using Spark.Core.Disposables; using Spark.Core.Disposables;
using System; using System;
@@ -40,7 +40,7 @@ namespace BCG {
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _send_period); MasterWorker.getInstance(false).addWorker(this, _send_period);
var server_rci = new RemoteConnectionInfo(_ip, _port); var server_rci = new RemoteConnectionInfo(_ip, _port);
bts.AsyncTCPConnector_AsyncWrapper.connect_to(server_rci, this); bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
@@ -76,4 +76,4 @@ namespace BCG {
} }
} }
@@ -1,5 +1,4 @@
#nullable enable namespace Bts.DataStream;
namespace Bts.DataStreamCore;
using BCG; using BCG;
@@ -1,5 +1,6 @@
using bts; using bts;
using Bts.DataStream;
using Bts.DataStreamCore; using Bts.DataStreamCore;
using Bts.DataStreamExpressions; using Bts.DataStreamExpressions;
using Bts.PubSub; using Bts.PubSub;
@@ -11,7 +12,6 @@ using Spark.Core.Disposables;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@@ -35,12 +35,12 @@ namespace BCG {
view_publisher = view_publisher_; view_publisher = view_publisher_;
} }
} }
static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) static ViewStream? build_stream(string stream_expression)
{ {
try try
{ {
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory); var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv; return rv;
} catch (Exception ex) { } catch (Exception ex) {
@@ -52,85 +52,43 @@ namespace BCG {
_stream_pub_map = new Dictionary<string, StreamPubUnit>(); _stream_pub_map = new Dictionary<string, StreamPubUnit>();
readonly IPubTableBroker _broker; readonly IPubTableBroker _broker;
readonly IManagedSubscriber _sub; readonly IManagedSubscriber _sub;
readonly IDataStreamFactory streamFactory;
readonly ITableStreamFactory tableStreamFactory;
readonly object syncRoot = new(); readonly object syncRoot = new();
IDisposable? cloudStreamsSub; IDisposable? cloudStreamsSub;
IDisposable? remoteRequestsSub; IDisposable? remoteRequestsSub;
State state; bool isDisposed;
public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, ITableStreamFactory tableStreamFactory, IEnumerable<IDataStreamFactory> streamFactories) {
public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
_sub = managedSubscriber; _sub = managedSubscriber;
_broker = pubTableBroker; _broker = pubTableBroker;
this.tableStreamFactory = tableStreamFactory;
var streamFactoryList = streamFactories.ToList();
streamFactory = streamFactoryList.Count == 1
? streamFactoryList[0]
: new CompositeDataStreamFactory(streamFactoryList);
}
enum State {
Idle = 0,
Running = 1,
Disposed = 2,
WaitingToDispose = 3,
} }
public Task StartAsync(CancellationToken cancellationToken) { public Task StartAsync(CancellationToken cancellationToken) {
lock (syncRoot) { lock (syncRoot) {
switch (state) { Log.assert(!isDisposed);
case State.Idle: Log.isnull(cloudStreamsSub);
break; Log.isnull(remoteRequestsSub);
case State.WaitingToDispose: cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
case State.Disposed: remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
throw new ObjectDisposedException(nameof(RemoteQueryManager)); RemoteRequest req = RemoteRequest.construct_from_bytes(data);
default: _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
return Task.CompletedTask; });
}
state = State.Running;
}
if (cloudStreamsSub is not null) {
throw new InvalidOperationException("cloudStreamsSub already created");
}
if (remoteRequestsSub is not null) {
throw new InvalidOperationException("remoteRequestsSub already created");
}
var cloudStream = tableStreamFactory.Create("cloud_streams")
?? throw new Exception("Failed to create TableStream 'cloud_stream'");
cloudStreamsSub = cloudStream.AsObservable().Subscribe(_on_str);
remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
});
lock (syncRoot) {
if (state == State.WaitingToDispose) {
state = State.Disposed;
}
}
if (state == State.Disposed) {
Disposable.dispose(ref cloudStreamsSub);
Disposable.dispose(ref remoteRequestsSub);
} }
return Task.CompletedTask; return Task.CompletedTask;
} }
public Task StopAsync(CancellationToken cancellationToken) { public Task StopAsync(CancellationToken cancellationToken) {
lock (syncRoot) { lock (syncRoot) {
switch (state) { if (!isDisposed) {
case State.Running: isDisposed = true;
break; Disposable.dispose(ref cloudStreamsSub);
default: Disposable.dispose(ref remoteRequestsSub);
return Task.CompletedTask;
} }
state = State.Disposed;
} }
Disposable.dispose(ref cloudStreamsSub);
Disposable.dispose(ref remoteRequestsSub);
return Task.CompletedTask; return Task.CompletedTask;
} }
private void _on_str(IDataPacket dp) void _on_str(IDataPacket dp) {
{
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
{ {
string name = r.get<string>("name"); string name = r.get<string>("name");
@@ -144,11 +102,7 @@ namespace BCG {
private void _spawn_remote_stream(string name, string stream_expression, bool persist) { private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
var stream = build_stream(streamFactory, stream_expression); ViewStream stream = build_stream(stream_expression);
if (stream is null) {
Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
return;
}
_expose_to_ws(name, stream); _expose_to_ws(name, stream);
if (persist) { if (persist) {
string tempDir = Path.GetTempPath(); string tempDir = Path.GetTempPath();
@@ -2,30 +2,17 @@ using BCG;
using bts; using bts;
using Bts.DataStreamCore;
using Spark.Core.Disposables;
using System; using System;
namespace BCG { namespace BCG {
/// <summary> // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
/// This is a stream so we can run it from the DSQA (since dsqa can only load streams). [DataStreamUserRespresentation("remoteQuery")]
/// <para>
/// Note: It's hidden because this stream is now created by an IDataStreamFactory
/// (see RemoteQueryStreamFactory.cs for details)
/// </para>
/// </summary>
[DataStreamUserRespresentation(false)]
public class RemoteQueryStream : ViewStream { public class RemoteQueryStream : ViewStream {
private readonly string _name, _stream_exp; private readonly string _name, _stream_exp;
readonly ITableStreamFactory tableStreamFactory; private PublisherCore _publisher_core = PublisherCore.Create();
readonly IPublisherCore _publisher_core; private ViewStream _source_str;
IDisposable? subscription;
public RemoteQueryStream(ITableStreamFactory tableStreamFactory, IPublisherCore publisherCore, params string[] lines) { public RemoteQueryStream(params string[] lines) {
this.tableStreamFactory = tableStreamFactory;
_publisher_core = publisherCore;
_stream_exp = get_expression(lines); _stream_exp = get_expression(lines);
_name = Guid.NewGuid().ToString(); _name = Guid.NewGuid().ToString();
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}"; _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
@@ -34,17 +21,17 @@ namespace BCG {
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
Log.isnull(subscription);
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name); RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
byte[] rem_req_bytes = rem_req.serialize(); byte[] rem_req_bytes = rem_req.serialize();
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes); _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
var sourceStream = tableStreamFactory.Create(_name); _source_str = new TableStream(_name);
sourceStream.onChange += _handle_data; _source_str.onChange += new ChangeDelegate(_handle_data);
subscription = Disposable.Create(() => sourceStream.onChange -= _handle_data);
} }
protected override void myOnLastSubscriber() => protected override void myOnLastSubscriber() {
Disposable.dispose(ref subscription); //maybe there should be a way to turn remote requests off ?
_source_str.onChange -= new ChangeDelegate(_handle_data);
}
static string get_expression(params string[] lines) { static string get_expression(params string[] lines) {
for (int i = 0; i < lines.Length; i++) { for (int i = 0; i < lines.Length; i++) {
@@ -2,9 +2,6 @@ namespace bts {
using System.IO; using System.IO;
using BCG; using BCG;
using System.IO;
using System.Threading.Tasks.Dataflow;
public class RemoteRequest { public class RemoteRequest {
public readonly string stream_expression; public readonly string stream_expression;
public readonly string published_table_name; public readonly string published_table_name;
@@ -28,4 +25,4 @@ namespace bts {
} }
} }
} }
@@ -1,33 +1,28 @@
using Spark.Core.Collections;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
namespace BCG { namespace BCG {
public class AddColumnsStream : ViewStream { public class AddColumnsStream : ViewStream {
public static Tuple<string, Type, object> parse_single_tok(string exp) { public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
string[] toks = exp.Split(':'); string[] toks = exp.Split(':');
string col_name = toks[0].Trim(); string col_name = toks[0].Trim();
string col_value = toks[1].Trim(); string col_value = toks[1].Trim();
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value); Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
return Tuple.Create(col_name, tup.Item1, tup.Item2); return (col_name, tup.Item1, tup.Item2);
} }
public static List<Tuple<string, Type, object>> parse_exp(string exp) { public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
string[] toks = exp.Split(","); exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
foreach (string s in toks) {
var tup = parse_single_tok(s.Trim());
rv.Add(tup);
}
return rv;
}
private Tuple<string, Type, object> _new_col; readonly List<(string Name, Type Type, object Value)> _new_col_map;
private List<Tuple<string, Type, object>> _new_col_map; IBlueTableSchema? _out_schema;
private IBlueTableSchema _out_schema;
public AddColumnsStream(ViewStream vs, string col_def) { public AddColumnsStream(ViewStream vs, string col_def) {
_new_col_map = parse_exp(col_def); _new_col_map = parse_exp(col_def);
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r)); ViewStream rs = new MapStream(vs, _get_schema, _map);
mySetRealSource(rs); mySetRealSource(rs);
} }
@@ -44,7 +39,7 @@ namespace BCG {
private IBlueRow _map(IBlueRow r) { private IBlueRow _map(IBlueRow r) {
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r); MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
foreach (var m in _new_col_map) { foreach (var m in _new_col_map) {
rv.setValue(m.Item1, m.Item3); rv.setValue(m.Name, m.Value);
} }
return rv; return rv;
} }
@@ -1,9 +1,10 @@
using System; using mbts;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
namespace BCG namespace BCG
{ {
@@ -33,7 +34,7 @@ namespace BCG
private void _stream_onChange(object newValue) private void _stream_onChange(object newValue)
{ {
DataPacket dp = newValue as DataPacket; var dp = (IDataPacket)newValue;
if (dp != null && dp.newRows.Count > 0) if (dp != null && dp.newRows.Count > 0)
{ {
_stream.onChange -= _stream_onChange; _stream.onChange -= _stream_onChange;
@@ -42,8 +43,8 @@ namespace BCG
} }
} }
public delegate void IncrementValues(object[] current_values, object[] values_to_inc); public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
public delegate string GetColumnPrefix(object[] transposing_col_values); public delegate string GetColumnPrefix(object?[] transposing_col_values);
public delegate string GetColumnSuffix(string col_name); public delegate string GetColumnSuffix(string col_name);
private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; }); private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
@@ -51,33 +52,31 @@ namespace BCG
private IncrementValues _increment_values; private IncrementValues _increment_values;
private GetColumnPrefix _get_column_prefix; private GetColumnPrefix _get_column_prefix;
private IBlueTableSchema _schema; IBlueTableSchema? _schema;
private string[] _group_by_cols; private string[] _group_by_cols;
private Pair<string, Type>[] _cols_per_transposed; private Pair<string, Type>[] _cols_per_transposed;
private string[] _transposing_cols; private string[] _transposing_cols;
private int[] _transposing_col_indices; int[]? _transposing_col_indices;
private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper; Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols, public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer) IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
: this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del, : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true) get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
{ } { }
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols, public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once) IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
{ {
if (get_column_suffix_del != null) if (get_column_suffix_del != null)
_get_column_suffix = get_column_suffix_del; _get_column_suffix = get_column_suffix_del;
if (transposed_col_value_comparer != null) if (transposed_col_value_comparer != null)
{ {
_transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) => _transpsed_col_value_comparer_wrapper = new((p1, p2) =>
{ transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
return transposed_col_value_comparer(p1.second, p2.second);
});
} }
_group_by_cols = group_by_cols.Split(' '); _group_by_cols = group_by_cols.Split(' ');
_transposing_cols = transposed_cols_group.Split(' '); _transposing_cols = transposed_cols_group.Split(' ');
@@ -106,7 +105,7 @@ namespace BCG
col_info.Add(new ColumnInfo(p.first, col_index++, p.second)); col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
foreach (string tc_prefix in tc_prefixes) foreach (string tc_prefix in tc_prefixes)
foreach (Pair<string, Type> p in _cols_per_transposed) foreach (Pair<string, Type> p in _cols_per_transposed)
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second)); col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name); IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
if (_schema == null) if (_schema == null)
_schema = schema; _schema = schema;
@@ -130,19 +129,14 @@ namespace BCG
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
{ {
Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>(); var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>(); var group_by_cols_value_dict = new Dictionary<int, object?[]>();
BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length); BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length]; Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
string table_name = string.Empty; string table_name = string.Empty;
if (grouped_rows.Count > 0) if (grouped_rows.Count > 0)
{ {
IBlueTableSchema schema = null; var schema = grouped_rows.Values.First()[0].schema;
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
{
schema = kvp.Value[0].schema;
break;
}
table_name = schema.tableName; table_name = schema.tableName;
_transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols); _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols); int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
@@ -155,23 +149,22 @@ namespace BCG
else else
return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name));
Dictionary<int, string> tc_prefixes = new Dictionary<int, string>(); Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>(); var tc_values = new List<Pair<int, object?[]>>();
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows) foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
{ {
//Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values
Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>(); var incrementing_values = new Dictionary<int, object?[]>();
foreach (IBlueRow r in kvp.Value) foreach (IBlueRow r in kvp.Value)
{ {
int hash = hasher.get_hash(r, _transposing_col_indices); int hash = hasher.get_hash(r, _transposing_col_indices);
object[] values; if (!incrementing_values.TryGetValue(hash, out var values))
if (!incrementing_values.TryGetValue(hash, out values))
{ {
incrementing_values.Add(hash, _get_per_transposing_col_values(r)); incrementing_values.Add(hash, _get_per_transposing_col_values(r));
string prefix = _get_column_prefix(_get_transposing_col_values(r)); string prefix = _get_column_prefix(_get_transposing_col_values(r));
if (!tc_prefixes.ContainsKey(hash)) if (!tc_prefixes.ContainsKey(hash))
{ {
tc_prefixes.Add(hash, prefix); tc_prefixes.Add(hash, prefix);
tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r))); tc_values.Add(new(hash, _get_transposing_col_values(r)));
} }
} }
else else
@@ -194,18 +187,18 @@ namespace BCG
_set_schema(prefixes, group_by_col_info, table_name); _set_schema(prefixes, group_by_col_info, table_name);
BlueTable result = new BlueTable(_schema); BlueTable result = new BlueTable(_schema);
int id = 1; int id = 1;
foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values) foreach (var kvp in all_tc_infos_and_values)
{ {
MutableBlueRow mbr = new MutableBlueRow(_schema); MutableBlueRow mbr = new MutableBlueRow(_schema);
mbr.setValue("id", id++); mbr.setValue("id", id++);
for (int i = 0; i < _group_by_cols.Length; i++) for (int i = 0; i < _group_by_cols.Length; i++)
mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]); mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value) foreach (var tc_cols in kvp.Value)
{ {
string prefix = tc_prefixes[tc_cols.Key]; string prefix = tc_prefixes[tc_cols.Key];
for (int i = 0; i < _cols_per_transposed.Length; i++) for (int i = 0; i < _cols_per_transposed.Length; i++)
{ {
string column = _get_full_column_name(prefix, _cols_per_transposed[i].first); string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
if(mbr.schema.columnNames.Contains(column)) if(mbr.schema.columnNames.Contains(column))
mbr.setValue(column, tc_cols.Value[i]); mbr.setValue(column, tc_cols.Value[i]);
} }
@@ -217,28 +210,25 @@ namespace BCG
#endregion #endregion
private object[] _get_transposing_col_values(IBlueRow row) object?[] _get_transposing_col_values(IBlueRow row) {
{ var values = new object?[_transposing_cols.Length];
object[] values = new object[_transposing_cols.Length];
for (int i = 0; i < _transposing_cols.Length; i++) for (int i = 0; i < _transposing_cols.Length; i++)
values[i] = row[_transposing_cols[i]]; values[i] = row[_transposing_cols[i]];
return values; return values;
} }
private object[] _get_per_transposing_col_values(IBlueRow row) object?[] _get_per_transposing_col_values(IBlueRow row) {
{ var values = new object?[_cols_per_transposed.Length];
object[] values = new object[_cols_per_transposed.Length];
for (int i = 0; i < _cols_per_transposed.Length; i++) for (int i = 0; i < _cols_per_transposed.Length; i++)
values[i] = row[_cols_per_transposed[i].first]; values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
return values; return values;
} }
private object[] _get_group_by_cols_values(IBlueRow row) object?[] _get_group_by_cols_values(IBlueRow row) {
{ var values = new object?[_group_by_cols.Length];
object[] values = new object[_group_by_cols.Length];
for (int i = 0; i < values.Length; i++) for (int i = 0; i < values.Length; i++)
values[i] = row[_group_by_cols[i]]; values[i] = row[_group_by_cols[i]];
return values; return values;
} }
} }
} }
@@ -2,7 +2,7 @@ using System;
namespace bts.BlueOrb { namespace bts.BlueOrb {
using BCG; using BCG;
[DataStreamUserRespresentation("oldCount")] [DataStreamUserRespresentation("CountStream")]
public class CountStream : ViewStream { public class CountStream : ViewStream {
private DataPacket myLastFiredDataPacket; private DataPacket myLastFiredDataPacket;
private IBlueTableSchema mySchema = new BlueTableSchema("count", private IBlueTableSchema mySchema = new BlueTableSchema("count",
@@ -15,6 +15,9 @@ namespace bts.BlueOrb {
myVS = vs; myVS = vs;
} }
public CountStream(string tableName)
: this(new TableStream(tableName)) { }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
myVS.onChange += new ChangeDelegate(myHandleChanges); myVS.onChange += new ChangeDelegate(myHandleChanges);
} }
@@ -1,22 +1,19 @@
using System; using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG namespace BCG
{ {
public class DerivativeStream : ViewStream public class DerivativeStream : ViewStream
{ {
IBlueTableSchema? mySchema;
private IBlueTableSchema mySchema;
private string myDerivandColumn; private string myDerivandColumn;
private string myOutputColumn; private string myOutputColumn;
private string myKeyColumn; private string myKeyColumn;
private DateTime myLastPacketArrival = DateTime.MinValue; private DateTime myLastPacketArrival = DateTime.MinValue;
private ViewStream myDataSource; private ViewStream myDataSource;
Dictionary<string, double>? myPreviousValues;
private IDictionary<string, double> myPreviousValues;
public DerivativeStream(ViewStream input, string keyColumn, string columnToDifferentiate, string outputColName) public DerivativeStream(ViewStream input, string keyColumn, string columnToDifferentiate, string outputColName)
{ {
@@ -39,9 +36,9 @@ namespace BCG
void myDataSource_onChange(object newValue) void myDataSource_onChange(object newValue)
{ {
DataPacket dp = (DataPacket)newValue; var dp = (IDataPacket)newValue;
if (myLastPacketArrival == DateTime.MinValue) if (myLastPacketArrival == DateTime.MinValue)
mySchema = myBuildOutputSchema(dp.schema); mySchema = myBuildOutputSchema(dp.Schema);
else else
{ {
DateTime now = DateTime.Now; DateTime now = DateTime.Now;
@@ -55,7 +52,7 @@ namespace BCG
else else
{ {
IBlueRowCollection deleted, added; IBlueRowCollection deleted, added;
BlueDataUtility.computeDifferences(thisVersion, ((DataPacket)myLastValue).newRows, out deleted, out added); BlueDataUtility.computeDifferences(thisVersion, ((IDataPacket)myLastValue).newRows, out deleted, out added);
myFireEvent(new DataPacket(thisVersion, added, deleted)); myFireEvent(new DataPacket(thisVersion, added, deleted));
} }
} }
@@ -64,12 +61,13 @@ namespace BCG
protected IBlueRow myMakeRow(IBlueRow r, double timeScale) protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
{ {
ExceptionAssert.NotNull(mySchema, myPreviousValues);
object[] outValues = new object[mySchema.columnInfo.Length]; object[] outValues = new object[mySchema.columnInfo.Length];
r.Values.CopyTo(outValues, 0); r.Values.CopyTo(outValues, 0);
object derivand = r[myDerivandColumn]; var derivand = r[myDerivandColumn];
double lastValue = default(double); var key = getRowKey(r);
if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value) if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale; outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
else else
outValues[mySchema.columnInfo.Length - 1] = 0.0; outValues[mySchema.columnInfo.Length - 1] = 0.0;
@@ -84,16 +82,19 @@ namespace BCG
cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double)); cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double));
return new BlueTableSchema("derviative of " + inSchema.tableName, cols); return new BlueTableSchema("derviative of " + inSchema.tableName, cols);
} }
protected void myStoreNewPacketInfo(DataPacket dp)
{ protected void myStoreNewPacketInfo(IDataPacket dp) {
myPreviousValues = new Dictionary<string, double>(); myPreviousValues = new Dictionary<string, double>();
myLastPacketArrival = DateTime.Now; myLastPacketArrival = DateTime.Now;
foreach (IBlueRow row in dp.newRows) foreach (IBlueRow row in dp.newRows)
{ {
object derivand = row[myDerivandColumn]; var derivand = row[myDerivandColumn];
if (derivand != null && derivand != DBNull.Value) if (derivand != null && derivand != DBNull.Value)
myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand); myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
} }
} }
string getRowKey(IBlueRow row) =>
row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
} }
} }
@@ -1,7 +1,9 @@
using System; using mbts;
using System.Collections;
using Spark.Core.Collections;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using mbts;
namespace BCG namespace BCG
{ {
@@ -12,9 +14,9 @@ namespace BCG
{ {
ViewStream myLStream, myRStream; ViewStream myLStream, myRStream;
DataPacket myLastR = null, myLastL = null; DataPacket? myLastR;
DataPacket? myLastL;
BlueTable myLastFired = null; BlueTable? myLastFired;
double myThreshold = .00005; double myThreshold = .00005;
@@ -24,14 +26,14 @@ namespace BCG
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) :
this(null, v1, v2, thresh, ColumnsToMatch) { } this(null, v1, v2, thresh, ColumnsToMatch) { }
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
{ {
myThreshold = thresh; myThreshold = thresh;
myLStream = v1; myLStream = v1;
myRStream = v2; myRStream = v2;
myColumnsToMatch = columnsToMatch; myColumnsToMatch = columnsToMatch;
} }
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch) public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
: this(db,v1,v2,.00005,columnsToMatch) : this(db,v1,v2,.00005,columnsToMatch)
{ {
} }
@@ -50,45 +52,22 @@ namespace BCG
myRStream.onChange -= myRChange; myRStream.onChange -= myRChange;
} }
struct ListComparer : IEqualityComparer<List<object>>
{
public bool Equals(List<object> x, List<object> y)
{
if (x.Count != y.Count)
return false;
for (int i = 0; i < x.Count; ++i)
if (!x[i].Equals(y[i]))
return false;
return true;
}
public int GetHashCode(List<object> obj)
{
int ret = 0;
for(int i = 0; i < obj.Count; ++i)
ret ^= obj[i].GetHashCode();
return ret;
}
}
void myHandleCallbacks() void myHandleCallbacks()
{ {
DataPacket packet = null; DataPacket? packet;
if (myLastL == null || myLastR == null) if (myLastL == null || myLastR == null)
{ {
packet = myLastL == null ? myLastR : myLastL; packet = myLastL == null ? myLastR : myLastL;
if (packet != null) if (packet is not null) {
myFireEvent(packet); myFireEvent(packet);
myLastFired = new BlueTable(packet.newRows);
myLastFired = new BlueTable(packet.newRows); }
return; return;
} }
BlueTable table = new BlueTable(myLastL.newRows.schema); BlueTable table = new BlueTable(myLastL.newRows.schema);
Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer()); var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
foreach(BlueRow row in myLastL.newRows) foreach(BlueRow row in myLastL.newRows)
{ {
@@ -118,11 +97,7 @@ namespace BCG
} }
catch (Exception) { } catch (Exception) { }
} }
rowPairs.TryGetValue(hash, out var pair);
Pair<BlueRow, BlueRow> pair;
rowPairs.TryGetValue(hash, out pair);
if (pair == null) if (pair == null)
pair = new Pair<BlueRow, BlueRow>(null, row); pair = new Pair<BlueRow, BlueRow>(null, row);
else else
@@ -138,8 +113,7 @@ namespace BCG
if (pair.first == null || pair.second == null) if (pair.first == null || pair.second == null)
{ {
BlueRow row = pair.first == null ? pair.second : pair.first; var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
Dictionary<string, object> newRow = new Dictionary<string, object>(); Dictionary<string, object> newRow = new Dictionary<string, object>();
foreach (ColumnInfo info in row.schema.columnInfo) foreach (ColumnInfo info in row.schema.columnInfo)
{ {
@@ -211,4 +185,4 @@ namespace BCG
} }
} }
@@ -14,10 +14,8 @@ namespace BCG
public DistinctRowEvaluator(string colMapExpression, string[] columnNames) public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
: base (colMapExpression, null, columnNames) {} : base (colMapExpression, null, columnNames) {}
public override IBlueRow? evaluate(IDataPacket packet) {
public override IBlueRow evaluate (DataPacket packet) myOutputTable ??= new GroupByOutputTable(packet.Schema);
{
myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
string hashString = GroupByStream.getRepString (packet, myGroupByCols); string hashString = GroupByStream.getRepString (packet, myGroupByCols);
if (packet.newRows.Count == 0) if (packet.newRows.Count == 0)
//if there are no rows for this group we remove the group from the output table //if there are no rows for this group we remove the group from the output table
@@ -5,7 +5,7 @@ namespace BCG
{ {
public class FileViewStream : ViewStream public class FileViewStream : ViewStream
{ {
private DataPacket myDp; DataPacket? myDp;
public FileViewStream (string dataFileName, public FileViewStream (string dataFileName,
string schemaFileName) string schemaFileName)
: this (dataFileName, schemaFileName, Int32.MaxValue) : this (dataFileName, schemaFileName, Int32.MaxValue)
@@ -1,8 +1,7 @@
using System; using BCG.Utilities.General;
using System.Collections.Generic;
using System.Linq; using System;
using System.Text;
using BCG.Utilities.General;
namespace BCG namespace BCG
{ {
public class DateBasedFilter: IDynamicStreamFilter, IWorker public class DateBasedFilter: IDynamicStreamFilter, IWorker
@@ -26,20 +25,14 @@ namespace BCG
myReference = myTodaysDate.AddDays(-1 * myNDays); myReference = myTodaysDate.AddDays(-1 * myNDays);
} }
public event FilterChangeDelegate onFilterChange; public event FilterChangeDelegate? onFilterChange;
public bool filter(IBlueRow r) public bool filter(IBlueRow r) =>
{ r[myDateColumn] switch {
if (DBNull.Value.Equals(r[myDateColumn])) DBNull or null => true,
{ DateTime t => t >= myReference,
//return false; _ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
return true; //it can be considered as our assumption whenever datecolumn is db null we return true };
}
DateTime t = (DateTime)r[myDateColumn];
bool returnVallue = t < myReference ? false : true;
return returnVallue;
}
public void doWork() public void doWork()
{ {
@@ -47,7 +40,7 @@ namespace BCG
{ {
myTodaysDate = DateTime.Today; myTodaysDate = DateTime.Today;
myReference = myTodaysDate.AddDays(-1 * myNDays); myReference = myTodaysDate.AddDays(-1 * myNDays);
onFilterChange(); onFilterChange?.Invoke();
} }
} }
} }
@@ -1,8 +1,6 @@
using System; using BCG.Utilities.General;
using System.Collections.Generic;
using System.Linq; using System;
using System.Text;
using BCG.Utilities.General;
namespace BCG namespace BCG
{ {
@@ -26,7 +24,7 @@ namespace BCG
#region IDynamicStreamFilter Members #region IDynamicStreamFilter Members
public event FilterChangeDelegate onFilterChange; public event FilterChangeDelegate? onFilterChange;
#endregion #endregion
@@ -34,7 +32,10 @@ namespace BCG
public bool filter(IBlueRow r) public bool filter(IBlueRow r)
{ {
DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date; DateTime rowDate = r[myDateColName] switch {
DBNull or null => DateTime.Now.Date,
object obj => (DateTime)obj,
};
DateTime today = DateTime.Now.Date; DateTime today = DateTime.Now.Date;
return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays); return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
} }
@@ -48,7 +49,7 @@ namespace BCG
if (DateTime.Now.Date != myLastCheckedTime.Date) if (DateTime.Now.Date != myLastCheckedTime.Date)
{ {
myLastCheckedTime = DateTime.Now; myLastCheckedTime = DateTime.Now;
onFilterChange(); onFilterChange?.Invoke();
} }
} }
@@ -1,18 +1,23 @@
using bts; using bts;
using Bts.DataStream;
using Bts.DataStream.Primitives.Filter; using Bts.DataStream.Primitives.Filter;
using Bts.DataStreamCore;
using mbts.config; using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System; using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
namespace BCG.StreamWorld.Addition { namespace BCG.StreamWorld.Addition {
[DataStreamUserRespresentation(false)] [DataStreamUserRespresentation(false)]
public class FilterStream : ViewStream { public class FilterStream : ViewStream {
private ViewStream myBaseStream; private ViewStream myBaseStream;
private BlueTable myBaseTable, myStateTable; BlueTable? myBaseTable;
BlueTable? myStateTable;
private IStreamFilter myFilter; private IStreamFilter myFilter;
public FilterStream(ViewStream vs) public FilterStream(ViewStream vs)
@@ -25,8 +30,8 @@ namespace BCG.StreamWorld.Addition {
myBaseStream = vs; myBaseStream = vs;
myFilter = filter; myFilter = filter;
vs.ErrorChanged += OnError; vs.ErrorChanged += OnError;
if (filter is IDynamicStreamFilter) if (filter is IDynamicStreamFilter dfilter)
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange); dfilter.onFilterChange += myHandleFilterChange;
} }
@@ -36,8 +41,8 @@ namespace BCG.StreamWorld.Addition {
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
myBaseStream.onChange -= new ChangeDelegate(myHandleChanges); myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
myBaseTable.clear(); myBaseTable?.clear();
myStateTable.clear(); myStateTable?.clear();
} }
bool filterRow(IBlueRow r) { bool filterRow(IBlueRow r) {
@@ -52,17 +57,14 @@ namespace BCG.StreamWorld.Addition {
private void myHandleChanges(object newValue) { private void myHandleChanges(object newValue) {
try { try {
DataPacket dp = newValue as DataPacket; var dp = (IDataPacket)newValue;
if (myStateTable == null) { var schema = dp.Schema;
bts.Log.assert(myBaseTable == null); init(schema);
myStateTable = new BlueTable(dp.schema); var dels = new BlueTable(schema);
myBaseTable = new BlueTable(dp.schema); var adds = new BlueTable(schema);
}
BlueTable dels = new BlueTable(dp.schema),
adds = new BlueTable(dp.schema);
foreach (IBlueRow r in dp.deletedRows) { foreach (IBlueRow r in dp.deletedRows) {
myBaseTable.deleteRow(r); myBaseTable.deleteRow(r);
if (myStateTable.containsID((int)r["id"])) { if (myStateTable.containsID(r.Id())) {
dels.addRow(r); dels.addRow(r);
myStateTable.deleteRow(r); myStateTable.deleteRow(r);
} }
@@ -85,6 +87,7 @@ namespace BCG.StreamWorld.Addition {
set { set {
myFilter = value; myFilter = value;
if (myBaseTable != null) { if (myBaseTable != null) {
ExceptionAssert.NotNull(myStateTable);
IBlueRowCollection oldState = new BlueTable(myStateTable); IBlueRowCollection oldState = new BlueTable(myStateTable);
myStateTable.clear(); myStateTable.clear();
// If the filter throws an exception, we can rethrow it as a StreamFilterException // If the filter throws an exception, we can rethrow it as a StreamFilterException
@@ -112,6 +115,17 @@ namespace BCG.StreamWorld.Addition {
//public string filterString { set { filter = new BooleanExpressionFilter(value); } } //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
private void myHandleFilterChange() { filter = myFilter; } private void myHandleFilterChange() { filter = myFilter; }
[MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
void init(IBlueTableSchema schema) {
if (myStateTable is null) {
Log.isnull(myBaseTable);
myStateTable = new BlueTable(schema);
myBaseTable = new BlueTable(schema);
} else {
Log.notnull(myBaseTable);
}
}
} }
} }
@@ -135,29 +149,24 @@ namespace BCG
} }
#region IStreamFilter Members #region IStreamFilter Members
public bool filter(IBlueRow dr) public bool filter(IBlueRow dr) =>
{ myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
bool rv = true; DBNull or null => string.Empty,
for (int i = 0; i < myCols.Length; i++) object obj => obj.ToString() ?? string.Empty
{ }));
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
if (!rv)
return rv;
}
return rv;
}
#endregion #endregion
} }
public class FilterStream : ViewStream public class FilterStream : ViewStream
{ {
private BlueTable myLastFiredRows; BlueTable? myLastFiredRows;
private IBlueRowCollection myLastReceivedRows; IBlueRowCollection? myLastReceivedRows;
private IStreamFilter myFilter; IStreamFilter? myFilter;
private ViewStream myParentStream; readonly ViewStream? myParentStream;
private string myFilterString; string? myFilterString;
private BCG.StreamWorld.Addition.FilterStream myFilterStream; readonly StreamWorld.Addition.FilterStream? myFilterStream;
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter; readonly bool myUseNewFilter = false;
IDisposable? subscription;
#region constructors #region constructors
public delegate bool OnFilter(IBlueRow r); public delegate bool OnFilter(IBlueRow r);
@@ -187,9 +196,10 @@ namespace BCG
{ {
myParentStream = vs; myParentStream = vs;
myFilter = filter; myFilter = filter;
myFilterString = filter.ToString(); myFilterString = filter.ToString() ?? string.Empty;
if (filter is IDynamicStreamFilter) if (filter is IDynamicStreamFilter dfilter) {
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange); dfilter.onFilterChange += myHandleFilterChange;
}
} }
} }
@@ -213,18 +223,20 @@ namespace BCG
#endregion #endregion
#region overridden methods #region overridden methods
protected override void myOnFirstSubscriber() protected override void myOnFirstSubscriber() {
{ myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); } ExceptionAssert.IsNull(subscription);
subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
}
protected override void myOnLastSubscriber() protected override void myOnLastSubscriber()
{ {
myLastFiredRows = null; myLastFiredRows = null;
myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks); Disposable.Dispose(ref subscription);
} }
protected override void myFireEvent(object o) protected override void myFireEvent(object o)
{ {
DataPacket dp = o as DataPacket; var dp = (IDataPacket)o;
if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0) if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
base.myFireEvent(o); base.myFireEvent(o);
} }
@@ -233,29 +245,30 @@ namespace BCG
private void myHandleCallbacks(object newValue) private void myHandleCallbacks(object newValue)
{ {
DataPacket retVal; DataPacket retVal;
DataPacket arrivedPacket = newValue as DataPacket; var arrivedPacket = (IDataPacket)newValue;
myLastReceivedRows = arrivedPacket.newRows; myLastReceivedRows = arrivedPacket.newRows;
try { try {
if (myLastFiredRows != null) if (myLastFiredRows != null)
{ {
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema), var addedRows = new BlueTable(myLastFiredRows.schema);
new BlueTable(myLastFiredRows.schema)); var deletedRows = new BlueTable(myLastFiredRows.schema);
retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
foreach (IBlueRow r in arrivedPacket.deletedRows) foreach (IBlueRow r in arrivedPacket.deletedRows)
if (myLastFiredRows.containsID((int)r["id"])) if (myLastFiredRows.containsID(r.Id()))
{ {
myLastFiredRows.deleteRow(r); myLastFiredRows.deleteRow(r);
(retVal.deletedRows as BlueTable).addRow(r); deletedRows.addRow(r);
} }
foreach (IBlueRow r in arrivedPacket.addedRows) foreach (IBlueRow r in arrivedPacket.addedRows)
if (filterRow(r)) if (filterRow(r))
{ {
myLastFiredRows.addRow(r); myLastFiredRows.addRow(r);
(retVal.addedRows as BlueTable).addRow(r); addedRows.addRow(r);
} }
} }
else else
{ {
myLastFiredRows = new BlueTable(arrivedPacket.schema); myLastFiredRows = new BlueTable(arrivedPacket.Schema);
retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema)); retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
foreach (IBlueRow r in arrivedPacket.newRows) foreach (IBlueRow r in arrivedPacket.newRows)
if (filterRow(r)) if (filterRow(r))
@@ -273,7 +286,7 @@ namespace BCG
{ {
if (myUseNewFilter) if (myUseNewFilter)
{ {
myFilterStream.filter = filter; requireNewFilter().filter = filter;
} }
else else
{ {
@@ -302,9 +315,9 @@ namespace BCG
get get
{ {
if (myUseNewFilter) if (myUseNewFilter)
return myFilterStream.filter; return requireNewFilter().filter;
else else
return myFilter; return requireOldFilter();
} }
} }
@@ -317,17 +330,25 @@ namespace BCG
private void myHandleFilterChange() private void myHandleFilterChange()
{ {
filter = myFilter; filter = requireOldFilter();
} }
bool filterRow(IBlueRow r) { bool filterRow(IBlueRow r) {
try { try {
return myFilter.filter(r); return requireOldFilter().filter(r);
} catch (Exception ex) { } catch (Exception ex) {
Log.Gen.error($"FilterStream delegate threw exception: {ex}"); Log.Gen.error($"FilterStream delegate threw exception: {ex}");
OnError(new StreamFilterException(StreamId, StreamName, null, ex, r)); OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
} }
return false; return false;
} }
[MemberNotNull(nameof(myFilterStream))]
StreamWorld.Addition.FilterStream requireNewFilter() =>
ExceptionAssert.MemberNotNull(myFilterStream);
[MemberNotNull(nameof(myFilter))]
IStreamFilter requireOldFilter() =>
ExceptionAssert.MemberNotNull(myFilter);
} }
} }
@@ -1,8 +1,6 @@
using System; using BCG.Utilities.General;
using System.Collections.Generic;
using System.Linq; using System;
using System.Text;
using BCG.Utilities.General;
namespace BCG namespace BCG
{ {
@@ -29,7 +27,7 @@ namespace BCG
myReference = myCurrentTime.AddSeconds(-1 * myNSecs); myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
} }
public event FilterChangeDelegate onFilterChange; public event FilterChangeDelegate? onFilterChange;
public bool filter(IBlueRow r) public bool filter(IBlueRow r)
{ {
@@ -38,9 +36,8 @@ namespace BCG
//it can be considered as our assumption whenever datecolumn is db null we return true //it can be considered as our assumption whenever datecolumn is db null we return true
return true; //dont use this for volparamsched use true return true; //dont use this for volparamsched use true
} }
DateTime t = (DateTime)r[myDateColumn]; var t = (DateTime)r[myDateColumn]!;
bool returnVallue = t < myReference ? false : true; return t >= myReference;
return returnVallue;
} }
@@ -51,8 +48,7 @@ namespace BCG
myCurrentTime = DateTime.Now; myCurrentTime = DateTime.Now;
myReference = myCurrentTime.AddSeconds(-1 * myNSecs); myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
myLastTimeFilteredApplied = DateTime.Now; myLastTimeFilteredApplied = DateTime.Now;
onFilterChange(); onFilterChange?.Invoke();
} }
} }
} }
@@ -1,11 +1,13 @@
using System; using BCG.Utilities.General;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Text;
using BCG.Utilities.General;
using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>; using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
using System.Diagnostics;
namespace BCG namespace BCG
{ {
@@ -40,12 +42,12 @@ namespace BCG
private IComparer<IBlueRow> myRightCompare; private IComparer<IBlueRow> myRightCompare;
protected FuzzyRowComparer myFuzzyCompare; protected FuzzyRowComparer myFuzzyCompare;
protected TimeSpan myTimeWindow; protected TimeSpan myTimeWindow;
protected IBlueTableSchema myLeftSchema; protected IBlueTableSchema? myLeftSchema;
protected IBlueTableSchema myRightSchema; protected IBlueTableSchema? myRightSchema;
protected IBlueTableSchema myOutputSchema; protected IBlueTableSchema? myOutputSchema;
private int myID = 0; private int myID = 0;
protected TimeWindowTable myStateRows; protected TimeWindowTable? myStateRows;
private DateTime myNextCull; private DateTime myNextCull;
const long MinCullIntervalMS = 1000; const long MinCullIntervalMS = 1000;
@@ -65,7 +67,7 @@ namespace BCG
this.myFuzzyCompare = fuzzyCompare; this.myFuzzyCompare = fuzzyCompare;
} }
public virtual IBlueTableSchema OutputSchema public virtual IBlueTableSchema? OutputSchema
{ {
get get
{ {
@@ -150,7 +152,7 @@ namespace BCG
int count; int count;
for (count = 0; count < list.Count; count++) for (count = 0; count < list.Count; count++)
{ {
if ((DateTime)list[count][timeColumn] > expiry) if ((DateTime)list[count][timeColumn]! > expiry)
{ {
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept. // Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
break; break;
@@ -172,16 +174,16 @@ namespace BCG
this.myFitness = fitness; this.myFitness = fitness;
} }
public IBlueRow myRightRow { get; private set; } public IBlueRow myRightRow { get; private set; }
public IBlueRow myLeftRow { get; set; } public IBlueRow? myLeftRow { get; set; }
public double myFitness { get; private set; } public double myFitness { get; private set; }
public IBlueRow myMergedRow { get; set; } public IBlueRow? myMergedRow { get; set; }
#region IComparable<RowMatch> Members #region IComparable<RowMatch> Members
public int CompareTo(RowMatch other) public int CompareTo(RowMatch? other) =>
{ other is null
return myFitness.CompareTo(other.myFitness); ? 1
} : myFitness.CompareTo(other.myFitness);
#endregion #endregion
} }
@@ -262,12 +264,12 @@ namespace BCG
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels) private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
{ {
ExceptionAssert.NotNull(myStateRows);
// We already emitted correction.MergedRow so we need to redact that and // We already emitted correction.MergedRow so we need to redact that and
// then emit correction.left as an add since it is no longer matched // then emit correction.left as an add since it is no longer matched
foreach (RowMatch correction in corrections.Distinct()) foreach (RowMatch correction in corrections.Distinct())
{ {
Debug.Assert(correction.myMergedRow != null); myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow));
this.myStateRows.RemoveRow(correction.myMergedRow);
adds.addRow(this.MergeRows(correction.myLeftRow, null)); adds.addRow(this.MergeRows(correction.myLeftRow, null));
dels.addRow(correction.myMergedRow); dels.addRow(correction.myMergedRow);
} }
@@ -285,7 +287,7 @@ namespace BCG
int r = 0; int r = 0;
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; ) for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
{ {
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn]) if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!)
{ {
// Advance Left // Advance Left
IEnumerable<RowMatch> rm = rightMatches[l]; IEnumerable<RowMatch> rm = rightMatches[l];
@@ -332,7 +334,7 @@ namespace BCG
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches) private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
{ {
IBlueRow merged = null; IBlueRow? merged = null;
if (rightMatches != null) if (rightMatches != null)
{ {
foreach (RowMatch match in rightMatches) foreach (RowMatch match in rightMatches)
@@ -366,9 +368,9 @@ namespace BCG
/// Override this function to provide a custom row merge. /// Override this function to provide a custom row merge.
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting. /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
/// </summary> /// </summary>
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right) protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
{ {
Dictionary<string, object> newRow = new Dictionary<string, object>(); Dictionary<string, object?> newRow = new();
if (left == null && right == null) throw new ArgumentException("oh come on."); if (left == null && right == null) throw new ArgumentException("oh come on.");
@@ -393,14 +395,15 @@ namespace BCG
} }
} }
newRow["id"] = this.myID++; newRow["id"] = this.myID++;
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn]; newRow[NEW_TIME_COLUMN] = left is not null
? left[myLeftTimeColumn]
: right![myRightTimeColumn];
return new BlueRow(this.OutputSchema, newRow); return new BlueRow(this.OutputSchema, newRow);
} }
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections) private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
{ {
List<RowMatch> matches = null; List<RowMatch>? matches = null;
// Search through all rows in list and compare with the fuzzy comparer // Search through all rows in list and compare with the fuzzy comparer
for (int x = 0; x < list.Count; x++) for (int x = 0; x < list.Count; x++)
@@ -421,8 +424,7 @@ namespace BCG
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch); matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
// Mark the best match for this particular right hand match // Mark the best match for this particular right hand match
RowMatch oldMatch; if (this.myBestMatch.TryGetValue(match, out var oldMatch))
if (this.myBestMatch.TryGetValue(match, out oldMatch))
{ {
if (currentMatch.myFitness < oldMatch.myFitness) if (currentMatch.myFitness < oldMatch.myFitness)
{ {
@@ -441,7 +443,7 @@ namespace BCG
} }
} }
return matches; return matches ?? Enumerable.Empty<RowMatch>();
} }
} }
} }
@@ -42,7 +42,7 @@ namespace BCG
return retVal; return retVal;
} }
public abstract IBlueRow evaluate (DataPacket dp) ; public abstract IBlueRow? evaluate(IDataPacket dp);
#endregion #endregion
#region Helper methods #region Helper methods
@@ -69,14 +69,7 @@ namespace BCG
protected IBlueRow myProject (IBlueRow r) {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);} protected IBlueRow myProject (IBlueRow r) {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
#endregion #endregion
private IBlueRow myProjectRow (IBlueRow r, string colMapExpression) IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
{ new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project"));
BlueTable auxTable = new BlueTable (r.schema);
auxTable.addRow (r);
BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable);
ProjectStream projStream = new ProjectStream (staticStream, colMapExpression);
return BlueDataUtility.extractRow (projStream.rows, 0);
}
} }
} }
@@ -8,12 +8,13 @@ namespace BCG {
internal class GroupByStream : MethodDrivenStream { internal class GroupByStream : MethodDrivenStream {
private class MyFuncGroupByEvaluator : IGroupByEvaluator { private class MyFuncGroupByEvaluator : IGroupByEvaluator {
private string[] _groupby_cols; private string[] _groupby_cols;
private Func<DataPacket, IBlueRow> _groupby_func; readonly Func<IDataPacket, IBlueRow> _groupby_func;
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) { public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
_groupby_cols = groupby_cols.ToArray(); _groupby_cols = groupby_cols.ToArray();
_groupby_func = groupby_func; _groupby_func = groupby_func;
} }
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); } public IBlueRow evaluate(IDataPacket packet) =>
_groupby_func(packet);
public string[] groupByColumns { get { return _groupby_cols; } } public string[] groupByColumns { get { return _groupby_cols; } }
@@ -32,7 +33,7 @@ namespace BCG {
private IBlueTableSchema myOutputSchema; private IBlueTableSchema myOutputSchema;
private int myIdGenerator = 0; private int myIdGenerator = 0;
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { } : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator) public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
@@ -187,11 +188,11 @@ namespace BCG {
} }
} }
public static string getRepString(DataPacket dp, string[] colNames) { public static string getRepString(IDataPacket dp, string[] colNames) {
return BlueDataUtility.rowToString(getRepRow(dp), colNames); return BlueDataUtility.rowToString(getRepRow(dp), colNames);
} }
public static IBlueRow getRepRow(DataPacket dp) { public static IBlueRow getRepRow(IDataPacket dp) {
if (dp.newRows.Count > 0) if (dp.newRows.Count > 0)
return BlueDataUtility.extractRow(dp.newRows, 0); return BlueDataUtility.extractRow(dp.newRows, 0);
else if (dp.addedRows.Count > 0) else if (dp.addedRows.Count > 0)
@@ -1,5 +1,3 @@
using System;
namespace BCG namespace BCG
{ {
/// <summary> /// <summary>
@@ -14,7 +12,7 @@ namespace BCG
/// deletetion) and the added and deletedRows being the increments. If a group has /// deletetion) and the added and deletedRows being the increments. If a group has
/// to be totally removed then a null should be returned /// to be totally removed then a null should be returned
/// </summary> /// </summary>
IBlueRow evaluate (DataPacket packet); IBlueRow? evaluate(IDataPacket packet);
/// <summary> /// <summary>
/// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator
@@ -14,9 +14,8 @@ namespace BCG {
public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
: base(colMapEpression, outputCol, groupByCols) { } : base(colMapEpression, outputCol, groupByCols) { }
public override IBlueRow evaluate(DataPacket dp) { public override IBlueRow? evaluate(IDataPacket dp) {
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable ??= new GroupByOutputTable(dp.Schema);
myOutputTable;
string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols); string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
IBlueRow currMaxRow = myOutputTable.getRow(hashString); IBlueRow currMaxRow = myOutputTable.getRow(hashString);
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn); IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
@@ -1,7 +1,3 @@
using System;
using BCG.Utilities;
namespace BCG { namespace BCG {
public class MinGroupByEvaluator : MaxGroupByEvaluator { public class MinGroupByEvaluator : MaxGroupByEvaluator {
public MinGroupByEvaluator(string outputCol, string[] groupByCols) public MinGroupByEvaluator(string outputCol, string[] groupByCols)
@@ -10,8 +6,8 @@ namespace BCG {
public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) : public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
base(colMapEpression, outputCol, groupByCols) { } base(colMapEpression, outputCol, groupByCols) { }
public override IBlueRow evaluate(DataPacket dp) { public override IBlueRow? evaluate(IDataPacket dp) {
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; myOutputTable ??= new GroupByOutputTable(dp.Schema);
string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols); string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
IBlueRow currMinRow = myOutputTable.getRow(stringRep); IBlueRow currMinRow = myOutputTable.getRow(stringRep);
IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn); IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
@@ -1,14 +1,10 @@
using System; using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG namespace BCG
{ {
public interface INonIncGroupByEvaluator public interface INonIncGroupByEvaluator
{ {
IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows); IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
string[] get_group_by_cols(); string[]? get_group_by_cols();
} }
} }
@@ -1,46 +1,51 @@
using System; using bts;
using bts.SilverPine;
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Text.Json;
using bts;
using bts.SilverPine;
namespace BCG { namespace BCG {
//[DataStreamUserRespresentation("AggregateStream")] //[DataStreamUserRespresentation("AggregateStream")]
public class NonIncGroupByStream : ViewStream { public class NonIncGroupByStream : ViewStream {
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator { private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
Func<string[]> _groupby_cols_func; Func<string[]?> _groupby_cols_func;
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func; Func<List<IBlueRow>, List<IBlueRow>?> _evaluate_func;
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func, public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) { Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
_groupby_cols_func = groupby_cols_func; _groupby_cols_func = groupby_cols_func;
_evaluate_func = evaluate_func; _evaluate_func = evaluate_func;
} }
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) { public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable rv = null; BlueTable? rv = null;
foreach (var kv in grouped_rows) { foreach (var kv in grouped_rows) {
List<IBlueRow> g_rows = kv.Value; List<IBlueRow> g_rows = kv.Value;
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows); var out_rows = _evaluate_func(g_rows);
if (out_rows != null && out_rows.Count > 0) { if (out_rows != null && out_rows.Count > 0) {
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema); rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
rv.addRows(out_rows);
} }
rv.addRows(out_rows);
} }
return rv; return rv;
} }
public string[] get_group_by_cols() { public string[]? get_group_by_cols() {
return _groupby_cols_func(); return _groupby_cols_func();
} }
} }
private ViewStream _main_stream; readonly ViewStream? _main_stream;
private INonIncGroupByEvaluator _groupby_func; readonly INonIncGroupByEvaluator? _groupby_func;
private BlueRowHasher _row_hasher; BlueRowHasher? _row_hasher;
private string[] _groupby_cols; readonly string[]? _groupby_cols;
private Stopwatch _timer = new(); private Stopwatch _timer = new();
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
IDisposable? subscription;
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) { public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
_main_stream = main_stream; _main_stream = main_stream;
@@ -49,7 +54,7 @@ namespace BCG {
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length); _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
} }
public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func, public NonIncGroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { } : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
@@ -58,7 +63,7 @@ namespace BCG {
public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) { public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text); MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
Func<string[]> groupby_cols_func = () => { Func<string[]?> groupby_cols_func = () => {
return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col }; return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
}; };
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => { Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
@@ -75,12 +80,17 @@ namespace BCG {
} }
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } protected override void myOnFirstSubscriber() {
ExceptionAssert.IsNull(subscription);
ExceptionAssert.NotNull(_main_stream).Subscribe(_main_stream_onchange);
}
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } protected override void myOnLastSubscriber() =>
Disposable.Dispose(ref subscription);
private void _main_stream_onchange(object nv) { private void _main_stream_onchange(object nv) {
DataPacket dp = nv as DataPacket; ExceptionAssert.NotNull(_groupby_func);
var dp = (IDataPacket)nv;
///grouping together ///grouping together
_timer.Restart(); _timer.Restart();
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ? IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -94,7 +104,7 @@ namespace BCG {
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime)); // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing /// reducing
_timer.Restart(); _timer.Restart();
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); var total_state = _groupby_func.evaluate(grouped_rows);
_timer.Stop(); _timer.Stop();
if (total_state is null) { if (total_state is null) {
Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update)."); Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
@@ -108,8 +118,12 @@ namespace BCG {
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}"); $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
} }
// TODO, // TODO,
if (total_state != null) if (total_state != null) {
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); var deletedRows = myLastValue is null
? new BlueTable(total_state.schema)
: ((IDataPacket)myLastValue).newRows;
myFireEvent(new DataPacket(total_state, total_state, deletedRows));
}
} }
} }
} }
@@ -1,6 +1,6 @@
using System; using System;
using BCG.Utilities;
using System.Collections; using System.Collections;
using System.Collections.Generic;
namespace BCG namespace BCG
{ {
@@ -11,12 +11,10 @@ namespace BCG
/// </summary> /// </summary>
public class RowDictionaryTable : IBlueRowCollection public class RowDictionaryTable : IBlueRowCollection
{ {
protected IDictionary myRowCollections = new Hashtable (); protected readonly Dictionary<string, BlueTable> myRowCollections = new();
protected BlueTable myLinearCollection; protected readonly BlueTable myLinearCollection;
protected string[] myCols; protected readonly string[] myCols;
protected IBlueTableSchema mySchema; protected IBlueTableSchema mySchema;
protected RowDictionaryTable () {}
public RowDictionaryTable (IBlueRowCollection rows, params string[] cols) public RowDictionaryTable (IBlueRowCollection rows, params string[] cols)
: this (rows.schema, cols) : this (rows.schema, cols)
@@ -40,7 +38,7 @@ namespace BCG
public virtual void addRow (IBlueRow row) public virtual void addRow (IBlueRow row)
{ {
BlueTable rows = (BlueTable)getGroup (row); var rows = getMutableGroup(row);
if (rows == null) if (rows == null)
{ {
//i.e we dont have any group for this row => we need to create a new group //i.e we dont have any group for this row => we need to create a new group
@@ -53,7 +51,7 @@ namespace BCG
public virtual void removeRow (IBlueRow row) public virtual void removeRow (IBlueRow row)
{ {
BlueTable rowGroup = (BlueTable)getGroup (row); var rowGroup = requireMutableGroup(row);
// BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0); // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
rowGroup.deleteRow (row); rowGroup.deleteRow (row);
myLinearCollection.deleteRow (row); myLinearCollection.deleteRow (row);
@@ -65,7 +63,7 @@ namespace BCG
public void removeGroup (IBlueRow representativeRow) public void removeGroup (IBlueRow representativeRow)
{ {
IBlueRowCollection rowGroup = getGroup (representativeRow); var rowGroup = requireGroup(representativeRow);
// BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0); // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
foreach (IBlueRow row in rowGroup) foreach (IBlueRow row in rowGroup)
myLinearCollection.deleteRow (row); myLinearCollection.deleteRow (row);
@@ -85,24 +83,31 @@ namespace BCG
/// <returns></returns> /// <returns></returns>
//public IBlueRowCollection getGroup (IBlueRow row) {return getGroup ( //public IBlueRowCollection getGroup (IBlueRow row) {return getGroup (
public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));} public IBlueRowCollection? getGroup (IBlueRow r) =>
getGroup(myRowToString(r));
public ICollection getGroups () {return myRowCollections.Values;} public ICollection getGroups () {return myRowCollections.Values;}
private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2) public IBlueRowCollection? getGroup(string repString) =>
{ getMutableGroup(repString);
bool retVal = true;
foreach (string col in myCols)
retVal &= r1[col].Equals(r2[col]);
return retVal;
}
public IBlueRowCollection getGroup (string repString) BlueTable? getMutableGroup(IBlueRow row) =>
{ getMutableGroup(myRowToString(row));
IBlueRowCollection retVal = myRowCollections[repString] != null ?
(IBlueRowCollection) myRowCollections[repString] : null; BlueTable? getMutableGroup(string group) =>
return retVal; myRowCollections.GetValueOrDefault(group);
}
IBlueRowCollection requireGroup(IBlueRow row) =>
requireGroup(myRowToString(row));
IBlueRowCollection requireGroup(string group) =>
getGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
BlueTable requireMutableGroup(IBlueRow row) =>
requireMutableGroup(myRowToString(row));
BlueTable requireMutableGroup(string group) =>
getMutableGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
private string myRowToString (IBlueRow r) {return BlueDataUtility.rowToString (r, myCols);} private string myRowToString (IBlueRow r) {return BlueDataUtility.rowToString (r, myCols);}
@@ -1,5 +1,3 @@
using System;
namespace BCG { namespace BCG {
/// <summary> /// <summary>
/// Computes the sum for a given column /// Computes the sum for a given column
@@ -10,8 +8,8 @@ namespace BCG {
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
: base(colMapEpression, outputCol, groupByCols) { } : base(colMapEpression, outputCol, groupByCols) { }
public override IBlueRow evaluate(DataPacket dp) { public override IBlueRow evaluate(IDataPacket dp) {
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; myOutputTable ??= new GroupByOutputTable(dp.Schema);
IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn); IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
return sumRow; return sumRow;
@@ -33,7 +31,7 @@ namespace BCG {
#region IGroupByEvaluator Members #region IGroupByEvaluator Members
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); } public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);
public string[] groupByColumns { get { return myGroupByCols; } } public string[] groupByColumns { get { return myGroupByCols; } }
@@ -51,9 +49,8 @@ namespace BCG {
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
: base(colMapEpression, outputCol, groupByCols) { } : base(colMapEpression, outputCol, groupByCols) { }
public override IBlueRow evaluate(DataPacket dp) { public override IBlueRow evaluate(IDataPacket dp) {
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; myOutputTable ??= new GroupByOutputTable(dp.Schema);
IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn); IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
return sumRow; return sumRow;
} }
@@ -68,7 +65,7 @@ namespace BCG {
#region IGroupByEvaluator Members #region IGroupByEvaluator Members
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); } public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);
public string[] groupByColumns { get { return myGroupByCols; } } public string[] groupByColumns { get { return myGroupByCols; } }
@@ -1,5 +1,3 @@
using System;
namespace BCG namespace BCG
{ {
[DataStreamUserRespresentation (false)] [DataStreamUserRespresentation (false)]
@@ -17,6 +15,7 @@ namespace BCG
protected override void myInitFiredPacket () protected override void myInitFiredPacket ()
{ {
requireSchema();
myLastFiredPacket = new DataPacket (); myLastFiredPacket = new DataPacket ();
myLastFiredPacket.newRows = new JoinForeignTable (mySchema); myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
myLastFiredPacket.addedRows = new BlueTable (mySchema); myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -25,6 +24,10 @@ namespace BCG
public override void onTableClear (bool isLeft) public override void onTableClear (bool isLeft)
{ {
if (myLastFiredPacket is null) {
return;
}
requireSchema();
//incase of natural join, left or right wont matter, the end result zeroes out //incase of natural join, left or right wont matter, the end result zeroes out
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new JoinForeignTable (mySchema); myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
@@ -1,5 +1,7 @@
using Spark.Core.Diagnostics;
using System; using System;
using System.Collections; using System.Diagnostics.CodeAnalysis;
namespace BCG namespace BCG
{ {
@@ -13,12 +15,10 @@ namespace BCG
protected JoinOperand myLeftOperand, myRightOperand; protected JoinOperand myLeftOperand, myRightOperand;
protected int myLeftJoinColOrdinal, myRightJoinColOrdinal; protected int myLeftJoinColOrdinal, myRightJoinColOrdinal;
protected ViewStream myLeftStream, myRightStream; protected ViewStream myLeftStream, myRightStream;
protected JoinTableSchema mySchema; protected JoinTableSchema? mySchema;
protected DataPacket myLastFiredPacket; protected DataPacket? myLastFiredPacket;
protected bool myOnceFired = false; protected bool myOnceFired = false;
public IBlueTableSchema schema {get{return mySchema;}}
public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight)
{ {
myLeftStream = vsLeft; myLeftStream = vsLeft;
@@ -27,6 +27,9 @@ namespace BCG
myRightOperand = new JoinOperand (jcRight, false); myRightOperand = new JoinOperand (jcRight, false);
} }
public IBlueTableSchema schema =>
requireSchema();
protected override void myOnFirstSubscriber() protected override void myOnFirstSubscriber()
{ {
myLeftOperand.master = this; myLeftOperand.master = this;
@@ -62,29 +65,32 @@ namespace BCG
{ {
myLeftJoinColOrdinal = myLeftOperand.joinColOridinal; myLeftJoinColOrdinal = myLeftOperand.joinColOridinal;
myRightJoinColOrdinal = myRightOperand.joinColOridinal; myRightJoinColOrdinal = myRightOperand.joinColOridinal;
myLeftOperand.RequireInitialized();
myRightOperand.RequireInitialized();
mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol, mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol,
myRightOperand.operandSchema, myRightOperand.joinCol); myRightOperand.operandSchema, myRightOperand.joinCol);
myInitFiredPacket (); myInitFiredPacket ();
} }
} }
[MemberNotNull(nameof(mySchema), nameof(myLastFiredPacket))]
protected virtual void myInitFiredPacket () protected virtual void myInitFiredPacket ()
{ {
var schema = requireSchema();
myLastFiredPacket = new DataPacket (); myLastFiredPacket = new DataPacket ();
myLastFiredPacket.newRows = new JoinTable (mySchema); myLastFiredPacket.newRows = new JoinTable (schema);
myLastFiredPacket.addedRows = new BlueTable (mySchema); myLastFiredPacket.addedRows = new BlueTable (schema);
myLastFiredPacket.deletedRows = new BlueTable (mySchema); myLastFiredPacket.deletedRows = new BlueTable (schema);
} }
public IBlueTableSchema getSchema (bool isLeft) public IBlueTableSchema? getSchema (bool isLeft) =>
{ isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
IBlueTableSchema retVal;
retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
return retVal;
}
public void endSession () public void endSession ()
{ {
if (myLastFiredPacket is null) {
return;
}
if (!myOnceFired) if (!myOnceFired)
{ {
myOnceFired = true; myOnceFired = true;
@@ -97,13 +103,17 @@ namespace BCG
protected virtual void myInit () protected virtual void myInit ()
{ {
myLeftOperand.init (); myLeftOperand.Reset ();
myRightOperand.init (); myRightOperand.Reset ();
myLastFiredPacket = null; myLastFiredPacket = null;
} }
public virtual void onTableClear (bool isLeft) public virtual void onTableClear (bool isLeft)
{ {
if (myLastFiredPacket is null) {
return;
}
requireSchema();
//incase of natural join, left or right wont matter, the end result zeroes out //incase of natural join, left or right wont matter, the end result zeroes out
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new JoinTable (mySchema); myLastFiredPacket.newRows = new JoinTable (mySchema);
@@ -112,15 +122,19 @@ namespace BCG
public virtual void deleteRows (IBlueRowCollection rows, bool isLeft) public virtual void deleteRows (IBlueRowCollection rows, bool isLeft)
{ {
var last = myLastFiredPacket;
if (last is null) {
return;
}
JoinOperand thisOp, otherOp; JoinOperand thisOp, otherOp;
int joinColOrdinal; int joinColOrdinal;
myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft); myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft);
foreach (IBlueRow row in rows) foreach (IBlueRow row in rows)
{ {
thisOp.deleteRow (row); thisOp.deleteRow (row);
if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0) if (otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)).Count == 0)
continue; //row does not participate in join continue; //row does not participate in join
((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket); ((JoinTable)last.newRows).deleteRow(isLeft, row, last);
} }
} }
@@ -135,10 +149,12 @@ namespace BCG
otherOp = myLeftOperand; otherOp = myLeftOperand;
joinColOrdinal = myRightJoinColOrdinal; joinColOrdinal = myRightJoinColOrdinal;
} }
var schema = requireSchema();
myLastFiredPacket ??= new DataPacket(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema));
foreach (IBlueRow row in rows) foreach (IBlueRow row in rows)
{ {
thisOp.addRow (row); thisOp.addRow (row);
RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]); RowList matchedNodes = otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal));
if (matchedNodes.Count == 0) if (matchedNodes.Count == 0)
continue; //this row did not participate in the join continue; //this row did not participate in the join
if (isLeft) if (isLeft)
@@ -146,10 +162,22 @@ namespace BCG
((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket); ((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket);
else else
foreach (RowNode matchedNode in matchedNodes) foreach (RowNode matchedNode in matchedNodes)
((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket); ((JoinTable)myLastFiredPacket.newRows).addRow(ExceptionAssert.NotNull(matchedNode.row), row, myLastFiredPacket);
} }
} }
[MemberNotNull(nameof(mySchema))]
protected JoinTableSchema requireSchema() =>
ExceptionAssert.NotNull(mySchema);
[MemberNotNull(nameof(myLastFiredPacket))]
protected DataPacket requireLastFiredPacket() {
if (myLastFiredPacket is null) {
myInitFiredPacket();
}
return myLastFiredPacket;
}
private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft) private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft)
{ {
if (isLeft) if (isLeft)
@@ -166,6 +194,9 @@ namespace BCG
} }
} }
object getJoinKey(IBlueRow row, int joinColOrdinal) =>
row[joinColOrdinal] ?? throw new ArgumentException("Row has null join key", nameof(row));
/**************************************TEST METHOD(S)*********************************************/ /**************************************TEST METHOD(S)*********************************************/
/// <summary> /// <summary>
/// Used for testing purposes only. This enables the test to work without a DataSource (PubServer, /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -173,8 +204,7 @@ namespace BCG
/// </summary> /// </summary>
/// <param name="leftPacket"></param> /// <param name="leftPacket"></param>
/// <param name="rightPacket"></param> /// <param name="rightPacket"></param>
public void test (DataPacket dp, bool isLeft) public void test(IDataPacket dp, bool isLeft) {
{
JoinOperand jo = isLeft ? myLeftOperand : myRightOperand; JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
jo.changeHandler (dp); jo.changeHandler (dp);
} }
@@ -1,5 +1,6 @@
using System; using Spark.Core.Diagnostics;
using System.Collections;
using System.Collections.Generic;
namespace BCG namespace BCG
{ {
@@ -8,7 +9,7 @@ namespace BCG
/// </summary> /// </summary>
internal class JoinDeletionIndex internal class JoinDeletionIndex
{ {
IDictionary myIndex = new Hashtable (); Dictionary<object, RowList> myIndex = new();
private string myOpIdName ; private string myOpIdName ;
private bool myIsLeft; private bool myIsLeft;
@@ -28,10 +29,8 @@ namespace BCG
/// <param name="row">JoinRow to add</param> /// <param name="row">JoinRow to add</param>
public void add (JoinRow row) public void add (JoinRow row)
{ {
object o = row[myOpIdName]; object o = ExceptionAssert.NotNull(row[myOpIdName]);
if (myIndex[o] == null) RowNode nodeAdded = myIndex.GetOrAdd(o, _ => new()).add(row);
myIndex[o] = new RowList ();
RowNode nodeAdded = ((RowList)myIndex[o]).add (row);
if (myIsLeft) if (myIsLeft)
row.leftNode = nodeAdded; row.leftNode = nodeAdded;
else else
@@ -43,7 +42,10 @@ namespace BCG
/// To remove a node, if node does not exist exception is thrown /// To remove a node, if node does not exist exception is thrown
/// </summary> /// </summary>
/// <param name="node"></param> /// <param name="node"></param>
public void remove (RowNode node) {((RowList)myIndex[node.row[myOpIdName]]).remove (node);} public void remove (RowNode node) {
var nodeId = ExceptionAssert.NotNull(node.row?[myOpIdName]);
myIndex[nodeId].remove(node);
}
public RowList getMatchingRows (object key) { public RowList getMatchingRows (object key) {
RowList retVal = (RowList)myIndex[key]; RowList retVal = (RowList)myIndex[key];
@@ -1,19 +1,18 @@
using System;
using BCG.Utilities;
using bts; using bts;
using Bts.DataStreamCore;
namespace BCG namespace BCG
{ {
internal class JoinForeignTable : JoinTable internal class JoinForeignTable : JoinTable
{ {
public JoinForeignTable (JoinTableSchema schema) : base (schema) {} public JoinForeignTable (JoinTableSchema schema) : base (schema) {}
public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp) public override void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
{ var newRow = new JoinRow(leftRow, rightRow, mySchema);
JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema); int leftId = leftRow.Id();
Log.assert(myIdIndex[leftRow["id"]] == null); Log.assert(myIdIndex[leftId] == null);
myIdIndex[leftRow["id"]] = newRow; myIdIndex[leftId] = newRow;
((BlueTable)(dp.addedRows)).addRow (newRow); ((BlueTable)(dp.addedRows)).addRow (newRow);
myLeftIndex.add (newRow); myLeftIndex.add (newRow);
myRightIndex.add (newRow); myRightIndex.add (newRow);
@@ -1,11 +1,12 @@
using System;
using System.Collections;
using BCG.Utilities;
using bts; using bts;
using Bts.ColumnMapExpressions; using Bts.ColumnMapExpressions;
using Spark.Core.Diagnostics;
using System;
using System.Collections;
namespace BCG namespace BCG
{ {
/// <summary> /// <summary>
@@ -37,7 +38,8 @@ namespace BCG
private JoinTableSchema myJoinSchema; private JoinTableSchema myJoinSchema;
private IBlueTableSchema myLeftSchema, myRightSchema; private IBlueTableSchema myLeftSchema, myRightSchema;
private string myLeftColExpression, myRightColExpression; private string myLeftColExpression, myRightColExpression;
private string[] myUserRules, myResolvedNames; string[] myUserRules;
string[]? myResolvedNames;
private IDictionary myFinalMap = new Hashtable (); private IDictionary myFinalMap = new Hashtable ();
@@ -87,16 +89,18 @@ namespace BCG
/// </summary> /// </summary>
private void myFormUserRules () private void myFormUserRules ()
{ {
var leftExp = ColumnMapExpression.Parse(myLeftColExpression) ?? throw new InvalidOperationException("Failed to parse left column expression");
var rightExp = ColumnMapExpression.Parse(myRightColExpression) ?? throw new InvalidOperationException("Failed to parse right column expression");
if (DataStreamLoadFileConfig.instance.useStaticParser) if (DataStreamLoadFileConfig.instance.useStaticParser)
{ {
myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression)); myFormUserRules(true, leftExp);
myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression)); myFormUserRules(false, rightExp);
} }
else else
{ {
// TODO (maybe): Actually use non-static parser (unclear what advantages it will have) // TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression)); myFormUserRules(true, leftExp);
myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression)); myFormUserRules(false, rightExp);
} }
} }
@@ -115,13 +119,7 @@ namespace BCG
OriginalNameInfo oni = myOriginalColInfo[i]; OriginalNameInfo oni = myOriginalColInfo[i];
if (oni.isLeft == isLeft) if (oni.isLeft == isLeft)
{ {
if (userMap[oni.colName] != null) myUserRules[i] = userMap[oni.colName] as string ?? oni.colName;
{
if ((string)userMap[oni.colName] != oni.colName)
myUserRules[i] = (string)userMap[oni.colName];
else
myUserRules[i] = oni.colName;
}
} }
} }
else else
@@ -131,10 +129,13 @@ namespace BCG
OriginalNameInfo oni = myOriginalColInfo[i]; OriginalNameInfo oni = myOriginalColInfo[i];
if (oni.isLeft == isLeft) if (oni.isLeft == isLeft)
{ {
if (userMap[oni.colName] == null) var colName = userMap[oni.colName] as string;
// myUserRules[i] = userMap[oni.colName] as string
if (colName is null) {
myUserRules[i] = oni.colName; myUserRules[i] = oni.colName;
else if ((string)userMap[oni.colName] != oni.colName) } else if (colName != oni.colName) {
myUserRules[i] = (string)userMap[oni.colName]; myUserRules[i] = colName;
}
} }
} }
} }
@@ -165,26 +166,12 @@ namespace BCG
private void myFormFinalMap () private void myFormFinalMap ()
{ {
ExceptionAssert.NotNull(myResolvedNames);
foreach (string resolvedName in myResolvedNames) foreach (string resolvedName in myResolvedNames)
if (resolvedName != null) if (resolvedName != null)
myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)]; myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)];
} }
/// <summary>
/// Utility method to invert a hashtable
/// </summary>
/// <param name="arg"></param>
/// <returns></returns>
private IDictionary reciprocateDictionary (IDictionary arg)
{
IDictionary retVal = new Hashtable ();
foreach (DictionaryEntry de in arg)
retVal[de.Value] = de.Key;
return retVal;
}
/// <summary> /// <summary>
/// Utility method to find all occurances of an obkect in an IList /// Utility method to find all occurances of an obkect in an IList
/// </summary> /// </summary>
@@ -194,9 +181,11 @@ namespace BCG
private ArrayList myFindAllInCollection (IList list, object obj) private ArrayList myFindAllInCollection (IList list, object obj)
{ {
ArrayList retVal = new ArrayList (); ArrayList retVal = new ArrayList ();
for (int i = 0; i < list.Count; i++) for (int i = 0; i < list.Count; i++) {
if (list[i] != null && list[i].Equals (obj)) if (Equals(list[i], obj)) {
retVal.Add (i); retVal.Add (i);
}
}
return retVal; return retVal;
} }
@@ -1,6 +1,7 @@
using Spark.Core.Diagnostics;
using System; using System;
using System.Collections; using System.Diagnostics.CodeAnalysis;
using BCG.Utilities;
namespace BCG namespace BCG
{ {
@@ -11,34 +12,46 @@ namespace BCG
public class JoinOperand public class JoinOperand
{ {
private string myJoinCol; private string myJoinCol;
private DataPacket myLastArrivedPacket; IDataPacket? myLastArrivedPacket;
private JoinOperand myOtherOperand; JoinOperand? myOtherOperand;
private bool myIsLeft; private bool myIsLeft;
private JoinOperandRowIndex myRowIndex; JoinOperandRowIndex? myRowIndex;
private InternalJoinStream myMaster; InternalJoinStream? myMaster;
private IBlueTableSchema myOpSchema;
public JoinOperand (string joinCol, bool isLeft) public JoinOperand (string joinCol, bool isLeft)
{ {
myJoinCol = joinCol; myJoinCol = joinCol;
myIsLeft = isLeft; myIsLeft = isLeft;
} }
internal void changeHandler (object o) [MemberNotNullWhen(true, nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
{ bool isInitialized {
DataPacket temp = (DataPacket)o; get {
if (myLastArrivedPacket != null) if (myLastArrivedPacket is null) {
mySubsequentChangeHandler ((DataPacket)o); return false;
else }
myFirstChangeHandler ((DataPacket)o); ExceptionAssert.NotNull(myMaster, myLastArrivedPacket, operandSchema, myOtherOperand, myRowIndex);
return true;
}
} }
private void myFirstChangeHandler (DataPacket arrivedDp) internal void changeHandler (object o)
{ {
var dp = (IDataPacket)o;
if (isInitialized) {
mySubsequentChangeHandler (dp);
} else {
myFirstChangeHandler(dp);
}
}
[MemberNotNull(nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
bool myFirstChangeHandler(IDataPacket arrivedDp) {
ExceptionAssert.NotNull(myMaster);
myLastArrivedPacket = arrivedDp; myLastArrivedPacket = arrivedDp;
myOpSchema = myLastArrivedPacket.newRows.schema; operandSchema = myLastArrivedPacket.newRows.schema;
myOtherOperand = myMaster.getOtherOperand (this); myOtherOperand = myMaster.getOtherOperand (this);
myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo); myRowIndex = new JoinOperandRowIndex (operandSchema[myJoinCol].colNo, operandSchema["id"].colNo);
if (myOtherOperand.hasReceivedData) if (myOtherOperand.hasReceivedData)
{ {
//the other operand has already received data, so the join would //the other operand has already received data, so the join would
@@ -49,14 +62,16 @@ namespace BCG
myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft); myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft);
myMaster.endSession (); myMaster.endSession ();
} }
else else {
myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows, myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows,
myOpSchema[myJoinCol].colNo, operandSchema[myJoinCol].colNo,
myOpSchema["id"].colNo); operandSchema["id"].colNo);
}
return true;
} }
private void mySubsequentChangeHandler (DataPacket arrivedDp) void mySubsequentChangeHandler (IDataPacket arrivedDp) {
{ RequireInitialized();
myLastArrivedPacket = arrivedDp; myLastArrivedPacket = arrivedDp;
if (!myOtherOperand.hasReceivedData) if (!myOtherOperand.hasReceivedData)
{ {
@@ -85,29 +100,46 @@ namespace BCG
myMaster.endSession (); myMaster.endSession ();
} }
internal void addRow (IBlueRow row) {myRowIndex.add (row);} internal void addRow (IBlueRow row) =>
ExceptionAssert.NotNull(myRowIndex).add(row);
internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);} internal void deleteRow (IBlueRow row) {
RequireInitialized();
myRowIndex.remove (row);
}
internal IBlueTableSchema operandSchema {get{return myOpSchema;}} internal IBlueTableSchema? operandSchema { get; private set; }
internal InternalJoinStream master {set{myMaster = value;}} internal InternalJoinStream master {set{myMaster = value;}}
internal string joinCol {get{return myJoinCol;}} internal string joinCol {get{return myJoinCol;}}
internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}} internal int joinColOridinal {
get {
RequireInitialized();
return operandSchema[myJoinCol].colNo;
}
}
internal RowList getMatchingRows (object key) {return (RowList) myRowIndex.getMatchingRows (key);} internal RowList getMatchingRows (object key) {
RequireInitialized();
return myRowIndex.getMatchingRows(key);
}
internal bool hasReceivedData {get{return myLastArrivedPacket != null;}} internal bool hasReceivedData {get{return myLastArrivedPacket != null;}}
internal void init () internal void Reset() {
{
///should be brought back to the the state before the first callback ///should be brought back to the the state before the first callback
///(ie the state after construction) ///(ie the state after construction)
myRowIndex = null; myRowIndex = null;
myLastArrivedPacket = null; myLastArrivedPacket = null;
} }
[MemberNotNull(nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
internal void RequireInitialized() {
if (!isInitialized) {
throw new InvalidOperationException("Internal state not properly initialized");
}
}
} }
} }
@@ -1,6 +1,6 @@
using System; using Spark.Core.Diagnostics;
using System.Collections;
using BCG.Utilities; using System.Collections.Generic;
namespace BCG namespace BCG
{ {
@@ -11,8 +11,10 @@ namespace BCG
/// </summary> /// </summary>
internal class JoinOperandRowIndex internal class JoinOperandRowIndex
{ {
private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable (); readonly Dictionary<object, RowList> myJoinColDict = new();
private int myJoinColOrdinal, myIdColOrdinal; readonly Dictionary<int, RowNode> myIdDict = new();
readonly int myJoinColOrdinal;
readonly int myIdColOrdinal;
/// <summary> /// <summary>
/// ///
@@ -34,19 +36,14 @@ namespace BCG
public void add (IBlueRow row) public void add (IBlueRow row)
{ {
RowList list; myIdDict[getId(row)] = myJoinColDict.GetOrAdd(getJoinKey(row), _ => new()).add(row);
if (myJoinColDict[row[myJoinColOrdinal]] == null)
myJoinColDict[row[myJoinColOrdinal]] = list = new RowList ();
else
list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
myIdDict[row[myIdColOrdinal]] = list.add (row);
} }
public void remove (IBlueRow row) public void remove (IBlueRow row)
{ {
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]]; var id = getId(row);
((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node); myJoinColDict[getJoinKey(row)].remove(myIdDict[id]);
myIdDict.Remove (row[myIdColOrdinal]); myIdDict.Remove(id);
} }
public RowList getMatchingRows (object key) public RowList getMatchingRows (object key)
@@ -62,5 +59,11 @@ namespace BCG
myJoinColDict.Clear (); myJoinColDict.Clear ();
myIdDict.Clear (); myIdDict.Clear ();
} }
object getJoinKey(IBlueRow row) =>
ExceptionAssert.NotNull(row[myJoinColOrdinal]);
int getId(IBlueRow row) =>
(int)row[myIdColOrdinal]!;
} }
} }
@@ -1,17 +1,21 @@
using System;
using BCG.Utilities;
using System.Collections;
using bts; using bts;
using Bts.DataStreamCore;
using System;
using System.Collections;
namespace BCG namespace BCG
{ {
internal class JoinRow : IBlueRow internal class JoinRow : IBlueRow
{ {
private RowNode myLeftNode, myRightNode; RowNode? myLeftNode;
protected IBlueRow myLeftRow, myRightRow; RowNode? myRightNode;
protected IBlueRow myLeftRow;
protected IBlueRow? myRightRow;
protected int myId; protected int myId;
protected JoinTableSchema mySchema; protected JoinTableSchema mySchema;
private object[] myValues; private object?[] myValues;
#region Constructor(s) #region Constructor(s)
@@ -21,8 +25,8 @@ namespace BCG
/// <param name="leftRow">left row of the join</param> /// <param name="leftRow">left row of the join</param>
/// <param name="rightRow">right row of the join</param> /// <param name="rightRow">right row of the join</param>
/// <param name="schema">schema of the joined row</param> /// <param name="schema">schema of the joined row</param>
public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) public JoinRow (IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema)
: this ((int)leftRow["id"], leftRow, rightRow, schema) {} : this(leftRow.Id(), leftRow, rightRow, schema) { }
/// <summary> /// <summary>
/// Constructs a row assigning it the ID provioded as the first argument /// Constructs a row assigning it the ID provioded as the first argument
@@ -31,8 +35,7 @@ namespace BCG
/// <param name="leftRow">left row of the join</param> /// <param name="leftRow">left row of the join</param>
/// <param name="rightRow">right row of the join</param> /// <param name="rightRow">right row of the join</param>
/// <param name="schema">schema of the joined row</param> /// <param name="schema">schema of the joined row</param>
public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) {
{
myId = id; myId = id;
myLeftRow = leftRow; myLeftRow = leftRow;
myRightRow = rightRow; myRightRow = rightRow;
@@ -48,13 +51,13 @@ namespace BCG
public bool isPartialJoin{get{return myRightRow == null;}} public bool isPartialJoin{get{return myRightRow == null;}}
public RowNode leftNode public RowNode? leftNode
{ {
get{return myLeftNode;} get{return myLeftNode;}
set{myLeftNode = value;} set{myLeftNode = value;}
} }
public RowNode rightNode public RowNode? rightNode
{ {
get{return myRightNode;} get{return myRightNode;}
set{myRightNode = value;} set{myRightNode = value;}
@@ -85,7 +88,7 @@ namespace BCG
/// The key can either be the string corrsponding to the column name or /// The key can either be the string corrsponding to the column name or
/// the order of the column /// the order of the column
/// </summary> /// </summary>
public virtual object this [object key] public virtual object? this[object key]
{ {
get get
{ {
@@ -96,7 +99,7 @@ namespace BCG
set {throw new NotSupportedException ("The IBlueRow is read only");} set {throw new NotSupportedException ("The IBlueRow is read only");}
} }
public object this [int index] public object? this[int index]
{ {
get get
{ {
@@ -119,7 +122,8 @@ namespace BCG
} }
} }
public object this [string colName] {get{return this[mySchema[colName].colNo];}} public object? this[string colName] =>
this[mySchema[colName].colNo];
public ICollection Keys{get{return mySchema.columnNames;}} public ICollection Keys{get{return mySchema.columnNames;}}
@@ -133,7 +137,8 @@ namespace BCG
} }
} }
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} public void Add (object key, object? value) =>
throw new NotSupportedException ("The IBlueRow is readonly");
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
@@ -1,20 +1,26 @@
using System;
using System.Data;
using System.Collections;
using System.Text.RegularExpressions;
using BCG.Utilities;
using BCG.Utilities.General;
using bts; using bts;
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System;
using System.Collections;
namespace BCG namespace BCG
{ {
public class JoinStream : ViewStream public class JoinStream : ViewStream
{ {
protected InternalJoinStream? myIjs; protected readonly InternalJoinStream? myIjs;
protected readonly ColumnMap? myLeftCm;
protected readonly ColumnMap? myRightCm;
protected readonly bool myIsProject = true;
protected ProjectStream? myPs; protected ProjectStream? myPs;
protected ColumnMap? myLeftCm, myRightCm;
protected bool myIsProject = true; readonly bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; IDisposable? internalSubscription;
IDisposable? projectSubscription;
protected JoinStream () {} protected JoinStream () {}
public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1, public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1,
@@ -111,47 +117,61 @@ namespace BCG
{ {
if (myPs == null) if (myPs == null)
{ {
JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression, ExceptionAssert.NotNull(myIjs, myLeftCm, myRightCm);
myIjs.getSchema (false), myRightCm.columnExpression); var leftSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Left schema is null");
var rightSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Right schema is null");
var joinSchema = (JoinTableSchema)myIjs.schema;
JoinNameResolver resolver = new JoinNameResolver(
joinSchema, leftSchema, myLeftCm.columnExpression, rightSchema, myRightCm.columnExpression);
IDictionary map = resolver.getMap (); IDictionary map = resolver.getMap ();
map["id"] = "id"; map["id"] = "id";
Disposable.Dispose(ref projectSubscription);
myPs = new ProjectStream (myIjs, map); myPs = new ProjectStream (myIjs, map);
myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler); projectSubscription = myPs.Subscribe(myProjectChangeHandler);
return; return;
} }
else else
Log.assert(false, "This call back should not have been called"); Log.assert(false, "This call back should not have been called");
} }
else else {
myFireEvent ((o as ICloneable).Clone()); myFireEvent(cloneDataPacket(o));
}
} }
private void myProjectChangeHandler (object o) private void myProjectChangeHandler (object o)
{ {
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1) ExceptionAssert.NotNull(myIjs);
myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler); if (myIjs.numberOfSubscribers > 1) {
myFireEvent ((o as ICloneable).Clone()); Disposable.Dispose(ref internalSubscription);
}
myFireEvent(cloneDataPacket(o));
} }
protected override void myOnFirstSubscriber () protected override void myOnFirstSubscriber ()
{ {
myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler); Log.isnull(internalSubscription);
internalSubscription = ExceptionAssert.NotNull(myIjs).Subscribe(myInternalJoinChangeHandler);
} }
protected override void myOnLastSubscriber () protected override void myOnLastSubscriber ()
{ {
// how does this relate to the -= in myProjectChangeHandler? Disposable.Dispose(ref internalSubscription);
myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
// if there is a project after the ijs we need to // if there is a project after the ijs we need to
// unsubscribe to it otherwise references will be kept and // unsubscribe to it otherwise references will be kept and
// a memory leak will occur // a memory leak will occur
if (myPs != null) Disposable.Dispose(ref projectSubscription);
myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
myPs = null; myPs = null;
} }
object cloneDataPacket(object dpObj) {
var rv = (dpObj as ICloneable)?.Clone();
if (rv is null) {
var dp = (IDataPacket)dpObj;
rv = new DataPacket(new BlueTable(dp.newRows), new BlueTable(dp.addedRows), new BlueTable(dp.deletedRows));
}
return rv;
}
/**************************************TEST METHOD(S)*********************************************/ /**************************************TEST METHOD(S)*********************************************/
/// <summary> /// <summary>
/// Used for testing purposes only. This enables the test to work without a DataSource (PubServer, /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -159,7 +179,9 @@ namespace BCG
/// </summary> /// </summary>
/// <param name="leftPacket"></param> /// <param name="leftPacket"></param>
/// <param name="rightPacket"></param> /// <param name="rightPacket"></param>
public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);} public void test (DataPacket dp, bool isLeft) {
ExceptionAssert.NotNull(myIjs).test(dp, isLeft);
}
/***********************************************************************************************/ /***********************************************************************************************/
} }
} }
@@ -1,14 +1,17 @@
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using System; using System;
using System.Collections; using System.Collections;
using System.IO; using System.Collections.Generic;
using BCG.Utilities.General;
namespace BCG namespace BCG
{ {
internal class JoinTable : IBlueRowCollection internal class JoinTable : IBlueRowCollection
{ {
protected JoinTableSchema mySchema; protected JoinTableSchema mySchema;
protected IDictionary myIdIndex = new Hashtable (); protected readonly Dictionary<int, IBlueRow> myIdIndex = new();
protected JoinDeletionIndex myLeftIndex, myRightIndex; protected JoinDeletionIndex myLeftIndex, myRightIndex;
private int myNewRowId; private int myNewRowId;
@@ -19,19 +22,16 @@ namespace BCG
myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false); myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
} }
public IBlueRow this [int id] public IBlueRow this [int id] {
{ get {
get var rowObj = myIdIndex[id]
{ ?? throw new BlueDataException($"ID {id} does not exist in table {mySchema.tableName}");
if (myIdIndex[id] == null) return (IBlueRow)rowObj;
throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
return (IBlueRow)myIdIndex[id];
} }
} }
#region JoinTable members #region JoinTable members
public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp) public virtual void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
{
JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema); JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
myIdIndex[myNewRowId ++] = newRow; myIdIndex[myNewRowId ++] = newRow;
((BlueTable)(dp.addedRows)).addRow (newRow); ((BlueTable)(dp.addedRows)).addRow (newRow);
@@ -40,7 +40,7 @@ namespace BCG
myRightIndex.add (newRow); myRightIndex.add (newRow);
} }
public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, DataPacket dp) public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, IDataPacket dp)
{ {
JoinDeletionIndex thisIndex = myRightIndex, otherIndex = myLeftIndex; JoinDeletionIndex thisIndex = myRightIndex, otherIndex = myLeftIndex;
if (isLeft) if (isLeft)
@@ -48,17 +48,17 @@ namespace BCG
thisIndex = myLeftIndex; thisIndex = myLeftIndex;
otherIndex = myRightIndex; otherIndex = myRightIndex;
} }
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]); RowList matchingNodes = thisIndex.getMatchingRows(deletedRow.Id());
foreach (RowNode node in matchingNodes) foreach (RowNode node in matchingNodes)
{ {
JoinRow row = (JoinRow)node.row; var row = ExceptionAssert.Argument.Is<JoinRow>(node.row);
RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode; RowNode? correspondingNode = isLeft ? row.rightNode : row.leftNode;
if (correspondingNode != null) if (correspondingNode != null)
otherIndex.remove (correspondingNode); //in case of partial left joinss otherIndex.remove (correspondingNode); //in case of partial left joinss
myIdIndex.Remove (row["id"]); myIdIndex.Remove(row.Id());
((BlueTable)dp.deletedRows).addRow (row); ((BlueTable)dp.deletedRows).addRow (row);
} }
thisIndex.removeAll (deletedRow["id"]); thisIndex.removeAll(deletedRow.Id());
} }
#endregion #endregion
#region ICollection members #region ICollection members
@@ -68,7 +68,11 @@ namespace BCG
public object SyncRoot{get{return this;}} public object SyncRoot{get{return this;}}
public void CopyTo (Array array, int index) {myIdIndex.Values.CopyTo (array, index);} public void CopyTo (Array array, int index) {
foreach (var row in myIdIndex.Values) {
array.SetValue(row, index++);
}
}
#endregion #endregion
#region IEnumerable members #region IEnumerable members
@@ -1,9 +1,8 @@
using System;
using System.Collections;
using BCG.Utilities;
using bts; using bts;
using System.Collections.Generic;
using System.Linq;
namespace BCG namespace BCG
{ {
/// <summary> /// <summary>
@@ -13,10 +12,11 @@ namespace BCG
public class JoinTableSchema : BlueTableSchema public class JoinTableSchema : BlueTableSchema
{ {
private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = ""; private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
private IDictionary myLeftNameToAlias = new Hashtable (); readonly Dictionary<string, string> myLeftNameToAlias = new();
private IDictionary myRightNameToAlias = new Hashtable (); readonly Dictionary<string, string> myRightNameToAlias = new();
private IDictionary myAliasToLeftName = new Hashtable(), readonly Dictionary<string, string> myAliasToLeftName = new();
myAliasToRightName = new Hashtable(); readonly Dictionary<string, string> myAliasToRightName = new();
/// <summary> /// <summary>
/// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of /// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
/// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id" /// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
@@ -85,9 +85,9 @@ namespace BCG
renamed = false; renamed = false;
} }
myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int)); myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"]; myIndexByColOrder[myIndexByColOrder.Length - 1] = myIndexByColName["id"];
myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias); myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias); myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
} }
public string leftJoinCol {get{return myLeftJc;}} public string leftJoinCol {get{return myLeftJc;}}
@@ -1,17 +1,21 @@
using bts; using bts;
using bts.collections; using bts.collections;
using Bts.DataStreamCore;
using mbts; using mbts;
using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Linq; using System.Diagnostics.CodeAnalysis;
using System.Security.Cryptography;
namespace BCG { namespace BCG {
public class NonIncJoinBookkeeper : IDisposable { public class NonIncJoinBookkeeper : IDisposable {
public Action on_new_data; public event Action? on_new_data;
private readonly NonIncJoinSpec _join_spec; private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position; private readonly int _join_position;
private readonly int _start_index_in_output; private readonly int _start_index_in_output;
@@ -20,13 +24,14 @@ namespace BCG {
private readonly bool _is_tag_along; private readonly bool _is_tag_along;
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
private IBlueTableSchema _schema = null; IBlueTableSchema? _schema;
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>(); private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>(); private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
private IBlueRow _solitary_row = null; IBlueRow? _solitary_row;
private int[] _join_col_indices = null, int[]? _join_col_indices;
_data_col_indices = null; int[]? _data_col_indices;
IDisposable? subscription;
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) { public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
_join_spec = the_join_spec; _join_spec = the_join_spec;
@@ -38,7 +43,7 @@ namespace BCG {
} }
private bool _filled_missing_data = false; private bool _filled_missing_data = false;
private void _fill_missing_data(DataPacket dp) { void _fill_missing_data(IDataPacket dp) {
if (_filled_missing_data) if (_filled_missing_data)
return; return;
Func<string[], bool> null_or_empty = (str_arr) => { Func<string[], bool> null_or_empty = (str_arr) => {
@@ -52,17 +57,17 @@ namespace BCG {
return true; return true;
}; };
Func<string[], string[]> remove_bad_cols = (col_names) => { Func<string[], string[]> remove_bad_cols = (col_names) => {
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames); HashSet<string> schema_col_map = new HashSet<string>(dp.Schema.columnNames);
List<string> cols_list = new List<string>(col_names); List<string> cols_list = new List<string>(col_names);
cols_list.RemoveWhere(c => !schema_col_map.Contains(c)); cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
return cols_list.ToArray(); return cols_list.ToArray();
}; };
if (null_or_empty(_join_spec.data_col_names)) { if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames; _join_spec.data_col_names = dp.Schema.columnNames;
} }
_join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names); _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
if (null_or_empty(_join_spec.data_col_names)) { if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames; _join_spec.data_col_names = dp.Schema.columnNames;
} }
if (null_or_empty(_join_spec.projected_col_names)) { if (null_or_empty(_join_spec.projected_col_names)) {
_join_spec.projected_col_names = _join_spec.data_col_names; _join_spec.projected_col_names = _join_spec.data_col_names;
@@ -70,20 +75,16 @@ namespace BCG {
} }
public void init() { public void init() {
_join_spec.stream.onChange += _on_stream_cb; ExceptionAssert.MemberIsNull(subscription);
subscription = ExceptionAssert.MemberNotNull(_join_spec.stream).Subscribe(_on_stream_cb);
} }
public void _on_stream_cb(object dp_obj) { public void _on_stream_cb(object dp_obj) {
//Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}"); //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
var timer = Stopwatch.StartNew(); var timer = Stopwatch.StartNew();
DataPacket dp = (DataPacket)dp_obj; var dp = (IDataPacket)dp_obj;
_fill_missing_data(dp); _fill_missing_data(dp);
if (_schema == null) { init(dp);
_schema = dp.schema;
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
}
if (_is_left_most) { if (_is_left_most) {
_row_list.Clear(); _row_list.Clear();
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
@@ -108,44 +109,68 @@ namespace BCG {
on_new_data(); on_new_data();
} }
[MemberNotNullWhen(true, nameof(_schema))]
public bool is_ready { get { return _schema != null; } } public bool is_ready { get { return _schema != null; } }
public void constribute_cols_to_schema(List<ColumnInfo> cis) { public void constribute_cols_to_schema(List<ColumnInfo> cis) {
requireReady();
for (int i = 0; i < _join_spec.data_col_names.Length; i++) for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
} }
public List<Pair<int, IBlueRow>> get_list() { return _row_list; } public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
public void copy_master_row_to_output_row(int row_index, object[] output_row) { public void copy_master_row_to_output_row(int row_index, object?[] output_row) =>
IBlueRow this_row = _row_list[row_index].second; copyRow(ExceptionAssert.NotNull(_row_list[row_index].second), output_row);
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
}
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) { public void copy_matching_row_to_output_row(int row_hash, object?[] output_row) {
if (_is_tag_along) { if (_is_tag_along) {
if (_solitary_row != null) if (_solitary_row != null)
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output); copyRow(_solitary_row, output_row);
else else
copy_dbnull_to_output_row(output_row); copy_dbnull_to_output_row(output_row);
} else { } else {
IBlueRow this_row = null; if (_row_map.TryGetValue(row_hash, out var this_row))
if (_row_map.TryGetValue(row_hash, out this_row)) copyRow(this_row, output_row);
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
else else
copy_dbnull_to_output_row(output_row); copy_dbnull_to_output_row(output_row);
} }
} }
private void copy_dbnull_to_output_row(object[] output_row) { void copy_dbnull_to_output_row(object?[] output_row) {
requireDataColIndices();
for (int i = 0; i < _data_col_indices.Length; i++) for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value; output_row[i + _start_index_in_output] = DBNull.Value;
} }
#region IDisposable #region IDisposable
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; } public void Dispose() =>
Disposable.Dispose(ref subscription);
#endregion #endregion
void copyRow(IBlueRow src, object?[] dst) {
requireDataColIndices();
BlueDataUtility.copy_to_row(src, _data_col_indices, dst, _start_index_in_output);
}
[MemberNotNull(nameof(_schema), nameof(_join_col_indices), nameof(_data_col_indices))]
void init(IDataPacket dp) {
_schema ??= dp.Schema;
_join_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
}
[MemberNotNull(nameof(_data_col_indices))]
int[] requireDataColIndices() =>
ExceptionAssert.NotNull(_data_col_indices);
[MemberNotNull(nameof(_schema))]
void requireReady() {
if (!is_ready) {
throw new InvalidOperationException("Not ready");
}
}
} }
} }
@@ -1,22 +1,25 @@
using System; using mbts;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts; using System;
using System.Collections.Generic;
namespace BCG { namespace BCG {
// this a collection of data, theres no joining logic in here // this a collection of data, theres no joining logic in here
public class NonIncJoinSpec { public class NonIncJoinSpec {
public ViewStream stream; public ViewStream? stream;
// join col names are the names on which the streams are joined (for equality), // join col names are the names on which the streams are joined (for equality),
// data col names are the names in the original stream // data col names are the names in the original stream
// projected col names are their new names, ie if its x:l_x, then x is in the // projected col names are their new names, ie if its x:l_x, then x is in the
// data_col_names and l_x in projected cols // data_col_names and l_x in projected cols
public string[] join_col_names, data_col_names, projected_col_names; public string[] join_col_names;
public string[] data_col_names;
public string[] projected_col_names;
public NonIncJoinSpec() { } public NonIncJoinSpec() {
join_col_names = Array.Empty<string>();
data_col_names = Array.Empty<string>();
projected_col_names = Array.Empty<string>();
}
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names) public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { } : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
@@ -5,18 +5,19 @@ using mbts;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
namespace BCG { namespace BCG {
[DataStreamUserRespresentation("fastJoin")] [DataStreamUserRespresentation("fastJoin")]
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker { public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
private readonly NonIncJoinSpec[] _join_specs; private readonly NonIncJoinSpec[] _join_specs;
private readonly INonIncJoinInlineMapper _output_mapper; readonly INonIncJoinInlineMapper? _output_mapper;
private readonly BlueRowHasher _row_hasher; private readonly BlueRowHasher _row_hasher;
private readonly TimeSpan _throttle_period; private readonly TimeSpan _throttle_period;
private readonly ViewStreamDatabase _vsd; private readonly ViewStreamDatabase _vsd;
private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>(); private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
private IBlueTableSchema _output_schema = null; IBlueTableSchema? _output_schema;
private DateTime _last_fire_time = DateTime.MinValue; private DateTime _last_fire_time = DateTime.MinValue;
private bool _is_dirty = false; private bool _is_dirty = false;
@@ -35,10 +36,10 @@ namespace BCG {
params Triplet<ViewStream, string, string>[] join_specs) params Triplet<ViewStream, string, string>[] join_specs)
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { } : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0)
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { } : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) { public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0) {
_join_specs = join_specs; _join_specs = join_specs;
_output_mapper = output_mapper; _output_mapper = output_mapper;
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length); _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -88,6 +89,7 @@ namespace BCG {
return rv; return rv;
} }
[MemberNotNull(nameof(_output_schema))]
private void _construct_schema() { private void _construct_schema() {
if (_output_schema == null) { if (_output_schema == null) {
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
@@ -2,11 +2,12 @@ namespace bts {
using BCG; using BCG;
using BCG.Utilities.General; using BCG.Utilities.General;
using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System; using System;
using System.Diagnostics; using System.Diagnostics;
using ZstdSharp.Unsafe;
// functon that joins the 2 rows and returns the outputput row. The function // functon that joins the 2 rows and returns the outputput row. The function
// can return a null which means the 2 rows dont need to be joined // can return a null which means the 2 rows dont need to be joined
using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>; using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
@@ -16,6 +17,10 @@ namespace bts {
private bool _dirty = false; private bool _dirty = false;
private Stopwatch _stopwatch = new Stopwatch(); private Stopwatch _stopwatch = new Stopwatch();
private ViewStreamDatabase _vsd = ViewStreamDatabase.instance; private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
IDisposable? subscription;
IBlueRowCollection? leftRows;
IBlueRowCollection? rightRows;
public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) { public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
_lvs = lvs; _lvs = lvs;
_rvs = rvs; _rvs = rvs;
@@ -23,46 +28,56 @@ namespace bts {
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
_lvs.onChange += _handle_data; _lvs.onChange += OnLeftChange;
_rvs.onChange += _handle_data; _rvs.onChange += OnRightChange;
subscription = Disposable.Create(Unsubscribe);
_vsd.addWorker(this); _vsd.addWorker(this);
void OnLeftChange(object dp_obj) {
leftRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
_dirty = true;
doWork();
}
void OnRightChange(object dp_obj) {
rightRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
_dirty = true;
doWork();
}
void Unsubscribe() {
_lvs.onChange -= OnLeftChange;
_rvs.onChange -= OnRightChange;
}
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
_lvs.onChange -= _handle_data; subscription?.Dispose();
_rvs.onChange -= _handle_data; subscription = null;
_vsd.removeWorker(this); _vsd.removeWorker(this);
} }
private void _handle_data(object dp_obj) {
_dirty = true;
doWork();
}
public void doWork() { public void doWork() {
Func<bool> all_data_revcd = () => { if (!_dirty || leftRows is null || rightRows is null) {
return _lvs.rows != null && _rvs.rows != null; return;
};
_stopwatch.Start();
if (_dirty && all_data_revcd()) {
_dirty = false;
BlueTable rv = null;
foreach (IBlueRow lr in _lvs.rows) {
foreach (IBlueRow rr in _rvs.rows) {
IBlueRow resr = _join_func(lr, rr);
if (resr == null)
continue;
rv = rv ?? new BlueTable(resr.schema);
rv.addRow(resr);
}
}
_stopwatch.Stop();
Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
myFireEvent(dp);
} }
_stopwatch.Start();
_dirty = false;
BlueTable? rv = null;
foreach (IBlueRow lr in leftRows) {
foreach (IBlueRow rr in rightRows) {
IBlueRow resr = _join_func(lr, rr);
if (resr == null)
continue;
rv = rv ?? new BlueTable(resr.schema);
rv.addRow(resr);
}
}
_stopwatch.Stop();
Log.Gen.info($"ProximityJoinStream left_count: {leftRows.Count} right_count: {rightRows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs");
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
myFireEvent(dp);
} }
} }
} }
@@ -1,3 +1,5 @@
using Spark.Core.Diagnostics;
using System; using System;
using System.Collections; using System.Collections;
@@ -9,7 +11,7 @@ namespace BCG
/// </summary> /// </summary>
internal class RowList : ICollection internal class RowList : ICollection
{ {
private RowNode myFirstPtr; RowNode? myFirstPtr;
private int myCount = 0; private int myCount = 0;
#region ICollection methods #region ICollection methods
@@ -67,19 +69,19 @@ namespace BCG
} }
else if (node.isFirst) else if (node.isFirst)
{ {
myFirstPtr = myFirstPtr.next; myFirstPtr = ExceptionAssert.NotNull(myFirstPtr).next;
myFirstPtr.previous = null; ExceptionAssert.NotNull(myFirstPtr).previous = null;
return; return;
} }
else if (node.isLast) else if (node.isLast)
{ {
node.previous.next = null; ExceptionAssert.NotNull(node.previous).next = null;
return; return;
} }
else if (!node.isFirst && !node.isLast) else if (!node.isFirst && !node.isLast)
{ {
node.previous.next = node.next; ExceptionAssert.NotNull(node.previous).next = node.next;
node.next.previous = node.previous; ExceptionAssert.NotNull(node.next).previous = node.previous;
return; return;
} }
else else
@@ -91,14 +93,16 @@ namespace BCG
internal class RowListEnumerator : IEnumerator internal class RowListEnumerator : IEnumerator
{ {
private RowNode myPointer; RowNode? myPointer;
private RowList myList; private RowList myList;
private int myPos = -1; private int myPos = -1;
public RowListEnumerator (RowList list) {myList = list;} public RowListEnumerator (RowList list) {myList = list;}
public object Current {get{return myPointer;}} public object? Current => myPointer;
public bool MoveNext() public bool MoveNext()
{ {
myPointer = myPos == -1 ? myList.myFirstPtr : myPointer.next; myPointer = myPos == -1
? myList.myFirstPtr
: ExceptionAssert.NotNull(myPointer).next;
myPos ++; myPos ++;
return myPointer != null; return myPointer != null;
} }
@@ -108,28 +112,24 @@ namespace BCG
internal class RowNode internal class RowNode
{ {
private IBlueRow myRow; public RowNode (IBlueRow row) =>
private RowNode myNext; this.row = row;
private RowNode myPrevious;
private RowList myList; public RowNode (IBlueRow row, RowList list)
: this(row) =>
public RowNode (IBlueRow row){myRow = row;} this.list = list;
public RowNode (IBlueRow row, RowList list) : this (row) {myList = list;}
public bool isLast {get{return myNext == null;}} public bool isLast => next is null;
public bool isFirst {get{return myPrevious == null;}} public bool isFirst => previous is null;
public RowList list {get{return myList;}} public RowList? list { get; }
public IBlueRow row public IBlueRow? row { get; set; }
{ public RowNode? next { get; set; }
get{return myRow;} public RowNode? previous { get; set; }
set{myRow = value;}
} public RowList RequireList() =>
public RowNode next { ExceptionAssert.MemberNotNull(list);
get{return myNext;}
set{myNext = value;} public IBlueRow RequireRow() =>
} ExceptionAssert.MemberNotNull(row);
public RowNode previous {
get{return myPrevious;}
set{myPrevious = value;}
}
} }
} }
@@ -1,125 +1,17 @@
using Bts.DataStreamCore;
using mbts;
using Spark.Core.Collections;
using Spark.Core.Diagnostics;
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using mbts; using System.Diagnostics.CodeAnalysis;
using Spark.Core.Collections;
namespace BCG namespace BCG
{ {
internal class SingleRowJoinDicionary<TKey, TValue> : IDictionary<TKey, TValue>
{
private TKey myKey;
private TValue myValue;
#region IDictionary<TKey,TValue> Members
public void Add(TKey key, TValue value)
{
myKey = key;
myValue = value;
}
public bool ContainsKey(TKey key)
{
return myKey.Equals(key);
}
public ICollection<TKey> Keys
{
get { return new TKey[] { myKey }; }
}
public bool Remove(TKey key)
{
bool rv = myKey.Equals(key);
myKey = default(TKey);
return rv;
}
public bool TryGetValue(TKey key, out TValue value)
{
bool rv = false;
value = (rv = myKey.Equals(key)) ? myValue : default(TValue);
return rv;
}
public ICollection<TValue> Values
{
get { throw new Exception("The method or operation is not implemented."); }
}
public TValue this[TKey key]
{
get
{
throw new Exception("The method or operation is not implemented.");
}
set
{
throw new Exception("The method or operation is not implemented.");
}
}
#endregion
#region ICollection<KeyValuePair<TKey,TValue>> Members
public void Add(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
public void Clear()
{
throw new Exception("The method or operation is not implemented.");
}
public bool Contains(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
{
throw new Exception("The method or operation is not implemented.");
}
public int Count
{
get { throw new Exception("The method or operation is not implemented."); }
}
public bool IsReadOnly
{
get { throw new Exception("The method or operation is not implemented."); }
}
public bool Remove(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
#region IEnumerable<KeyValuePair<TKey,TValue>> Members
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
#region IEnumerable Members
IEnumerator IEnumerable.GetEnumerator()
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
}
#region Helper class to deal with stream projections #region Helper class to deal with stream projections
public class JoinProjectHelper public class JoinProjectHelper
@@ -167,7 +59,7 @@ namespace BCG
{ {
Pair<string, bool> p = myAliasToOriginal[col]; Pair<string, bool> p = myAliasToOriginal[col];
isLeft = p.second; isLeft = p.second;
return p.first; return ExceptionAssert.NotNull(p.first);
} }
} }
@@ -178,10 +70,11 @@ namespace BCG
public class SuperJoinRow : IBlueRow public class SuperJoinRow : IBlueRow
{ {
private JoinProjectHelper myHelper; private JoinProjectHelper myHelper;
private IBlueRow myLeftRow, myRightRow; IBlueRow myLeftRow;
IBlueRow? myRightRow;
int myID; int myID;
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper) public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
{ {
myID = id; myID = id;
myLeftRow = leftRow; myLeftRow = leftRow;
@@ -189,21 +82,22 @@ namespace BCG
myHelper = joinProjectHelper; myHelper = joinProjectHelper;
} }
public int leftID { get { return (int)myLeftRow["id"]; } } public int leftID => myLeftRow.Id();
public int rightID { get { return (int)myRightRow["id"]; } } public int rightID => myRightRow.Id();
public IBlueRow leftRow { get { return myLeftRow; } } public IBlueRow leftRow { get { return myLeftRow; } }
public IBlueRow rightRow { get { return myRightRow; } } public IBlueRow? rightRow => myRightRow;
#region IBlueRow Members #region IBlueRow Members
public IBlueTableSchema schema { get { return myHelper.schema; } } public IBlueTableSchema schema { get { return myHelper.schema; } }
public object this[int index] { get { return this[schema[index].colName]; } } public object? this[int index] =>
this[schema[index].colName];
public object this[string colName] public object? this[string colName]
{ {
get get
{ {
@@ -229,7 +123,7 @@ namespace BCG
public bool Contains(object key) public bool Contains(object key)
{ {
return myLeftRow.Contains(key) || myRightRow.Contains(key); return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
} }
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
@@ -242,11 +136,11 @@ namespace BCG
public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); } public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); }
public System.Collections.ICollection Values public ICollection Values
{ {
get get
{ {
List<object> rv = new List<object>(); var rv = new List<object?>(Keys.Count);
foreach (string col in Keys) foreach (string col in Keys)
rv.Add(this[col]); rv.Add(this[col]);
return rv; return rv;
@@ -291,9 +185,10 @@ namespace BCG
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>(); myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false; private bool myManageUnmappedLeftRows = false;
protected string myLeftProj = String.Empty, myRightProj = String.Empty; protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema myLeftSchema, myRightSchema; protected IBlueTableSchema? myLeftSchema;
protected JoinProjectHelper myJoinProjectHelper; protected IBlueTableSchema? myRightSchema;
protected BlueTable myState; protected JoinProjectHelper? myJoinProjectHelper;
protected BlueTable? myState;
protected int myID; protected int myID;
protected ViewStream myLeftStream, myRightStream; protected ViewStream myLeftStream, myRightStream;
protected string myLeftKeyCol, myRightKeyCol; protected string myLeftKeyCol, myRightKeyCol;
@@ -346,23 +241,19 @@ namespace BCG
private void myHandleLeftStreamChanges(object newValue) private void myHandleLeftStreamChanges(object newValue)
{ {
myHandleStreamChanges(newValue as DataPacket, true); myHandleStreamChanges((IDataPacket)newValue, true);
} }
private void myHandleRightStreamChanges(object newValue) private void myHandleRightStreamChanges(object newValue)
{ {
myHandleStreamChanges(newValue as DataPacket, false); myHandleStreamChanges((IDataPacket)newValue, false);
} }
private void myHandleStreamChanges(DataPacket dp, bool isLeft) private void myHandleStreamChanges(IDataPacket dp, bool isLeft)
{ {
myInitSchema(dp.schema, isLeft); myInitSchema(dp.Schema, isLeft);
BlueTable dels = null, adds = null; var dels = new BlueTable(myState.schema);
if (myState != null) var adds = new BlueTable(myState.schema);
{
dels = new BlueTable(myState.schema);
adds = new BlueTable(myState.schema);
}
myDeleteRows(dp.deletedRows, isLeft, myState, dels); myDeleteRows(dp.deletedRows, isLeft, myState, dels);
myAddRows(dp.addedRows, isLeft, myState, adds); myAddRows(dp.addedRows, isLeft, myState, adds);
if (myState != null) if (myState != null)
@@ -373,6 +264,7 @@ namespace BCG
} }
} }
[MemberNotNull(nameof(myState))]
private void myInitSchema(IBlueTableSchema schema, bool isLeft) private void myInitSchema(IBlueTableSchema schema, bool isLeft)
{ {
if (myState == null) if (myState == null)
@@ -402,6 +294,7 @@ namespace BCG
private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut) private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
{ {
ExceptionAssert.NotNull(myJoinProjectHelper);
object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol, object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol; otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows, Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
@@ -410,8 +303,8 @@ namespace BCG
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows; otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in addedRows) foreach (IBlueRow r in addedRows)
{ {
object currentKey = r[thisKeyCol]; object currentKey = ExceptionAssert.NotNull(r[thisKeyCol]);
int currentID = (int)r["id"]; int currentID = r.Id();
///add it to 1. ///add it to 1.
if (!thisKeyToRows.ContainsKey(currentKey)) if (!thisKeyToRows.ContainsKey(currentKey))
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>()); thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
@@ -422,14 +315,14 @@ namespace BCG
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values; ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
foreach (IBlueRow mr in matchingRows) foreach (IBlueRow mr in matchingRows)
{ {
int mrID = (int)mr["id"]; int mrID = mr.Id();
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper); SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
state.addRow(rr); state.addRow(rr);
addedRowsOut.addRow(rr); addedRowsOut.addRow(rr);
///add to 3. ///add to 3.
if (!thisIDToResultRows.ContainsKey(currentID)) if (!thisIDToResultRows.ContainsKey(currentID))
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>()); thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
thisIDToResultRows[currentID].Add((int)rr["id"], rr); thisIDToResultRows[currentID].Add(rr.Id(), rr);
///add to 4. ///add to 4.
if (!otherIDToResultRows.ContainsKey(mrID)) if (!otherIDToResultRows.ContainsKey(mrID))
{ {
@@ -437,7 +330,7 @@ namespace BCG
if (myManageUnmappedLeftRows && !isLeft) if (myManageUnmappedLeftRows && !isLeft)
myTakeCareOfUnmatachedLeftRows(mr, false); myTakeCareOfUnmatachedLeftRows(mr, false);
} }
otherIDToResultRows[mrID].Add((int)rr["id"], rr); otherIDToResultRows[mrID].Add(rr.Id(), rr);
} }
} }
else else
@@ -458,22 +351,22 @@ namespace BCG
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows; otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in deletedRows) foreach (IBlueRow r in deletedRows)
{ {
int id = (int)r["id"]; int id = r.Id();
object key = r[thisKeyCol]; object key = ExceptionAssert.NotNull(r[thisKeyCol]);
///delete it from 1. ///delete it from 1.
thisKeyToRows[key].Remove(id); thisKeyToRows[key].Remove(id);
if (thisKeyToRows[key].Count == 0) if (thisKeyToRows[key].Count == 0)
thisKeyToRows.Remove(key); thisKeyToRows.Remove(key);
///form return rows and delete from 3. ///form return rows and delete from 3.
if (thisIDToResultRows.ContainsKey(id)) if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRowMap))
{ {
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection; var removedResultRows = (ICollection)removedResultRowMap.Values;
thisIDToResultRows.Remove(id); thisIDToResultRows.Remove(id);
foreach (SuperJoinRow rr in removedResultRows) foreach (SuperJoinRow rr in removedResultRows)
{ {
///delete from 4. ///delete from 4.
int otherId = isLeft ? rr.rightID : rr.leftID; int otherId = isLeft ? rr.rightID : rr.leftID;
otherIDToResultRows[otherId].Remove((int)rr["id"]); otherIDToResultRows[otherId].Remove(rr.Id());
if (otherIDToResultRows[otherId].Count == 0) if (otherIDToResultRows[otherId].Count == 0)
otherIDToResultRows.Remove(otherId); otherIDToResultRows.Remove(otherId);
delRowsOut.addRow(rr); delRowsOut.addRow(rr);
@@ -493,7 +386,7 @@ namespace BCG
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add) private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
{ {
int lid = (int)leftRow["id"]; int lid = leftRow.Id();
if (add) if (add)
myLeftUnmatchedRowsAdds.Add(lid, leftRow); myLeftUnmatchedRowsAdds.Add(lid, leftRow);
else else
@@ -501,7 +394,7 @@ namespace BCG
if (myLeftUnmatchedRowsAdds.ContainsKey(lid)) if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
myLeftUnmatchedRowsAdds.Remove(lid); myLeftUnmatchedRowsAdds.Remove(lid);
else else
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow); myLeftUnmatchedRowsDels.Add(lid, leftRow);
} }
} }
} }
@@ -522,44 +415,49 @@ namespace BCG
protected override void myFireEvent(object o) protected override void myFireEvent(object o)
{ {
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket); myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, (IDataPacket)o);
} }
private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels, void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels, Dictionary<int, IBlueRow> unmatchedLeftRowAdds, IDataPacket resultPacket) {
Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket) ExceptionAssert.NotNull(myJoinProjectHelper);
{ var dels = resultPacket.deletedRows;
BlueTable dels = resultPacket.deletedRows as BlueTable, var adds = resultPacket.addedRows;
adds = resultPacket.addedRows as BlueTable;
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) if (unmatchedLeftRowsDels.Count > 0) {
{ ExceptionAssert.NotNull(myState);
///an lr will be here if its actually deleted or var mutableDels = (BlueTable)dels;
///a right row is added, so now it has a partner foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
///and is no longer unmatched. ///an lr will be here if its actually deleted or
///in both cases the half row that this lr ///a right row is added, so now it has a partner
///created needs to be removed from this ///and is no longer unmatched.
///class's state and added to dels ///in both cases the half row that this lr
///it means a right is added ///created needs to be removed from this
int lid = (int)lr["id"]; ///class's state and added to dels
SuperJoinRow rr = myLeftIDToHalfRows[lid]; ///it means a right is added
myLeftIDToHalfRows.Remove(lid); int lid = lr.Id();
myState.deleteRow(rr); SuperJoinRow rr = myLeftIDToHalfRows[lid];
dels.addRow(rr); myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
mutableDels.addRow(rr);
}
} }
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) if (unmatchedLeftRowAdds.Count > 0) {
{ ExceptionAssert.NotNull(myState);
///an lr will be here if its added with no var mutableAdds = (BlueTable)adds;
///matching right, or a right matching row foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
///is gone so it is unmatched now. ///an lr will be here if its added with no
///in both cases a half row needs to be ///matching right, or a right matching row
///added to this class's state and added ///is gone so it is unmatched now.
///to adds ///in both cases a half row needs to be
int lid = (int)lr["id"]; ///added to this class's state and added
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper); ///to adds
myLeftIDToHalfRows.Add(lid, rr); int lid = lr.Id();
myState.addRow(rr); SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
adds.addRow(rr); myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
mutableAdds.addRow(rr);
}
} }
base.myFireEvent(new DataPacket(myState, adds, dels)); base.myFireEvent(new DataPacket(myState, adds, dels));
} }
Loaded 100 of 1488 files, more files were not shown because too many files have changed in this diff. Show more