From 5fa284cad330473c2aba8e4706532fd5fdad554a Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 28 Jun 2024 11:39:45 -0400 Subject: [PATCH] refactor: Use DependencyInjection paradigm in DataStreamSuite This allows us to create loosely coupled services with automatic dependency resolution --- .vscode/launch.json | 33 +++ res.json | 0 .../Bts.DataStreamWebApp.Client/src/index.css | 2 +- .../Bts.DataStreamWebApp.Server.csproj | 1 + .../Configurations/DevServer.json | 31 +++ .../DataStreamSubscriptionController.cs | 10 +- .../DataStreamWebSocketServer.cs | 11 +- .../Messages/SubscribeRequest.cs | 11 +- .../Bts.DataStreamWebApp.Server/Program.cs | 119 +++++----- .../Services/DataStreamWebAppService.cs | 65 ++++++ .../appsettings.Production.json | 6 + .../appsettings.json | 33 ++- .../Bts.Extensions.DependencyInjection.csproj | 13 ++ .../IDependOn.cs | 11 + .../IDependOnKeyed.cs | 6 + .../ServiceCollectionExtensions.cs | 130 +++++++++++ .../DataStream/DataStream.csproj | 1 + .../DataStream/DataStreamActivator.cs | 35 +++ .../DataStream/IViewStreamFactory.cs | 7 + .../DataStream/Misc/PandasScriptStream.cs | 7 +- .../DataStream/Misc/PandasStream.cs | 14 +- .../Misc/TCPConnectionTestStream.cs | 8 +- .../Offloading/CloudStreamManager.cs | 32 ++- .../Offloading/RemoteQueryStream.cs | 13 +- .../Primitives/Filter/DateBasedFilter.cs | 7 +- .../Primitives/Filter/FilterStream.cs | 18 ++ .../Primitives/Filter/SlidingWindowFilter.cs | 13 +- .../Primitives/Filter/TimeBasedFilter.cs | 37 +-- .../Filter/TimeBasedFilterStream.cs | 14 ++ .../Join/NonIncremental/NonIncJoinStream.cs | 14 +- .../Misc/DataPacketRecorderStream.cs | 14 +- .../Primitives/Misc/PubSubTestStream.cs | 21 +- .../Primitives/Misc/SimpleTCPEchoStream.cs | 4 +- .../Primitives/Misc/SnapshotReplayStream.cs | 16 +- .../DataStream/Primitives/Misc/TestStream.cs | 105 +++++---- .../Primitives/Misc/TickingTestStream.cs | 7 +- .../Primitives/Misc/WebScrapingStream.cs | 21 +- .../Primitives/MongoDB/MongoDBStream.cs | 60 ++--- .../TextFileStreams/TextWriterStream.cs | 10 +- .../Primitives/ThreadSafe/ThreadSafeStream.cs | 9 +- .../DataStream/Primitives/Time/DayStream.cs | 15 +- .../Primitives/Time/MillisecondStream.cs | 2 +- .../Primitives/Time/SecondStream.cs | 10 +- .../DataStream/Primitives/Time/TickStream.cs | 19 +- .../Primitives/Time/TimeUnitStream.cs | 9 +- .../DataStream/Publish/CircularPublisher.cs | 13 +- .../DataStream/Publish/PublisherStream.cs | 17 +- .../DataStream/Publish/ViewPublisher.cs | 7 +- .../DataStream/Services/DataStreamServices.cs | 94 ++++++++ .../DataStreamCore/ConfiguredViewStream.cs | 7 + .../DataStreamCore/DataStream.cs | 6 +- .../DataStreamCore/DataStreamCore.csproj | 2 + .../DynamicLoading/PrimitiveManager.cs | 116 ++++++---- .../DataStreamCore/IViewStreamDatabase.cs | 7 + .../DataStreamCore/IViewStreamWorker.cs | 13 ++ .../Messaging/Publish/IPubTableBroker.cs | 8 + .../Messaging/Publish/PubTableBroker.cs | 11 +- .../Messaging/Publish/SinglePubTableBroker.cs | 6 +- .../UniversalStreamIODeserializer.cs | 3 +- .../PubsubServerStreamTagHandler.cs | 30 ++- .../DataStreamCore/ProxyStream.cs | 2 +- .../Services/DataStreamCoreServices.cs | 24 ++ .../Services/ViewStreamDatabaseService.cs | 34 +++ .../ViewStreamDatabaseServiceConfig.cs | 10 + .../DataStreamCore/TableStream.cs | 52 ++--- .../DataStreamCore/ViewStream.cs | 76 +++++++ .../DataStreamCore/ViewStreamDatabase.cs | 67 ++---- .../DataStreamExpressions/Assignment.cs | 8 +- .../DataStreamExpression.cs | 10 +- .../PrimitiveExpression.cs | 22 +- .../StatementExpression.cs | 8 +- .../ViewNameExpression.cs | 12 +- .../Configurations/MarketDataPublisher.json | 215 ++++++++++-------- .../DataStreamPublisher.csproj | 2 + .../DataStreamPublisher/Program.cs | 168 ++++---------- .../Services/DataStreamPublisherServices.cs | 109 +++++++++ .../Services/PublisherService.cs | 156 +++++++++++++ .../Services/PublisherServiceConfig.cs | 12 + .../DataStreamQueryAnalyzer.json | 32 +-- .../DataStreamQueryAnalyzer/MainForm.cs | 2 +- src/DataStreamSuite/DataStreamSuite.sln | 6 + .../PubSubServerManagedTCPConnection.cs | 1 - .../PubSubServerConnectionFactory.cs | 4 +- .../PubSub/Client/IPubSubServerTagHandler.cs | 1 + .../PubSub/Client/PubSubTagAttribute.cs | 11 + .../Client/PublishingUtilities/IPublisher.cs | 19 -- .../Client/PublishingUtilities/Publisher.cs | 66 ------ .../PubSub/Client/PubsubTagHandlerFactory.cs | 55 +++-- .../PubSub/Client/Routing/PubSubRoute.cs | 11 +- .../PubSub/Client/Routing/PubSubRouter.cs | 29 ++- .../Client/Routing/PubSubRouterConfig.cs | 40 ++++ .../ManagedSubscriber.cs | 22 +- .../PubSub/Client/TagTypeToAssembly.cs | 55 +++-- .../PubSub/Config/PubSubConfig.cs | 41 ++-- .../PubSub/Misc/AnonymousWorker.cs | 15 ++ .../PubSub/Misc/IParameterizedWorker.cs | 10 + .../PubSub/Misc/IWorkScheduler.cs | 10 + .../PubSub/Misc/MasterWorker.cs | 11 +- .../PubSub/Misc/WorkSchedulerExtensions.cs | 25 ++ src/DataStreamSuite/PubSub/PubSub.csproj | 5 + src/DataStreamSuite/PubSub/PublishedCache.cs | 14 +- src/DataStreamSuite/PubSub/PublisherCore.cs | 30 ++- .../Server/Communication/IPubsubMessageHub.cs | 2 +- .../ManagedTCP/ManagedTCPMessageHub.cs | 10 +- .../PubSub/Server/IInternalPublisher.cs | 7 + .../PubSub/Server/IInternalPublisherQueue.cs | 16 ++ .../PubSub/Server/IPubSubMessageReactor.cs | 5 + .../PubSub/Server/IPubSubState.cs | 33 +++ .../PubSub/Server/IPubSubTagHandlerFactory.cs | 7 + .../PubSub/Server/InternalPublisher.cs | 25 ++ .../PubSub/Server/InternalPublisherQueue.cs | 21 ++ .../SystemHeartbeatPublisher.cs | 20 +- .../SystemStatsPublisher.cs | 44 ++-- .../SystemSubscriptionMapPublisher.cs | 53 +++-- .../PubSub/Server/PubSubMessageReactor.cs | 38 +++- .../PubSub/Server/PubSubState.cs | 22 +- .../PubSub/Server/SingleTagData.cs | 40 ++-- .../PubSub/Services/MasterWorkerConfig.cs | 7 + .../PubSub/Services/MasterWorkerService.cs | 87 +++++++ .../PubSub/Services/PubSubServerConfig.cs | 25 ++ .../PubSub/Services/PubSubServerService.cs | 123 ++++++++++ .../PubSub/Services/PubSubServices.cs | 70 ++++++ src/DataStreamSuite/PubSub/SubscriberCore.cs | 23 +- .../PubSub/Testing/TestPublisher.cs | 53 ----- .../PubSub/Testing/TestSubscriber.cs | 33 --- .../Configurations/PubSubServer.json | 123 +++++----- src/DataStreamSuite/PubSubServer/Program.cs | 107 +++------ .../PubSubServer/PubSubServerConfig.cs | 65 ------ .../Business/MarketData/BBOStream.cs | 27 +-- .../Business/MarketData/BookStream.cs | 24 +- .../MarketData/Charting/TradeBarsStream.cs | 24 +- .../Business/MarketData/FillsStream.cs | 104 +++++++++ .../Business/MarketData/InstrumentsStream.cs | 30 +-- .../Business/MarketData/PriceStatsStream.cs | 39 +--- .../Business/MarketData/TimeSalesStream.cs | 42 ++-- .../System/PS3/PubsubRoutesStream.cs | 33 ++- .../PS3/Stats/PubSubServerTagStatsStream.cs | 35 +-- .../Subscription/PubSubServerStateStream.cs | 36 +-- .../PS3/Subscription/PubsubHeartbeatStream.cs | 30 +-- .../spark_api/Services/SparkApiServices.cs | 118 ++++++++++ src/client/spark_api/Services/SparkdConfig.cs | 11 + .../spark_api/Services/SparkdCredentials.cs | 7 + .../spark_api/Services/SparkdLoginService.cs | 34 +++ src/client/spark_api/api/ServiceContext.cs | 11 +- src/client/spark_api/spark_api.csproj | 1 + src/client/spark_core/Services/SparkConfig.cs | 6 + .../spark_core/Services/SparkCoreServices.cs | 15 ++ .../spark_core/Services/SparkService.cs | 27 +++ src/client/spark_core/logging/Log.cs | 12 + src/client/spark_core/spark_core.csproj | 1 + .../utils/Exceptions/ExceptionAssert.cs | 7 +- src/client/spark_core/utils_api/ICloneable.cs | 3 +- 152 files changed, 3052 insertions(+), 1550 deletions(-) create mode 100644 res.json create mode 100644 src/DataStreamSuite/Bts.DataStreamWebApp.Server/Services/DataStreamWebAppService.cs create mode 100644 src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Production.json create mode 100644 src/DataStreamSuite/Bts.Extensions.DependencyInjection/Bts.Extensions.DependencyInjection.csproj create mode 100644 src/DataStreamSuite/Bts.Extensions.DependencyInjection/IDependOn.cs create mode 100644 src/DataStreamSuite/Bts.Extensions.DependencyInjection/IDependOnKeyed.cs create mode 100644 src/DataStreamSuite/Bts.Extensions.DependencyInjection/ServiceCollectionExtensions.cs create mode 100644 src/DataStreamSuite/DataStream/DataStreamActivator.cs create mode 100644 src/DataStreamSuite/DataStream/IViewStreamFactory.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilterStream.cs create mode 100644 src/DataStreamSuite/DataStream/Services/DataStreamServices.cs create mode 100644 src/DataStreamSuite/DataStreamCore/ConfiguredViewStream.cs create mode 100644 src/DataStreamSuite/DataStreamCore/IViewStreamDatabase.cs create mode 100644 src/DataStreamSuite/DataStreamCore/IViewStreamWorker.cs create mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Publish/IPubTableBroker.cs create mode 100644 src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs create mode 100644 src/DataStreamSuite/DataStreamCore/Services/ViewStreamDatabaseService.cs create mode 100644 src/DataStreamSuite/DataStreamCore/Services/ViewStreamDatabaseServiceConfig.cs create mode 100644 src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs create mode 100644 src/DataStreamSuite/DataStreamPublisher/Services/PublisherService.cs create mode 100644 src/DataStreamSuite/DataStreamPublisher/Services/PublisherServiceConfig.cs create mode 100644 src/DataStreamSuite/PubSub/Client/PubSubTagAttribute.cs delete mode 100644 src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs delete mode 100644 src/DataStreamSuite/PubSub/Client/PublishingUtilities/Publisher.cs create mode 100644 src/DataStreamSuite/PubSub/Client/Routing/PubSubRouterConfig.cs create mode 100644 src/DataStreamSuite/PubSub/Misc/AnonymousWorker.cs create mode 100644 src/DataStreamSuite/PubSub/Misc/IParameterizedWorker.cs create mode 100644 src/DataStreamSuite/PubSub/Misc/IWorkScheduler.cs create mode 100644 src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs create mode 100644 src/DataStreamSuite/PubSub/Server/IInternalPublisher.cs create mode 100644 src/DataStreamSuite/PubSub/Server/IInternalPublisherQueue.cs create mode 100644 src/DataStreamSuite/PubSub/Server/IPubSubMessageReactor.cs create mode 100644 src/DataStreamSuite/PubSub/Server/IPubSubState.cs create mode 100644 src/DataStreamSuite/PubSub/Server/IPubSubTagHandlerFactory.cs create mode 100644 src/DataStreamSuite/PubSub/Server/InternalPublisher.cs create mode 100644 src/DataStreamSuite/PubSub/Server/InternalPublisherQueue.cs create mode 100644 src/DataStreamSuite/PubSub/Services/MasterWorkerConfig.cs create mode 100644 src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs create mode 100644 src/DataStreamSuite/PubSub/Services/PubSubServerConfig.cs create mode 100644 src/DataStreamSuite/PubSub/Services/PubSubServerService.cs create mode 100644 src/DataStreamSuite/PubSub/Services/PubSubServices.cs delete mode 100644 src/DataStreamSuite/PubSub/Testing/TestPublisher.cs delete mode 100644 src/DataStreamSuite/PubSub/Testing/TestSubscriber.cs delete mode 100644 src/DataStreamSuite/PubSubServer/PubSubServerConfig.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/FillsStream.cs create mode 100644 src/client/spark_api/Services/SparkApiServices.cs create mode 100644 src/client/spark_api/Services/SparkdConfig.cs create mode 100644 src/client/spark_api/Services/SparkdCredentials.cs create mode 100644 src/client/spark_api/Services/SparkdLoginService.cs create mode 100644 src/client/spark_core/Services/SparkConfig.cs create mode 100644 src/client/spark_core/Services/SparkCoreServices.cs create mode 100644 src/client/spark_core/Services/SparkService.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index b0be6a96a..7e8a17da7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -21,6 +21,20 @@ "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy", } }, + { + "name": "Bts.DataStreamWebApp.Server (Skip Build)", + "type": "coreclr", + "request": "launch", + "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll", + // "args": [ + // "--config", + // "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json", + // ], + "env": { + "ASPNETCORE_ENVIRONMENT": "Development", + "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy", + } + }, { "name": "DataStreamPublisher", "type": "coreclr", @@ -100,6 +114,25 @@ "stopAtEntry": false, "console": "integratedTerminal" }, + { + "name": "PubSubServer (Skip Build)", + "type": "coreclr", + "request": "launch", + "program": "", + "linux": { + "program": "PubSubServer", + }, + "windows": { + "program": "PubSubServer.exe", + }, + "args": [ + "--config", + "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json" + ], + "cwd": "build/Debug/bin/net7.0", + "stopAtEntry": false, + "console": "integratedTerminal" + }, { "name": "Spark", "type": "coreclr", diff --git a/res.json b/res.json new file mode 100644 index 000000000..e69de29bb diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css b/src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css index e19c83c22..732cec168 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css @@ -4,7 +4,7 @@ font-weight: 400; color-scheme: light dark; - color: rgba(255, 255, 255, 0.87); + color: rgba(0, 0, 0, 0.87); background-color: #242424; font-synthesis: none; diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj index d5be7c5a9..cfbe63769 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj @@ -27,6 +27,7 @@ +