From 856cd552eb38310f807d462f8053927b486b798b Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 31 Jul 2024 00:01:37 -0400 Subject: [PATCH] refactor: Add Api Server service modules for Dependency Injection * Part of a broader paradigm shift towards Dependency Injection and Factory Pattern. * Adds "api-server" service to docker compose * Replaces MarketDataClient w/ IMarketDataClient in a few streams * Replaces ViewStreamDatabase w/ IWorkScheduler in a few streams --- .vscode/launch.json | 54 +++ .vscode/tasks.json | 134 ++++++ compose.yaml | 12 + spark_client.sln | 27 ++ .../DynamicLoading/MarketDataStreamFactory.cs | 4 +- .../DataStreamQueryAnalyzer/Program.cs | 17 +- .../CrossExchangeArbDiscoveryStream.cs | 12 +- .../CrossExchangeSymbolMatcherStream.cs | 11 +- .../BlueOrb/MarketData/Live/LiveBBOStream.cs | 17 +- .../MarketData/Live/LiveFuturesBookStream.cs | 11 +- .../Live/LiveFuturesContractStream.cs | 9 +- .../BlueOrb/SyntheticSpreadBBOStream.cs | 12 +- .../Analytics/SpatialArbScoreStream.cs | 16 +- .../Analytics/SpatialArbStatsStream.cs | 19 +- .../Analytics/SpatialFuturePairsStream.cs | 21 +- .../Analytics/SpreadPerformanceStream.cs | 38 +- ...eadStatArbsWeightLinearRegressionStream.cs | 17 +- .../MarketData/Analytics/SpreadStatsStream.cs | 17 +- .../Business/MarketData/BBOStream.cs | 39 +- .../Business/MarketData/BookStream.cs | 49 +- .../Business/MarketData/InstrumentsStream.cs | 47 +- .../Business/MarketData/PriceStatsStream.cs | 34 +- .../MarketData/SyntheticSpreadStream.cs | 33 +- .../Business/MarketData/TimeSalesStream.cs | 24 +- .../spark/Model/SecurityResolverExporter.cs | 7 +- .../Market/TimeAndSales/TimeAndSalesRowVM.cs | 43 +- .../Market/TimeAndSales/TimeAndSalesVM.cs | 9 +- src/client/spark_api/Admin/AdminClient.cs | 96 ++-- .../spark_api/Admin/EmptyAdminClient.cs | 16 + src/client/spark_api/Admin/IAdminClient.cs | 31 ++ .../spark_api/MarketData/IMarketDataClient.cs | 50 ++ .../spark_api/MarketData/ISecurityResolver.cs | 15 + .../spark_api/MarketData/MarketDataClient.cs | 435 ++++++++++-------- .../MarketData/MarketDataClientConfig.cs | 23 + .../MarketData/PassiveSecurityResolver.cs | 202 ++++++++ .../MarketData/PollablePriceStats.cs | 31 ++ .../MarketData/PollablePriceTicker.cs | 48 ++ src/client/spark_api/MarketData/TradeInfo.cs | 58 ++- .../MarketData/TradeMessageExtensions.cs | 8 + .../spark_api/MarketData/TradesSource.cs | 3 +- src/client/spark_api/Orders/IOrdersClient.cs | 58 +++ src/client/spark_api/Orders/OrdersClient.cs | 94 ++-- .../spark_api/Reactive/SecurityResolver.cs | 44 +- .../Services/CachedServiceProvider.cs | 33 ++ .../spark_api/Services/SparkApiServices.cs | 35 +- src/client/spark_api/api/ServiceContext.cs | 3 +- src/client/spark_api_server/BookDumper.cs | 254 ++++++---- .../spark_api_server/BookDumperConfig.cs | 13 + .../Configurations/spark_api_server.json | 61 --- src/client/spark_api_server/Dockerfile | 17 + src/client/spark_api_server/Program.cs | 173 +++---- src/client/spark_api_server/SecurityDumper.cs | 172 +++++-- .../spark_api_server/SecurityDumperConfig.cs | 8 + .../appsettings.Development.json | 15 + .../spark_api_server/appsettings.Staging.json | 5 + src/client/spark_api_server/appsettings.json | 83 ++++ src/client/spark_api_server/spark.conf | 15 + .../spark_api_server/spark_api_server.csproj | 8 + src/client/spark_core/Update/TableUpdate.cs | 12 +- src/client/spark_core/Update/Update.cs | 6 +- src/client/spark_core/logging/Log.cs | 8 + .../spark_core/logging/SparkLoggerAdapter.cs | 65 +++ .../spark_core/logging/SparkLoggerProvider.cs | 13 + .../spark_sdk/samples/MarketDataSample.cs | 25 +- src/client/spark_sdk/samples/OrderSample.cs | 25 +- .../spark_sdk/samples/SdkSampleService.cs | 11 +- src/client/spark_sdk/samples/UISample.cs | 10 +- .../Core/IWebsocketConnectionGateway.cs | 12 +- .../spark_websocket/Core/WebsocketClientID.cs | 9 +- .../Core/WebsocketConnectNotification.cs | 11 + .../Core/WebsocketConnectionGateway.cs | 74 +-- .../Core/WebsocketConnectionUpdate.cs | 73 +++ .../Core/WebsocketConnectionUpdateType.cs | 9 + .../spark_websocket/Core/WebsocketDataSync.cs | 58 ++- .../Core/WebsocketDisconnectNotification.cs | 8 + .../Core/WebsocketJsonParsers.cs | 27 +- .../Core/WebsocketRestHandler.cs | 18 +- .../Core/WebsocketSubscriptionHandler.cs | 17 +- .../spark_websocket/JsonRpc/JsonRpcId.cs | 50 ++ .../Services/ISparkApiWebSocketServer.cs | 7 + .../Services/SparkApiWebSocketServer.cs | 39 ++ .../Services/SparkWebSocketServices.cs | 20 + 82 files changed, 2368 insertions(+), 1079 deletions(-) create mode 100644 src/client/spark_api/Admin/EmptyAdminClient.cs create mode 100644 src/client/spark_api/Admin/IAdminClient.cs create mode 100644 src/client/spark_api/MarketData/IMarketDataClient.cs create mode 100644 src/client/spark_api/MarketData/ISecurityResolver.cs create mode 100644 src/client/spark_api/MarketData/MarketDataClientConfig.cs create mode 100644 src/client/spark_api/MarketData/PassiveSecurityResolver.cs create mode 100644 src/client/spark_api/MarketData/PollablePriceStats.cs create mode 100644 src/client/spark_api/MarketData/PollablePriceTicker.cs create mode 100644 src/client/spark_api/Orders/IOrdersClient.cs create mode 100644 src/client/spark_api/Services/CachedServiceProvider.cs create mode 100644 src/client/spark_api_server/BookDumperConfig.cs delete mode 100644 src/client/spark_api_server/Configurations/spark_api_server.json create mode 100644 src/client/spark_api_server/Dockerfile create mode 100644 src/client/spark_api_server/SecurityDumperConfig.cs create mode 100644 src/client/spark_api_server/appsettings.Development.json create mode 100644 src/client/spark_api_server/appsettings.Staging.json create mode 100644 src/client/spark_api_server/appsettings.json create mode 100644 src/client/spark_api_server/spark.conf create mode 100644 src/client/spark_core/logging/SparkLoggerAdapter.cs create mode 100644 src/client/spark_core/logging/SparkLoggerProvider.cs create mode 100644 src/client/spark_websocket/Core/WebsocketConnectNotification.cs create mode 100644 src/client/spark_websocket/Core/WebsocketConnectionUpdate.cs create mode 100644 src/client/spark_websocket/Core/WebsocketConnectionUpdateType.cs create mode 100644 src/client/spark_websocket/Core/WebsocketDisconnectNotification.cs create mode 100644 src/client/spark_websocket/JsonRpc/JsonRpcId.cs create mode 100644 src/client/spark_websocket/Services/ISparkApiWebSocketServer.cs create mode 100644 src/client/spark_websocket/Services/SparkApiWebSocketServer.cs create mode 100644 src/client/spark_websocket/Services/SparkWebSocketServices.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index f9c7bc317..b62e10a01 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,6 +14,49 @@ "type": "coreclr", "request": "attach" }, + { + "name": "Api Server", + "type": "coreclr", + "request": "launch", + "program": "", + "preLaunchTask": "Build Api Server (Debug)", + "windows": { + "program": "spark_api_server.exe", + }, + "linux": { + "program": "spark_api_server", + }, + "args": [ + "run", + ], + "cwd": "build/Debug/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}", + }, + "stopAtEntry": false, + "console": "integratedTerminal" + }, + { + "name": "Api Server (Skip Build)", + "type": "coreclr", + "request": "launch", + "program": "", + "windows": { + "program": "spark_api_server.exe", + }, + "linux": { + "program": "spark_api_server", + }, + "args": [ + "run", + ], + "cwd": "build/Debug/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}", + }, + "stopAtEntry": false, + "console": "integratedTerminal" + }, { "name": "DataStreamPublisher", "type": "coreclr", @@ -286,6 +329,17 @@ }, ], "inputs": [ + { + "id": "apiServerEnvironment", + "type": "pickString", + "description": "Api Server Configuration", + "options": [ + "Development", + "Local", + "Staging", + ], + "default": "Development", + }, { "id": "dataStreamPublisherConfig", "type": "pickString", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 8e9de2a28..1b71f0249 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -351,6 +351,129 @@ "id": "wrench" }, }, + { + "label": "Run Api Server", + "dependsOn": "Build Api Server", + "type": "process", + "command": "", + "windows": { + "command": "spark_api_server.exe", + }, + "linux": { + "command": "spark_api_server", + }, + "args": [ + "run", + ], + "options": { + "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}", + } + }, + "icon": { + "id": "run", + "color": "terminal.ansiGreen", + }, + }, + { + "label": "Run Api Server (Debug)", + "dependsOn": "Build Api Server (Debug)", + "type": "process", + "command": "", + "windows": { + "command": "spark_api_server.exe", + }, + "linux": { + "command": "spark_api_server", + }, + "args": [ + "run", + ], + "options": { + "cwd": "${workspaceFolder}/build/Debug/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}", + } + }, + "icon": { + "id": "run", + "color": "terminal.ansiGreen", + }, + }, + { + "label": "Run Api Server (Release)", + "dependsOn": "Build Api Server (Release)", + "type": "process", + "command": "", + "windows": { + "command": "spark_api_server.exe", + }, + "linux": { + "command": "spark_api_server", + }, + "args": [ + "run", + ], + "options": { + "cwd": "${workspaceFolder}/build/Release/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}", + } + }, + "icon": { + "id": "run", + "color": "terminal.ansiGreen", + }, + }, + { + "label": "Run Api Server (Debug | Skip Build)", + "type": "process", + "command": "", + "windows": { + "command": "spark_api_server.exe", + }, + "linux": { + "command": "spark_api_server", + }, + "args": [ + "run", + ], + "options": { + "cwd": "${workspaceFolder}/build/Debug/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}", + } + }, + "icon": { + "id": "run", + "color": "terminal.ansiGreen", + }, + }, + { + "label": "Run Api Server (Release | Skip Build)", + "type": "process", + "command": "", + "windows": { + "command": "spark_api_server.exe", + }, + "linux": { + "command": "spark_api_server", + }, + "args": [ + "run", + ], + "options": { + "cwd": "${workspaceFolder}/build/Release/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}", + } + }, + "icon": { + "id": "run", + "color": "terminal.ansiGreen", + }, + }, { "label": "Run DataStreamQueryAnalyzer", "dependsOn": "Build DataStream", @@ -844,6 +967,17 @@ }, ], "inputs": [ + { + "id": "apiServerEnvironment", + "type": "pickString", + "description": "Api Server Configuration", + "options": [ + "Development", + "Local", + "Staging", + ], + "default": "Development", + }, { "id": "buildConfiguration", "type": "pickString", diff --git a/compose.yaml b/compose.yaml index 2e91c7da7..dc38087a9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -174,6 +174,18 @@ services: - .env - ${MD_PUBLISHER__CONFIG_DIR}/.env + api-server: + depends_on: + - sparkd + ports: + - 8008:8008 + - 9443:9443 + build: + dockerfile: src/client/spark_api_server/Dockerfile + command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ] + environment: + - ASPNETCORE_ENVIRONMENT=Staging + volumes: sparkd_data: name: "sparkd_${SPARKD__CONFIG_PORT:-8000}" diff --git a/spark_client.sln b/spark_client.sln index e771263ce..fad889740 100644 --- a/spark_client.sln +++ b/spark_client.sln @@ -33,6 +33,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStre EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -361,6 +363,30 @@ Global {5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU {5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU {5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.ActiveCfg = Release|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.Build.0 = Release|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.ActiveCfg = Release|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.Build.0 = Release|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.ActiveCfg = Release|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.Build.0 = Release|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU + {90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -368,6 +394,7 @@ Global GlobalSection(NestedProjects) = preSolution {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {5DE3B533-DCE6-4BF4-8F7E-08C440E177B4} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} + {90A48718-F6A7-4747-97EA-89632B451008} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/MarketDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/MarketDataStreamFactory.cs index a4b68b872..135d362fe 100644 --- a/src/DataStreamSuite/DataStreamCore/DynamicLoading/MarketDataStreamFactory.cs +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/MarketDataStreamFactory.cs @@ -3,6 +3,8 @@ namespace Bts.DataStreamCore; using BCG; +using Bts.PubSub; + using System; using System.Collections.Generic; using System.Linq; @@ -31,7 +33,7 @@ public class MarketDataStreamFactory { var paramInjections = new List<(int Index, object Value)>(2) { (mdClientIdx, mdClient) }; - if (ctrParams.TryGetFirst(p => p.ParameterType == typeof(ViewStreamDatabase), out var vsdParam, out int vsdIdx)) { + if (ctrParams.TryGetFirst(p => p.ParameterType.IsAssignableTo(typeof(IWorkScheduler)), out var vsdParam, out int vsdIdx)) { paramInjections.Add((vsdIdx, ViewStreamDatabase.instance)); } if (nArgs != ctrParams.Length - paramInjections.Count) { diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs index 777f181f4..cd5b65f20 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs @@ -2,14 +2,14 @@ using BCG; using bts; using bts.api.core; -using bts.md; using Bts.DataStreamQueryAnalyzer; using Bts.PubSub; using CommandLine; -using mbts.config; +using Spark.Api.Services; +using Spark.Core.Services; using System; using System.Threading.Tasks; @@ -26,8 +26,17 @@ static class Program { cfg.MergeWithConfigFile(); cfg.MergeWith(DataStreamQueryAnalyzerConfig.Default()); PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); - await Configuration.InitializeAsync(cfg.Sparkd); - PrimitiveManager.SetMarketDataClient(new MarketDataClient(new(TimeSpan.FromSeconds(0.5)))); + var configLoader = new SparkConfigLoader(new SparkConfigLoaderOpts() { ConfigServer = cfg.Sparkd }); + var serviceContext = new ServiceContext(configLoader); + var credentials = new SparkApiCredentials() { + User = cfg.User, + Password = cfg.Password + }; + var loginService = new SparkdLoginService(serviceContext, credentials, configLoader); + var permissions = new UserDataPermissions(loginService); + var mdClientConfig = new MarketDataClientConfig() { HotPollInterval = TimeSpan.FromSeconds(0.5) }; + var mdClient = new MarketDataClient(configLoader, loginService, permissions, mdClientConfig); + PrimitiveManager.SetMarketDataClient(mdClient); Log.Gen.info("Starting DataStreamQueryAnalyzer"); var mainForm = new MainForm(); if (cfg.StreamFile is not null) { diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeArbDiscoveryStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeArbDiscoveryStream.cs index 8d99612be..ed4be0675 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeArbDiscoveryStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeArbDiscoveryStream.cs @@ -3,10 +3,10 @@ namespace bts.BlueOrb { using BCG; - using bts.api; - using bts.md; - using Bts.DataStreamCore; + using Bts.PubSub; + + using Spark.Api.MarketData; /* ---------------------------------------------------------------------- @@ -62,9 +62,9 @@ r = TimeSeriesStream(r) [LiveMarketDataStream] public class CrossExchangeArbDiscoveryStream : ViewStream { - public CrossExchangeArbDiscoveryStream([MarketDataClientParam] MarketDataClient mdClient, ViewStreamDatabase db) { + public CrossExchangeArbDiscoveryStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) { ViewStream r; - r = new CrossExchangeSymbolMatcherStream(mdClient,db); + r = new CrossExchangeSymbolMatcherStream(mdClient, scheduler); r = new RowBasedAppendStream(r, r => new SyntheticSpreadBBOStream( @@ -76,7 +76,7 @@ r = TimeSeriesStream(r) r.get("left_cpp"), r.get("right_cpp"), mdClient, - db), + scheduler), "left_sec", "left_venue","right_sec", "right_venue" ); r = new NewProjectStream (r, diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeSymbolMatcherStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeSymbolMatcherStream.cs index 9bc0e41bf..274ab7225 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeSymbolMatcherStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeSymbolMatcherStream.cs @@ -3,15 +3,14 @@ namespace bts.BlueOrb { using BCG; - using bts.api; - using bts.md; - using Bts.DataStreamCore; + using Bts.PubSub; + + using Spark.Api.MarketData; - [LiveMarketDataStream] public class CrossExchangeSymbolMatcherStream : ViewStream { - public CrossExchangeSymbolMatcherStream([MarketDataClientParam] MarketDataClient mdClient, ViewStreamDatabase db) { + public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) { ViewStream r; ViewStream okx_usdc; ViewStream okx_usdt; @@ -22,7 +21,7 @@ namespace bts.BlueOrb { ViewStream perps; ViewStream not_perps; - r = new InstrumentsStream(); + r = new InstrumentsStream(mdClient, scheduler); r = new DateTimeExpanstionStream(r, "expiration"); okx_usdc = new FilterStream(r, diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveBBOStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveBBOStream.cs index 4b73a2c48..7e14103bf 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveBBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveBBOStream.cs @@ -1,22 +1,23 @@ namespace bts.BlueOrb { using BCG; - using bts.md; - using Bts.DataStream; using Bts.DataStreamCore; + using Bts.PubSub; using Bts.SharedStreams.MarketData; + using Spark.Api.MarketData; + [LiveMarketDataStream] [DataStreamUserRespresentation("LiveBBOStream")] public class LiveBBOStream : ViewStream { - public LiveBBOStream([MarketDataClientParam] MarketDataClient mdClient, ViewStreamDatabase vsd) - : this(mdClient, vsd, null, null, null) { } + public LiveBBOStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) + : this(mdClient, scheduler, null, null, null) { } - public LiveBBOStream([MarketDataClientParam] MarketDataClient mdClient, ViewStreamDatabase vsd, string? venueStr, string? nameOrAsset) - : this(mdClient, vsd, venueStr, null, nameOrAsset) { } + public LiveBBOStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler, string? venueStr, string? nameOrAsset) + : this(mdClient, scheduler, venueStr, null, nameOrAsset) { } - public LiveBBOStream([MarketDataClientParam] MarketDataClient mdClient, ViewStreamDatabase vsd, string? venueStr, string? secTypeStr, string? nameOrAsset) { + public LiveBBOStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler, string? venueStr, string? secTypeStr, string? nameOrAsset) { var opts = new BookStreamParams() { NameOrAsset = nameOrAsset, }; @@ -29,7 +30,7 @@ var schemaBd = opts.GetSchemaBuilder(); var adapter = schemaBd.BboAdapter(); var secFilter = opts.GetSecurityFilter(); - mySetRealSource(mdClient.books(secFilter).AsDataPackets(adapter).OnWorkScheduler(vsd).AsViewStream()); + mySetRealSource(mdClient.Books(secFilter).AsDataPackets(adapter).OnWorkScheduler(scheduler).AsViewStream()); } } } diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs index 470e67c23..ef6c6c13c 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs @@ -1,13 +1,18 @@ namespace bts.BlueOrb { using BCG; + using Bts.DataStreamCore; + using Bts.PubSub; + + using Spark.Api.MarketData; + using System; [DataStreamUserRespresentation("LiveFuturesBooktream")] public class LiveFuturesBooktream : ViewStream { - public LiveFuturesBooktream(string future_target) { - ViewStream contract_str = new LiveFuturesContractStream(future_target); - ViewStream bbo_str = new BookStream(); + public LiveFuturesBooktream([MarketDataClientParam] IMarketDataClient mdClient, string future_target, IWorkScheduler scheduler) { + ViewStream contract_str = new LiveFuturesContractStream(mdClient, future_target, scheduler); + ViewStream bbo_str = new BookStream(mdClient, scheduler); ViewStream rs = new SimpleJoinStream(contract_str, "sec_key", bbo_str, "sec_key"); rs = new ProjectStream(rs, "left_sec:sec, left_sec_key:sec_key, bid_price, bid_sz:bid_size, ask_price, ask_sz:ask_size"); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs index 74b780cca..4a16164b9 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs @@ -5,12 +5,17 @@ using bts.api; + using Bts.DataStreamCore; + using Bts.PubSub; + + using Spark.Api.MarketData; + [DataStreamUserRespresentation("LiveFuturesContractStream")] public class LiveFuturesContractStream : ViewStream { - public LiveFuturesContractStream(string target) { + public LiveFuturesContractStream([MarketDataClientParam] IMarketDataClient mdClient, string target, IWorkScheduler scheduler) { var fc = new ContractUtils.FutureContract(target); // temporary - ViewStream ins_str = new InstrumentsStream(); + ViewStream ins_str = new InstrumentsStream(mdClient, scheduler); //new bts.BlueOrb.CSVReaderStream(@"C:\Temp\instruments.csv"); var rs = new LiveFuturesContractStream(ins_str, fc); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/SyntheticSpreadBBOStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/SyntheticSpreadBBOStream.cs index 73d32971c..0d4dd5d3f 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/SyntheticSpreadBBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/SyntheticSpreadBBOStream.cs @@ -3,10 +3,10 @@ namespace bts.BlueOrb { using BCG; - using bts.api; - using bts.md; - using Bts.DataStreamCore; + using Bts.PubSub; + + using Spark.Api.MarketData; /* ---------------------------------------------------------------------- @@ -64,8 +64,8 @@ namespace bts.BlueOrb { [LiveMarketDataStream] [DataStreamUserRespresentation("SyntheticSpreadBBOStream")] public class SyntheticSpreadBBOStream : ViewStream { - public SyntheticSpreadBBOStream(string leg_1, string venue_1,string leg_2, string venue_2, [MarketDataClientParam] MarketDataClient mdClient, ViewStreamDatabase vsd) : this(leg_1, venue_1, leg_2, venue_2, 1, -1, 1, 1, mdClient, vsd) { } - public SyntheticSpreadBBOStream(string leg_1, string venue_1, string leg_2, string venue_2, double pm1, double pm2, double cpp1, double cpp2, [MarketDataClientParam] MarketDataClient mdClient, ViewStreamDatabase vsd) : this(new LiveBBOStream(mdClient, vsd, venue_1,leg_1), new LiveBBOStream(mdClient, vsd, venue_2,leg_2), pm1, pm2, cpp1, cpp2) { } + public SyntheticSpreadBBOStream(string leg_1, string venue_1,string leg_2, string venue_2, [MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) : this(leg_1, venue_1, leg_2, venue_2, 1, -1, 1, 1, mdClient, scheduler) { } + public SyntheticSpreadBBOStream(string leg_1, string venue_1, string leg_2, string venue_2, double pm1, double pm2, double cpp1, double cpp2, [MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) : this(new LiveBBOStream(mdClient, scheduler, venue_1,leg_1), new LiveBBOStream(mdClient, scheduler, venue_2,leg_2), pm1, pm2, cpp1, cpp2) { } public SyntheticSpreadBBOStream(ViewStream bbo_leg_1, ViewStream bbo_leg_2, double pm1, double pm2, double cpp1, double cpp2) { ViewStream r; double qm1 =1; @@ -106,4 +106,4 @@ namespace bts.BlueOrb { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs index 5d847ea59..d9531377e 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs @@ -1,15 +1,19 @@ namespace bts.BlueOrb { - using bts.SilverPine; using BCG; + + using Spark.Api.MarketData; + using System; - [DataStreamUserRespresentation("SpatialArbScoreStream")] + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"SpatialArbScoreStream"*/ false)] public class SpatialArbScoreStream : ViewStream { - public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz) - : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { } + public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz, IMarketDataClient mdClient) + : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), mdClient, ViewStreamDatabase.instance) { } - public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) { - ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd); + public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, IMarketDataClient mdClient, ViewStreamDatabase vsd) { + ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, mdClient, vsd); rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd); rs = new SortStream(rs, "-n_trade_ticks"); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs index 2985d8e81..9be8a5da7 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs @@ -1,17 +1,16 @@ namespace bts.BlueOrb { using BCG; - using SharpCompress.Common.Rar; + using Spark.Api.MarketData; using System; using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Threading.Tasks; // For all the pairs generated by SpatialFuturePairsStream, this stream // generates the statistics, by appending together a SpreadStatsStream - [DataStreamUserRespresentation("SpatialArbStatsStream")] + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"SpatialArbStatsStream"*/ false)] public class SpatialArbStatsStream : ViewStream{ private ViewStream _spatial_pairs_stream; private ViewStreamDatabase _vsd; @@ -20,15 +19,15 @@ private ViewStream _bbo_stream; private readonly TimeSpan _window_sz, _sampling_period_sz; - public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds) - : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { } + public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds, IMarketDataClient mdClient) + : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), mdClient, ViewStreamDatabase.instance) { } - public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) { + public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, IMarketDataClient mdClient, ViewStreamDatabase vsd) { _vsd = vsd; _window_sz = window_sz; _sampling_period_sz = sampling_period_sz; - _spatial_pairs_stream = new SpatialFuturePairsStream(vsd); - _bbo_stream = new BBOStream(vsd); + _spatial_pairs_stream = new SpatialFuturePairsStream(mdClient, vsd); + _bbo_stream = new BBOStream(mdClient, vsd); } protected override void myOnFirstSubscriber() { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs index 4c59e7cea..265323b45 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs @@ -3,22 +3,27 @@ using bts.api; + using Spark.Api.MarketData; + using System; using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Threading.Tasks; - [DataStreamUserRespresentation("SpatialFuturePairsStream")] + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"SpatialFuturePairsStream"*/ false)] public class SpatialFuturePairsStream : ViewStream { private ViewStream _main_stream; private IBlueTableSchema _schema; private HashSet _key_map = new HashSet(); - public SpatialFuturePairsStream() - : this(ViewStreamDatabase.instance) { } - public SpatialFuturePairsStream(ViewStreamDatabase vsd) { - _main_stream = new InstrumentsStream(vsd); + public SpatialFuturePairsStream(IMarketDataClient mdClient) + : this(mdClient, ViewStreamDatabase.instance) { } + + public SpatialFuturePairsStream(IMarketDataClient mdClient, ViewStreamDatabase vsd) + : this(new InstrumentsStream(mdClient, vsd), vsd) { } + + public SpatialFuturePairsStream(ViewStream instruments, ViewStreamDatabase vsd) { + _main_stream = instruments; _main_stream = new FilterStream(_main_stream, "sec_type='Future'"); _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32)); _schema = _schema.add_column(TypeCode.String, "s1"); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs index 76fca12e3..ce5d46b5a 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs @@ -1,22 +1,44 @@ namespace bts.BlueOrb { using BCG; + using Spark.Api.MarketData; + using System.Diagnostics; + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(false)] public class SpreadPerformanceStream : ViewStream { ViewStream _spread_price_stream = null; private readonly IBlueTableSchema _schema; private readonly int _window_secs; private readonly double _buy_threshold, _sell_threshold, _flat_threshold; private ViewStream _bbo_stream; - public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs, - double buy_threshold, double sell_threshold, double flat_threshold) - : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { } + public SpreadPerformanceStream( + string s1, + string s2, + double w1, + double w2, + int window_secs, + double buy_threshold, + double sell_threshold, + double flat_threshold, + IMarketDataClient mdClient + ) : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, mdClient, ViewStreamDatabase.instance) { } - public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs, - double buy_threshold, double sell_threshold, double flat_threshold, - ViewStreamDatabase vsd) { - _bbo_stream = new BBOStream(vsd); + public SpreadPerformanceStream( + string s1, + string s2, + double w1, + double w2, + int window_secs, + double buy_threshold, + double sell_threshold, + double flat_threshold, + IMarketDataClient mdClient, + ViewStreamDatabase vsd + ) { + _bbo_stream = new BBOStream(mdClient, vsd); _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32), Tuple.Create("t0", TypeCode.DateTime), Tuple.Create("t1", TypeCode.DateTime), @@ -56,7 +78,7 @@ namespace bts.BlueOrb { _buy_threshold = buy_threshold; _sell_threshold = sell_threshold; _flat_threshold = flat_threshold; - _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd); + _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream); _spread_price_stream = new TimeSeriesStream(_spread_price_stream); _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd); } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs index 3ad739079..396c94d03 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs @@ -1,16 +1,21 @@ namespace bts.BlueOrb{ using BCG; + + using Spark.Api.MarketData; + + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(false)] public class SpreadStatArbsWeightLinearRegressionStream : ViewStream { private IBlueTableSchema _schema; private ViewStream _leg_price_stream; private readonly double _window_secs; private readonly string _s1, _s2; - public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs) - : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { } + public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, IMarketDataClient mdClient) + : this(s1, s2, window_secs, sample_period_secs, mdClient, ViewStreamDatabase.instance) { } - - public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) { + public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, IMarketDataClient mdClient, ViewStreamDatabase vsd) { _window_secs = window_secs; _s1 = s1; _s2 = s2; @@ -25,8 +30,8 @@ namespace bts.BlueOrb{ Tuple.Create("s2", TypeCode.String), Tuple.Create("w2", TypeCode.String)); - _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2)); - _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd); + _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2, mdClient, vsd)); + _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(sample_period_secs), vsd); } protected override void myOnFirstSubscriber() { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs index 20630b236..a0afc7ba1 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs @@ -1,15 +1,16 @@ namespace bts.BlueOrb { using BCG; - using MathNet.Numerics.LinearAlgebra.Factorization; + using Spark.Api.MarketData; using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; - using System.Text; - using System.Threading.Tasks; - [DataStreamUserRespresentation("SpreadStatsStream")] + + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"SpreadStatsStream"*/ false)] // This is the fundamental stream that computes the stats (including buy and sell zones) // of a spread. the legs and the weights are passed as args public class SpreadStatsStream : ViewStream { @@ -17,8 +18,10 @@ private readonly IBlueTableSchema _schema; private readonly TimeSpan _window_sz; private readonly string _spread; - public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs) - : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { } + + public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, IMarketDataClient mdClient) + : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(mdClient, ViewStreamDatabase.instance), ViewStreamDatabase.instance) { } + public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) { _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds); _spread = $"{w1} X {s1} - {w2} X {s2}"; @@ -60,7 +63,7 @@ _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks"); _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price"); _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks"); - _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd); + _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream); _spread_price_stream = new TimeSeriesStream(_spread_price_stream); double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100); TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs index 31df64695..6b925884b 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs @@ -1,18 +1,16 @@ namespace bts.BlueOrb { + using bts.api.core; + + using BCG; using BCG.Utilities.General; - using bts.md; - using BCG; + using Spark.Api.MarketData; using Spark.Core.Reactive; using System; using System.Collections.Generic; using System.Linq; using System.Reactive.Linq; - using bts.api.core; - using Spark.Api.MarketData; - - using SecurityKey = System.UInt32; public class FakeBBOStream : ViewStream, IWorker { private string _symbol; @@ -100,13 +98,15 @@ namespace bts.BlueOrb { } } - [DataStreamUserRespresentation("bbo")] + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"bbo"*/ false)] public class BBOStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; private ViewStreamDatabase? _vsd; - IReadOnlyDictionary? _cached_state = null; - MarketDataClient? _md_client; + IReadOnlyDictionary? _cached_state = null; + IMarketDataClient? _md_client; internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo", Tuple.Create("id", TypeCode.Int32), Tuple.Create("sec_key", TypeCode.UInt32), @@ -118,23 +118,19 @@ namespace bts.BlueOrb { Tuple.Create("ask_sz", TypeCode.Single)); // TODO: add cons for symbol, exchange etc - public BBOStream() - : this(ViewStreamDatabase.instance) { - } - - public BBOStream(ViewStreamDatabase vsd) { + public BBOStream(IMarketDataClient mdClient, ViewStreamDatabase vsd) { bts.Log.Gen.info("BBOStream.cons"); _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + _md_client = mdClient; //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd); //mySetRealSource(rs); } protected override void myOnFirstSubscriber() { _vsd.addWorker(this); - _md_client.books().Subscribe(_on_books); + _md_client.Books().Subscribe(_on_books); _vsd?.addWorker(this); - _md_client?.books().Subscribe(_on_books); + _md_client?.Books().Subscribe(_on_books); } private void _log_updates(TableUpdate update) { @@ -165,13 +161,12 @@ namespace bts.BlueOrb { if (_cached_state == null) return; BlueTable new_state = new BlueTable(_bbo_schema); - foreach (var kv in _cached_state) { - if (!kv.Value.security.HasFuture) + foreach (var (skey, bt) in _cached_state) { + if (!bt.security.HasFuture) { continue; - SecurityKey skey = kv.Key; - var sec = kv.Value.security; + } + var sec = bt.security; int id = Convert.ToInt32(skey); - BookTuple bt = kv.Value; var venue = sec.Exchange; IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size, bt.book.asks[0].price, bt.book.asks[0].size); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs index 03ce1f0c6..ba32e4f7c 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs @@ -1,29 +1,31 @@ namespace bts.BlueOrb{ - using BCG.Utilities.General; using BCG; + using BCG.Utilities.General; - using bts.md; + using bts.api.core; + using Bts.DataStreamCore; + using Bts.PubSub; + + using Spark.Api.MarketData; using Spark.Core.Reactive; using System; using System.Collections.Generic; + using System.Diagnostics; using System.Linq; using System.Reactive.Linq; - using bts.api.core; - using Spark.Api.MarketData; - using SecurityKey = System.UInt32; - using System.Diagnostics; - - [DataStreamUserRespresentation("books")] + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"books"*/ false)] public class BookStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; - private ViewStreamDatabase _vsd; + readonly IWorkScheduler scheduler; private int _id = 0; - IReadOnlyDictionary? _cached_state = null; - MarketDataClient _md_client; + IReadOnlyDictionary? _cached_state = null; + IMarketDataClient _md_client; readonly Stopwatch _timer = new(); private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books", Tuple.Create("id", TypeCode.Int32), @@ -36,20 +38,15 @@ namespace bts.BlueOrb{ Tuple.Create("ask_sz", TypeCode.Single)); // TODO: add cons for symbol, exchange etc - public BookStream() - : this(ViewStreamDatabase.instance) { - } - - public BookStream(ViewStreamDatabase vsd) { + public BookStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) { bts.Log.Gen.info("BookStream.cons"); - _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + this.scheduler = scheduler; + _md_client = mdClient; } protected override void myOnFirstSubscriber() { - _vsd.addWorker(this); - _md_client.books().Subscribe(_on_books); - + scheduler.Add(this); + _md_client.Books().Subscribe(_on_books); } private void _log_updates(TableUpdate update) { @@ -70,11 +67,11 @@ namespace bts.BlueOrb{ } protected override void myOnLastSubscriber() { - _vsd.removeWorker(this); + scheduler.Remove(this); } - private Dictionary> _id_map = new Dictionary>(); - private int _get_id(SecurityKey skey, int depth) { + private Dictionary> _id_map = new Dictionary>(); + private int _get_id(uint skey, int depth) { var d_map = _id_map.GetOrAdd(skey, _ => new() { {depth, _id++} }); if (!d_map.TryGetValue(depth, out int rv)) { rv = _id++; @@ -90,9 +87,7 @@ namespace bts.BlueOrb{ if (_cached_state == null) return; BlueTable new_state = new BlueTable(_bbo_schema); - foreach (var kv in _cached_state) { - SecurityKey skey = kv.Key; - BookTuple bt = kv.Value; + foreach (var (skey, bt) in _cached_state) { for (int level = 0; level < bt.book.bids.Count; level++) { int id = _get_id(skey, level); IBlueRow r = new BlueRow(_bbo_schema, id, skey, bt.security.ToString(), Convert.ToUInt32(level), bt.book.bids[level].price, bt.book.bids[0].size, diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs index 3845eb2e9..be31e30f5 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs @@ -1,29 +1,30 @@ namespace bts.BlueOrb { + using BCG; using BCG.Utilities.General; using bts.api; - using bts.md; - using BCG; + using bts.api.core; + using Bts.DataStreamCore; + using Bts.PubSub; + + using Spark.Api.MarketData; using Spark.Core.Reactive; using System; - using System.Collections.Generic; - - using System.Linq; using System.Reactive.Linq; - using SecurityKey = System.UInt32; - - [DataStreamUserRespresentation("InstrumentsStream")] + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"InstrumentsStream"*/ false)] public class InstrumentsStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; - private ViewStreamDatabase _vsd; + readonly IWorkScheduler workScheduler; private int _id = 0; IReadOnlyDictionary _cached_state = null; - MarketDataClient _md_client; + IMarketDataClient _md_client; private bool _dirty = false; private readonly IBlueTableSchema _instrument_schema = BlueDataUtility.form_schema("instruments", Tuple.Create("id", TypeCode.Int32), @@ -40,19 +41,15 @@ namespace bts.BlueOrb { ); // TODO: add cons for symbol, exchange etc - public InstrumentsStream() - : this(ViewStreamDatabase.instance) { - } - - public InstrumentsStream(ViewStreamDatabase vsd) { + public InstrumentsStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler workScheduler) { bts.Log.Gen.info("InstrumentsStream.cons"); - _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + this.workScheduler = workScheduler; + _md_client = mdClient; } protected override void myOnFirstSubscriber() { - _vsd.addWorker(this); - _md_client.securities().Subscribe(_on_instruments); + workScheduler.Add(this); + _md_client.Securities().Subscribe(_on_instruments); } @@ -62,16 +59,16 @@ namespace bts.BlueOrb { } } - private void _on_instruments(IReadOnlyDictionary state) { + private void _on_instruments(TableUpdate table) { _dirty = true; //_log_updates(state); // just cacching in here, will populate the rows in _poll_scan and fire the datapacket - bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {state.Count()}"); - _cached_state = state; + bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {table.state.Count}"); + _cached_state = table.state; } protected override void myOnLastSubscriber() { - _vsd.removeWorker(this); + workScheduler.Remove(this); } private void _scan_instruments() { @@ -81,10 +78,8 @@ namespace bts.BlueOrb { if (_cached_state == null) return; BlueTable new_state = new BlueTable(_instrument_schema); - foreach (var kv in _cached_state) { - SecurityKey skey = kv.Key; + foreach (var (skey, sec) in _cached_state) { int id = Convert.ToInt32(skey); - SecurityImmutable sec = kv.Value; bts.api.SecurityType sec_type = sec.security_type(); object exp = DBNull.Value; if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option) diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs index 80d9742da..8c7b3d87c 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs @@ -1,44 +1,28 @@ namespace BCG { using BCG.Utilities.General; - using bts.api; - using bts.api.domain; - using bts.api.synthetic; - using bts.md; - using bts.spark; - using bts.utils; - - using mbts; - using mbts.co2; - using mbts.config; - using mbts.spark; - using mbts.ui; - - using Spark.Core.Data; using Spark.Core.Disposables; using Spark.Core.Reactive; using System; - using System.Collections.Concurrent; using System.Collections.Generic; - using System.Collections.ObjectModel; using System.Linq; using System.Reactive.Linq; using bts.api.core; using Spark.Api.MarketData; using SecurityKey = System.UInt32; - using System.Data; - using System.Reflection.Metadata; - [DataStreamUserRespresentation("price_stats")] + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"price_stats"*/ false)] public class PriceStatsStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; private ViewStreamDatabase _vsd; private int _id = 0; IReadOnlyDictionary _cached_state = null; - MarketDataClient _md_client; + IMarketDataClient _md_client; private bool _dirty = false; private IDisposable _sub = null; private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo", @@ -52,19 +36,15 @@ namespace BCG { IDisposable? subscription; // TODO: add cons for symbol, exchange etc - public PriceStatsStream() - : this(ViewStreamDatabase.instance) { - } - - public PriceStatsStream(ViewStreamDatabase vsd) { + public PriceStatsStream(IMarketDataClient mdClient, ViewStreamDatabase vsd) { bts.Log.Gen.info("PriceStatsStream.cons"); _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + _md_client = mdClient; } protected override void myOnFirstSubscriber() { _vsd.addWorker(this); - subscription = _md_client.price_stats().Subscribe(_on_price_stats); + subscription = _md_client.PriceStats().Subscribe(_on_price_stats); } private void _log_updates(TableUpdate update) { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs index 932a853fc..382e0d6e9 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs @@ -1,6 +1,8 @@ using System; using BCG; +using Spark.Api.MarketData; + namespace bts.BlueOrb { /* ---------------------------------------------------------------------- @@ -34,18 +36,23 @@ namespace bts.BlueOrb { // Combines the market data of spread legs into one stream, lined up nicely in a row. // this is useful for operations like finding weights, that need leg market data organized together succintly. + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(false)] public class SyntheticSpreadLegsStream : ViewStream { - public SyntheticSpreadLegsStream(string s1, string s2) : - this(s1, s2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) {} + public SyntheticSpreadLegsStream(string s1, string s2, IMarketDataClient mdClient) + : this(s1, s2, mdClient, ViewStreamDatabase.instance) { } - public SyntheticSpreadLegsStream(string s1, string s2, ViewStream bbo_stream, ViewStreamDatabase vsb) - { + public SyntheticSpreadLegsStream(string s1, string s2, IMarketDataClient mdClient, ViewStreamDatabase vsd) + : this(s1, s2, new BBOStream(mdClient, vsd)) { } + + public SyntheticSpreadLegsStream(string s1, string s2, ViewStream bbo_stream) { ViewStream vs1; ViewStream vs2; ViewStream rs; - vs1 = new FilterStream (new BBOStream (), + vs1 = new FilterStream (bbo_stream, $"sec = '{s1}'"); vs1 = new AddColumnsStream (vs1, "ljc:0"); @@ -62,11 +69,19 @@ namespace bts.BlueOrb { mySetRealSource(rs); } } + + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(false)] public class SyntheticSpreadStream : ViewStream{ - public SyntheticSpreadStream (string s1, string s2, double w1, double w2) - : this(s1, s2, w1, w2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance){} - public SyntheticSpreadStream (string s1, string s2, double w1, double w2, ViewStream bbo_stream, ViewStreamDatabase vsd){ - ViewStream rs = new SyntheticSpreadLegsStream(s1, s2, bbo_stream, vsd); + public SyntheticSpreadStream(string s1, string s2, double w1, double w2, IMarketDataClient mdClient) + : this(s1, s2, w1, w2, mdClient, ViewStreamDatabase.instance) { } + + public SyntheticSpreadStream(string s1, string s2, double w1, double w2, IMarketDataClient mdClient, ViewStreamDatabase vsd) + : this(s1, s2, w1, w2, new BBOStream(mdClient, vsd)) { } + + public SyntheticSpreadStream(string s1, string s2, double w1, double w2, ViewStream bbo_stream) { + ViewStream rs = new SyntheticSpreadLegsStream(s1, s2, bbo_stream); Func get_schema = (inp_schema) => { IBlueTableSchema rv = inp_schema; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs index 153da7e76..308db0781 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs @@ -1,9 +1,6 @@ namespace BCG { using BCG.Utilities.General; - using bts.api; - using bts.md; - using mbts.spark; using Spark.Core.Reactive; @@ -16,17 +13,16 @@ namespace BCG { using System.Reactive.Linq; using Spark.Api.MarketData; - using SecurityKey = System.UInt32; - using bts; - - [DataStreamUserRespresentation("timesales")] + // TODO: Use dependency injection to instantiate this stream, and others like it that depend on + // IMarketDataClient. Until then, hide it from DSQA. + [DataStreamUserRespresentation(/*"timesales"*/ false)] public class TimeSalesStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; private ViewStreamDatabase _vsd; private int _id = 0; //IReadOnlyDictionary _cached_state = null; - MarketDataClient _md_client; + IMarketDataClient _md_client; private bool _dirty = false; private ConcurrentQueue _cached_trades = new ConcurrentQueue(); private CircularRowCollection _circulare_table; @@ -40,22 +36,18 @@ namespace BCG { Tuple.Create("price", TypeCode.Double), Tuple.Create("qty", TypeCode.UInt32), Tuple.Create("aggressor", TypeCode.String)); - public TimeSalesStream(int buffer_size) - : this(buffer_size, ViewStreamDatabase.instance) { - } - - public TimeSalesStream(int buffer_size, ViewStreamDatabase vsd) { + public TimeSalesStream(int buffer_size, IMarketDataClient mdClient, ViewStreamDatabase vsd) { _buffer_size = buffer_size; _circulare_table = new CircularRowCollection(_schema, buffer_size); bts.Log.Gen.info("TimeSalesStream.cons"); _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + _md_client = mdClient; } protected override void myOnFirstSubscriber() { _vsd.addWorker(this); - _md_client.trades(new TradesSubscriptionFilter()).Subscribe(_on_trades); + _md_client.Trades().Subscribe(_on_trades); //_md_client.trades(null).Subscribe(_on_trades); } @@ -101,4 +93,4 @@ namespace BCG { } } } -} \ No newline at end of file +} diff --git a/src/client/spark/Model/SecurityResolverExporter.cs b/src/client/spark/Model/SecurityResolverExporter.cs index 9bbf04c69..c7fb01972 100644 --- a/src/client/spark/Model/SecurityResolverExporter.cs +++ b/src/client/spark/Model/SecurityResolverExporter.cs @@ -3,6 +3,7 @@ namespace Spark.Model { using bts; using bts.api; using bts.co2; + using bts.collections; using mbts.spark; @@ -10,13 +11,17 @@ namespace Spark.Model { using Spark.Core; using System.Composition; + using System.Reactive.Linq; class SecurityResolverExporter : VirtualSingleton { readonly LazyOnce? _resolver; [ImportingConstructor] public SecurityResolverExporter(IBus bus, MarketDataRelay md_relay) => - _resolver = new(() => new(md_relay, bus.subscribe(SparkTags.SecuritiesTag))); + _resolver = new(() => new( + md_relay, + bus.subscribe(SparkTags.SecuritiesTag) + .Select(secMds => MappedDictionary.create(secMds, s => s.security)))); [Export] public SecurityResolver resolver => diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesRowVM.cs b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesRowVM.cs index d672ec309..6e1460f51 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesRowVM.cs +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesRowVM.cs @@ -1,11 +1,9 @@ namespace bts.shell.time_and_sales { using bts.api; - using bts.api.domain; using bts.co2; using bts.utils; using mbts; - using mbts.analytics; using mbts.spark; using mbts.ui.controls; using mbts.ui.shell; @@ -27,53 +25,40 @@ .to_comparer() .to_safe(); - readonly TradeMessageImmutable _trade; - readonly GreeksTuple _greeks_tuple; + readonly TradeInfo _trade; - public TimeAndSalesRowVM( - int row_key, - SecurityMD security_md, - TradeMessageImmutable trade, - double volume, - GreeksTuple greeks_tuple - ) { + public TimeAndSalesRowVM(int row_key, TradeInfo trade) { this.row_key = row_key; - this.security_md = security_md; _trade = trade; - this.volume = volume; - _greeks_tuple = greeks_tuple; } public event Action invalidated; public int row_key { get; } - public uint trade_key => _trade.Key; + public uint trade_key => (uint)_trade.RecordId; public uint security_key => security.Id; - public SecurityMD security_md { get; } - public SecurityImmutable security => security_md.security; + public SecurityImmutable security => _trade.security; public DateTime time => - new LLDateTime((ulong)_trade.Time).to_datetime().ToLocalTime(); + _trade.Timestamp.ToLocalTime(); - public double price => _trade.price(); - public double size => _trade.Size; - public double display_size => _qty_display.to_display_value(size); - public string size_text => _qty_display.format(size); + public double price => _trade.Price; + public double size => _trade.Quantity; + public double display_size => _trade.qty_display.to_display_value(size); + public string size_text => _trade.qty_display.format(size); public AggressorSide aggressor_side => _trade.AggressorSide; public double volume { get; } // greeks - public double vol => _greeks_tuple.vol; - public double display_vol => _greeks_tuple.vol * 100; - public double driver => _greeks_tuple.driver; - public double theo => _greeks_tuple.normalized_theo; + public double vol => _trade.greeks_tuple?.vol ?? double.NaN; + public double display_vol => vol * 100; + public double driver => _trade.greeks_tuple?.driver ?? double.NaN; + public double theo => _trade.greeks_tuple?.normalized_theo ?? double.NaN; public double buy_edge => price - theo; public double sell_edge => theo - price; - public double delta => _greeks_tuple.delta; + public double delta => _trade.greeks_tuple?.delta ?? double.NaN; public double display_delta => delta * 100; - TickDisplaySettingsImmutable _qty_display => security_md.quantity_display; - public void raise_invalidated() => invalidated?.Invoke(); diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs index 78d90181d..c3b453336 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs @@ -1,11 +1,9 @@ namespace bts.shell.time_and_sales { - using bts.api; using bts.co2; using bts.collections; using bts.utils; using mbts; - using mbts.analytics; using mbts.persist; using mbts.spark; using mbts.spark.filters; @@ -467,12 +465,7 @@ int _row_id_counter; public TimeAndSalesRowVM create(TradeInfo trade_info) => - new( - _row_id_counter++, - trade_info.security_md, - trade_info.trade, - trade_info.volume, - trade_info.greeks_tuple ?? GreeksTuple.empty); + new(_row_id_counter++, trade_info); } } } diff --git a/src/client/spark_api/Admin/AdminClient.cs b/src/client/spark_api/Admin/AdminClient.cs index f335059cd..a5e6dd9be 100644 --- a/src/client/spark_api/Admin/AdminClient.cs +++ b/src/client/spark_api/Admin/AdminClient.cs @@ -3,8 +3,10 @@ namespace Spark.Api.Admin { using bts; using bts.api.admin; using bts.api.core; - using bts.utils; + using Microsoft.Extensions.Options; + + using Spark.Api.Services; using Spark.Core.Disposables; using Spark.Core.Reactive; @@ -13,33 +15,42 @@ namespace Spark.Api.Admin { using System.Linq; using System.Reactive.Linq; - public sealed class AdminClient : IDisposable { - public readonly UserTuple user; + public sealed class AdminClient : IAdminClient, IDisposable { + readonly IObservable> rawUsers; + readonly IObservable> rawAccounts; IDisposable? _dispose; - public AdminClient(AdminService service, UserTuple user) { - this.user = user; + public AdminClient(AdminService service, IOptions credentials) + : this(service, credentials.Value) { } + + public AdminClient(AdminService service, SparkApiCredentials credentials) + : this(service, credentials.User ?? string.Empty) { } + + public AdminClient(AdminService service, UserTuple user) + : this(service, user.name) { } + + public AdminClient(AdminService service, string userName) { + // this.user = user; var bootstraps = new DisposableBuilder(); + var adminDataTables = bootstraps.emplace(service.subscribei_admin_data().Table().Cache().ToObservable()); + rawAccounts = bootstraps.emplace(getAdminBasedDataTables(adminDataTables, a => a.Accounts, a => a.Id).Cache().ToObservable()); + rawUsers = bootstraps.emplace(getAdminBasedDataTables(adminDataTables, a => a.Users, u => u.Id).Cache().ToObservable()); + // accounts accounts = bootstraps.emplace( - service.subscribei_admin_data() + adminDataTables .Select(get_account_map) .Table(a => a.id) .Cache() .ToObservable()); // users - users = user.permissions switch { - UserTuple.Permissions.Admin => bootstraps.emplace( - service.subscribei_admin_data() - .Select(get_user_map) - .Table(UserTuple.key_fn) - .Cache() - .ToObservable()), - _ => Observable.Return( - TableUpdate.mk_empty(u => u.id).with_added(new[] { user })) - }; + var userTables = adminDataTables.Select(get_user_map).Table(UserTuple.key_fn); + if (userName != UserTuple.admin.name) { + userTables = userTables.Filter(u => u.name == userName); + } + users = bootstraps.emplace(userTables.Cache().ToObservable()); _dispose = bootstraps; static BTSUserImmutable? find_user(string name, AdminDataImmutable admin_data) => @@ -48,17 +59,17 @@ namespace Spark.Api.Admin { rv.UserData.HasAccountPermissions ? rv : null; - Dictionary get_user_map(Update up) => - up.all.FirstOrNull() + Dictionary get_user_map(TableUpdate up) => + up.state.Values.FirstOrNull() ?.Users ?.ToDictionary(u => u.Id, u => new UserTuple(u.Id, u.Name)) ?? new(0); - Dictionary get_account_map(Update update) { - if (!update.all.TryGetFirst(out var admin_data)) { + Dictionary get_account_map(TableUpdate update) { + if (!update.state.Values.TryGetFirst(out var admin_data)) { return new(0); } - var user_in = find_user(user.name, admin_data); + var user_in = find_user(userName, admin_data); if (!user_in.HasValue) { return new(0); } @@ -77,23 +88,36 @@ namespace Spark.Api.Admin { public IObservable> users { get; } public IObservable> accounts { get; } - public IObservable> permission_filter( - IObservable> positions, Func get_account_id) where TKey : IEquatable => - user.permissions == UserTuple.Permissions.Admin ? - positions : - positions.Filter(accounts.Select( - t => new Func(p => t.state.ContainsKey(get_account_id(p))))); - - public IObservable> permission_filter( - IObservable> positions, Func get_account_name) where TKey : IEquatable => - user.permissions == UserTuple.Permissions.Admin ? - positions : - positions.Filter(accounts.Select(t => { - var hash = t.state.Values.Select(a => a.name).ToHashSet(); - return new Func(v => hash.Contains(get_account_name(v))); - })); + IObservable> IAdminClient.Users => rawUsers; + IObservable> IAdminClient.Accounts => rawAccounts; public void Dispose() => Disposable.dispose(ref _dispose); + + static IObservable> getAdminBasedDataTables(IObservable> adminDataUpdates, Func> getItems, Func keySelector) where TKey : notnull, IEquatable { + return Observable.Create>(Subscribe); + + IDisposable Subscribe(IObserver> observer) { + var lastOut = TableUpdate.mk_empty(keySelector); + var sub = adminDataUpdates.Subscribe(OnAdminData); + return sub; + + void OnAdminData(TableUpdate update) { + var bd = lastOut.to_builder(); + bd.clear(); + if (update.state.Values.TryGetFirst(out var data)) { + var items = getItems(data); + foreach (var item in items) { + bd.add_or_update(item); + } + } + var nextOut = bd.to_table(); + if (nextOut.has_changes()) { + observer.OnNext(nextOut); + lastOut = nextOut; + } + } + } + } } } diff --git a/src/client/spark_api/Admin/EmptyAdminClient.cs b/src/client/spark_api/Admin/EmptyAdminClient.cs new file mode 100644 index 000000000..d3d5437f7 --- /dev/null +++ b/src/client/spark_api/Admin/EmptyAdminClient.cs @@ -0,0 +1,16 @@ +#nullable enable +namespace Spark.Api.Admin; + +using bts.api.admin; +using bts.api.core; + +using System; +using System.Reactive.Linq; + +public class EmptyAdminClient : IAdminClient { + public IObservable> Users => + Observable.Return(TableUpdate.mk_empty(u => u.Id)); + + public IObservable> Accounts => + Observable.Return(TableUpdate.mk_empty(a => a.Id)); +} diff --git a/src/client/spark_api/Admin/IAdminClient.cs b/src/client/spark_api/Admin/IAdminClient.cs new file mode 100644 index 000000000..8f3bd29c9 --- /dev/null +++ b/src/client/spark_api/Admin/IAdminClient.cs @@ -0,0 +1,31 @@ +#nullable enable +namespace Spark.Api.Admin; + +using bts.api.admin; +using bts.api.core; + +using System; +using System.Linq; +using System.Reactive.Linq; + +public interface IAdminClient { + IObservable> Users { get; } + IObservable> Accounts { get; } +} + +public static class AdminClientExtensions { + public static IObservable> permission_filter( + this IAdminClient adminClient, + IObservable> positions, + Func get_account_id) where TKey : IEquatable => + positions.Filter(adminClient.Accounts.Select( + t => new Func(p => t.state.ContainsKey(get_account_id(p))))); + + public static IObservable> permission_filter( + this IAdminClient adminClient, + IObservable> positions, Func get_account_name) where TKey : IEquatable => + positions.Filter(adminClient.Accounts.Select(t => { + var hash = t.state.Values.Select(a => a.Name).ToHashSet(); + return new Func(v => hash.Contains(get_account_name(v))); + })); +} diff --git a/src/client/spark_api/MarketData/IMarketDataClient.cs b/src/client/spark_api/MarketData/IMarketDataClient.cs new file mode 100644 index 000000000..aa356c876 --- /dev/null +++ b/src/client/spark_api/MarketData/IMarketDataClient.cs @@ -0,0 +1,50 @@ +#nullable enable +namespace Spark.Api.MarketData; + +using bts.api; +using bts.api.core; + +using mbts.spark; + +using System; +using System.Collections.Generic; + +public interface IMarketDataClient { + IObservable> Securities(Func? includeSecurity); + IObservable> Books(Func? includeSecurity); + IObservable> Trades(Func? includeSecurity); + IObservable> PriceStats(Func? includeSecurity); + IObservable> PriceTicker(Func? includeSecurity); +} + +public static class MarketDataClientExtensions { + public static IObservable> Securities(this IMarketDataClient marketDataClient) => + marketDataClient.Securities(null); + + public static IObservable> Securities(this IMarketDataClient marketDataClient, SecurityCriteria? securityCriteria) => + marketDataClient.Securities(securityCriteria?.is_defined() ?? false ? securityCriteria.includes : null); + + public static IObservable> Books(this IMarketDataClient marketDataClient) => + marketDataClient.Books(null); + + public static IObservable> Books(this IMarketDataClient marketDataClient, SecurityCriteria? securityCriteria) => + marketDataClient.Books(securityCriteria?.is_defined() ?? false ? securityCriteria.includes : null); + + public static IObservable> Trades(this IMarketDataClient marketDataClient) => + marketDataClient.Trades(null); + + public static IObservable> Trades(this IMarketDataClient marketDataClient, SecurityCriteria? securityCriteria) => + marketDataClient.Trades(securityCriteria?.is_defined() ?? false ? securityCriteria.includes : null); + + public static IObservable> PriceStats(this IMarketDataClient marketDataClient) => + marketDataClient.PriceStats(null); + + public static IObservable> PriceStats(this IMarketDataClient marketDataClient, SecurityCriteria? securityCriteria) => + marketDataClient.PriceStats(securityCriteria?.is_defined() ?? false ? securityCriteria.includes : null); + + public static IObservable> PriceTicker(this IMarketDataClient marketDataClient) => + marketDataClient.PriceTicker(null); + + public static IObservable> PriceTicker(this IMarketDataClient marketDataClient, SecurityCriteria? securityCriteria) => + marketDataClient.PriceTicker(securityCriteria?.is_defined() ?? false ? securityCriteria.includes : null); +} diff --git a/src/client/spark_api/MarketData/ISecurityResolver.cs b/src/client/spark_api/MarketData/ISecurityResolver.cs new file mode 100644 index 000000000..1cca2b01a --- /dev/null +++ b/src/client/spark_api/MarketData/ISecurityResolver.cs @@ -0,0 +1,15 @@ +#nullable enable +namespace Spark.Api.MarketData; + +using bts.api; +using bts.api.core; + +using System; + +public interface ISecurityResolver { + IObservable> Resolve( + IObservable> source, Func getSecurityId) where TKey : IEquatable; + + IObservable> Resolve( + IObservable> source, Func getSecurityId) where TKey : IEquatable; +} diff --git a/src/client/spark_api/MarketData/MarketDataClient.cs b/src/client/spark_api/MarketData/MarketDataClient.cs index ccc5120ae..704320583 100644 --- a/src/client/spark_api/MarketData/MarketDataClient.cs +++ b/src/client/spark_api/MarketData/MarketDataClient.cs @@ -1,245 +1,276 @@ #nullable enable -namespace bts.md { - using bts.api; - using bts.api.core; - using bts.collections; - using bts.utils; +namespace Spark.Api.Services; - using mbts; - using mbts.co2; - using mbts.spark; +using bts; +using bts.api; +using bts.api.core; +using bts.co2; +using bts.collections; - using Spark.Api.MarketData; - using Spark.Core; - using Spark.Core.Reactive; +using mbts; +using mbts.co2; +using mbts.spark; - using System; - using System.Collections.Generic; - using System.Linq; - using System.Reactive.Linq; +using Microsoft.Extensions.Options; - public class MarketDataClient { - readonly MDSource _source; - readonly TimeSpan _poll_interval; - readonly TimeSpan _slow_poll_interval; - readonly uint _num_depth_levels; +using Spark.Api.MarketData; +using Spark.Core.Disposables; +using Spark.Core.Reactive; +using Spark.Core.Services; - public MarketDataClient(MDSource source) - : this(source, 5, TimeSpan.FromSeconds(0.1), TimeSpan.FromSeconds(1)) { } +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Threading; +using System.Threading.Tasks; - public MarketDataClient( - MDSource source, - uint num_depth_levels, - TimeSpan poll_interval, - TimeSpan slow_poll_interval) { - _source = source; - _poll_interval = poll_interval; - if (slow_poll_interval < poll_interval) { - Log.Api.warn("MarketDataClient 'slow_poll_interval' smaller than 'poll_interval'"); - } - _slow_poll_interval = slow_poll_interval; - _num_depth_levels = num_depth_levels; - } +public class MarketDataClient : IMarketDataClient { + readonly ISubject mdConsumerSrc = new ReplaySubject(1); + readonly IObservable> securitiesSrc; + readonly List _resources = new(); + readonly MarketDataClientConfig config; + int idCounter; + // The Co2RelayControl manages the grpc subscription to md under the hood (in c++). + // Even though it's not directly used by this class, we store it in a member var to + // keep the subscription alive. + Co2RelayControl? relay; + volatile bool isDisposed; - public IObservable> securities( - SecurityCriteria? criteria = null) => - criteria.subscribe(_source.subscribe_securities()) - .Select(secs => secs.ToDictionary(s => s.Id)); + public MarketDataClient(ISparkConfigLoader configLoader, ISparkdLoginService loginService, IUserDataPermissions userDataPermissions, IOptions config) + : this(configLoader, loginService, userDataPermissions, config.Value) { } - public IObservable> securities_incremental(SecurityCriteria? criteria = null) => - securities_incremental(criteria is null ? null : criteria.includes); - - public IObservable> securities_incremental(Func? filter) { - var secs_src = _source.subscribe_securities().ToSelector( - sec_mds => MappedDictionary.create(sec_mds, s => s.security)); - if (filter is not null) { - secs_src = secs_src.extend( - secs => new FilteredDictionary(secs, filter)); - } - return secs_src.Table(s => s.Id); - } - - public IObservable> books(SecurityCriteria? criteria = null) => - books(criteria is null ? null : - new Func(s => criteria.includes(s))); - - public IObservable> books(Func? filter) { - // This MDSource's IObservable provides callbacks when a security is added or removed, but - // doesn't detect book changes. To avoid maintaining complicated state, create a brand new - // IObservable of book updates whenever the securities change. - var secs = _source.subscribe_securities(); - var filtered_secs = filter is null ? - secs.Select(mds => mds.Values) : - secs.Select(mds => mds.Values.Where(md => filter(md.security))); - return filtered_secs.Select(fixed_sec_market_data).FlattenTableSeries(); - - IObservable> fixed_sec_market_data( - IEnumerable security_mds) { - // Convert 'security_mds' to a list of PollableBooks. - var cached_books = security_mds - .Select(s => new PollableBook(s, _num_depth_levels)) - .ToList(); - - return Observable.Interval(_poll_interval).Scan( - TableUpdate.mk_empty(get_sec_id), - poll_books); - - uint get_sec_id(BookTuple book) => - book.security.Id; - - TableUpdate poll_books(TableUpdate table, long _) { - var bd = table.to_builder(); - foreach (var cbook in cached_books) { - if (cbook.poll_changes()) { - bd.add_or_update(cbook.to_book_tuple()); + public MarketDataClient(ISparkConfigLoader configLoader, ISparkdLoginService loginService, IUserDataPermissions userDataPermissions, MarketDataClientConfig config) { + this.config = config; + _resources.Add(mdConsumerSrc + .CombineLatest(Observable.FromAsync(() => loginService.LoginAsync().AsTask())) + .Subscribe(tup => { + var (consumer, loginRes) = tup; + Log.notnull(loginRes.Token); + var relay = new Co2RelayControl(consumer, config.RelayThrottle); + relay.preload_securities((MDRequest.Types.Flags)0xff).ignore(); + relay.set_token(loginRes.Token.Value); + this.relay = relay; + })); + securitiesSrc = mdConsumerSrc + .SelectSeries(consumer => Log.notnull(consumer.on_security) + .Buffer(config.SecurityPollInterval) + .Select(buffer => buffer.SelectMany(_ => _).Where(SecFilter)) + .Where(buffer => buffer.Any()) + .Scan( + new ConcurrentDictionary(), + (cache, new_secs) => { + foreach (var sec in new_secs) { + var sec_md = new SecurityMD(new(sec.security), sec.handle); + cache.AddOrUpdate(sec.security.Id, sec_md, (_, _) => sec_md); } - } - return bd.to_table(); + return cache; + })) + .Replay(1) + .ToObservable(); + // Log md errors + _resources.Add(securitiesSrc.Subscribe(_ => { }, e => Log.Md.error(e.ToString()))); + // Init the consumer in background. Once it's ready, md will start flowing. + Task.Run(InitConsumer).ignore(); + + bool SecFilter(SecurityAndHandle sec) => + sec.security.ContractCase != Security.ContractOneofCase.Synthetic || + userDataPermissions.Allow(sec.security.Synthetic.UserId); + + void InitConsumer() { + // Wait until config has been loaded so MDConsumer knows which symbols to load + // (otherwise it just loads defaults). + var configTask = configLoader.InitializeAsync(CancellationToken.None); + var lastLogTime = DateTime.MinValue; + var logInterval = TimeSpan.FromSeconds(1); + var stopwatch = Stopwatch.StartNew(); + for (var sw = Stopwatch.StartNew(); !configTask.IsCompleted; Thread.Sleep(100)) { + if (isDisposed) { + mdConsumerSrc.OnCompleted(); + return; + } + if (stopwatch.Elapsed > logInterval) { + Log.Api.warn($$"""[md_cli_waiting_for_config.json]{ "ts": "{{Log.FormatTimestamp()}}" }"""); + stopwatch.Restart(); } } + mdConsumerSrc.OnNext(new()); } + } - public IObservable> trades(TradesSubscriptionFilter filter) { - var sec_mds = filter.security_criteria == null ? - _source.subscribe_securities() : - filter.security_criteria.subscribe_md_map(_source.subscribe_securities()); - return TradesSource.create(_source.consumer, sec_mds, _poll_interval); + public IObservable> Securities(Func? includeSecurity) { + var src = securitiesSrc.ToSelector(secMds => MappedDictionary.create(secMds, s => s.security)); + if (includeSecurity is not null) { + src = src.extend(secs => new FilteredDictionary(secs, includeSecurity)); } + return src.Table(s => s.Id); + } - public IObservable> price_ticker( - SecurityCriteria? criteria = null) { - // This MDSource's IObservable provides callbacks when a security is added or removed, but - // doesn't detect ticker changes. To avoid maintaining complicated state, create a brand new - // IObservable of ticker updates whenever the securities change. - return criteria.subscribe_md(_source.subscribe_securities()) - .Select(fixed_sec_price_tickers) - .FlattenTableSeries(); + public IObservable> Books(Func? includeSecurity) { + // This MDSource's IObservable provides callbacks when a security is added or removed, but + // doesn't detect book changes. To avoid maintaining complicated state, create a brand new + // IObservable of book updates whenever the securities change. + var filteredSecs = includeSecurity is null ? + securitiesSrc.Select(mds => mds.Values) : + securitiesSrc.Select(mds => mds.Values.Where(md => includeSecurity(md.security))); + return filteredSecs.Select(fixed_sec_market_data).FlattenTableSeries(); - IObservable> fixed_sec_price_tickers( - IEnumerable security_mds) { - // Convert 'security_mds' to a list of PollablePriceTickers. - var cached_tickers = security_mds - .Select(s => new PollablePriceTicker(s)) - .ToList(); + IObservable> fixed_sec_market_data( + IEnumerable security_mds) { + // Convert 'security_mds' to a list of PollableBooks. + var cached_books = security_mds + .Select(s => new PollableBook(s, config.NumDepthLevels)) + .ToList(); - return Observable.Interval(_poll_interval).Scan( - TableUpdate.mk_empty(get_sec_id), - poll_tickers); + return Observable.Interval(config.HotPollInterval).Scan( + TableUpdate.mk_empty(GetSecId), + PollBooks); - uint get_sec_id(PriceTicker ticker) => - ticker.security.Id; + uint GetSecId(BookTuple book) => + book.security.Id; - TableUpdate poll_tickers(TableUpdate table, long _) { - bool is_first_update = table.state.Count == 0; - var bd = table.to_builder(); - foreach (var cticker in cached_tickers) { - if (cticker.poll_changes() || is_first_update) { - bd.add_or_update(cticker.to_ticker()); - } + TableUpdate PollBooks(TableUpdate table, long _) { + var bd = table.to_builder(); + foreach (var cbook in cached_books) { + if (cbook.poll_changes()) { + bd.add_or_update(cbook.to_book_tuple()); } - return bd.to_table(); } + return bd.to_table(); } } + } - public IObservable> price_stats( - SecurityCriteria? criteria = null) { - // This MDSource's IObservable provides callbacks when a security is added or removed, but - // doesn't detect stats changes. To avoid maintaining complicated state, create a brand new - // IObservable of stats updates whenever the securities change. - return criteria.subscribe_md(_source.subscribe_securities()) - .Select(fixed_sec_price_stats) - .FlattenTableSeries(); - - IObservable> fixed_sec_price_stats( - IEnumerable security_mds) { - // Convert 'security_mds' to a list of PollablePriceStats. - var cached_statss = security_mds - .Select(s => new PollablePriceStats(s)) - .ToList(); - - return Observable.Interval(_slow_poll_interval).Scan( - TableUpdate.mk_empty(get_sec_id), - poll_stats); - - uint get_sec_id(PriceStats stats) => - stats.security.Id; - - TableUpdate poll_stats(TableUpdate table, long _) { - bool is_first_update = table.state.Count == 0; - var bd = table.to_builder(); - foreach (var cstats in cached_statss) { - if (cstats.poll_changes() || is_first_update) { - bd.add_or_update(cstats.to_stats()); - } + public IObservable> Trades(Func? includeSecurity) { + IObservable> filteredSecs = includeSecurity is null ? + securitiesSrc : + securitiesSrc.Select(mds => new FilteredDictionary(mds, md => includeSecurity(md.security))); + return mdConsumerSrc.SelectSeries(consumer => filteredSecs.Scan( + (channels: ImmutableDictionary.Empty, active: new List(0)), + (channel_data, map) => { + var (channels, active) = channel_data; + var products = map.Values.Select(s => s.security.product()).Distinct(); + active = new List(); + foreach (var p in products) { + if (!channels.TryGetValue(p, out var c)) { + c = new ChannelConsumer( + this, + consumer.get_channel(p.Symbol, p.Type, p.Exchange), + map.Values.Where(s => s.security.product().Equals(p))); + channels = channels.Add(p, c); } - return bd.to_table(); + active.Add(c); } + return (channels, active); + }) + .SelectSeries(channel_data => { + var (channels, active) = channel_data; + return Observable.Interval(config.TradePollInterval) + .Select(_ => active.SelectMany(c => c.read_trades()).ToList()) + .Where(trades => trades.Count > 0); + })); + } + + public IObservable> PriceStats(Func? includeSecurity) { + // This MDSource's IObservable provides callbacks when a security is added or removed, but + // doesn't detect stat changes. To avoid maintaining complicated state, create a brand new + // IObservable of stat updates whenever the securities change. + var filteredSecs = includeSecurity is null ? + securitiesSrc.Select(mds => mds.Values) : + securitiesSrc.Select(mds => mds.Values.Where(md => includeSecurity(md.security))); + return filteredSecs.Select(FixedSecStats).FlattenTableSeries(); + + IObservable> FixedSecStats(IEnumerable securityMds) { + // Convert 'securityMds' to a list of PollablePriceStats. + var cachedStats = securityMds + .Select(s => new PollablePriceStats(s)) + .ToList(); + + return Observable.Interval(config.ColdPollInterval).Scan( + TableUpdate.mk_empty(GetSecId), + PollStats); + + uint GetSecId(PriceStats stats) => + stats.security.Id; + + TableUpdate PollStats(TableUpdate table, long _) { + var bd = table.to_builder(); + foreach (var cstats in cachedStats) { + if (cstats.PollChanges()) { + bd.add_or_update(cstats.ToStats()); + } + } + return bd.to_table(); } } + } - class PollablePriceTicker { - readonly SecurityMD _security_md; - ulong _bbo_poll_time = 0; - ulong _last_poll_time = 0; - NativePriceSize _bid; - NativePriceSize _ask; - NativePriceSize _last; - PriceTicker _ticker; - ulong _volume; + public IObservable> PriceTicker(Func? includeSecurity) { + // This MDSource's IObservable provides callbacks when a security is added or removed, but + // doesn't detect ticker changes. To avoid maintaining complicated state, create a brand new + // IObservable of ticker updates whenever the securities change. + var filteredSecs = includeSecurity is null ? + securitiesSrc.Select(mds => mds.Values) : + securitiesSrc.Select(mds => mds.Values.Where(md => includeSecurity(md.security))); + return filteredSecs.Select(FixedSecTickers).FlattenTableSeries(); - public PollablePriceTicker(SecurityMD security_md) => - _security_md = security_md; + IObservable> FixedSecTickers(IEnumerable securityMds) { + // Convert 'securityMds' to a list of PollablePriceTickers. + var cachedTickers = securityMds + .Select(s => new PollablePriceTicker(s)) + .ToList(); - public PriceTicker to_ticker() => - _ticker; + return Observable.Interval(config.ColdPollInterval).Scan( + TableUpdate.mk_empty(GetSecId), + PollTickers); - public bool poll_changes() { - var md = _security_md.md; - md.get_bbo(ref _bbo_poll_time, out _bid, out _ask); - md.get_last_trade(ref _last_poll_time, out _last, out _, out _volume); - return Swap.toggle_to(ref _ticker, _mk_ticker()); - } + uint GetSecId(PriceTicker ticker) => + ticker.security.Id; - PriceTicker _mk_ticker() { - var md = _security_md.md; - return new( - _security_md.security, - _security_md.quantity_display, - md.session_high, - md.session_low, - _volume, - _bid.price, - _bid.size, - _ask.price, - _ask.size, - md.fixing_price, - _last.price, - _last.size); + TableUpdate PollTickers(TableUpdate table, long _) { + var bd = table.to_builder(); + foreach (var cticker in cachedTickers) { + if (cticker.PollChanges()) { + bd.add_or_update(cticker.ToTicker()); + } + } + return bd.to_table(); } } + } - class PollablePriceStats { - readonly SecurityMD _security_md; - PriceStats _stats; + public void Dispose() { + GC.SuppressFinalize(this); + isDisposed = true; + _resources.DisposeAll(true); + } - public PollablePriceStats(SecurityMD security_md) { - _security_md = security_md; - _stats = _mk_stats(); - } + class ChannelConsumer { + readonly MarketDataClient owner; + readonly MDConsumerChannel _channel; + readonly Dictionary _security_mds = new(0); + long _cursor = 0; - public PriceStats to_stats() => - _stats; + public ChannelConsumer(MarketDataClient owner, MDConsumerChannel channel) { + this.owner = owner; + _channel = channel; + } - public bool poll_changes() => - Swap.toggle_to(ref _stats, _mk_stats()); - PriceStats _mk_stats() { - var md = _security_md.md; - return new( - _security_md.security, - md.open_price, - md.close_price, - md.settlement_price); + public ChannelConsumer(MarketDataClient owner, MDConsumerChannel channel, IEnumerable security_mds) { + this.owner = owner; + _channel = channel; + _security_mds = security_mds.ToDictionaryCrushed(s => s.id); + } + + public IEnumerable read_trades() { + foreach (var tm in _channel.read_trades(ref _cursor)) { + if (tm != null && _security_mds.TryGetValue(tm.Key, out var smd)) + yield return new TradeInfo( + smd.security, tm.Timestamp(), tm.price(), tm.Size, tm.AggressorSide, null, recordId: Interlocked.Increment(ref owner.idCounter)); } } } diff --git a/src/client/spark_api/MarketData/MarketDataClientConfig.cs b/src/client/spark_api/MarketData/MarketDataClientConfig.cs new file mode 100644 index 000000000..83fc75595 --- /dev/null +++ b/src/client/spark_api/MarketData/MarketDataClientConfig.cs @@ -0,0 +1,23 @@ +#nullable enable +namespace Spark.Api.Services; + +using System; + +public class MarketDataClientConfig { + public uint NumDepthLevels { get; set; } = 5; + public TimeSpan HotPollInterval { get; set; } = TimeSpan.FromSeconds(0.1); + public TimeSpan ColdPollInterval { get; set; } = TimeSpan.FromSeconds(1); + public TimeSpan TradePollInterval { get; set; } = TimeSpan.FromSeconds(0.4); + public TimeSpan RelayThrottle { get; set; } = TimeSpan.FromSeconds(0.1); + public TimeSpan SecurityPollInterval { get; set; } = TimeSpan.FromSeconds(1); + + public static MarketDataClientConfig Default => + new() { + NumDepthLevels = 5, + HotPollInterval = TimeSpan.FromSeconds(0.1), + ColdPollInterval = TimeSpan.FromSeconds(1), + TradePollInterval = TimeSpan.FromSeconds(0.4), + RelayThrottle = TimeSpan.FromSeconds(0.1), + SecurityPollInterval = TimeSpan.FromSeconds(1), + }; +} diff --git a/src/client/spark_api/MarketData/PassiveSecurityResolver.cs b/src/client/spark_api/MarketData/PassiveSecurityResolver.cs new file mode 100644 index 000000000..c4a5845ea --- /dev/null +++ b/src/client/spark_api/MarketData/PassiveSecurityResolver.cs @@ -0,0 +1,202 @@ +#nullable enable +namespace Spark.Api.MarketData; + +using bts.api; +using bts.api.core; + +using Spark.Core.Disposables; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Reactive.Linq; + +/// +/// Joins tabular data with a securities stream. It's "passive" in the sense that it doesn't attempt +/// to resolve security ids that aren't in the stream. For example, the original SecurityResolver +/// would check the database for expired contracts for orders/fills left in the system after exp. +/// +/// The securities stream +public class PassiveSecurityResolver : ISecurityResolver { + readonly IObservable> securities; + + public PassiveSecurityResolver(IObservable> securities) => + this.securities = securities; + + public PassiveSecurityResolver(IMarketDataClient mdClient) + : this(mdClient.Securities()) { } + + public IObservable> Resolve(IObservable> source, Func getSecurityId) where TKey : IEquatable { + return Observable.Create>(Subscribe); + + IDisposable Subscribe(IObserver> observer) { + object syncRoot = new(); + Func? keySelector = null; + IReadOnlyDictionary secMap = new Dictionary(0); + var state = ImmutableDictionary.Empty; + var securitiesSub = securities.Subscribe(OnSecurities); + var sourceSub = source.Subscribe(OnData); + return Disposable.Create(securitiesSub, sourceSub); + + void OnSecurities(TableUpdate table) { + lock (syncRoot) { + secMap = table.state; + if (keySelector is null) { + return; + } + ImmutableDictionary.Builder? bd = null; + List>? changes = null; + foreach (var (key, valueTup) in state) { + if (valueTup.Security.HasValue) { + continue; + } + var sec = GetSecurity(valueTup.Value); + if (!sec.HasValue) { + continue; + } + bd ??= state.ToBuilder(); + changes ??= new(1); + bd[key] = (sec, valueTup.Value); + changes.Add(new(valueTup, (sec, valueTup.Value))); + } + if (bd is not null) { + state = bd.ToImmutable(); + observer.OnNext(new(state.Values, null, changes!, null, KeySelectorWrapper)); + } + + TKey KeySelectorWrapper((SecurityImmutable? Security, T Value) tup) => + keySelector(tup.Value); + } + } + + void OnData(Update update) { + lock (syncRoot) { + keySelector ??= update.key_extractor; + ImmutableDictionary.Builder stateBd = state.ToBuilder(); + List> changes = new(0); + List<(SecurityImmutable? Security, T Value)> adds = new(0); + List<(SecurityImmutable? Security, T Value)> removes = new(0); + lock (syncRoot) { + foreach (var v in update.removed) { + var k = keySelector(v); + removes.Add(state[k]); + stateBd.Remove(k); + } + foreach (var (_, newVal) in update.changes) { + var k = keySelector(newVal); + var sec = GetSecurity(newVal); + changes.Add(new(state[k], (sec, newVal))); + stateBd[k] = (sec, newVal); + } + foreach (var v in update.added) { + var k = keySelector(v); + var sec = GetSecurity(v); + stateBd.Add(k, (sec, v)); + adds.Add((sec, v)); + } + if (stateBd is not null) { + state = stateBd.ToImmutable(); + observer.OnNext(new(state.Values, null, changes!, null, KeySelectorWrapper)); + } + } + + TKey KeySelectorWrapper((SecurityImmutable? Security, T Value) tup) => + keySelector(tup.Value); + } + } + + SecurityImmutable? GetSecurity(T value) { + var id = getSecurityId(value); + return secMap.TryGetValue(id, out var sec) ? sec : null; + } + } + } + + public IObservable> Resolve(IObservable> source, Func getSecurityId) where TKey : IEquatable { + return Observable.Create>(Subscribe); + IDisposable Subscribe(IObserver> observer) { + object syncRoot = new(); + Func? keySelector = null; + IReadOnlyDictionary secMap = new Dictionary(0); + var state = ImmutableDictionary.Empty; + var securitiesSub = securities.Subscribe(OnSecurities); + var sourceSub = source.Subscribe(OnData); + return Disposable.Create(securitiesSub, sourceSub); + + void OnSecurities(TableUpdate table) { + lock (syncRoot) { + secMap = table.state; + if (keySelector is null) { + return; + } + ImmutableDictionary.Builder? bd = null; + List>? changes = null; + foreach (var (key, valueTup) in state) { + if (valueTup.Security.HasValue) { + continue; + } + var sec = GetSecurity(valueTup.Value); + if (!sec.HasValue) { + continue; + } + bd ??= state.ToBuilder(); + changes ??= new(1); + bd[key] = (sec, valueTup.Value); + changes.Add(new(valueTup, (sec, valueTup.Value))); + } + if (bd is not null) { + state = bd.ToImmutable(); + observer.OnNext(new(state, null, changes!, null, KeySelectorWrapper)); + } + + TKey KeySelectorWrapper((SecurityImmutable? Security, T Value) tup) => + keySelector(tup.Value); + } + } + + void OnData(TableUpdate update) { + keySelector ??= update.key_extractor; + if (keySelector is null) { + throw new InvalidOperationException("TableUpdate key_extractor is null"); + } + lock (syncRoot) { + ImmutableDictionary.Builder stateBd = state.ToBuilder(); + List> changes = new(0); + List<(SecurityImmutable? Security, T Value)> adds = new(0); + List<(SecurityImmutable? Security, T Value)> removes = new(0); + lock (syncRoot) { + foreach (var v in update.removed) { + var k = keySelector(v); + removes.Add(state[k]); + stateBd.Remove(k); + } + foreach (var (_, newVal) in update.changes) { + var k = keySelector(newVal); + var sec = GetSecurity(newVal); + changes.Add(new(state[k], (sec, newVal))); + stateBd[k] = (sec, newVal); + } + foreach (var v in update.added) { + var k = keySelector(v); + var sec = GetSecurity(v); + stateBd.Add(k, (sec, v)); + adds.Add((sec, v)); + } + if (stateBd is not null) { + state = stateBd.ToImmutable(); + observer.OnNext(new(state, null, changes!, null, KeySelectorWrapper)); + } + } + + TKey KeySelectorWrapper((SecurityImmutable? Security, T Value) tup) => + keySelector(tup.Value); + } + } + + SecurityImmutable? GetSecurity(T value) { + var id = getSecurityId(value); + return secMap.TryGetValue(id, out var sec) ? sec : null; + } + } + } +} diff --git a/src/client/spark_api/MarketData/PollablePriceStats.cs b/src/client/spark_api/MarketData/PollablePriceStats.cs new file mode 100644 index 000000000..cdad3aa92 --- /dev/null +++ b/src/client/spark_api/MarketData/PollablePriceStats.cs @@ -0,0 +1,31 @@ +#nullable enable +namespace Spark.Api.MarketData; + +using mbts; + +using Spark.Core; + +class PollablePriceStats { + readonly SecurityMD securityMd; + PriceStats stats; + + public PollablePriceStats(SecurityMD securityMd) { + this.securityMd = securityMd; + stats = mkStats(); + } + + public PriceStats ToStats() => + stats; + + public bool PollChanges() => + Swap.toggle_to(ref stats, mkStats()); + + PriceStats mkStats() { + var md = securityMd.md; + return new( + securityMd.security, + md.open_price, + md.close_price, + md.settlement_price); + } +} diff --git a/src/client/spark_api/MarketData/PollablePriceTicker.cs b/src/client/spark_api/MarketData/PollablePriceTicker.cs new file mode 100644 index 000000000..b6f838877 --- /dev/null +++ b/src/client/spark_api/MarketData/PollablePriceTicker.cs @@ -0,0 +1,48 @@ +#nullable enable +namespace Spark.Api.MarketData; + +using mbts; +using mbts.co2; + +using Spark.Core; + +class PollablePriceTicker { + readonly SecurityMD securityMd; + ulong bboPollTime = 0; + ulong lastPollTime = 0; + NativePriceSize bid; + NativePriceSize ask; + NativePriceSize last; + PriceTicker ticker; + ulong volume; + + public PollablePriceTicker(SecurityMD securityMd) => + this.securityMd = securityMd; + + public PriceTicker ToTicker() => + ticker; + + public bool PollChanges() { + var md = securityMd.md; + md.get_bbo(ref bboPollTime, out bid, out ask); + md.get_last_trade(ref lastPollTime, out last, out _, out volume); + return Swap.toggle_to(ref ticker, mkTicker()); + } + + PriceTicker mkTicker() { + var md = securityMd.md; + return new( + securityMd.security, + securityMd.quantity_display, + md.session_high, + md.session_low, + volume, + bid.price, + bid.size, + ask.price, + ask.size, + md.fixing_price, + last.price, + last.size); + } +} diff --git a/src/client/spark_api/MarketData/TradeInfo.cs b/src/client/spark_api/MarketData/TradeInfo.cs index 2f57ab8b3..26246e1af 100644 --- a/src/client/spark_api/MarketData/TradeInfo.cs +++ b/src/client/spark_api/MarketData/TradeInfo.cs @@ -6,47 +6,75 @@ using mbts.analytics; using Spark.Core; +using System; + namespace mbts.spark { public readonly struct TradeInfo : bts.api.IJsonFormatter { - public readonly SecurityMD security_md; public readonly TradeMessageImmutable trade; public readonly GreeksTuple? greeks_tuple; public readonly double volume; + public readonly int RecordId; public TradeInfo( - SecurityMD security_md, - TradeMessageImmutable trade, + SecurityImmutable security, + DateTime timestamp, + double price, + double quantity, + AggressorSide aggressorSide, + TickDisplaySettingsImmutable? quantityDisplay = null, GreeksTuple? greeks_tuple = null, - double volume = double.NaN) { - this.security_md = security_md; - this.trade = trade; + double volume = double.NaN, + int recordId = 0 + ) : this() { + this.security = security; + Timestamp = timestamp; + Price = price; + Quantity = quantity; + AggressorSide = aggressorSide; this.greeks_tuple = greeks_tuple; this.volume = volume; + qty_display = quantityDisplay ?? security.QuantityDisplay(); + RecordId = recordId; } - public SecurityImmutable security => security_md.security; - public TickDisplaySettingsImmutable qty_display => security_md.quantity_display; + public SecurityImmutable security { get; } + public DateTime Timestamp { get; } + public double Price { get; } + public double Quantity { get; } + public AggressorSide AggressorSide { get; } + public TickDisplaySettingsImmutable qty_display { get; } public static System.Object ToJsonObj(TradeInfo obj) => new { security = obj.security.Exchange.ToString() + "." + obj.security.ToString(), price = obj.trade.price(), timestamp = obj.trade.Time, quantity = obj.volume, - agg_side = obj.trade.AggressorSide.ToString() + agg_side = obj.trade.AggressorSide.ToString(), + rec_id = obj.RecordId, }; } public static class TradeInfoExtensions { public static TradeInfo with( this in TradeInfo trade_info, - SecurityMD? security_md = null, - TradeMessageImmutable? trade = null, + SecurityImmutable? security = null, + DateTime? timestamp = null, + double? price = null, + double? quantity = null, + AggressorSide? aggressorSide = null, Optional greeks_tuple = default, - double? volume = null) => + double? volume = null, + TickDisplaySettingsImmutable? qty_display = null, + int? recordId = null) => new( - security_md ?? trade_info.security_md, - trade ?? trade_info.trade, + security ?? trade_info.security, + timestamp ?? trade_info.Timestamp, + price ?? trade_info.Price, + quantity ?? trade_info.Quantity, + aggressorSide ?? trade_info.AggressorSide, + qty_display ?? trade_info.qty_display, greeks_tuple.value_or(trade_info.greeks_tuple), - volume ?? trade_info.volume); + volume ?? trade_info.volume, + recordId ?? trade_info.RecordId); } } diff --git a/src/client/spark_api/MarketData/TradeMessageExtensions.cs b/src/client/spark_api/MarketData/TradeMessageExtensions.cs index f6669862d..54a5f9988 100644 --- a/src/client/spark_api/MarketData/TradeMessageExtensions.cs +++ b/src/client/spark_api/MarketData/TradeMessageExtensions.cs @@ -1,10 +1,18 @@ using bts.co2; +using mbts.utils; + using System; namespace mbts.spark { static class TradeMessageExtensions { + public static double price(this TradeMessage trade) => + trade.Mantissa * Math.Pow(10, trade.Exponent); + public static double price(this TradeMessageImmutable trade) => trade.Mantissa * Math.Pow(10, trade.Exponent); + + public static DateTime Timestamp(this TradeMessage trade) => + new LLDateTime((ulong)trade.Time).to_datetime(); } } diff --git a/src/client/spark_api/MarketData/TradesSource.cs b/src/client/spark_api/MarketData/TradesSource.cs index 247f70777..88393633d 100644 --- a/src/client/spark_api/MarketData/TradesSource.cs +++ b/src/client/spark_api/MarketData/TradesSource.cs @@ -1,7 +1,6 @@ namespace mbts.spark { using bts.api; using bts.api.core; - using bts.co2; using bts.collections; using bts.utils; @@ -102,7 +101,7 @@ foreach (var tm in _channel.read_trades(ref _cursor)) { if (tm != null && _security_mds.TryGetValue(tm.Key, out var smd)) yield return new TradeInfo( - smd, new TradeMessageImmutable(tm)); + smd.security, tm.Timestamp(), tm.price(), tm.Size, tm.AggressorSide); } } } diff --git a/src/client/spark_api/Orders/IOrdersClient.cs b/src/client/spark_api/Orders/IOrdersClient.cs new file mode 100644 index 000000000..8a4df63d8 --- /dev/null +++ b/src/client/spark_api/Orders/IOrdersClient.cs @@ -0,0 +1,58 @@ +#nullable enable +namespace Spark.Api.Orders; + +using bts.api.core; +using bts.api.orderman; +using bts.api.position; + +using mbts.spark; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +public interface IOrdersClient { + Task GetRegisteredDynamicOrderTypesAsync(); + Task EnterOrderAsync(OrderDefinition order, double price, double quantity); + Task DeleteOrderAsync(uint orderId); + Task SetPrecomputedTheosAsync(IEnumerable ptp_list); + IObservable> OrdersWithRoot(OrdersSubscriptionFilter? filter); + IObservable> Fills(FillsSubscriptionFilter? filter); + IObservable> Positions(); + Task UpdateOrderAsync(CustomOrderDefinition customDef, double price, double quantity, uint orderId); +} + +public static class OrdersClientExtensions { + public static async Task TryEnterOrderAsync(this IOrdersClient ordersClient, OrderDefinition order, double price, double quantity) { + try { + return await ordersClient.EnterOrderAsync(order, price, quantity); + } catch { + var r = new Response { + ErrorCode = OrderError.Undefined + }; + return r; + } + } + + public static async Task TryDeleteOrderAsync(this IOrdersClient ordersClient, uint orderId) { + try { + return await ordersClient.DeleteOrderAsync(orderId); + } catch { + var r = new Response { + ErrorCode = OrderError.Undefined + }; + return r; + } + } + + public static async Task TryUpdateOrderAsync(this IOrdersClient ordersClient, CustomOrderDefinition customDef, double price, double quantity, uint orderId) { + try { + return await ordersClient.UpdateOrderAsync(customDef, price, quantity, orderId); + } catch { + var r = new Response { + ErrorCode = OrderError.Undefined + }; + return r; + } + } +} diff --git a/src/client/spark_api/Orders/OrdersClient.cs b/src/client/spark_api/Orders/OrdersClient.cs index a99ccb6af..2dad35af1 100644 --- a/src/client/spark_api/Orders/OrdersClient.cs +++ b/src/client/spark_api/Orders/OrdersClient.cs @@ -1,7 +1,6 @@ #nullable enable namespace Spark.Api.Orders { using bts; - using bts.api; using bts.api.core; using bts.api.orderman; @@ -11,6 +10,7 @@ namespace Spark.Api.Orders { using mbts.spark; using Spark.Api.Admin; + using Spark.Api.MarketData; using Spark.Api.Reactive; using Spark.Core.Disposables; using Spark.Core.Reactive; @@ -22,7 +22,7 @@ namespace Spark.Api.Orders { using System.Reactive.Linq; using System.Threading.Tasks; - public class OrdersClient : IDisposable { + public class OrdersClient : IOrdersClient, IDisposable { readonly IObservable> _orders; readonly IObservable> _orders_with_root; readonly IObservable> _fills; @@ -34,8 +34,8 @@ namespace Spark.Api.Orders { public OrdersClient( OrderService order_service, PositionService position_service, - AdminClient admin_client, - SecurityResolver sec_resolver, + IAdminClient admin_client, + ISecurityResolver sec_resolver, TimeSpan? interval = null ) { _order_service = order_service; @@ -95,23 +95,23 @@ namespace Spark.Api.Orders { .ToObservable()); var manual_fill_tuples_account_join = manual_fills_securities.LeftJoin( - admin_client.accounts, + admin_client.Accounts, mf => mf.val.RecordId, - a => a.id, + a => a.Id, mf => mf.val.AccountId, - (mf, mf_id, a, a_id) => new ManualFillTuple(mf.val, mf.sec, a, null), + (mf, mf_id, a, a_id) => new ManualFillTuple(mf.val, mf.sec, new(a.Id, a.Name), null), (mf, mf_id, a_id) => new ManualFillTuple(mf.val, mf.sec, null, null), mf => mf.id); _manual_fills = bootstraps.emplace( manual_fill_tuples_account_join .LeftJoin( - admin_client.users, + admin_client.Users, mf => mf.id, - u => u.id, + u => u.Id, mf => mf.manual_fill.UserId, (mf, mf_id, u, u_id) => new ManualFillTuple( - mf.manual_fill, mf.security, mf.account, u), + mf.manual_fill, mf.security, mf.account, new(u.Id, u.Name)), (mf, mf_id, u_id) => mf, mf => mf.id) .Cache() @@ -119,25 +119,18 @@ namespace Spark.Api.Orders { _dispose = bootstraps; } - public async Task TryEnterOrderAsync( + public async Task EnterOrderAsync( OrderDefinition Order, double price, double quantity ) { - try { - Result r = await _order_service.create_order_async(new Create { - Quantity = (uint)Math.Floor(quantity), - FractionalQuantity = new FractionalQuantity { Fraction = quantity - Math.Floor(quantity) }, - Price = price, - NewOrder = Order + Result r = await _order_service.create_order_async(new Create { + Quantity = (uint)Math.Floor(quantity), + FractionalQuantity = new FractionalQuantity { Fraction = quantity - Math.Floor(quantity) }, + Price = price, + NewOrder = Order - }); - return r.value; - } catch { - var r = new Response { - ErrorCode = OrderError.Undefined - }; - return r; - } + }); + return r.value; } public async Task TrySetPrecomputedTheosAsync( List ptp_list @@ -151,44 +144,31 @@ namespace Spark.Api.Orders { }; return r2; } + Task IOrdersClient.SetPrecomputedTheosAsync(IEnumerable ptp_list) => + TrySetPrecomputedTheosAsync(ptp_list as List ?? new(ptp_list)); public async Task GetRegisteredDynamicOrderTypesAsync(){ Result r = await _order_service.get_registered_dynamic_order_types_async(new bts.api.VoidMessage()); return r.value; } - public async Task TryUpdateOrderAsync( + public async Task UpdateOrderAsync( CustomOrderDefinition custom, double price, double quantity, uint order_id ) { - try { - Result r = await _order_service.update_order_async(new Update { - Quantity = (uint)Math.Floor(quantity), - FractionalQuantity = new FractionalQuantity { Fraction = quantity - Math.Floor(quantity) }, - Price = price, - OrderKey = order_id, - Custom = custom - }); - return r.value; - } catch { - var r = new Response { - ErrorCode = OrderError.Undefined - }; - return r; - } + Result r = await _order_service.update_order_async(new Update { + Quantity = (uint)Math.Floor(quantity), + FractionalQuantity = new FractionalQuantity { Fraction = quantity - Math.Floor(quantity) }, + Price = price, + OrderKey = order_id, + Custom = custom + }); + return r.value; } - public async Task TryDeleteOrderAsync(uint order_id - ) { - try { - var r = await _order_service.cancel_order_async(new Cancel { - OrderKey = order_id - }); - return r.value; - } catch { - var r = new Response { - ErrorCode = OrderError.Undefined - }; - return r; - } + public async Task DeleteOrderAsync(uint order_id) { + var r = await _order_service.cancel_order_async(new Cancel { + OrderKey = order_id + }); + return r.value; } public IObservable> orders( OrdersSubscriptionFilter? filter = null) => @@ -203,9 +183,15 @@ namespace Spark.Api.Orders { public IObservable> Positions() => positions; + IObservable> IOrdersClient.OrdersWithRoot(OrdersSubscriptionFilter? filter) => + orders_with_root(filter); + public IObservable> fills( FillsSubscriptionFilter? filter = null) => filter is null ? _fills : _fills.Filter(filter.includes); + + IObservable> IOrdersClient.Fills(FillsSubscriptionFilter? filter) => + fills(filter); public IObservable> manual_fills( ManualFillsSubscriptionFilter? filter = null) => diff --git a/src/client/spark_api/Reactive/SecurityResolver.cs b/src/client/spark_api/Reactive/SecurityResolver.cs index 08ca89253..8386a80d4 100644 --- a/src/client/spark_api/Reactive/SecurityResolver.cs +++ b/src/client/spark_api/Reactive/SecurityResolver.cs @@ -2,10 +2,10 @@ using bts.api; using bts.api.core; using bts.co2; +using bts.collections; using bts.md; -using mbts; - +using Spark.Api.MarketData; using Spark.Core.Disposables; using System; @@ -20,32 +20,32 @@ using System.Threading.Tasks; using security_key = System.UInt32; namespace Spark.Api.Reactive { - public class SecurityResolver : IDisposable { + public class SecurityResolver : ISecurityResolver, IDisposable { readonly MarketDataRelay _md; IDisposable _dispose; - IReadOnlyDictionary _live_securities; + IReadOnlyDictionary _live_securities; readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); readonly Subject> _cache_subject = new Subject>(); public SecurityResolver(MarketDataRelay md, MDSource securities) - : this(md, securities.subscribe_securities()) { } + : this(md, securities.subscribe_securities().Select(secs => MappedDictionary.create(secs, s => s.security))) { } public SecurityResolver( - MarketDataRelay md, IObservable> securities) { + MarketDataRelay md, IObservable> securities) { _md = md; on_securities = securities; _dispose = securities.Subscribe(dict => _live_securities = dict); _cache_subject.OnNext(_cache); } - IObservable> on_securities; + IObservable> on_securities; - SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary live = null) { + SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary live = null) { // Try live securities first live = live ?? _live_securities; if (live != null) { - if (live.TryGetValue(key, out SecurityMD value)) { - return value.security; + if (live.TryGetValue(key, out var value)) { + return value; } } @@ -57,7 +57,7 @@ namespace Spark.Api.Reactive { return null; } - async Task get_security(security_key key, IReadOnlyDictionary live) { + async Task get_security(security_key key, IReadOnlyDictionary live) { var local_sec = try_get_local(key, live); if (local_sec.HasValue) { return local_sec; @@ -80,7 +80,7 @@ namespace Spark.Api.Reactive { return null; } - async Task> get_securities(IEnumerable keys, IReadOnlyDictionary live) { + async Task> get_securities(IEnumerable keys, IReadOnlyDictionary live) { var result = new Dictionary(); var missing = new SecurityKeyList(); foreach (var key in keys) { @@ -133,12 +133,12 @@ namespace Spark.Api.Reactive { } SecurityImmutable try_cache_security(SecurityImmutable s) { - if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) { + if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out var live)) { _cache.TryAdd(s.Id, s); _cache_subject.OnNext(_cache); return s; } - return live.security; + return live; } /////////////// @@ -176,6 +176,9 @@ namespace Spark.Api.Reactive { state.Values, added, changes, removed, key_extractor); } + IObservable> ISecurityResolver.Resolve(IObservable> source, Func getSecurityId) => + join(source, getSecurityId); + public IObservable join( IObservable source, Func skeyfn, @@ -196,7 +199,7 @@ namespace Spark.Api.Reactive { var changed = new List>(); foreach (var entry in state.Values) { if (entry.s == null) { - var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary); + var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary); if (security.HasValue) { changed.Add( new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); @@ -283,6 +286,9 @@ namespace Spark.Api.Reactive { t => (t.state.Values, t.added, t.changes, t.removed, t.key_extractor), (state, added, changes, removed, key) => new TableUpdate<(SecurityImmutable?, T), TKey>( state, added, changes, removed, key)); + + IObservable> ISecurityResolver.Resolve(IObservable> source, Func getSecurityId) => + join_table(source, getSecurityId); #endregion } @@ -309,13 +315,13 @@ namespace Spark.Api.Reactive { .SelectKeyValue(tup => result_selector(tup.val, tup.sec), TResult.key_fn); public static IObservable> - join(this IObservable> source, SecurityResolver store, Func skeyfn) where TKey : IEquatable { - return store.join(source, skeyfn); + join(this IObservable> source, ISecurityResolver store, Func skeyfn) where TKey : IEquatable { + return store.Resolve(source, skeyfn); } public static IObservable> - join_table(this IObservable> source, SecurityResolver store, Func skeyfn) where TKey : IEquatable { - return store.join_table(source, skeyfn); + join_table(this IObservable> source, ISecurityResolver store, Func skeyfn) where TKey : IEquatable { + return store.Resolve(source, skeyfn); } } } diff --git a/src/client/spark_api/Services/CachedServiceProvider.cs b/src/client/spark_api/Services/CachedServiceProvider.cs new file mode 100644 index 000000000..a6316a01d --- /dev/null +++ b/src/client/spark_api/Services/CachedServiceProvider.cs @@ -0,0 +1,33 @@ +#nullable enable +namespace Spark.Api.Services; + +using bts.api.core; + +using System; +using System.Collections.Generic; + +/// +/// A wrapper for that creates keyed services +/// lazily and caches them so they're only created once. +/// +public class CachedServiceProvider : Api.IServiceProvider { + readonly object syncRoot = new(); + readonly Api.IServiceProvider serviceProvider; + readonly Dictionary> cachedServices = new(); + + public CachedServiceProvider(Api.IServiceProvider serviceProvider) => + this.serviceProvider = serviceProvider; + + public T GetService(string? qualifier) where T : class, IApiClient { + Lazy? service = null; + lock (syncRoot) { + service = cachedServices.GetOrAdd(new(typeof(T), qualifier), ServiceFactory); + } + return (T)service.Value; + + Lazy ServiceFactory(ServiceKey key) => + new(() => serviceProvider.GetService(key.Qualifier)); + } + + readonly record struct ServiceKey(Type Type, string? Qualifier); +} diff --git a/src/client/spark_api/Services/SparkApiServices.cs b/src/client/spark_api/Services/SparkApiServices.cs index 85b26452c..58b00820a 100644 --- a/src/client/spark_api/Services/SparkApiServices.cs +++ b/src/client/spark_api/Services/SparkApiServices.cs @@ -10,9 +10,9 @@ using bts.md; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Spark.Api.Admin; +using Spark.Api.MarketData; using Spark.Api.Orders; using Spark.Api.Reactive; using Spark.Core.Services; @@ -44,25 +44,32 @@ public static class SparkApiServices { .AddSingleton(sp => sp.GetRequiredService()) .AddSingleton() .AddSingleton(sp => sp.GetRequiredService()) - .AddHostedService(sp => sp.GetRequiredService()) - .AddSingleton(); - if (configurationSection.GetValue("EnableOrders", true)) { + .AddHostedService(sp => sp.GetRequiredService()); + bool isMarketDataEnabled = configurationSection.GetValue("EnableMarketData", true); + bool isOrdersEnabled = configurationSection.GetValue("EnableOrders", true); + if (isMarketDataEnabled) { + services.AddSparkdService() + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton() + .AddSingleton() + .AddSingleton(); + } + if (isOrdersEnabled) { + if (!isMarketDataEnabled) { + services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); + } services.AddSparkdService(); - services.TryAddSingleton(); - services.AddSingleton(); + services.AddSingleton(); } if (configurationSection.GetValue("EnablePosition", true)) { services.AddSparkdService(); } - if (configurationSection.GetValue("EnableMarketData", true)) { - services.AddSparkdService() - .AddSingleton() - .AddSingleton() - .AddSingleton(); - } if (configurationSection.GetValue("EnableAdmin", true)) { - services.AddSparkdService(); - services.AddSparkdService(); + services.AddSparkdService() + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); } return services; } diff --git a/src/client/spark_api/api/ServiceContext.cs b/src/client/spark_api/api/ServiceContext.cs index 8f38f0c21..9e37f34e9 100644 --- a/src/client/spark_api/api/ServiceContext.cs +++ b/src/client/spark_api/api/ServiceContext.cs @@ -94,7 +94,7 @@ namespace bts.api.core { if (result.value.IsValid) { _token = result.value.Token; user = new(result.value.UserId, username); - return new() { reason = result.value.Reason, success = true, user = user }; + return new() { reason = result.value.Reason, success = true, user = user, Token = result.value.Token }; } else { return new() { reason = result.value.Reason }; } @@ -143,5 +143,6 @@ namespace bts.api.core { public bool success { get; init; } public string? reason { get; init; } public UserTuple? user { get; init; } + public AuthTokenImmutable? Token { get; init; } } } diff --git a/src/client/spark_api_server/BookDumper.cs b/src/client/spark_api_server/BookDumper.cs index 482d7eaeb..703623860 100644 --- a/src/client/spark_api_server/BookDumper.cs +++ b/src/client/spark_api_server/BookDumper.cs @@ -1,101 +1,189 @@ -namespace spark_api_server { - using bts; - using bts.api.core; - using bts.md; +namespace Spark.Api.Server; - using mbts; - using mbts.spark; +using bts; +using bts.api; +using bts.api.core; +using bts.md; - using Spark.Api.MarketData; - using Spark.Api.Server; - using Spark.Core; - using Spark.Core.Reactive; +using Bts.DataStreamCore.Services; +using Bts.Extensions.Hosting; +using Bts.PubSub.Services; - using System; - using System.Collections.Generic; - using System.Linq; - using System.Reactive.Linq; - using System.Text; - using System.Threading.Tasks; +using mbts; +using mbts.spark; - static class BookDumper { - public static async Task run(DumpBooksOptions opts) { - Log.notnull(opts.user); - Log.notnull(opts.password); - var service_ctx = ServiceContext.create(opts.sparkd); - if (!await service_ctx.try_login(opts.user, opts.password)) { - return 1; - } - var md_source = new MDSource(TimeSpan.FromSeconds(0.5)); - var md_client = new MarketDataClient(md_source); - var name_set = opts.securities.empty_if_null().ToHashSet(); - var books_src = md_client.books(s => name_set.Contains(s.ToString()!)) - .Cache().ToObservable(); - var sub = books_src.Subscribe(on_books); - await books_src.LastOrDefaultAsync(); - return 0; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; - void on_books(TableUpdate books) { - var updated_books = books.added.Concat(books.changed); - var desc = "\n".join(updated_books.Select(fmt_book)); - Console.WriteLine(desc); +using Spark.Api.MarketData; +using Spark.Api.Services; +using Spark.Core; +using Spark.Core.Disposables; +using Spark.Core.Reactive; +using Spark.Core.Services; - string fmt_book(BookTuple book_tuple) { - var sb = new StringBuilder(); - // Write the 2 line header (the security, followed by a separator line) - sb.AppendLine(book_tuple.security.ToString()); - var total_width = opts.quantity_width * 2 + opts.price_width + 2; - sb.AppendLine(new string('*', total_width)); +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; - // Format the book levels like the ladder. For example (w/ 2 price levels): - // ask_qty ask_price - // ask_qty ask_price - // ------------------------- - // bid_price bid_qty - // bid_price bid_qty - var bk = book_tuple.book; - var ask_lines = bk.asks - .TakeWhile((l, i) => i < opts.num_levels && l.size > 0) - .Reverse() - .Select(fmt_ask); - var bid_lines = bk.bids - .TakeWhile((l, i) => i < opts.num_levels && l.size > 0) - .Select(fmt_bid); - foreach (var l in ask_lines) { - sb.AppendLine(l); - } +public class BookDumper { + public static async Task Run(DumpBooksOptions opts) { + var args = Environment.GetCommandLineArgs(); + var host = HostBuilderExtensions.CreateBtsBuilder(args) + .ConfigureServices(ConfigureServices) + .Build(); + try { + await host.RunAsync(); + } catch (Exception ex) { + Console.Error.Write($"BookDumper crashed: {ex}"); + return 1; + } + return 0; + } - // write line to separate bids and asks (the length is the sum of col widths: - // qty + 1 + price + 1 + qty (the 1's represent space between cols) - sb.AppendLine(new string('-', total_width)); + public static void ConfigureServices(HostBuilderContext context, IServiceCollection services) { + var config = context.Configuration; + services + .AddLogging(ConfigureLogging) + .AddSpark(config.GetSection("Spark")) + .AddSparkApi(config.GetSection("SparkApi")) + .AddPubSubClient(config.GetSection("PubSub")) + .AddViewStreamDatabase() + .AddMasterWorker(config.GetSection("MasterWorker")) + .Configure(config.GetSection("SecurityDumper")) + .AddSingleton() + .AddHostedService(sp => sp.GetRequiredService()); - // Each bid level begins at the price column. Pad accordingly. - var bid_padding = new string(' ', opts.quantity_width + 1); - foreach (var l in bid_lines) { - sb.AppendLine(bid_padding + l); - } - sb.AppendLine(); - return sb.ToString(); + static void ConfigureLogging(ILoggingBuilder loggingBuilder) => + loggingBuilder.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) + .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) + .AddFilter("NToastNotify", Microsoft.Extensions.Logging.LogLevel.Warning) + .AddProvider(new SparkLoggerProvider()) + .AddSimpleConsole(opts => { + opts.SingleLine = true; + opts.TimestampFormat = "yyyy-MM-dd HH:mm:ss.ffffff|"; + }) + .AddDebug(); + } +} - string fmt_ask(PriceSize ps) { - var qty_text = fmt_qty(ps).PadRight(opts.quantity_width); - var price_text = fmt_price(ps).PadRight(opts.price_width); - return $"{qty_text} {price_text}"; - } +class BookDumperService : IHostedService { + readonly BookDumperConfig config; + readonly IMarketDataClient mdClient; + readonly ILogger logger; + IDisposable? subscription; - string fmt_bid(PriceSize ps) { - var price_text = fmt_price(ps).PadRight(opts.price_width); - var qty_text = fmt_qty(ps).PadRight(opts.quantity_width); - return $"{price_text} {qty_text}"; - } + public BookDumperService(IOptions config, IMarketDataClient mdClient, ILogger logger) + : this(config.Value, mdClient, logger) { } - string fmt_price(PriceSize ps) => - book_tuple.security.spec_or_default().format(ps.price); + public BookDumperService(BookDumperConfig config, IMarketDataClient mdClient, ILogger logger) { + this.config = config; + this.mdClient = mdClient; + this.logger = logger; + } - string fmt_qty(PriceSize ps) => - book_tuple.quantity_display.format((double)ps.size); + public Task StartAsync(CancellationToken cancellationToken) { + logger.LogInformation("Starting"); + Log.isnull(subscription); + var venues = new List(); + var venueStr = config.Venue?.ToString() ?? string.Empty; + if (Enum.TryParse(venueStr, true, out Venue venue) || Enum.TryParse(venueStr, true, out venue)) { + venues.Add(venue); + } + var secTypes = new List(); + var secTypeStr = config.SecurityType?.ToString() ?? string.Empty; + if (Enum.TryParse(secTypeStr, true, out SecurityType secType) || Enum.TryParse(secTypeStr, true, out secType)) { + secTypes.Add(secType); + } + var assets = new List(); + if (!string.IsNullOrEmpty(config.Asset)) { + assets.Add(config.Asset); + } + var names = new List(); + if (!string.IsNullOrEmpty(config.SecurityName)) { + names.Add(config.SecurityName); + } + var criteria = new SecurityCriteria( + new SecurityCriteriaClause[] { + new(exchanges: venues.ToArray(), security_types: secTypes.ToArray(), symbols: assets.ToArray(), names: names.ToArray()) + }); + logger.LogDebug("Subscribing to books. {criteria}", criteria.ToString()); + subscription = mdClient.Books(criteria).Subscribe(OnBooks); + return Task.CompletedTask; + + void OnBooks(TableUpdate books) { + logger.LogDebug("BookDumper OnBooks. NumAdded={numAdded}, NumTotal={numTotal}", books.added.Count(), books.state.Count); + var updated_books = books.added.Concat(books.changed); + var desc = "\n".join(updated_books.Select(FmtBook)); + Console.WriteLine(desc); + + string FmtBook(BookTuple book_tuple) { + var sb = new StringBuilder(); + // Write the 2 line header (the security, followed by a separator line) + sb.AppendLine(book_tuple.security.ToString()); + var total_width = config.QuantityWidth * 2 + config.PriceWidth + 2; + sb.AppendLine(new string('*', total_width)); + + // Format the book levels like the ladder. For example (w/ 2 price levels): + // ask_qty ask_price + // ask_qty ask_price + // ------------------------- + // bid_price bid_qty + // bid_price bid_qty + var bk = book_tuple.book; + var ask_lines = bk.asks + .TakeWhile((l, i) => i < config.NumLevels && l.size > 0) + .Reverse() + .Select(fmt_ask); + var bid_lines = bk.bids + .TakeWhile((l, i) => i < config.NumLevels && l.size > 0) + .Select(fmt_bid); + foreach (var l in ask_lines) { + sb.AppendLine(l); } + + // write line to separate bids and asks (the length is the sum of col widths: + // qty + 1 + price + 1 + qty (the 1's represent space between cols) + sb.AppendLine(new string('-', total_width)); + + // Each bid level begins at the price column. Pad accordingly. + var bid_padding = new string(' ', config.QuantityWidth + 1); + foreach (var l in bid_lines) { + sb.AppendLine(bid_padding + l); + } + sb.AppendLine(); + return sb.ToString(); + + string fmt_ask(PriceSize ps) { + var qty_text = fmt_qty(ps).PadRight(config.QuantityWidth); + var price_text = fmt_price(ps).PadRight(config.PriceWidth); + return $"{qty_text} {price_text}"; + } + + string fmt_bid(PriceSize ps) { + var price_text = fmt_price(ps).PadRight(config.PriceWidth); + var qty_text = fmt_qty(ps).PadRight(config.QuantityWidth); + return $"{price_text} {qty_text}"; + } + + string fmt_price(PriceSize ps) => + book_tuple.security.spec_or_default().format(ps.price); + + string fmt_qty(PriceSize ps) => + book_tuple.quantity_display.format((double)ps.size); } } } + + public Task StopAsync(CancellationToken cancellationToken) { + logger.LogInformation("Stopping"); + Disposable.dispose(ref subscription); + return Task.CompletedTask; + } } diff --git a/src/client/spark_api_server/BookDumperConfig.cs b/src/client/spark_api_server/BookDumperConfig.cs new file mode 100644 index 000000000..2de3a6474 --- /dev/null +++ b/src/client/spark_api_server/BookDumperConfig.cs @@ -0,0 +1,13 @@ +namespace Spark.Api.Server; + +public class BookDumperConfig { + public string? Venue { get; set; } + public string? SecurityType { get; set; } + public string? Asset { get; set; } + public string? SecurityName { get; set; } + public bool Verbose { get; set; } + + public int PriceWidth { get; set; } = 14; + public int QuantityWidth { get; set; } = 12; + public int NumLevels { get; set; } = 5; +} diff --git a/src/client/spark_api_server/Configurations/spark_api_server.json b/src/client/spark_api_server/Configurations/spark_api_server.json deleted file mode 100644 index f8b799711..000000000 --- a/src/client/spark_api_server/Configurations/spark_api_server.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "user": "bts", - "pubsub": { - "milliseconds_between_sub_info_updates": 1000, - "milliseconds_between_time_slice_updates": 5000, - "milliseconds_between_no_buffer_space_retries": 100, - "retries_on_no_buffer_space": 1, - "do_it_all_seconds_between_logs": 0, - "heartbeat_interval": 15000, - "missing_heartbeats_before_gc": 50, - "use_managed_sockets": true, - "state_update_period": 5000, - "statistics_update_period": 5000, - "disposed_connection_check_period": 1000, - "message_processing_warning_threshold": 1000, - "server_port": 12345, - "publisher_routes": [ - { - "name": "default", - "server_address": "bts-cloud://localhost:12345", - "tag": ".*" - } - ], - "subscriber_routes": [ - { - "name": "default", - "server_address": "bts-cloud://localhost:12345", - "tag": ".*" - } - ], - "max_packet_size": 500000000, - "blue_comm_must_die": true, - "crushable_tags": [], - "tag_type_to_assembly_map": [ - { - "handler_class": "BCG.PubSubServerTagHandler", - "assembly": "DataStreamCore.dll" - } - ], - "server_queue": "PubSubQueue", - "port_range": { - "first": 30400, - "last": 31700 - }, - "bind_to_random_port": false, - "debug_tags": "^NOTAGS$", - "latency_report_threshold_ms": 1500, - "use_substations": false, - "use_dynamic_clients": false, - "is_substation": false, - "internally_publish_sub_info": true, - "use_managed_clients_in_native_code": false, - "compressed_tags": "^NOTAGS$", - "tags_to_report_compression_ratio": "^NOTAGS$", - "disable_msmq": false, - "use_prepublished_server_state": false, - "ip_translation": null, - "gc_activation_threshold": 0, - "wait_period_after_gc": 1000 - } -} diff --git a/src/client/spark_api_server/Dockerfile b/src/client/spark_api_server/Dockerfile new file mode 100644 index 000000000..87210b36c --- /dev/null +++ b/src/client/spark_api_server/Dockerfile @@ -0,0 +1,17 @@ +FROM bluetradingsystems/linux_dotnet7 + +ARG SOURCE_DIR="build/Release/bin/net8.0/linux-x64/publish" +ARG CONF_FILE="src/client/spark_api_server/spark.conf" + +ENV DOCKERIZE_VERSION=v0.7.0 +USER root +RUN apt-get update \ + && apt-get install -y wget \ + && wget -O - https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar xzf - -C /usr/local/bin \ + && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/* + +USER bts +RUN dotnet dev-certs https --trust +COPY --chown=bts:bts "${SOURCE_DIR}" "bin" +COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/spark_api_server/conf/spark.conf" +WORKDIR "/home/bts/bin" diff --git a/src/client/spark_api_server/Program.cs b/src/client/spark_api_server/Program.cs index 95a829479..54c4f7727 100644 --- a/src/client/spark_api_server/Program.cs +++ b/src/client/spark_api_server/Program.cs @@ -1,121 +1,70 @@ -namespace spark_api_server { - using BCG; +using bts; - using Bts.PubSub; - using Bts.Websocket; +using Bts.DataStreamCore.Services; +using Bts.Extensions.Hosting; +using Bts.PubSub.Services; - using bts; - using bts.api.core; +using CommandLine; - using CommandLine; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; - using Spark.Api.Server; - using Spark.Api.Services; - using Spark.Core.Services; +using Spark.Api.Server; +using Spark.Api.Services; +using Spark.Core; +using Spark.Core.Services; +using Spark.WebSocket.Services; - using System; - using System.Reactive.Linq; - using System.Threading.Tasks; - using System.Threading; - using bts.md; - using MathNet.Numerics.Optimization; +using System; +using System.Threading.Tasks; - class Program { - [Flags] - enum Errors { - None = 0, - Listen = 1, - // Add errors as needed - } +var res = Parser.Default.ParseArguments(args); +await res.MapResult>( + Run, SecurityDumper.Run, BookDumper.Run, errs => Task.FromResult(1)); - static async Task Main(string[] args) { - var res = Parser.Default.ParseArguments(args); - return await res.MapResult>( - _run, SecurityDumper.run, BookDumper.run, errs => Task.FromResult(1)); - } - - static async Task _run(RunOptions opts) { - Console.WriteLine("Starting spark_api_server... "); - opts.MergeWithConfigFile(); - opts.MergeWith(RunOptions.Default()); - Log.notnull(opts.user); - Log.notnull(opts.password); - PubSubConfig.SetGlobalConfig(Log.notnull(opts.PubSub)); - var cfgLoader = new SparkConfigLoader(new SparkConfigLoaderOpts() { ConfigServer = opts.sparkd } ); - var serviceCtx = new ServiceContext(cfgLoader); - var loginService = new SparkdLoginService( - serviceCtx, new SparkApiCredentials() { User = opts.user, Password = opts.password }, cfgLoader); - await loginService.LoginAsync(); - PrimitiveManager.SetMarketDataClient(new MarketDataClient(new(TimeSpan.FromSeconds(0.5)))); - - var userDataPermissions = new UserDataPermissions(loginService); - var sync = new bts.websocket_core.WebsocketDataSync(serviceCtx, userDataPermissions); - var gwy = new bts.websocket_core.WebsocketConnectionGateway(); - var prms = new WebsocketServerConfig { - DefaultEndPoint = "/", - Host = opts.host, - Scheme = "http", - Port = opts.port, - KeepAliveIntervalSecs = 0.1, - }; - gwy.init(sync); - await sync.initAsync(gwy); - var errors = Errors.None; - Log.Api.info("Starting websocket server..."); - // try { - // await gwy.start_listening(prms); - // Log.Api.info("Websocket server stopped without error"); - // } catch (Exception ex) { - // Log.Api.error($"Websocket server threw ex: {ex}"); - // errors |= Errors.Listen; - // } - //REMOVE IN PRODUCTION AND UNCOMMENT ABOVE CODE - var cancellationTokenSource = new CancellationTokenSource(); - var listen = Task.Run(async () => { - Log.Api.info("Starting websocket server..."); - try { - await gwy.start_listening(prms); - Log.Api.info("Websocket server stopped without error"); - } catch (Exception ex) { - Log.Api.error($"Websocket server threw ex: {ex}"); - errors |= Errors.Listen; - } - - }); - var vsdTask = Task.Run(() => { - Log.Gen.info("Starting ViewStreamDatabase work loop"); - var cancellationToken = cancellationTokenSource.Token; - var counter = 0; - const int ERROR_LIMIT = 4; - while (!cancellationToken.IsCancellationRequested) { - try { - ViewStreamDatabase.instance.doWork(); - counter = 0; - } catch (Exception ex) { - counter++; - Log.Gen.error(ex.Message); - } - if (counter >= ERROR_LIMIT) { - throw new Exception("counter limit reached"); - } - Thread.Sleep(100); - } - }, cancellationTokenSource.Token); - // var task = Task.Run(() => { - // while (!cancellationTokenSource.IsCancellationRequested) { - // // Do something that can be interrupted - // } - // }, cancellationTokenSource.Token); - - // Console.WriteLine("Press any key to stop the server..."); - // Console.ReadKey(); - // cancellationTokenSource.Cancel(); - await Task.WhenAny(listen, vsdTask); - //UPTILL HERE - // These tasks aren't designed to finish naturally. If the code reaches here, it means one - // of them threw an exception. Cancel the other task to exit (somewhat) gracefully. - cancellationTokenSource.Cancel(); - return (int)errors; - } +static Task Run(RunOptions opts) { + const string appName = "spark_api_server"; + var builder = WebApplication.CreateBuilder(new WebApplicationOptions { + Args = Environment.GetCommandLineArgs(), + ApplicationName = appName, + }); + HostBuilderExtensions.ConfigureApp(builder.Configuration, builder.Environment.EnvironmentName, appName); + ConfigureServices(builder.Services, builder.Configuration); + var app = builder.Build(); + app.UseWebSockets(app.Configuration.GetValue("WebSocketOptions") ?? new() { KeepAliveInterval = TimeSpan.FromMinutes(2) }); + var gateway = new Lazy(app.Services.GetRequiredService); + app.Map("/", HandleRequest); + var errors = Errors.None; + Log.Api.info("Starting websocket server..."); + try { + app.Run(); + Log.Api.info("Websocket server stopped without error"); + } catch (Exception ex) { + Log.Api.error($"Websocket server threw ex: {ex}"); + errors |= Errors.Listen; } + return Task.FromResult((int)errors); + + Task HandleRequest(HttpContext context) => + gateway.Value.HandleRequestAsync(context); +} + +static void ConfigureServices(IServiceCollection services, IConfiguration config) { + services.Configure(config.GetSection("WebSocketOptions")) + .AddLogging(bd => HostBuilderExtensions.ConfigureLogging(bd, new SparkLoggerProvider())) + .AddSpark(config.GetSection("Spark")) + .AddSparkApi(config.GetSection("SparkApi")) + .AddPubSubClient(config.GetSection("PubSub")) + .AddViewStreamDatabase() + .AddMasterWorker(config.GetSection("MasterWorker")) + .AddWebSocketServer(); +} + +[Flags] +enum Errors { + None = 0, + Listen = 1, + // Add errors as needed } diff --git a/src/client/spark_api_server/SecurityDumper.cs b/src/client/spark_api_server/SecurityDumper.cs index 0ac4d2a10..477cc6604 100644 --- a/src/client/spark_api_server/SecurityDumper.cs +++ b/src/client/spark_api_server/SecurityDumper.cs @@ -1,59 +1,135 @@ -namespace spark_api_server { - using bts; - using bts.api; - using bts.api.core; - using bts.md; +namespace Spark.Api.Server; - using mbts.spark; +using bts; +using bts.api; +using bts.api.core; - using Spark.Api.Server; - using Spark.Core.Reactive; +using Bts.DataStreamCore.Services; +using Bts.Extensions.Hosting; +using Bts.PubSub.Services; - using System; - using System.Linq; - using System.Reactive.Linq; - using System.Threading.Tasks; +using mbts.spark; - static class SecurityDumper { - public static async Task run(DumpSecuritiesOptions opts) { - Log.notnull(opts.user); - Log.notnull(opts.password); - var service_ctx = ServiceContext.create(opts.sparkd); - if (!await service_ctx.try_login(opts.user, opts.password)) { - return 1; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Spark.Api.MarketData; +using Spark.Api.Services; +using Spark.Core; +using Spark.Core.Disposables; +using Spark.Core.Reactive; +using Spark.Core.Services; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Threading; +using System.Threading.Tasks; + +public class SecurityDumper { + public static async Task Run(DumpSecuritiesOptions opts) { + var args = Environment.GetCommandLineArgs(); + var host = HostBuilderExtensions.CreateBtsBuilder(args) + .ConfigureServices(ConfigureServices) + .Build(); + try { + await host.RunAsync(); + } catch (Exception ex) { + Console.Error.Write($"SecurityDumper crashed: {ex}"); + return 1; + } + return 0; + } + + public static void ConfigureServices(HostBuilderContext context, IServiceCollection services) { + var config = context.Configuration; + services + .AddLogging(ConfigureLogging) + .AddSpark(config.GetSection("Spark")) + .AddSparkApi(config.GetSection("SparkApi")) + .AddPubSubClient(config.GetSection("PubSub")) + .AddViewStreamDatabase() + .AddMasterWorker(config.GetSection("MasterWorker")) + .Configure(config.GetSection("SecurityDumper")) + .AddSingleton() + .AddHostedService(sp => sp.GetRequiredService()); + + static void ConfigureLogging(ILoggingBuilder loggingBuilder) => + loggingBuilder.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) + .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) + .AddFilter("NToastNotify", Microsoft.Extensions.Logging.LogLevel.Warning) + .AddProvider(new SparkLoggerProvider()) + .AddSimpleConsole(opts => { + opts.SingleLine = true; + opts.TimestampFormat = "yyyy-MM-dd HH:mm:ss.ffffff|"; + }) + .AddDebug(); + } +} + +class SecurityDumperService : IHostedService { + readonly SecurityDumperConfig config; + readonly IMarketDataClient mdClient; + readonly ILogger logger; + IDisposable? subscription; + + public SecurityDumperService(IOptions config, IMarketDataClient mdClient, ILogger logger) + : this(config.Value, mdClient, logger) { } + + public SecurityDumperService(SecurityDumperConfig config, IMarketDataClient mdClient, ILogger logger) { + this.config = config; + this.mdClient = mdClient; + this.logger = logger; + } + + public Task StartAsync(CancellationToken cancellationToken) { + logger.LogDebug("Starting"); + Log.isnull(subscription); + var venues = new List(); + var venueStr = config.Venue?.ToString() ?? string.Empty; + if (Enum.TryParse(venueStr, true, out Venue venue) || Enum.TryParse(venueStr, true, out venue)) { + venues.Add(venue); + } + var secTypes = new List(); + var secTypeStr = config.SecurityType?.ToString() ?? string.Empty; + if (Enum.TryParse(secTypeStr, true, out SecurityType secType) || Enum.TryParse(secTypeStr, true, out secType)) { + secTypes.Add(secType); + } + var assets = new List(); + if (!string.IsNullOrEmpty(config.Asset)) { + assets.Add(config.Asset); + } + var criteria = new SecurityCriteria( + new SecurityCriteriaClause[] { + new(exchanges: venues.ToArray(), security_types: secTypes.ToArray(), symbols: assets.ToArray()) + }); + logger.LogDebug("Subscribing to securities. {criteria}", criteria.ToString()); + subscription = mdClient.Securities(criteria).Subscribe(OnSecurities); + return Task.CompletedTask; + + void OnSecurities(TableUpdate securities) { + logger.LogDebug("SecurityDumper OnSecurities. NumAdded={numAdded}, NumTotal={numTotal}", securities.added.Count(), securities.state.Count); + foreach (var sec in securities.added) { + Console.WriteLine(Fmt(sec)); } - var md_source = new MDSource(TimeSpan.FromSeconds(0.5)); - var md_client = new MarketDataClient(md_source); - var criteria = new SecurityCriteria( - new SecurityCriteriaClause[] { - new( - exchanges: opts.venues?.ToArray(), - security_types: opts.types?.ToArray(), - symbols: opts.assets?.ToArray()) - }); - var secs_src = md_client.securities_incremental(criteria); - var sub = secs_src.Subscribe(on_securities); - var wait_src = secs_src; - if (opts.timeout.HasValue) { - wait_src = wait_src.StopAfter(opts.timeout.Value); - } - await wait_src.LastOrDefaultAsync(); - return 0; - void on_securities(TableUpdate securities) { - foreach (var sec in securities.added) { - Console.WriteLine(fmt(sec)); - } - - string fmt(SecurityImmutable sec) { - if (!opts.verbose) { - return sec.ToString()!; - } - var fmt_settings = Google.Protobuf.JsonFormatter.Settings.Default; - var formatter = new Google.Protobuf.JsonFormatter(fmt_settings); - return formatter.Format(sec.CloneToMutable()); + string Fmt(SecurityImmutable sec) { + if (!config.Verbose) { + return sec.ToString()!; } + var fmtSettings = Google.Protobuf.JsonFormatter.Settings.Default; + var formatter = new Google.Protobuf.JsonFormatter(fmtSettings); + return formatter.Format(sec.CloneToMutable()); } } } + + public Task StopAsync(CancellationToken cancellationToken) { + logger.LogInformation("Stopping"); + Disposable.dispose(ref subscription); + return Task.CompletedTask; + } } diff --git a/src/client/spark_api_server/SecurityDumperConfig.cs b/src/client/spark_api_server/SecurityDumperConfig.cs new file mode 100644 index 000000000..b9ca9c080 --- /dev/null +++ b/src/client/spark_api_server/SecurityDumperConfig.cs @@ -0,0 +1,8 @@ +namespace Spark.Api.Server; + +public class SecurityDumperConfig { + public string? Venue { get; set; } + public string? SecurityType { get; set; } + public string? Asset { get; set; } + public bool Verbose { get; set; } +} diff --git a/src/client/spark_api_server/appsettings.Development.json b/src/client/spark_api_server/appsettings.Development.json new file mode 100644 index 000000000..f95035d9c --- /dev/null +++ b/src/client/spark_api_server/appsettings.Development.json @@ -0,0 +1,15 @@ +{ + "Kestrel": { + "Endpoints": { + "Http": { + "Url": "http://localhost:8080" + }, + "Https": { + "Url": "https://localhost:8443" + } + } + }, + "Spark": { + "ConfigServer": "http://localhost:8000/config" + } +} diff --git a/src/client/spark_api_server/appsettings.Staging.json b/src/client/spark_api_server/appsettings.Staging.json new file mode 100644 index 000000000..894ba08e2 --- /dev/null +++ b/src/client/spark_api_server/appsettings.Staging.json @@ -0,0 +1,5 @@ +{ + "Spark": { + "ConfigServer": "http://sparkd:8000/config" + } +} diff --git a/src/client/spark_api_server/appsettings.json b/src/client/spark_api_server/appsettings.json new file mode 100644 index 000000000..3e02447f9 --- /dev/null +++ b/src/client/spark_api_server/appsettings.json @@ -0,0 +1,83 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "Kestrel": { + "Endpoints": { + "Http": { + "Url": "http://0.0.0.0:8080" + }, + "Https": { + "Url": "https://0.0.0.0:8443" + } + } + }, + "WebSocketOptions": { + "KeepAliveInterval": "00:02:00", + "ReceiveBufferSize": 4096 + }, + "SparkApi": { + "Credentials": { + "User": "bts", + "Password": "" + } + }, + "PubSub": { + "HeartbeatInterval": 15000, + "MissingHeartbeatsBeforeGc": 50, + "DisposedConnectionCheckPeriod": 1000, + "ServerPort": 22019, + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://127.0.0.1:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://127.0.0.1:22019", + "TagRegExp": ".*" + } + ] + }, + "TagTypeToAssemblyMap": { + "Entries": [ + { + "HandlerClass": "BCG.PubSubServerTagHandler", + "Assembly": "DataStreamCore.dll" + } + ] + }, + "TagHandlerFactory": { + "Handlers": [ + { + "TagType": "DataStream", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerStreamTagHandler" + }, + { + "TagType": "System_Sub", + "Assembly": "PubSub", + "Class": "BCG.SystemSubscriptionMapPublisher" + }, + { + "TagType": "System_Stats", + "Assembly": "PubSub", + "Class": "BCG.SystemStatsPublisher" + }, + { + "TagType": ".*", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerGenericDataHandler" + } + ] + } + } +} diff --git a/src/client/spark_api_server/spark.conf b/src/client/spark_api_server/spark.conf new file mode 100644 index 000000000..81b3e4f58 --- /dev/null +++ b/src/client/spark_api_server/spark.conf @@ -0,0 +1,15 @@ +Logging { + value { + console { + default_level: "Info" + } + syslog { + default_level: "Info" + } + file { + default_level: "Info" + } + initial_queue_size: 120001 + max_queue_growth: 4 + } +} diff --git a/src/client/spark_api_server/spark_api_server.csproj b/src/client/spark_api_server/spark_api_server.csproj index 4d0df7247..3804381b3 100644 --- a/src/client/spark_api_server/spark_api_server.csproj +++ b/src/client/spark_api_server/spark_api_server.csproj @@ -21,6 +21,14 @@ + + + + + + + + diff --git a/src/client/spark_core/Update/TableUpdate.cs b/src/client/spark_core/Update/TableUpdate.cs index 888ddcfda..055158a8d 100644 --- a/src/client/spark_core/Update/TableUpdate.cs +++ b/src/client/spark_core/Update/TableUpdate.cs @@ -18,14 +18,14 @@ namespace bts.api.core { public TableUpdate( ImmutableDictionary init, - IEnumerable added, - IEnumerable> changes, - IEnumerable removed, + IEnumerable? added, + IEnumerable>? changes, + IEnumerable? removed, Func? key_extractor) { _state = init; - this.added = added; - this.changes = changes; - this.removed = removed; + this.added = added ?? Array.Empty(); + this.changes = changes ?? Array.Empty>(); + this.removed = removed ?? Array.Empty(); this.key_extractor = key_extractor; } diff --git a/src/client/spark_core/Update/Update.cs b/src/client/spark_core/Update/Update.cs index 0259bcf24..59c00fdfb 100644 --- a/src/client/spark_core/Update/Update.cs +++ b/src/client/spark_core/Update/Update.cs @@ -12,9 +12,9 @@ namespace bts.api.core { public Update( IEnumerable all, - IEnumerable added, - IEnumerable> changes, - IEnumerable removed, + IEnumerable? added, + IEnumerable>? changes, + IEnumerable? removed, Func key_extractor) { this.added = added ?? Array.Empty(); this.removed = removed ?? Array.Empty(); diff --git a/src/client/spark_core/logging/Log.cs b/src/client/spark_core/logging/Log.cs index 6978d72bf..4592cfc5d 100644 --- a/src/client/spark_core/logging/Log.cs +++ b/src/client/spark_core/logging/Log.cs @@ -11,6 +11,8 @@ namespace bts { internal static event Action? OnSystemCrashing; + public const string TimestampFormat = "yyyy-MM-dd HH:mm:ss.ffffff"; + public static Logger Sdk => new(LogSubsystem.Sdk); public static Logger Http => new(LogSubsystem.Http); @@ -27,6 +29,12 @@ namespace bts { public static Logger Testing => new(LogSubsystem.Testing); public static Logger DataFlow => new(LogSubsystem.DataFlow); + public static string FormatTimestamp() => + DateTime.Now.ToString(TimestampFormat); + + public static string FormatTimestamp(DateTime ts) => + ts.ToString(TimestampFormat); + [Conditional("DEBUG")] public static void assert_d( bool test, diff --git a/src/client/spark_core/logging/SparkLoggerAdapter.cs b/src/client/spark_core/logging/SparkLoggerAdapter.cs new file mode 100644 index 000000000..8400c8fd2 --- /dev/null +++ b/src/client/spark_core/logging/SparkLoggerAdapter.cs @@ -0,0 +1,65 @@ +#nullable enable +namespace Spark.Core; + +using bts; + +using Microsoft.Extensions.Logging; + +using System; + +using MsLogLevel = Microsoft.Extensions.Logging.LogLevel; + +using SparkLogCategory = bts.LogSubsystem; +using SparkLogLevel = bts.LogLevel; +using SparkLogger = bts.Logger; + +class SparkLoggerAdapter : ILogger { + readonly string categoryName; + readonly MsLogLevel level; + + public SparkLoggerAdapter(string categoryName, MsLogLevel level = MsLogLevel.Debug) { + this.categoryName = categoryName; + this.level = level; + } + + public IDisposable? BeginScope(TState state) where TState : notnull => + null; + + public bool IsEnabled(MsLogLevel logLevel) => + logLevel >= level; + + public void Log(MsLogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + getSparkLogger(categoryName).log_at(toSparkLogLevel(logLevel), formatter(state, exception)); + + static SparkLogLevel toSparkLogLevel(MsLogLevel logLevel) => logLevel switch { + MsLogLevel.Trace or MsLogLevel.Debug => SparkLogLevel.Diag, + MsLogLevel.Information => SparkLogLevel.Info, + MsLogLevel.Warning => SparkLogLevel.Warn, + MsLogLevel.Error => SparkLogLevel.Error, + MsLogLevel.Critical => SparkLogLevel.Fatal, + MsLogLevel.None => SparkLogLevel.NoLog, + _ => throw new ArgumentException($"Invalid logLevel {logLevel}", nameof(logLevel)), + }; + + static SparkLogCategory toSparkCategory(string categoryName) => + Enum.TryParse(categoryName, out var subsys) ? subsys : SparkLogCategory.gen; + + static SparkLogger getSparkLogger(string categoryName) => + toSparkCategory(categoryName) switch { + SparkLogCategory.Sdk => bts.Log.Sdk, + SparkLogCategory.Http => bts.Log.Http, + SparkLogCategory.gen => bts.Log.Gen, + SparkLogCategory.Config => bts.Log.Config, + SparkLogCategory.QBus => bts.Log.QBus, + SparkLogCategory.md => bts.Log.Md, + SparkLogCategory.Api => bts.Log.Api, + SparkLogCategory.Persist => bts.Log.Persist, + SparkLogCategory.User => bts.Log.User, + SparkLogCategory.OrderMan => bts.Log.OrderMan, + SparkLogCategory.Assert => bts.Log.Assert, + SparkLogCategory.Ui => bts.Log.Ui, + SparkLogCategory.Testing => bts.Log.Testing, + SparkLogCategory.DataFlow => bts.Log.DataFlow, + _ => throw new ArgumentException($"Invalid log category '{categoryName}'", categoryName), + }; +} diff --git a/src/client/spark_core/logging/SparkLoggerProvider.cs b/src/client/spark_core/logging/SparkLoggerProvider.cs new file mode 100644 index 000000000..04f0b4ffa --- /dev/null +++ b/src/client/spark_core/logging/SparkLoggerProvider.cs @@ -0,0 +1,13 @@ +#nullable enable +namespace Spark.Core; + +using Microsoft.Extensions.Logging; + +using System; + +public sealed class SparkLoggerProvider : ILoggerProvider { + public ILogger CreateLogger(string categoryName) => + new SparkLoggerAdapter(categoryName); + + void IDisposable.Dispose() { } +} diff --git a/src/client/spark_sdk/samples/MarketDataSample.cs b/src/client/spark_sdk/samples/MarketDataSample.cs index f23d5e9a9..d805e5ee4 100644 --- a/src/client/spark_sdk/samples/MarketDataSample.cs +++ b/src/client/spark_sdk/samples/MarketDataSample.cs @@ -1,25 +1,24 @@ -using System; +using bts.api; +using bts.api.core; + +using mbts.spark; + +using Spark.Api.MarketData; + +using System; using System.Linq; using System.Collections.Generic; -using System.Collections.Concurrent; -using System.Threading; -using System.Threading.Tasks; using System.Reactive.Linq; using System.Reactive.Concurrency; -using bts.api; -using bts.api.core; -using bts.md; -using mbts.spark; - namespace bts.sdk { public class MarketDataSample : IDisposable { - readonly MarketDataClient _md; + readonly IMarketDataClient _md; readonly List _disps = new List(); - public MarketDataSample(MarketDataClient client) + public MarketDataSample(IMarketDataClient client) { _md = client; } @@ -58,7 +57,7 @@ namespace bts.sdk new(new[] { ven }, new[] { st }, new[] { symbol }) }); - _disps.Add(_md.books(criteria).Subscribe(x => + _disps.Add(_md.Books(criteria).Subscribe(x => { print_update(x.added.Concat(x.changed)); })); @@ -74,7 +73,7 @@ namespace bts.sdk var criteria = new SecurityCriteria(new SecurityCriteriaClause[] { new(new[] { ven }, new[] { st }, new[] { symbol }) }); - var md_source = _md.books(criteria) + var md_source = _md.Books(criteria) .Filter(b => b.security.security_desc() == security_desc); _disps.Add(md_source.Subscribe(x => { diff --git a/src/client/spark_sdk/samples/OrderSample.cs b/src/client/spark_sdk/samples/OrderSample.cs index c8cfd652a..d8272bd16 100644 --- a/src/client/spark_sdk/samples/OrderSample.cs +++ b/src/client/spark_sdk/samples/OrderSample.cs @@ -1,26 +1,21 @@ -using System; -using System.Linq; -using System.Collections.Generic; -using System.Collections.Concurrent; -using System.Threading; -using System.Threading.Tasks; -using System.Reactive.Linq; - -using bts.api; -using bts.md; -using bts.api.core; +using bts.api; using bts.api.orderman; -using bts.api.user; using bts.api.admin; -using bts.co2; + using Spark.Api.Orders; +using Spark.Api.MarketData; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; namespace bts.sdk { public class OrderSample : IDisposable { - readonly MarketDataClient _md; + readonly IMarketDataClient _md; readonly OrdersClient _orders_client; readonly List _disps = new List(); IReadOnlyDictionary _securities = @@ -32,7 +27,7 @@ namespace bts.sdk OrdersClient orders_client, OrderService orders, AdminService admin, - MarketDataClient md_client, + IMarketDataClient md_client, IObservable> securities_src ) { _orders_client = orders_client; diff --git a/src/client/spark_sdk/samples/SdkSampleService.cs b/src/client/spark_sdk/samples/SdkSampleService.cs index 0c036bb11..e1d6a3aca 100644 --- a/src/client/spark_sdk/samples/SdkSampleService.cs +++ b/src/client/spark_sdk/samples/SdkSampleService.cs @@ -1,4 +1,3 @@ - namespace bts.sdk { using bts.api.admin; using bts.api.core; @@ -12,7 +11,6 @@ namespace bts.sdk { using Spark.Api.Admin; using Spark.Api.Orders; - using Spark.Api.Reactive; using Spark.Api.Services; using Spark.Core.Services; using Spark.Sdk; @@ -94,13 +92,15 @@ namespace bts.sdk { Console.WriteLine("{0}", resp); } - public async Task StartAsync(CancellationToken cancellationToken) { + public Task StartAsync(CancellationToken cancellationToken) { BTS.init(); Log.Sdk.info("Welcome to the BTS Client Side SDK example"); - await loginService.LoginAsync(); var md_source = new MDSource(userDataPermissions); - var md_client = new MarketDataClient(md_source); + var md_client = new MarketDataClient( + sparkConfigLoader, + loginService, userDataPermissions, + MarketDataClientConfig.Default); using (var disp = new CompositeDisposable(md_source)) { if (sdkConfig.MarketData is not null) { var s = new MarketDataSample(md_client); @@ -121,6 +121,7 @@ namespace bts.sdk { wait_for_stop(); } + return Task.CompletedTask; } public Task StopAsync(CancellationToken cancellationToken) => diff --git a/src/client/spark_sdk/samples/UISample.cs b/src/client/spark_sdk/samples/UISample.cs index 97ae9fded..425e3c04f 100644 --- a/src/client/spark_sdk/samples/UISample.cs +++ b/src/client/spark_sdk/samples/UISample.cs @@ -1,15 +1,13 @@ -using System; -using System.Linq; +using Spark.Api.MarketData; -using bts.api; -using bts.md; +using System; namespace bts.sdk.ui { public class UISample : IDisposable { - readonly MarketDataClient _md; - public UISample(MarketDataClient client) + readonly IMarketDataClient _md; + public UISample(IMarketDataClient client) { _md = client; } diff --git a/src/client/spark_websocket/Core/IWebsocketConnectionGateway.cs b/src/client/spark_websocket/Core/IWebsocketConnectionGateway.cs index e7e699d22..7b5c5fe4b 100644 --- a/src/client/spark_websocket/Core/IWebsocketConnectionGateway.cs +++ b/src/client/spark_websocket/Core/IWebsocketConnectionGateway.cs @@ -1,8 +1,4 @@ -using Bts.Websocket; - -using System; -using System.Collections.Generic; -using System.Reflection; +using Microsoft.AspNetCore.Http; namespace bts.websocket_core { // The implementation of this interface controls all low level @@ -18,12 +14,10 @@ namespace bts.websocket_core { // will only be called once, and the provided sync will be used // to send connection, disconnection and data back to the sync public void init(IWebsocketDataSync sync); - // This function tells the class to start listening to client connections. - // All the parameters (ports to open, security parameters etc) are - // passed through the WebsocketServerParams arguments - public Task start_listening(WebsocketServerConfig server_params); // Messages are sent to the client through this public void send_to_clients( IEnumerable messages); + + Task HandleRequest(HttpContext context); } } diff --git a/src/client/spark_websocket/Core/WebsocketClientID.cs b/src/client/spark_websocket/Core/WebsocketClientID.cs index 2e80c4ea4..ca83ac1bb 100644 --- a/src/client/spark_websocket/Core/WebsocketClientID.cs +++ b/src/client/spark_websocket/Core/WebsocketClientID.cs @@ -1,3 +1,5 @@ +using bts.websocket_core.json_rpc; + using System; using System.Collections.Generic; using System.Reflection; @@ -7,5 +9,10 @@ namespace bts.websocket_core { public class WebsocketClientID : UniqueID { public WebsocketClientID() : base(Guid.NewGuid().ToString()) { } + + public WebsocketClientID(string str) : base(str) { } + + public static WebsocketClientID? Parse(JsonRpcId id) => + new(id.ToString()!); } -} \ No newline at end of file +} diff --git a/src/client/spark_websocket/Core/WebsocketConnectNotification.cs b/src/client/spark_websocket/Core/WebsocketConnectNotification.cs new file mode 100644 index 000000000..240983959 --- /dev/null +++ b/src/client/spark_websocket/Core/WebsocketConnectNotification.cs @@ -0,0 +1,11 @@ +namespace bts.websocket_core; + +public class WebsocketConnectNotification { + public WebsocketConnectNotification(WebsocketClientID id, WebsocketIncomingMessage? message = null) { + Id = id; + Message = message; + } + + public WebsocketClientID Id { get; } + public WebsocketIncomingMessage? Message { get; } +} diff --git a/src/client/spark_websocket/Core/WebsocketConnectionGateway.cs b/src/client/spark_websocket/Core/WebsocketConnectionGateway.cs index 5261d26e4..18cefef3d 100644 --- a/src/client/spark_websocket/Core/WebsocketConnectionGateway.cs +++ b/src/client/spark_websocket/Core/WebsocketConnectionGateway.cs @@ -1,12 +1,10 @@ using Bts.Websocket; +using Microsoft.AspNetCore.Http; + using System.Net.WebSockets; using System.Text; using System.Collections.Concurrent; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; namespace bts.websocket_core { public enum MessageType { @@ -328,8 +326,6 @@ namespace bts.websocket_core { } - private WebApplication? app = null; - private WebsocketServerConfig? _server_params = null; private WebSocketConnectionStore Connections { get; } = new(); private IWebsocketDataSync? _sync = null; private IWebsocketDataSync Sync { @@ -417,63 +413,6 @@ namespace bts.websocket_core { if (_WebSocketHandler.AddToQueue(id, Tuple.Create(MessageType.Close, ""))) logger.info(" added to que"); } - /// - /// Starts listening for incoming websocket connections. need to call init before this - /// - /// The parameters for the websocket server includind the URL and endpoint, - public Task start_listening(WebsocketServerConfig server_params) { - // start listening, assert that init has been called already - bts.Log.notnull(_sync); - bts.Log.isnull(_server_params); - bts.Log.isnull(app); - - - _server_params = server_params; - var builder = WebApplication.CreateBuilder(); - - var localServerParams = _server_params.Clone(); - localServerParams.Scheme = "http"; - var localUri = localServerParams.Uri(); - builder.WebHost.UseUrls(localUri); - - var publicServerParams = _server_params.Clone(); - publicServerParams.Scheme = "https"; - publicServerParams.Host = "192.168.50.94"; - var publicUri = publicServerParams.Uri(); - builder.WebHost.UseUrls(publicUri); - - // var MyAllowSpecificOrigins = "_myAllowSpecificOrigins"; - builder.Services.AddLogging(builder => { - builder.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning) - .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning) - .AddFilter("NToastNotify", Microsoft.Extensions.Logging.LogLevel.Warning) - .AddConsole(); - }); - // builder.Services.AddCors(options => { - // options.AddPolicy(MyAllowSpecificOrigins, - // policy => policy - // .WithOrigins("http://localhost:3000") - // .AllowAnyHeader() - // .AllowAnyMethod()); - // }); - app = builder.Build(); - app.UseWebSockets(_server_params.get_ws_options()); - // app.Use(async (context, next) => { - // context.Response.Headers.Add("Access-Control-Allow-Origin", "*"); - // context.Response.Headers.Add("Access-Control-Allow-Headers", "*"); - // await next(); - // }); - app.Map(_server_params.DefaultEndPoint ?? string.Empty, async context => { - if (context.WebSockets.IsWebSocketRequest) { - using (var ws = await context.WebSockets.AcceptWebSocketAsync()) { - await _WebSocketHandler.ProcessWebSocketConnectionAsync(ws); - } - } else { - context.Response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest; - } - }); - return app.RunAsync(); - } /// /// Sends a message to a list of websocket clients. @@ -487,5 +426,14 @@ namespace bts.websocket_core { break; } } + + public async Task HandleRequest(HttpContext context) { + if (!context.WebSockets.IsWebSocketRequest) { + context.Response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest; + return; + } + using var ws = await context.WebSockets.AcceptWebSocketAsync(); + await _WebSocketHandler.ProcessWebSocketConnectionAsync(ws); + } } } diff --git a/src/client/spark_websocket/Core/WebsocketConnectionUpdate.cs b/src/client/spark_websocket/Core/WebsocketConnectionUpdate.cs new file mode 100644 index 000000000..8beacea87 --- /dev/null +++ b/src/client/spark_websocket/Core/WebsocketConnectionUpdate.cs @@ -0,0 +1,73 @@ +namespace bts.websocket_core; + +using bts.websocket_core.json_rpc; + +public class WebsocketConnectionUpdate { + readonly WebsocketClientID? connection; + readonly WebsocketIncomingMessage? incomingMessage; + readonly WebsocketOutgoingMessage? outgoingMessage; + readonly WebsocketConnectNotification? connect; + readonly WebsocketDisconnectNotification? disconnect; + + WebsocketConnectionUpdate( + WebsocketClientID connection, + WebsocketIncomingMessage? incomingMessage = null, + WebsocketOutgoingMessage? outgoingMessage = null, + WebsocketConnectNotification? connect = null, + WebsocketDisconnectNotification? disconnect = null) { + this.connection = connection; + this.incomingMessage = incomingMessage; + this.outgoingMessage = outgoingMessage; + this.connect = connect; + this.disconnect = disconnect; + } + + public static WebsocketConnectionUpdate Create( + WebsocketClientID connection, WebsocketConnectNotification connect) => + new(connection, connect: connect); + + public static WebsocketConnectionUpdate Create( + WebsocketClientID connection, WebsocketDisconnectNotification disconnect) => + new(connection, disconnect: disconnect); + + public static WebsocketConnectionUpdate Create( + WebsocketClientID connection, WebsocketIncomingMessage incomingMessage) => + new(connection, incomingMessage: incomingMessage); + + public static WebsocketConnectionUpdate Create( + WebsocketClientID connection, WebsocketOutgoingMessage outgoingMessage) => + new(connection, outgoingMessage: outgoingMessage); + + public WebsocketClientID Connection => + connection ?? throw new InvalidOperationException(); + + public WebsocketConnectionUpdateType Type { + get { + if (incomingMessage is not null) { + return WebsocketConnectionUpdateType.IncomingMessage; + } + if (outgoingMessage is not null) { + return WebsocketConnectionUpdateType.OutgoingMessage; + } + if (connect is not null) { + return WebsocketConnectionUpdateType.Connect; + } + if (disconnect is not null) { + return WebsocketConnectionUpdateType.Disconnect; + } + return WebsocketConnectionUpdateType.Unspecified; + } + } + + public WebsocketIncomingMessage IncomingMessage => + incomingMessage ?? throw new InvalidOperationException(); + + public WebsocketOutgoingMessage OutgoingMessage => + outgoingMessage ?? throw new InvalidOperationException(); + + public WebsocketConnectNotification Connect => + connect ?? throw new InvalidOperationException(); + + public WebsocketDisconnectNotification Disconnect => + disconnect ?? throw new InvalidOperationException(); +} diff --git a/src/client/spark_websocket/Core/WebsocketConnectionUpdateType.cs b/src/client/spark_websocket/Core/WebsocketConnectionUpdateType.cs new file mode 100644 index 000000000..ccd9f704b --- /dev/null +++ b/src/client/spark_websocket/Core/WebsocketConnectionUpdateType.cs @@ -0,0 +1,9 @@ +namespace bts.websocket_core; + +public enum WebsocketConnectionUpdateType { + Unspecified, + Connect, + IncomingMessage, + OutgoingMessage, + Disconnect +}; diff --git a/src/client/spark_websocket/Core/WebsocketDataSync.cs b/src/client/spark_websocket/Core/WebsocketDataSync.cs index e5010dbf9..39ab58f1d 100644 --- a/src/client/spark_websocket/Core/WebsocketDataSync.cs +++ b/src/client/spark_websocket/Core/WebsocketDataSync.cs @@ -1,22 +1,20 @@ namespace bts.websocket_core { + using bts.api; using bts.api.admin; using bts.api.core; using bts.api.orderman; using bts.api.position; using bts.api.synthetic; - using bts.api; using bts.co2; using bts.md; - using mbts; - using Newtonsoft.Json; using Spark.Api.Admin; using Spark.Api.MarketData; using Spark.Api.Orders; - using Spark.Api.Reactive; - using Spark.Api.Services; + using Spark.Core.Reactive; + using System.Collections.Concurrent; /// @@ -25,13 +23,14 @@ namespace bts.websocket_core { /// public class WebsocketDataSync : IWebsocketDataSync { readonly AdminService adminService; - readonly ServiceContext _service_ctx; - readonly IUserDataPermissions userDataPermissions; + readonly Spark.Api.IServiceProvider _service_ctx; + readonly ISecurityResolver securityResolver; - public WebsocketDataSync(ServiceContext service_ctx, IUserDataPermissions userDataPermissions) { + public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IMarketDataClient mdClient, ISecurityResolver securityResolver) { _service_ctx = service_ctx; - this.userDataPermissions = userDataPermissions; - adminService = service_ctx.locate(); + this.securityResolver = securityResolver; + this.mdClient = mdClient; + adminService = service_ctx.GetService(); } /// @@ -124,9 +123,9 @@ namespace bts.websocket_core { // } // } - MarketDataClient? mdClient; - MarketDataClient Md_Client { - get => Log.notnull(mdClient); + IMarketDataClient? mdClient; + IMarketDataClient Md_Client { + get => Log.notnull(mdClient); set { Log.isnull(mdClient); mdClient = value; @@ -187,7 +186,7 @@ namespace bts.websocket_core { } } - IReadOnlyDictionary _securities = new Dictionary(); + IReadOnlyDictionary _securities = new Dictionary(); IDisposable? _securities_sub; /// @@ -197,21 +196,16 @@ namespace bts.websocket_core { public async Task initAsync(WebsocketConnectionGateway gtwy) { Gateway = gtwy; clients = new ClientsStore(); - MD_Source = new MDSource(userDataPermissions); - Md_Client = new(MD_Source); - _securities_sub = MD_Source.subscribe_securities().Subscribe(sec_map => _securities = sec_map); - var order_service = _service_ctx.locate(); - var position_service = _service_ctx.locate(); - var admin_service = _service_ctx.locate(); - var synth_sec_service = _service_ctx.locate(); + var sec_table_src = Md_Client.Securities().Cache().ToObservable(); + _securities_sub = sec_table_src.Subscribe(secTable => _securities = secTable.state); + var order_service = _service_ctx.GetService(); + var position_service = _service_ctx.GetService(); + var admin_service = _service_ctx.GetService(); + var synth_sec_service = _service_ctx.GetService(); var admin_client = new AdminClient(admin_service, UserTuple.admin); - var md_relay = _service_ctx.locate(); - var sec_resolver = new SecurityResolver(md_relay, MD_Source.subscribe_securities()); + var md_relay = _service_ctx.GetService(); Orders_Client = new OrdersClient( - order_service, position_service, admin_client, sec_resolver, TimeSpan.FromSeconds(0.1)); - var sec_table_src = MD_Source.subscribe_securities() - .Table(s => s.security.Id) - .Select(s => s.security, s => s.Id); + order_service, position_service, admin_client, securityResolver, TimeSpan.FromSeconds(0.1)); var r = await Orders_Client.GetRegisteredDynamicOrderTypesAsync(); Dynamic_Descriptors = r .DynamicOrderDesc @@ -224,13 +218,13 @@ namespace bts.websocket_core { } private uint get_security(Venue v, string sec_name) { var securites = _securities; - if (securites.Values.TryGetFirst(sc_md => + if (securites.Values.TryGetFirst(sec => ( - sc_md.security.Exchange == v && - sc_md.security.ToString() == sec_name), - out SecurityMD sc_md + sec.Exchange == v && + sec.ToString() == sec_name), + out var sec )) { - return sc_md.id; + return sec.Id; } throw new ArgumentException($"{sec_name} at {v} not found in securites"); } diff --git a/src/client/spark_websocket/Core/WebsocketDisconnectNotification.cs b/src/client/spark_websocket/Core/WebsocketDisconnectNotification.cs new file mode 100644 index 000000000..4449e462f --- /dev/null +++ b/src/client/spark_websocket/Core/WebsocketDisconnectNotification.cs @@ -0,0 +1,8 @@ +namespace bts.websocket_core; + +public class WebsocketDisconnectNotification { + public WebsocketDisconnectNotification(WebsocketClientID id) => + Id = id; + + public WebsocketClientID Id { get; } +} diff --git a/src/client/spark_websocket/Core/WebsocketJsonParsers.cs b/src/client/spark_websocket/Core/WebsocketJsonParsers.cs index 9c91c4d1d..5c0279433 100644 --- a/src/client/spark_websocket/Core/WebsocketJsonParsers.cs +++ b/src/client/spark_websocket/Core/WebsocketJsonParsers.cs @@ -3,9 +3,6 @@ namespace bts.websocket_core { using bts.api.core; using bts.api.orderman; - using bts.md; - - using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -45,28 +42,28 @@ namespace bts.websocket_core { public Venue venue{get;init;} public String exchange_security_name{get;init;} public String driver_exchange_security_name{get;init;} - public static bool GetTheoPoint(PrecomputedTheoPointWrapper inp, [NotNullWhen(true)] out PrecomputedTheoPoint? p, IReadOnlyDictionary _securites){ + public static bool GetTheoPoint(PrecomputedTheoPointWrapper inp, [NotNullWhen(true)] out PrecomputedTheoPoint? p, IReadOnlyDictionary _securites){ p = null; - if (!_securites.Values.TryGetFirst(sc_md => + if (!_securites.Values.TryGetFirst(sec => ( - sc_md.security.Exchange == inp.venue && - sc_md.security.ToString() == inp.exchange_security_name), - out mbts.SecurityMD sec + sec.Exchange == inp.venue && + sec.ToString() == inp.exchange_security_name), + out var sec )) { return false; } - if (!_securites.Values.TryGetFirst(sc_md => + if (!_securites.Values.TryGetFirst(sec => ( - sc_md.security.Exchange == inp.venue && - sc_md.security.ToString() == inp.driver_exchange_security_name), - out mbts.SecurityMD driver_sec + sec.Exchange == inp.venue && + sec.ToString() == inp.driver_exchange_security_name), + out var driver_sec )) { return false; } p = new(){ - SecId = sec.id, + SecId = sec.Id, Driver=inp.driver, - DriverSecId=driver_sec.id, + DriverSecId=driver_sec.Id, Theo=inp.theo, Delta=inp.delta, Gamma=inp.gamma, @@ -221,4 +218,4 @@ namespace bts.websocket_core { } -} \ No newline at end of file +} diff --git a/src/client/spark_websocket/Core/WebsocketRestHandler.cs b/src/client/spark_websocket/Core/WebsocketRestHandler.cs index bbed3d239..fd76b05b0 100644 --- a/src/client/spark_websocket/Core/WebsocketRestHandler.cs +++ b/src/client/spark_websocket/Core/WebsocketRestHandler.cs @@ -369,7 +369,7 @@ namespace bts.websocket_core { static async Task<(bool, OrderDefinition?, string)> TryGetOrderDefAsync( WebsocketRequest req, - IReadOnlyDictionary securities, + IReadOnlyDictionary securities, AdminService adminService, IReadOnlyDictionary Dynamic_Descriptors ) { @@ -408,11 +408,11 @@ namespace bts.websocket_core { var _securites = securities; - if (!_securites.Values.TryGetFirst(sc_md => + if (!_securites.Values.TryGetFirst(sec => ( - sc_md.security.Exchange == exchange && - sc_md.security.ToString() == exchange_securty_name.Value.Val), - out SecurityMD sc_md + sec.Exchange == exchange && + sec.ToString() == exchange_securty_name.Value.Val), + out var sec )) { return (false, null, "security not found"); } @@ -425,7 +425,7 @@ namespace bts.websocket_core { return (true, new OrderDefinition { TimeInForce = TimeInForce.Gtc, - SecurityKey = sc_md.id, + SecurityKey = sec.Id, Side = (Side)side, UserId = user_id, AccountId = account_id, @@ -434,7 +434,7 @@ namespace bts.websocket_core { User = user.Name }, status); } - static bool TryGetPoints(WebsocketRequest req, [NotNullWhen(true)] out List? theos, out string message, IReadOnlyDictionary securities) { + static bool TryGetPoints(WebsocketRequest req, [NotNullWhen(true)] out List? theos, out string message, IReadOnlyDictionary securities) { theos = null; if (!WebsocketJsonParsers.TryParse(req.parameters, "points", out WebsocketJsonParsers.PrecomputedTheoPointWrapper[]? wrapper_theos, out message) || wrapper_theos == null) { return false; @@ -451,7 +451,7 @@ namespace bts.websocket_core { return true; } - public static async Task<(bool success, uint? id, string status)> SetTheosAsync(WebsocketRequest req, OrdersClient oc, IReadOnlyDictionary securities) { + public static async Task<(bool success, uint? id, string status)> SetTheosAsync(WebsocketRequest req, OrdersClient oc, IReadOnlyDictionary securities) { if (req.parameters == null) { return (false, null, "parameters not present"); } @@ -499,7 +499,7 @@ namespace bts.websocket_core { public static async Task<(bool success, uint? orderId, string status)> EnterOrderAsync( WebsocketRequest request, OrdersClient ordersClient, - IReadOnlyDictionary securities, + IReadOnlyDictionary securities, AdminService adminService, IReadOnlyDictionary Dynamic_Descriptors) { OrderDefinition? orderDefinition = null; diff --git a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs index 91f376f41..a33db825b 100644 --- a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs +++ b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs @@ -3,7 +3,6 @@ namespace bts.websocket_core { using bts.api; using bts.api.orderman; - using bts.md; using Bts.DataStream; using Bts.DataStreamCore; @@ -42,7 +41,7 @@ namespace bts.websocket_core { public IDisposable? dis; public void Dispose() => dis?.Dispose(); Action _sm; - private MarketDataClient _MdClient; + private IMarketDataClient _MdClient; private OrdersClient _OrdersClient; public Action subscribe; void set_invalid(string message) { @@ -63,35 +62,35 @@ namespace bts.websocket_core { } private static Action GetSecuritiesSubscriber(SecurityCriteria cri) { return (WebsocketSubscriptionHandler hdlr) => { - hdlr.dis = hdlr._MdClient.securities_incremental(cri).Subscribe( + hdlr.dis = hdlr._MdClient.Securities(cri).Subscribe( upd => hdlr.SendMessage(WebsocketJsonParsers.to_json(upd))); }; } private static Action GetTradesSubscriber(TradesSubscriptionFilter cri) { return (WebsocketSubscriptionHandler hdlr) => { - hdlr.dis = hdlr._MdClient.trades(cri).Subscribe( + hdlr.dis = hdlr._MdClient.Trades(cri.security_criteria).Subscribe( upd => hdlr.SendMessage(WebsocketJsonParsers.to_json(upd))); }; } private static Action GetBookSubscriber(SecurityCriteria cri) { return (WebsocketSubscriptionHandler hdlr) => { - hdlr.dis = hdlr._MdClient.books(cri).Subscribe( + hdlr.dis = hdlr._MdClient.Books(cri).Subscribe( upd => hdlr.SendMessage(WebsocketJsonParsers.to_json(upd))); }; } private static Action GetPriceStatsSubscriber(SecurityCriteria cri) { return (WebsocketSubscriptionHandler hdlr) => { - hdlr.dis = hdlr._MdClient.price_stats(cri).Subscribe( + hdlr.dis = hdlr._MdClient.PriceStats(cri).Subscribe( upd => hdlr.SendMessage(WebsocketJsonParsers.to_json(upd))); }; } private static Action GetPriceTickerSubscriber(SecurityCriteria cri) { return (WebsocketSubscriptionHandler hdlr) => { - hdlr.dis = hdlr._MdClient.price_ticker(cri).Subscribe( + hdlr.dis = hdlr._MdClient.PriceTicker(cri).Subscribe( upd => hdlr.SendMessage(WebsocketJsonParsers.to_json(upd))); }; @@ -310,7 +309,7 @@ namespace bts.websocket_core { public static bool TryCreate( WebsocketRequest req, - MarketDataClient md, + IMarketDataClient md, OrdersClient oc, Action sm, [NotNullWhen(true)] out WebsocketSubscriptionHandler? hdlr, @@ -397,7 +396,7 @@ namespace bts.websocket_core { private WebsocketSubscriptionHandler( string id, - MarketDataClient md, + IMarketDataClient md, OrdersClient oc, Action sm, Action sub) { diff --git a/src/client/spark_websocket/JsonRpc/JsonRpcId.cs b/src/client/spark_websocket/JsonRpc/JsonRpcId.cs new file mode 100644 index 000000000..c22efe0fe --- /dev/null +++ b/src/client/spark_websocket/JsonRpc/JsonRpcId.cs @@ -0,0 +1,50 @@ +namespace bts.websocket_core.json_rpc; + +using System.Text.Json; +using System.Text.Json.Serialization; + +[JsonConverter(typeof(JsonRpcIdConverter))] +public readonly struct JsonRpcId { + readonly long? int64; + readonly string? str; + + JsonRpcId(long int64) => + this.int64 = int64; + + JsonRpcId(string str) => + this.str = str; + + public static JsonRpcId Create() => + new(DateTime.Now.Ticks); + + public override string? ToString() => + int64?.ToString() ?? str; + + class JsonRpcIdConverter : JsonConverter { + public override bool CanConvert(Type typeToConvert) => + typeToConvert == typeof(JsonRpcId); + + public override JsonRpcId Read( + ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (typeToConvert != typeof(JsonRpcId)) { + throw new ArgumentException($"Unable to parse {typeToConvert}"); + } + return reader.TokenType switch { + JsonTokenType.String => new(reader.GetString()!), + JsonTokenType.Number => new(reader.GetInt64()), + _ => throw new ArgumentException($"Unable to parse {typeToConvert} from {reader.TokenType}"), + }; + } + + public override void Write( + Utf8JsonWriter writer, JsonRpcId value, JsonSerializerOptions options) { + if (value.int64.HasValue) { + writer.WriteNumberValue(value.int64.Value); + } + if (value.str is not null) { + writer.WriteStringValue(value.str); + } + } + } + +} diff --git a/src/client/spark_websocket/Services/ISparkApiWebSocketServer.cs b/src/client/spark_websocket/Services/ISparkApiWebSocketServer.cs new file mode 100644 index 000000000..1311821de --- /dev/null +++ b/src/client/spark_websocket/Services/ISparkApiWebSocketServer.cs @@ -0,0 +1,7 @@ +namespace Spark.WebSocket.Services; + +using Microsoft.AspNetCore.Http; + +public interface ISparkApiWebSocketServer { + Task HandleRequestAsync(HttpContext context); +} diff --git a/src/client/spark_websocket/Services/SparkApiWebSocketServer.cs b/src/client/spark_websocket/Services/SparkApiWebSocketServer.cs new file mode 100644 index 000000000..e7d3006e0 --- /dev/null +++ b/src/client/spark_websocket/Services/SparkApiWebSocketServer.cs @@ -0,0 +1,39 @@ +namespace Spark.WebSocket.Services; + +using bts.md; +using bts.websocket_core; + +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +using Spark.Api.MarketData; + +using System.Threading; +using System.Threading.Tasks; + +public class SparkApiWebSocketServer : BackgroundService, ISparkApiWebSocketServer { + readonly WebsocketDataSync dataSync; + readonly WebsocketConnectionGateway gateway; + readonly ILogger logger; + + public SparkApiWebSocketServer( + Api.IServiceProvider serviceProvider, + IMarketDataClient mdClient, + ISecurityResolver securityResolver, + ILogger logger + ) { + this.logger = logger; + dataSync = new(serviceProvider, mdClient, securityResolver); + gateway = new(); + gateway.init(dataSync); + } + + public Task HandleRequestAsync(HttpContext httpContext) => + gateway.HandleRequest(httpContext); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + logger.LogInformation("Initializing DataSync..."); + await dataSync.initAsync(gateway); + } +} diff --git a/src/client/spark_websocket/Services/SparkWebSocketServices.cs b/src/client/spark_websocket/Services/SparkWebSocketServices.cs new file mode 100644 index 000000000..6bffdd66e --- /dev/null +++ b/src/client/spark_websocket/Services/SparkWebSocketServices.cs @@ -0,0 +1,20 @@ +namespace Spark.WebSocket.Services; + +using Microsoft.Extensions.DependencyInjection; + +using Spark.Api.Services; + +public static class SparkWebSocketServices { + public static IServiceCollection AddWebSocketServer(this IServiceCollection services) => + services.AddSingleton() + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddHostedService(sp => sp.GetRequiredService()); + // .AddCors( + // options => options.AddPolicy( + // "_myAllowSpecificOrigins", + // policy => policy + // .WithOrigins("http://localhost:3000") + // .AllowAnyHeader() + // .AllowAnyMethod())); +}