diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs index 7f7e5dd19..667e7cf91 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs @@ -1,14 +1,21 @@ namespace Bts.DataStreamWebApp.Server.Controllers; +using Bts.DataStream; + using Microsoft.AspNetCore.Mvc; [Route("/ws")] public class DataStreamSubscriptionController : ControllerBase { readonly ILogger logger; + readonly IDataStreamFactory streamFactory; - public DataStreamSubscriptionController(ILogger logger) { + public DataStreamSubscriptionController(ILogger logger, IEnumerable streamFactories) { this.logger = logger; logger.LogInformation("Creating {ControllerType}", GetType().Name); + var streamFactoryList = streamFactories.ToList(); + streamFactory = streamFactoryList.Count == 1 + ? streamFactoryList[0] + : new CompositeDataStreamFactory(streamFactoryList); } [Route("/ws")] @@ -21,7 +28,7 @@ public class DataStreamSubscriptionController : ControllerBase { try { using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync(); logger.LogInformation("Websocket connection established."); - using var server = new DataStreamWebSocketServer(webSocket, logger); + using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger); await server.Run(cancellationToken); } catch (Exception ex) { logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex); diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs index f3c35e8a5..01fbb222c 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs @@ -2,6 +2,7 @@ namespace Bts.DataStreamWebApp.Server.Controllers; using bts.websocket_core.json_rpc; +using Bts.DataStream; using Bts.DataStreamWebApp.Server.Messages; using System.Net.WebSockets; @@ -13,9 +14,11 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable { readonly ILogger logger; readonly WebSocket webSocket; readonly Dictionary subscriptions = new(); + readonly IDataStreamFactory streamFactory; - public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) { + public DataStreamWebSocketServer(WebSocket webSocket, IDataStreamFactory streamFactory, ILogger logger) { this.webSocket = webSocket; + this.streamFactory = streamFactory; this.logger = logger; } @@ -67,7 +70,7 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable { } var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)"); disposeSubscription(subId); - if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) { + if (!subReq.ParseQuery(streamFactory, out var vs, out parseErr, out errCode)) { logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr); var rpcError = JsonRpcResponse.CreateError( req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr }); diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs index fb608ecf9..f51cdd84d 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs @@ -4,6 +4,7 @@ using BCG; using bts.websocket_core.json_rpc; +using Bts.DataStream; using Bts.DataStreamExpressions; using System.Diagnostics.CodeAnalysis; @@ -37,7 +38,12 @@ public class SubscribeRequest { return true; } - internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) { + internal bool ParseQuery( + IDataStreamFactory streamFactory, + [NotNullWhen(true)] out ViewStream? stream, + [NotNullWhen(false)] out SubscriptionError? error, + out ErrorCode errorCode + ) { stream = null; errorCode = ErrorCode.None; error = null; @@ -54,7 +60,7 @@ public class SubscribeRequest { } Exception? exception = null; try { - stream = exp.getView(new Dictionary(0)); + stream = exp.getView(new Dictionary(0), streamFactory); } catch (Exception ex) { exception = ex; } diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs index 24b90d867..940fc56ba 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs @@ -1,5 +1,6 @@ using Bts.DataStreamCore.Services; using Bts.PubSub.Services; +using Bts.SharedStreams.Services; using Spark.Api.Services; using Spark.Core.Services; @@ -78,5 +79,9 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config .AddSparkApi(config.GetSection("SparkApi")) .AddMasterWorker(config.GetSection("MasterWorker")) .AddViewStreamDatabase() + .AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams")) + .AddOhlcvStreams(config.GetSection("OhlcvStream")) + .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream")) + .AddCompositeDataStreamFactory() .AddPubSubClient(config.GetSection("PubSub")); } diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.NaturalLanguageCustom.json b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.NaturalLanguageCustom.json new file mode 100644 index 000000000..6b369e52b --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.NaturalLanguageCustom.json @@ -0,0 +1,45 @@ +{ + "NaturalLanguagePrompt": { + "prompts": [ + { + "category": "Impact of financial events on the market", + "description": "Show how trading metrics (e.g., price movement, volume) are affected by events related to finance (rate announcements, earnings reports, etc.).", + "prompt": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings", + "examples": [ + { + "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings", + "dsl": [ + "time_idx=EventTimeIndex.OpecMeetings(which: \"last 5\", duration: \"+5min\")", + "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")", + "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])" + ] + }, + { + "natural_language": "How do OPEC meetings affect the price of Crude futures? Examine the 10 minutes after each meeting this year.", + "dsl": [ + "time_idx=EventTimeIndex.OpecMeetings(duration: \"+10min\", predicate: \"timestamp.year==2024\")", + "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")", + "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])" + ] + }, + { + "natural_language": "Show me how the SP 500 price moves after FOMC rate announcements that day", + "dsl": [ + "time_idx=EventTimeIndex.FomcRateAnnouncements(which: \"timestamp.year==2024\", duration: \"+10min\")", + "contract_idx=ActiveFutureContracts(predicate: \"product==ES\")", + "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])" + ] + }, + { + "natural_language": "Analyze the impact of the FED's interest rate hikes in 2022 on the 10-year Treasury yield, focusing on the 2-week period following each hike.", + "dsl": [ + "time_idx=FedInterestRateAnnouncements(predicate: \"timestamp.year==2022 && change_pct>0\", duration: \"+2 weeks\")", + "contract_idx=ActiveFutureContracts(predicate: \"product==ZN\")", + "Bars(time_index: time_idx, contract_index: contract_idx)" + ] + } + ] + } + ] + } +} diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json index 1daf11ff3..90675f1c3 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json @@ -22,6 +22,384 @@ "ViewStreamDatabase": { "Interval": "00:00:00.1" }, + "PrimitiveStream": { + "Assemblies": [ + "DataStream.dll", + "SharedStreams.dll" + ], + "ExcludedTypes": [ + "SettlementPriceStream", + "OHLCVStream" + ] + }, + "OhlcvStream": { + "Database": { + "Mbo": { + "Path": "/home/bts/.local/share/bts/silver_pine/Databento/mbo/${VENUE}/${SYMBOL}/store.db", + "Query": "select * from ${SYMBOL}_mbo_${DATE}", + "QueryParams": [ + "SYMBOL", + "DATE" + ] + }, + "Bbo1s": { + "Path": "/home/bts/.local/share/bts/silver_pine/Databento/bbo-1s/${VENUE}/${SYMBOL}/store.db", + "Query": "select * from ${SYMBOL}_bbo1s_${DATE}", + "QueryParams": [ + "SYMBOL", + "DATE" + ] + }, + "Bbo1m": { + "Path": "/home/bts/.local/share/bts/silver_pine/Databento/bbo-1m/${VENUE}/${SYMBOL}/store.db", + "Query": "select * from ${SYMBOL}_bbo1m_${DATE}", + "QueryParams": [ + "SYMBOL", + "DATE" + ] + }, + "Ohlcv1d": { + "Path": "/home/bts/.local/share/bts/silver_pine/database/stream_data.db", + "Table": "${SYMBOL}_ohlcv_1day", + "Query": "select * from ${SYMBOL}_ohlcv_1day", + "QueryParams": [ + "SYMBOL" + ] + } + }, + "Databases": [ + { + "Path": "/home/bts/.local/share/bts/silver_pine/database/stream_data.db", + "Venues": [ + "GLBX" + ], + "Types": [ + "Future" + ], + "Symbols": [ + "ES" + ], + "Datasets": { + "mbo": { + } + } + }, + { + "Path": "/home/bts/.local/share/bts/silver_pine/Databento/bbo/GLBX/ZN/store.db", + "Tables": [ + { + "DataType": "mbo", + "Venue": "GLBX", + "SecurityType": "Future", + "Symbol": "ZN", + "TableName": "ZN_bbo1m_20241001", + "TradingDates": [ + "2024-10-01" + ] + } + ] + } + ] + }, + "NaturalLanguageStream": { + "prompts": [ + { + "category": "Settlement", + "MulitLineDescription": [ + "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). ", + "The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). ", + "The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement ", + "(symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have ", + "datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know ", + "what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however ", + "you should use the Today's real date" + ], + "prompt": "What was the settlement for SP 500 Future on June 8th?", + "examples": [ + { + "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.", + "dsl": "Settlement(\"ES\", 4/22/2023)" + }, + { + "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.", + "dsl": "Settlement(\"SIZ4\", 4/22/2023)" + }, + { + "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.", + "dsl": "Settlement(\"CL\", 10/10/2024)" + }, + { + "natural_language": "Find today's settlement of the 10 year treasury future.", + "dsl": "Settlement(\"ZN\", 10/11/2024)" + }, + { + "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.", + "dsl": "Settlement(\"NGZ4\", 9/6/2024)" + }, + { + "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.", + "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)" + }, + { + "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25 spread for the first two weeks of September.", + "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)" + }, + { + "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.", + "dsl": "Settlement(\"NG\", 9/6/2024)" + } + ] + }, + { + "category": "BBO", + "MulitLineDescription": [ + "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ", + "It accepts the following parameters: ", + "symbol (a product code such as ZN for 10 year T-Note futures), ", + "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ", + "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ", + "interval (sample frequency of the data), ", + "begin and end (inclusive beginning and end of the time window to query, respectively), ", + "inclusive_end (a bool flag that, when true, makes the end time inclusive). ", + "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ", + "Depending on the wording of the request, some arguments may be omitted. ", + "The begin, end, and interval args should be expressed in iso 8601 format. ", + "Interpret dates according to the trading schedule (rather than calendar date), whenever possible. ", + "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ", + "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00." + ], + "prompt": "Show me bbo for SP 500 futures on Sep 23rd of this year.", + "examples": [ + { + "natural_language": "Get SP500 future bbo from Wednesday.", + "dsl": "Bbo(symbol: \"ES\", venue: \"GLBX\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")" + }, + { + "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.", + "dsl": "Bbo(symbol: \"ES\", venue: \"GLBX\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00\", interval: \"PT1S\", exclusive_end: true)" + }, + { + "natural_language": "Show me L1 data for Crude oil futures this week", + "dsl": "Bbo(symbol: \"CL\", venue: \"GLBX\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")" + }, + { + "natural_language": "Fetch hourly bbo for Gold futures this week", + "dsl": "Bbo(symbol: \"GC\", venue: \"GLBX\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")" + }, + { + "natural_language": "What was the minute by minute bbo for Tesla yesterday?", + "dsl": "Bbo(symbol: \"TSLA\", venue: NASD)" + } + ] + }, + { + "category": "VWAP", + "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.", + "prompt": "What was the VWAP for SP 500 Future over the past 30 days?", + "examples": [ + { + "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.", + "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)" + }, + { + "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.", + "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)" + }, + { + "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.", + "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)" + }, + { + "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.", + "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)" + } + ] + }, + { + "category": "Price Diff", + "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024", + "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?", + "examples": [ + { + "natural_language": "What is the difference in ZN price from July 10th to July 14th?", + "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)" + }, + { + "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?", + "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)" + }, + { + "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?", + "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)" + } + ] + }, + { + "category": "Absolute Volume", + "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc). The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ", + "prompt": "Which SP 500 Future traded the most between 7/14/2024 and 7/10/2024", + "examples": [ + { + "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024", + "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)" + }, + { + "natural_language": "Which Gold Spread traded the most between Today and Sept 20th", + "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)" + }, + { + "natural_language": "Which 10 year Spread traded the highest volume last week?", + "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)" + }, + { + "natural_language": "Which Corn Spread traded the most today?", + "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)" + }, + { + "natural_language": "Which Gold Future traded the most today?", + "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)" + } + ] + }, + { + "category": "Roll", + "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ", + "prompt": "When does the roll typically occur In SP 500?", + "examples": [ + { + "natural_language": "When does the roll typically occur In CL?", + "dsl": "FuturesRoll(\"CL\")" + }, + { + "natural_language": "When does the roll typically occur for 10 year treasury?", + "dsl": "FuturesRoll(\"ZN\")" + }, + { + "natural_language": "When does the roll usually occur for Gold Future", + "dsl": "FuturesRoll(\"GC\")" + }, + { + "natural_language": "At what date does the future rollover happens for SP500?", + "dsl": "FuturesRoll(\"ES\")" + }, + { + "natural_language": "When does the future rollover happens for the Corn Future?", + "dsl": "FuturesRoll(\"ZC\")" + } + ] + }, + { + "category": "Daily Move", + "description": "This is a few shot learning example for an NL to DSL. ", + "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?", + "examples": [ + { + "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close", + "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)" + }, + { + "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close", + "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)" + }, + { + "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open", + "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)" + }, + { + "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close", + "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)" + } + ] + }, + { + "category": "Average Volume", + "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ", + "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?", + "examples": [ + { + "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ", + "dsl": "FuturesRoll(\"ES\")" + }, + { + "natural_language": "Usually when does the open interest move from one contract to another in WTI?", + "dsl": "FuturesRoll(\"CL\")" + } + ] + }, + { + "category": "Events", + "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.", + "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?", + "examples": [ + { + "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?", + "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)" + }, + { + "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?", + "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)" + }, + { + "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.", + "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)" + }, + { + "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.", + "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)" + } + ] + }, + { + "IsEnabled": false, + "category": "Market by Order", + "MulitLineDescription": [ + "This is a few shot learning example for an NL to DSL. For the purpose of these few shot prompts, assume today is 10/3/2024.", + "" + ], + "prompt": "Show order flow imbalance between buy and sell orders for WTI between 9:30 and 9:31", + "examples": [ + { + "natural_language": "" + } + ] + }, + { + "category": "Indicator, Standard Deviation", + "MulitLineDescription": [ + "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when ", + "the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, ", + "where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and ", + "amounts of the last 4+ standard deviation moves in WTI futures\" would be: ", + "r=standardDeviation(\"ES\", \"90d\")\n", + "r=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\n", + "r=filter(r,\"abs_daily_change >= comp\"). ", + "The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, ", + "and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, ", + "abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters ", + "(like a sql where) of rows where abs_daily_change >= comp" + ], + "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP futures ?", + "examples": [ + { + "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?", + "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")" + }, + { + "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation", + "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")" + }, + { + "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation", + "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")" + }, + { + "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation", + "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")" + } + ] + } + ] + }, "SparkApi": { "IsEnabled": false, "Credentials": { @@ -77,7 +455,7 @@ { "TagType": ".*", "Assembly": "DataStreamCore", - "Class": "bts.PubsubServerGenericDataHandler" + "Class": "bts.PubsubServerGenericDataHandler" } ] } diff --git a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs index c652e15e8..15f8586c0 100644 --- a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs +++ b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs @@ -3,24 +3,25 @@ namespace bts.SilverPine { using BCG; using System.Collections.Generic; using Bts.DataStreamExpressions; + using Bts.DataStream; // All this does is take in a dsl expression, parse it and fire the data that come out of running it. // So for example, if an expresison is passed [DataStreamUserRespresentation("dsl")] public class DSLStream : ViewStream { private readonly string _dsl; - public DSLStream(string dsl) { + public DSLStream(string dsl, IDataStreamFactory streamFactory) { Log.Gen.info($"DSLStream.cons dsl: {dsl}"); _dsl = dsl; - ViewStream rs = build_stream(dsl); + var rs = build_stream(streamFactory, dsl); rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\""); mySetRealSource(rs); } - static ViewStream? build_stream(string stream_expression) { + static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) { try { var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); + var rv = exp.getView(new Dictionary(0), streamFactory); return rv; } catch (Exception ex) { @@ -29,4 +30,4 @@ namespace bts.SilverPine { } } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs index db91adb68..b99e41985 100644 --- a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs +++ b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs @@ -34,12 +34,12 @@ namespace BCG { view_publisher = view_publisher_; } } - static ViewStream? build_stream(string stream_expression) + static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) { try { var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); + var rv = exp.getView(new Dictionary(0), streamFactory); return rv; } catch (Exception ex) { @@ -51,15 +51,17 @@ namespace BCG { _stream_pub_map = new Dictionary(); readonly IPubTableBroker _broker; readonly IManagedSubscriber _sub; + readonly IDataStreamFactory streamFactory; readonly object syncRoot = new(); IDisposable? cloudStreamsSub; IDisposable? remoteRequestsSub; bool isDisposed; - public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) { + public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory) { _sub = managedSubscriber; _broker = pubTableBroker; + this.streamFactory = streamFactory; } public Task StartAsync(CancellationToken cancellationToken) { @@ -102,7 +104,11 @@ namespace BCG { private void _spawn_remote_stream(string name, string stream_expression, bool persist) { bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); - ViewStream stream = build_stream(stream_expression); + var stream = build_stream(streamFactory, stream_expression); + if (stream is null) { + Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}"); + return; + } _expose_to_ws(name, stream); if (persist) { stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs new file mode 100644 index 000000000..28ab1b035 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs @@ -0,0 +1,311 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using mbts.utils; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +public class BlueRowStore : IBlueRow, IReadOnlyDictionary { + readonly string? name; + IBlueTableSchema? _schema; + + public BlueRowStore() { } + + public BlueRowStore( + int id, + (string Name, T1 Value) item1, + (string Name, T2 Value) item2, + (string Name, T3 Value) item3, + (string Name, T4 Value) item4, + (string Name, T5 Value) item5, + (string Name, T6 Value) item6 + ) { + Id = id; + Item1 = item1; + Item2 = item2; + Item3 = item3; + Item4 = item4; + Item5 = item5; + Item6 = item6; + } + + public int Id { get; set; } + public (string Name, T1 Value) Item1 { get; set; } + public (string Name, T2 Value) Item2 { get; set; } + public (string Name, T3 Value) Item3 { get; set; } + public (string Name, T4 Value) Item4 { get; set; } + public (string Name, T5 Value) Item5 { get; set; } + public (string Name, T6 Value) Item6 { get; set; } + + public int Count => 5; + public IReadOnlyList Keys => new KeyList(this); + public IReadOnlyList Values => new ValueList(this); + + public IBlueTableSchema schema => + _schema ??= new BlueTableSchema(name, new ColumnInfo[]{ + new("id", 0, typeof(int)), + new(Item1.Name, 1, typeof(T1)), + new(Item2.Name, 2, typeof(T2)), + new(Item3.Name, 3, typeof(T3)), + new(Item4.Name, 4, typeof(T4)), + new(Item5.Name, 5, typeof(T5)), + new(Item6.Name, 6, typeof(T6)), + }); + + ICollection IDictionary.Keys => new KeyList(this); + ICollection IDictionary.Values => new ValueList(this); + + IEnumerable IReadOnlyDictionary.Keys => Keys; + IEnumerable IReadOnlyDictionary.Values => Values; + + bool ICollection.IsSynchronized => false; + object ICollection.SyncRoot => this; + + public object? this[string colName] { + get { + if (colName == "id") { + return Id; + } + if (colName == Item1.Name) { + return Item1.Value; + } + if (colName == Item2.Name) { + return Item2.Value; + } + if (colName == Item3.Name) { + return Item3.Value; + } + if (colName == Item4.Name) { + return Item4.Value; + } + if (colName == Item5.Name) { + return Item5.Value; + } + if (colName == Item6.Name) { + return Item6.Value; + } + throw new KeyNotFoundException(); + } + set { + if (colName == "id") { + Id = ExceptionAssert.Cast(value).To(); + } + if (colName == Item1.Name) { + Item1 = (Item1.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item2.Name) { + Item2 = (Item2.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item3.Name) { + Item3 = (Item3.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item4.Name) { + Item4 = (Item4.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item5.Name) { + Item5 = (Item5.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item6.Name) { + Item6 = (Item6.Name, ExceptionAssert.Cast(value).To()); + } + throw new KeyNotFoundException(); + } + } + + public object? this[int index] => index switch { + 0 => Id, + 1 => Item1.Value, + 2 => Item2.Value, + 3 => Item3.Value, + 4 => Item4.Value, + 5 => Item5.Value, + 6 => Item6.Value, + _ => throw new IndexOutOfRangeException() + }; + + object? IDictionary.this[object key] { + get => this[ExceptionAssert.Cast(key).To()]; + set => this[ExceptionAssert.Cast(key).To()] = value; + } + + public void CopyTo(Array array, int index) { + array.SetValue(Id, index++); + array.SetValue(Item1.Value, index++); + array.SetValue(Item2.Value, index++); + array.SetValue(Item3.Value, index++); + array.SetValue(Item4.Value, index++); + array.SetValue(Item5.Value, index++); + array.SetValue(Item6.Value, index++); + } + + public bool Contains(object key) => + key is string keystr && ContainsKey(keystr); + + public bool ContainsKey(string key) => + Keys.Contains(key); + + public void SetId(int id) => + Id = id; + + public void SetValue1(T1 value) => + Item1 = (Item1.Name, value); + + public void SetValue1(string colName, T1 value) => + Item1 = (colName, value); + + public void SetValue2(T2 value) => + Item2 = (Item2.Name, value); + + public void SetValue2(string colName, T2 value) => + Item2 = (colName, value); + + public void SetValue3(T3 value) => + Item3 = (Item3.Name, value); + + public void SetValue3(string colName, T3 value) => + Item3 = (colName, value); + + public void SetValue4(T4 value) => + Item4 = (Item4.Name, value); + + public void SetValue4(string colName, T4 value) => + Item4 = (colName, value); + + public void SetValue5(T5 value) => + Item5 = (Item5.Name, value); + + public void SetValue5(string colName, T5 value) => + Item5 = (colName, value); + + public void SetValue6(T6 value) => + Item6 = (Item6.Name, value); + + public void SetValue6(string colName, T6 value) => + Item6 = (colName, value); + + public bool TryGetValue(string key, [MaybeNullWhen(false)] out object? value) { + if (!Keys.TryFindIndex(key, out int index)) { + value = null; + return false; + } + value = this[index]; + return true; + } + + void IDictionary.Add(object key, object? value) => + throw new NotSupportedException(); + + void IDictionary.Clear() => + throw new NotSupportedException(); + + void IDictionary.Remove(object key) => + throw new NotSupportedException(); + + IDictionaryEnumerator IDictionary.GetEnumerator() => + throw new NotImplementedException(); + + IEnumerator IEnumerable.GetEnumerator() => + throw new NotImplementedException(); + + public IEnumerator> GetEnumerator() { + throw new NotImplementedException(); + } + + class KeyList : ICollection, IReadOnlyList { + readonly BlueRowStore store; + + public KeyList(BlueRowStore store) => + this.store = store; + + public int Count => 5; + bool ICollection.IsSynchronized => false; + object ICollection.SyncRoot => this; + + public string this[int index] => + index switch { + 0 => "id", + 1 => store.Item1.Name, + 2 => store.Item2.Name, + 3 => store.Item3.Name, + 4 => store.Item4.Name, + 5 => store.Item5.Name, + 6 => store.Item6.Name, + _ => throw new IndexOutOfRangeException() + }; + + public void CopyTo(Array array, int index) { + array.SetValue(store.Id, index++); + array.SetValue(store.Item1.Value, index++); + array.SetValue(store.Item2.Value, index++); + array.SetValue(store.Item3.Value, index++); + array.SetValue(store.Item4.Value, index++); + array.SetValue(store.Item5.Value, index++); + array.SetValue(store.Item6.Value, index++); + } + + public IEnumerator GetEnumerator() { + yield return "id"; + yield return store.Item1.Name; + yield return store.Item2.Name; + yield return store.Item3.Name; + yield return store.Item4.Name; + yield return store.Item5.Name; + yield return store.Item6.Name; + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } + + class ValueList : ICollection, IReadOnlyList { + readonly BlueRowStore store; + + public ValueList(BlueRowStore store) => + this.store = store; + + public object? this[int index] => + index switch { + 0 => store.Id, + 1 => store.Item1.Value, + 2 => store.Item2.Value, + 3 => store.Item3.Value, + 4 => store.Item4.Value, + 5 => store.Item5.Value, + 6 => store.Item6.Value, + _ => throw new IndexOutOfRangeException() + }; + + public int Count => 5; + public bool IsSynchronized => false; + public object SyncRoot => this; + + public void CopyTo(Array array, int index) { + array.SetValue(store.Id, index++); + array.SetValue(store.Item1.Value, index++); + array.SetValue(store.Item2.Value, index++); + array.SetValue(store.Item3.Value, index++); + array.SetValue(store.Item4.Value, index++); + array.SetValue(store.Item5.Value, index++); + array.SetValue(store.Item6.Value, index++); + } + + public IEnumerator GetEnumerator() { + yield return store.Id; + yield return store.Item1.Value; + yield return store.Item2.Value; + yield return store.Item3.Value; + yield return store.Item4.Value; + yield return store.Item5.Value; + yield return store.Item6.Value; + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs new file mode 100644 index 000000000..1497d21ae --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs @@ -0,0 +1,29 @@ +#nullable enable +namespace Bts.DataStream; + +using BCG; + +using bts.SilverPine; + +using System.Collections.Generic; +using System.Linq; + +public class CompositeDataStreamFactory : IDataStreamFactory { + readonly List streamFactories; + + public CompositeDataStreamFactory(IEnumerable streamFactories) { + this.streamFactories = streamFactories.ToList(); + foreach (var rsf in this.streamFactories.OfType()) { + rsf.OnCompositeFactory(this); + } + } + + public IEnumerable GetSupportedStreams() => + streamFactories.SelectMany(f => f.GetSupportedStreams()); + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) => + streamFactories.Where(f => f.CanCreate(streamId)) + .Select(f => f.Create(streamId, args)) + .ExceptNull() + .FirstOrDefault(); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs new file mode 100644 index 000000000..6edb20565 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs @@ -0,0 +1,52 @@ +#nullable enable +namespace bts.SilverPine; + +using BCG; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +public interface IDataStreamMetadata { + public string Name { get; } + public IEnumerable AlternateNames { get; } +} + +public class DataStreamMetadata : IDataStreamMetadata { + public string? Name { get; set; } + public List? AlternateNames { get; set; } + + string IDataStreamMetadata.Name => Name ?? string.Empty; + IEnumerable IDataStreamMetadata.AlternateNames => + AlternateNames ?? new(0); + + public static DataStreamMetadata FromType(Type type) { + var friendlyName = type.GetCustomAttribute()?.friendlyName; + var rv = new DataStreamMetadata() { + Name = friendlyName ?? type.Name, + AlternateNames = new(2) { + Log.notnull(type.FullName) + } + }; + if (friendlyName is not null) { + rv.AlternateNames.Add(type.Name); + } + return rv; + } +} + +public static class DataStreamMetadataExtensions { + public static bool IsMatch(this IDataStreamMetadata dsm, string name, bool checkAlternateNames = true, bool ignoreCase = true, bool ignoreUnderscores = true) { + var strComp = ignoreCase ? StringComparison.CurrentCultureIgnoreCase : StringComparison.CurrentCulture; + var nameWithoutUnderscores = new Lazy(() => RemoveUnderscores(name)); + return IsNameMatch(dsm.Name) || (checkAlternateNames && dsm.AlternateNames.Any(IsNameMatch)); + + static string RemoveUnderscores(string s) => + s.Replace("_", ""); + + bool IsNameMatch(string otherName) => + string.Equals(name, otherName, strComp) + || (ignoreUnderscores && string.Equals(nameWithoutUnderscores.Value, RemoveUnderscores(otherName), strComp)); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs new file mode 100644 index 000000000..cbcd65cd1 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs @@ -0,0 +1,27 @@ +#nullable enable +namespace Bts.DataStream; + +using BCG; + +using bts.SilverPine; + +using System.Collections.Generic; +using System.Linq; + +public interface IDataStreamFactory { + IEnumerable GetSupportedStreams(); + + ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args); + + bool CanCreate(string streamId) => + GetSupportedStreams().Any(s => s.IsMatch(streamId)); +} + +/// +/// This interface provides a way for IDataStreamFactory's that create ViewStreams +/// that depend specifically on the composite IDataStreamFactory. This was designed +/// specifically for the NLStreamDataFactory. +/// +public interface IRecursiveDataStreamFactory : IDataStreamFactory { + void OnCompositeFactory(IDataStreamFactory compositeFactory); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs index 8d81e65ba..d658e6fec 100644 --- a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs @@ -9,31 +9,36 @@ namespace BCG { using System.Reflection; using System.IO; using System.Collections.Generic; + using System.Linq; + using bts.collections; + using System.Diagnostics.CodeAnalysis; + using bts.SilverPine; + using Spark.Core.Collections; + using Bts.DataStreamCore; + using Microsoft.Extensions.Options; - public class PrimitiveManager { + public class PrimitiveManager : IDataStreamFactory { private static PrimitiveManager mySelf; readonly List myAssemblyNames = new(); readonly Dictionary myTypeLookupTable = new(); - private PrimitiveManager() { - myLoadAllTypes(); - } + public PrimitiveManager(IOptions config) : this(config.Value) { } - private void myLoadAllTypes() { + public PrimitiveManager(PrimitiveManagerConfig config) { AppDomain currentDomain = AppDomain.CurrentDomain; currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver); loadAssembly(this.GetType().Assembly.FullName); - var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" }; - var config_assemblies = Configuration.get().Assemblies; - bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}"); - foreach (string assemblyName in assemblies/*config_assemblies*/) { + foreach (string assemblyName in config.Assemblies.empty_if_null()) { bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}"); string resolvedAssemblyPath = assemblyName; if (!System.IO.Path.IsPathRooted(assemblyName)) { resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName)); } - loadAssembly(resolvedAssemblyPath); + loadAssembly(resolvedAssemblyPath, IncludeType); } + + bool IncludeType(Type t) => + !config.ExcludeTypes.empty_if_null().Any(tname => tname == t.Name); } private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) { @@ -47,7 +52,7 @@ namespace BCG { /// Loads all the Stream types associated with the assembly /// /// - public void loadAssembly(string assemblyName) { + public void loadAssembly(string assemblyName, Func typeFilter = null) { bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}"); if (myAssemblyNames.Contains(assemblyName)) return; @@ -56,7 +61,11 @@ namespace BCG { GetType().Assembly : Assembly.LoadFrom(assemblyName); - foreach (Type type in loadedAssembly.GetTypes()) + IEnumerable types = loadedAssembly.GetTypes(); + if (typeFilter is not null) { + types = types.Where(typeFilter); + } + foreach (Type type in types) if (!type.IsInterface && !type.IsAbstract && type.IsSubclassOf(typeof(ViewStream)) @@ -73,7 +82,7 @@ namespace BCG { break; } if (isVisible) { - Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName); + Log.assert_d(!myTypeLookupTable.ContainsKey(userRep)); myTypeLookupTable[userRep] = type; } } @@ -89,22 +98,30 @@ namespace BCG { return retVal; } + public IEnumerable GetSupportedTypes() => + myTypeLookupTable.Values.Cast(); + public IEnumerable GetSupportedStreams() => + GenericDictionaryWrapper.Create(myTypeLookupTable).Select(kv => new DataStreamMetadata() { + Name = kv.Key, + AlternateNames = new() { kv.Value.Name, kv.Value.FullName } + }); - public static PrimitiveManager instance { - get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; } + public Type ResolveType(string streamId) => + TryResolveType(streamId, out var t) ? t + : throw new ArgumentException($"Failed to resolve ViewStream type '{streamId}'", nameof(streamId)); + + public bool TryResolveType(string streamId, [NotNullWhen(true)] out Type result) { + result = GetSupportedStreams() + .Where(s => s.IsMatch(streamId)) + .Select(s => s.Name) + .Prepend(streamId) + .Select(myTypeLookupTable.GetValueOrDefault) + .ExceptNull() + .FirstOrDefault(); + return result is not null; } - /// - /// Returns the Type corresponding to the primitive name - /// - /// - /// - public Type find(string friendlyName) => - myTypeLookupTable.TryGetValue(friendlyName, out var t) - ? t - : throw new ArgumentException($"'{friendlyName}' is not the friendly name of any of the loaded types", nameof(friendlyName)); - /// /// Constructs and returns a ViewStream of type corresponding to the friendly name /// @@ -118,8 +135,110 @@ namespace BCG { constructorArgs = new object[args.Count]; args.CopyTo(constructorArgs, 0); } - var type = find(friendlyName); - return Activator.CreateInstance(type, constructorArgs) as ViewStream; + if (myTypeLookupTable[friendlyName] == null) + throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types"); + try { + return + constructorArgs == null ? + Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream : + Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream; + } catch (Exception ex) { + Log.Gen.warn($"PrimitiveManager failed to construct ViewStream type '{friendlyName}': {ex}"); + } + return null; + } + + public ViewStream Create(string streamId, IEnumerable<(string Name, int? Index, object Value)> args) => + TryResolveType(streamId, out var type) ? Create(type, args) : null; + + /// + /// Constructs and returns a ViewStream of type corresponding to the friendly name + /// + /// friendly primitive name + /// args can be null or an empty ICollection if the paramterless constructor + /// is the desired one + /// + public ViewStream Create(Type type, IEnumerable<(string Name, int? Index, object Value)> args) { + var ctrs = type.GetConstructors(BindingFlags.Instance | BindingFlags.Public); + var argList = args.ToList(); + const int MAX_ALLOWED_ARGS = 64; + if (argList.Count > MAX_ALLOWED_ARGS) { + throw new ArgumentException( + $"ViewStream arg count ({argList.Count}) exceeds max allowed ({MAX_ALLOWED_ARGS})", nameof(args)); + } + if (argList.Any(a => !(a.Name?.Length > 0) && !a.Index.HasValue)) { + throw new ArgumentException( + "One or more ViewStream args lack both Name and Index (unable to assign a constructor param)", nameof(args)); + } + var indexedArgs = argList.Where(a => a.Index.HasValue).ToList(); + var namedArgs = argList.Where(a => a.Name?.Length > 0).ToList(); + if (!namedArgs.Select(a => a.Name).IsDistinct()) { + throw new ArgumentException("ViewStream arg names are not unique"); + } + if (!indexedArgs.Select(a => a.Index.Value).IsDistinct()) { + throw new ArgumentException("ViewStream arg indices are not unique"); + } + // Constructor params with default values may be omitted from the input 'args'. Therefore, + // 'minArgCount' >= 'argList.Count'. + var minArgCount = Math.Max(indexedArgs.Select(a => a.Index.Value).MaxOrDefault(), argList.Count); + foreach (var ctr in ctrs) { + if (TryAssignConstructorArgs(ctr, minArgCount, argList, indexedArgs, namedArgs, out var ctrArgs)) { + return ctr.Invoke(ctrArgs) as ViewStream; + } + } + return null; + + static bool TryAssignConstructorArgs( + ConstructorInfo ctr, + int minArgCount, + List<(string Name, int? Index, object Value)> argList, + List<(string Name, int? Index, object Value)> indexedArgs, + List<(string Name, int? Index, object Value)> namedArgs, + [NotNullWhen(true)] out object[] ctrArgs + ) { + ctrArgs = null; + var ctrParams = ctr.GetParameters(); + if (ctrParams.Length < minArgCount) { + return false; + } + var ctrArgList = new List(); + int remInputArgs = argList.Count; + for (int i = 0; i < ctrParams.Length; i++) { + var ctrParam = ctrParams[i]; + var paramType = ctrParams[i].ParameterType; + if (indexedArgs.TryGetFirst(a => CanAssignInputArg(ctrParam, a), out var a)) { + remInputArgs--; + ctrArgList.Add(a.Value); + } else if (ctrParam.HasDefaultValue) { + ctrArgList.Add(ctrParam.DefaultValue); + } else { + return false; + } + } + if (remInputArgs > 0) { + return false; + } + Log.assert(ctrArgList.Count == ctrParams.Length); + ctrArgs = ctrArgList.ToArray(); + return true; + } + + static bool CanAssignInputArg(ParameterInfo pi, (string Name, int? Index, object Value) arg) { + if (!string.IsNullOrEmpty(arg.Name) && arg.Name != pi.Name) { + return false; + } + if (arg.Index.HasValue && arg.Index.Value != pi.Position) { + return false; + } + if (arg.Value is null) { + if (!pi.ParameterType.IsValueType && !pi.ParameterType.IsNullable()) { + return false; + } + } else if (!pi.ParameterType.IsAssignableFrom(arg.Value.GetType())) { + return false; + } + return true; + } } /// diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs new file mode 100644 index 000000000..881433190 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs @@ -0,0 +1,9 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System.Collections.Generic; + +public class PrimitiveManagerConfig { + public List? Assemblies { get; set; } + public List? ExcludeTypes { get; set; } +} diff --git a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs index fc036feb4..4769af3ed 100644 --- a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs +++ b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs @@ -3,6 +3,9 @@ namespace Bts.DataStreamCore.Services; using BCG; using BCG.Utilities.General; +using Bts.DataStream; + +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; public static class DataStreamCoreServices { @@ -14,4 +17,11 @@ public static class DataStreamCoreServices { ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService())); return ViewStreamDatabase.instance; }); + + public static IServiceCollection AddPrimitiveDataStreamFactory(this IServiceCollection services, IConfiguration configurationSection) => + services.Configure(configurationSection) + .AddSingleton(); + + public static IServiceCollection AddCompositeDataStreamFactory(this IServiceCollection services) => + services.AddSingleton(); } diff --git a/src/DataStreamSuite/DataStreamExpressions/Assignment.cs b/src/DataStreamSuite/DataStreamExpressions/Assignment.cs index 86590fad7..1d0ccb9ee 100644 --- a/src/DataStreamSuite/DataStreamExpressions/Assignment.cs +++ b/src/DataStreamSuite/DataStreamExpressions/Assignment.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStream; + class Assignment : StatementExpression { readonly StatementExpression expression; readonly string id; @@ -19,8 +21,8 @@ class Assignment : StatementExpression { return result; } - public override ViewStream? myGetView(IDictionary env) { - var result = expression.getView(env); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) { + var result = expression.getView(env, streamFactory); env[id] = expression; return result; } diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs index 7f84a8fad..708371ee4 100644 --- a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStream; + using System.Text; public class DataStreamExpression { @@ -31,10 +33,10 @@ public class DataStreamExpression { return sb.ToString(); } - public ViewStream? getView(IDictionary env) { + public ViewStream? getView(IDictionary env, IDataStreamFactory streamFactory) { for (int i = 0; i < Statements.Count - 1; i++) { - Statements[i].getView(env); + Statements[i].getView(env, streamFactory); } - return Statements[^1].getView(env); + return Statements[^1].getView(env, streamFactory); } } diff --git a/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs b/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs index 1c87372e6..c52956201 100644 --- a/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs @@ -2,7 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; -using System.Collections; +using Bts.DataStream; + using System.Collections.Immutable; public class PrimitiveExpression : StatementExpression { @@ -20,7 +21,7 @@ public class PrimitiveExpression : StatementExpression { } public override string print(int startCol, IList activeBindings) { - string prelude = "new " + PrimitiveManager.instance.find(myName).Name + " ("; + string prelude = "new " + myName + " ("; return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")"; } @@ -31,21 +32,19 @@ public class PrimitiveExpression : StatementExpression { return prelude + myPrintViewArgs(startCol + prelude.Length) + ")"; } - public override ViewStream myGetView(IDictionary env) { - ArrayList resolvedArgs = new ArrayList(); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) { + var resolvedArgs = myArgs.Select((arg, i) => ((string?)arg.Name, string.IsNullOrEmpty(arg.Name) ? (int?)i : null, ResolveArg(arg))); + return streamFactory.Create(myName, resolvedArgs); - foreach (var arg in myArgs) { + object? ResolveArg(Argument arg) { if (arg.StatementExp is not null) { - resolvedArgs.Add(arg.StatementExp.myGetView(env)); - continue; + return arg.StatementExp.myGetView(env, streamFactory); } if (arg.QuotedView is not null) { - resolvedArgs.Add(arg.QuotedView.Value); - continue; + return arg.QuotedView.Value; } - resolvedArgs.Add(arg.Obj); + return arg.Obj; } - return PrimitiveManager.instance.build(myName, resolvedArgs); } string myPrintSingleArg(int atColumn, object? o, IList activeBindings) { diff --git a/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs b/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs index 211c290bb..49f3a889c 100644 --- a/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStream; + public abstract class StatementExpression { ViewStream? view; @@ -13,10 +15,10 @@ public abstract class StatementExpression { public virtual string? printView(int startCol) => null; - public ViewStream? getView(IDictionary env) => - view ??= myGetView(env); + public ViewStream? getView(IDictionary env, IDataStreamFactory streamFactory) => + view ??= myGetView(env, streamFactory); - public virtual ViewStream? myGetView(IDictionary env) => + public virtual ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) => null; public static string makeCSString(string s) { diff --git a/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs b/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs index 5d4e3f6e2..ae823573d 100644 --- a/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStream; + public class ViewNameExpression : StatementExpression { readonly string id; @@ -14,6 +16,6 @@ public class ViewNameExpression : StatementExpression { public override string printView(int startCol) => id; - public override ViewStream? myGetView(IDictionary env) => - env.TryGetValue(id, out var exp) ? exp.getView(env) : new TableStream(id); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) => + env.TryGetValue(id, out var exp) ? exp.getView(env, streamFactory) : new TableStream(id); } diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs index 1e09e7781..87c165906 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs @@ -4,6 +4,7 @@ using BCG; using bts; +using Bts.DataStream; using Bts.DataStreamCore; using Bts.DataStreamExpressions; using Bts.PubSub; @@ -28,13 +29,15 @@ public class DataStreamPublisherService : IHostedService, IDisposable { readonly DataStreamPublisherConfig config; readonly IPubTableBroker pubTableBroker; readonly IWorkScheduler workScheduler; + readonly IDataStreamFactory streamFactory; List? resources = new(); - public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions config) - : this(pubTableBroker, workScheduler, config.Value) { } + public DataStreamPublisherService(IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory, IWorkScheduler workScheduler, IOptions config) + : this(pubTableBroker, streamFactory, workScheduler, config.Value) { } - public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) { + public DataStreamPublisherService(IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory, IWorkScheduler workScheduler, DataStreamPublisherConfig config) { this.pubTableBroker = pubTableBroker; + this.streamFactory = streamFactory; this.workScheduler = workScheduler; this.config = config; } @@ -48,7 +51,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable { Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json)."); continue; } - var vs = CreateViewStream(publishedStream); + var vs = CreateViewStream(streamFactory, publishedStream); if (vs is null) { continue; } @@ -72,7 +75,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable { resources?.Clear(); } - static ViewStream? CreateViewStream(PublishedStream ps) { + static ViewStream? CreateViewStream(IDataStreamFactory streamFactory, PublishedStream ps) { Log.Gen.info( $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); if (string.IsNullOrEmpty(ps.Name)) { @@ -80,7 +83,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable { $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); return null; } - var vs = buildStream(ps.Expression); + var vs = buildStream(streamFactory, ps.Expression); if (vs is null) { Log.Gen.warn( $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); @@ -89,14 +92,14 @@ public class DataStreamPublisherService : IHostedService, IDisposable { return vs; } - static ViewStream? buildStream(string? streamExpression) { + static ViewStream? buildStream(IDataStreamFactory streamFactory, string? streamExpression) { if (streamExpression is null) { return null; } try { Log.Gen.info($"Building ViewStream: {streamExpression}"); var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression); - return exp.getView(new Dictionary(0)); + return exp.getView(new Dictionary(0), streamFactory); } catch (Exception ex) { Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}"); return null; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs index 7291a773b..c74f06498 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -111,7 +111,14 @@ namespace bts.SilverPine { : this(symbol, Enum.Parse(bar_size)) { } - - + internal OHLCVStream(ViewStream source, OHLCVBarSize bar_size) { + ViewStream rs = source; + rs = new SchemaGuesserStream(rs); + if (bar_size == OHLCVBarSize.ONE_DAY) { + rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); + } + rs = new NewProjectStream(rs, "symbol:contract"); + mySetRealSource(rs); + } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs index 383a7becb..020550d41 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs @@ -50,4 +50,4 @@ namespace bts.SilverPine { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs b/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs new file mode 100644 index 000000000..27faaa1bb --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs @@ -0,0 +1,77 @@ +namespace bts.SilverPine; + +using Spark.Core.Json; + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +public class NLPromptConfig { + static NLPromptConfig? instance; + + public static NLPromptConfig Instance => + instance ?? throw new InvalidOperationException( + "SetGlobalConfig must be called to initialize config static singleton"); + + public List prompts { get; } = new(0); + + public static void SetGlobalConfig(NLPromptConfig cfg) { + // This should only be called once + Log.isnull(instance); + instance = Log.notnull(cfg); + } + + public static NLPromptConfig Default() => new(); + + // Represents a pair of a natural language query and its dsl + public class FewShotExample { + public string? natural_language { get; set; } + public string? dsl { get; set; } + } + + public class FewShotPrompt { + public string? category { get; set; } + + public string? description { get; set; } + + public bool IsEnabled { get; set; } = true; + + // MultiLineDescription is simply here for convenience. It is functionally redundant, as it + // serves the same purpose as 'description'. If 'description' is too long, you can break it + // up into an array of strings and assign it to MultiLineDescription instead. + public List? MultiLineDescription { get; set; } + + // the actual prompt given by the user/CME + public string? prompt { get; set; } + + public List examples { get; } = new(0); + + public override string ToString() { + var desc = description ?? string.Concat(MultiLineDescription.empty_if_null()); + string rv = $"System: {desc}\n"; + int ex_no = 0; + foreach (FewShotExample ex in examples ?? new(0)) { + rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n"; + rv += $" DSL: {ex.dsl}\n"; + + } + return rv; + } + + // Reads normal strings and arrays of lines + public class FewShotPromptLinesConverter : JsonConverter { + public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.ReadString(out var str)) { + return str; + } + if (reader.ReadArray(out var array)) { + return string.Join('\n', array.Select(v => v?.AsString())); + } + throw new JsonException("Failed to parse FewShotPrompt lines"); + } + + public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => + writer.WriteStringValue(value); + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs b/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs new file mode 100644 index 000000000..0c56f5e6a --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs @@ -0,0 +1,42 @@ +namespace bts.SilverPine; + +using BCG; + +using Bts.DataStream; + +using mbts.utils; + +using System.Collections.Generic; + +public class NLStreamFactory : IRecursiveDataStreamFactory { + IDataStreamFactory? compositeFactory; + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.NaturalLanguage.IsMatch(streamId)) { + if (compositeFactory is null) { + Log.DataFlow.warn("Failed to create NLStream because 'compositeFactory' is null."); + return null; + } + if (args.SingleOrDefault().Value is string prompt) { + return new NLStream(prompt, compositeFactory); + } + } + return null; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.NaturalLanguage; + } + + public void OnCompositeFactory(IDataStreamFactory compositeFactory) => + this.compositeFactory = compositeFactory; + + static class SupportedStreams { + public static readonly DataStreamMetadata NaturalLanguage = new() { + Name = "nl", + AlternateNames = new() { "naturalLanguage" } + }; + + public static string NlName => ExceptionAssert.not_null(NaturalLanguage.Name); + } +} diff --git a/src/DataStreamSuite/SharedStreams/NL/NLToDSLStream.cs b/src/DataStreamSuite/SharedStreams/NL/NLToDSLStream.cs index b056334ee..403178c11 100644 --- a/src/DataStreamSuite/SharedStreams/NL/NLToDSLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/NLToDSLStream.cs @@ -40,4 +40,4 @@ namespace bts.SilverPine { } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/NL/NLToIndexedFewShotPromptStream.cs b/src/DataStreamSuite/SharedStreams/NL/NLToIndexedFewShotPromptStream.cs index e8f3fbd8c..28e05ab4f 100644 --- a/src/DataStreamSuite/SharedStreams/NL/NLToIndexedFewShotPromptStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/NLToIndexedFewShotPromptStream.cs @@ -7,38 +7,16 @@ namespace bts.SilverPine { using System.Text.Json; + using Root = NLPromptConfig; + using FewShotExample = NLPromptConfig.FewShotExample; + using FewShotPrompt = NLPromptConfig.FewShotPrompt; [DataStreamUserRespresentation("nl_to_indexed_fewshot_prompt")] // given an nl prompt, retreives the correct fewshot prompt and converts it into a form // that be sent to openAI as a prompt public class NLToIndexedFewShotPromptStream : ViewStream { - // Represents a pair of a natural language query and its dsl - public class FewShotExample { - public string natural_language { get; set; } - public string dsl { get; set; } - } - // Class to represent each prompt category - public class FewShotPrompt { - public string category { get; set; } - // this is the explaination of the DSL - public string description { get; set; } - // th actual prompt given by the user/CME - public string prompt { get; set; } - public List examples { get; set; } - - public override string ToString() { - string rv = $"System: {description}\n"; - int ex_no = 0; - foreach (FewShotExample ex in examples) { - rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n"; - rv += $" DSL: {ex.dsl}\n"; - - } - return rv; - } - } // this is how it should be prepended to the actual prompt: // System: This is a few-shot learning example for converting natural language prompts to a DSL function for settlement data retrieval. The DSL function can take either a single date or a date range. Assume today's date is 10/11/2024. The function is structured as follows: @@ -66,21 +44,6 @@ namespace bts.SilverPine { // [Your Real Prompt Goes Here] - - // Root class to map the JSON structure - public class Root { - public List prompts { get; set; } - } - - public static Root read_json_file() { - // TODO - string json_file_path = @"C:\Users\m_tah\development\spark\src\DataStreamSuite\SharedStreams\NL\FewShotPrompts.json"; - string json_string = File.ReadAllText(json_file_path); - var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; - Root root = JsonSerializer.Deserialize(json_string, options); - return root; - } - private readonly string _nl_prompt; private string _fewshot_index_prompt; private string _fewshot_index_reponse; @@ -101,7 +64,7 @@ namespace bts.SilverPine { } protected override void myOnFirstSubscriber() { - Root root = read_json_file(); + var root = NLPromptConfig.Instance; string meta_prompt = build_fewshot_index_prompt(root, _nl_prompt); _fewshot_index_prompt = meta_prompt; Log.Gen.info($"NLToDSLStream.myOnFirstSubscriber _fewshot_index_prompt: {_fewshot_index_prompt}"); @@ -138,7 +101,7 @@ namespace bts.SilverPine { string[] toks = response.Split('.'); index = Int32.Parse(toks[0]); } - var root = read_json_file(); + var root = NLPromptConfig.Instance; FewShotPrompt fs_prompt = root.prompts[index]; string final_prompt = _build_fewshot_prompt(fs_prompt, _nl_prompt); _fire_event(final_prompt); @@ -166,6 +129,10 @@ namespace bts.SilverPine { string meta_promp = "Take a look at the following prompt examples:\n"; int rv = 0; foreach (FewShotPrompt fs_entry in root.prompts) { + if (!fs_entry.IsEnabled) { + continue; + } + var prompt = fs_entry.prompt; meta_promp += $"{rv++}. {fs_entry.prompt}\n"; } meta_promp += "Based on the above prompts, tell me which prompt (give me answer as the index only, and nothing else) does the following prompt most closely matches to: \n"; @@ -194,4 +161,4 @@ namespace bts.SilverPine { return rv; } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/NL/OhlcvStreamConfig.cs b/src/DataStreamSuite/SharedStreams/NL/OhlcvStreamConfig.cs new file mode 100644 index 000000000..e436278c4 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/NL/OhlcvStreamConfig.cs @@ -0,0 +1,32 @@ +namespace bts.SilverPine; + +// public class DateRange { +// public DateTime Begin { get; set; } +// public DateTime End { get; set; } +// } + +public class OhlcvDatabaseConfig { + public class DatasetConfig { + public string? Path { get; set; } + public string? Query { get; set; } + public List? QueryParams { get; set; } + } + + public DatasetConfig? Bbo1s { get; set; } + public DatasetConfig? Bbo1m { get; set; } + public DatasetConfig? Ohlcv1s { get; set; } + public DatasetConfig? Ohlcv1m { get; set; } + public DatasetConfig? Ohlcv1h { get; set; } + public DatasetConfig? Ohlcv1d { get; set; } + public DatasetConfig? Mbo { get; set; } + + // public List? Symbols { get; set; } + // public List? Venues { get; set; } + // public List? Types { get; set; } + // public List? DateRanges { get; set; } + // public string? Path { get; set; } +} + +public class OhlcvStreamConfig { + public OhlcvDatabaseConfig? Database { get; set; } +} diff --git a/src/DataStreamSuite/SharedStreams/Services/NaturalLanguageStreamService.cs b/src/DataStreamSuite/SharedStreams/Services/NaturalLanguageStreamService.cs new file mode 100644 index 000000000..1d940b339 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Services/NaturalLanguageStreamService.cs @@ -0,0 +1,43 @@ +// namespace Bts.SharedStreams.Services; + +// using BCG; +// using BCG.Utilities.General; + +// using bts.SilverPine; + + +// using Microsoft.Extensions.Hosting; +// using Microsoft.Extensions.Options; + +// using System; +// using System.Collections.Generic; +// using System.Threading; +// using System.Threading.Tasks; + +// /// +// /// An that imports a collection of s and +// /// calls each in a loop. +// /// +// public class NaturalLanguageStreamService : BackgroundService { +// readonly IWorker masterWorker; +// readonly NLPromptConfig config; + +// public NaturalLanguageStreamService(IOptions config) +// : this(config.Value) { } + +// public NaturalLanguageStreamService(NLPromptConfig config) { +// this.config = config; +// } + +// protected override Task ExecuteAsync(CancellationToken cancellationToken) { +// return Task.Factory.StartNew( +// Execute, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default); + +// void Execute() { +// while (!cancellationToken.IsCancellationRequested) { +// masterWorker.doWork(); +// Thread.Sleep(config.Interval); +// } +// } +// } +// } diff --git a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs new file mode 100644 index 000000000..74ccf4e9c --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs @@ -0,0 +1,33 @@ +namespace Bts.SharedStreams.Services; + +using bts.SilverPine; + +using Bts.DataStream; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +public static class SharedStreamsServices { + public static IServiceCollection AddOhlcvStreams(this IServiceCollection services, IConfigurationSection configurationSection) => + services.Configure(configurationSection) + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + // .AddSingleton() + // .AddSingleton(sp => sp.GetRequiredService()) + ; + + public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) { + services.Configure(configurationSection); + NLPromptConfig.SetGlobalConfig(configurationSection.Get() ?? NLPromptConfig.Default()); + // TODO: integrate config into a NLStream provider (so it can provide config values + // instead of the ViewStream's getting them from a static singleton) + return services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamFactory.cs new file mode 100644 index 000000000..065f07460 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamFactory.cs @@ -0,0 +1,78 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.api.core; +using bts.SilverPine.StreamFactory.Logging; +using bts.SilverPine.Tables; + +using Bts.DataStream; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using System; +using System.Collections.Generic; +using System.Data; + +public class BboStreamFactory : IDataStreamFactory { + readonly DatabaseConfig config; + readonly ILogger logger; + + public BboStreamFactory(IOptions config, ILogger logger) + : this(config.Value, logger) { } + + public BboStreamFactory(DatabaseConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.Bbo.IsMatch(streamId)) { + return createBbo(args); + } + return null; + } + + public IObservable>? CreateTables( + BboStreamParams args) { + args.Validate(); + return new HistoricDataLoader(config, logger).LoadTablesObservable( + args.DataType, args.Venue, args.Symbol, args.BeginDate, args.EndDate, args.Interval, GetTimestamp, RowFactory); + + DateTime? GetTimestamp(IBlueRow r) { + var tsObj = r["ts_event"]; + return tsObj is not DateTime ts && !DateTime.TryParse(tsObj as string, out ts) ? null : ts; + } + + BboRow RowFactory(IBlueRow row, int groupKey, DateTime timestamp) => + new() { + Id = groupKey, + Timestamp = timestamp, + Symbol = args.Symbol, + Venue = args.Venue, + BidPrice = row.GetDouble("bid_px_00") ?? double.NaN, + BidQuantity = row.GetDouble("bid_sz_00") ?? double.NaN, + AskPrice = row.GetDouble("ask_px_00") ?? double.NaN, + AskQuantity = row.GetDouble("ask_sz_00") ?? double.NaN, + }; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.Bbo; + } + + ViewStream? createBbo(IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!StreamFactoryArgStore.TryGetParams(args, out BboStreamParams? bsp, out var err)) { + logger.LogInvalidParams("bbo", err); + return null; + } + return CreateTables(bsp)?.AsViewStream(BboRow.Schema); + } + + static class SupportedStreams { + public static readonly DataStreamMetadata Bbo = new() { + Name = "bbo" + }; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamParams.cs new file mode 100644 index 000000000..c6a2de513 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamParams.cs @@ -0,0 +1,56 @@ +namespace bts.SilverPine; + +using System.Diagnostics.CodeAnalysis; + +public class BboStreamParams { + [StreamArgument("venue", "exchange", "exch", IsRequired = true)] + public string? Venue { get; set; } + + [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)] + public string? Symbol { get; set; } + + [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)] + public DateTime BeginDate { get; set; } + + [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)] + public DateTime EndDate { get; set; } + + [StreamArgument("interval", IsRequired = true)] + public TimeSpan Interval { get; set; } + + // Normally, the dataType is inferred from Interval, though it can be explicitly specified as well. + public string? DataType { get; set; } + + [MemberNotNull(nameof(Venue), nameof(Symbol), nameof(DataType))] + public BboStreamParams Validate() { + if (Venue is null) { + throw new InvalidOperationException("Venue is null"); + } + if (Symbol is null) { + throw new InvalidOperationException("Symbol is null"); + } + if (Interval <= TimeSpan.Zero) { + throw new InvalidOperationException("Interval <= TimeSpan.Zero"); + } + DataType ??= DataTypeFromInterval(Interval); + if (DataType is null) { + throw new InvalidOperationException( + $"DataType is null. Interval {Interval} did not map to well-known data type (bbo1s, bbo1m, bbo1h, bbo1d)."); + } + return this; + } + + public static string? DataTypeFromInterval(TimeSpan interval) { + if (interval == TimeSpan.FromSeconds(1)) { + return "bbo1s"; + } else if (interval == TimeSpan.FromMinutes(1)) { + return "bbo1m"; + } else if (interval == TimeSpan.FromHours(1)) { + return "bbo1h"; + } else if (interval == TimeSpan.FromDays(1)) { + return "bbo1d"; + } else { + return null; + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataLoader.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataLoader.cs new file mode 100644 index 000000000..02f69138c --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataLoader.cs @@ -0,0 +1,110 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.api.core; + +using Bts.DataStream; + +using Microsoft.Extensions.Logging; + +using System.Data; +using System.Data.SQLite; +using System.Reactive.Linq; +using System.Reflection; + +public class HistoricDataLoader { + readonly DatabaseConfig dbConfig; + readonly ILogger logger; + + public HistoricDataLoader(DatabaseConfig dbConfig, ILogger logger) { + this.dbConfig = dbConfig; + this.logger = logger; + } + + public IEnumerable LoadQueryMetadata() { + try { + var connectionStr = $"Data Source={dbConfig.Path};Version=3"; + var dataTable = new DataTable(); + using var conn = new SQLiteConnection($"Data Source={dbConfig.Path};Version=3"); + conn.Open(); + using var sqlCmd = new SQLiteCommand(dbConfig.Query, conn); + using var adapter = new SQLiteDataAdapter(sqlCmd); + adapter.Fill(dataTable); + + return dataTable.Rows.OfType() + .Select(r => new StreamQueryMetadata() { + Symbol = r["symbol"] as string, + DataType = r["data_type"] as string, + BeginDate = DateTime.Parse(r["begin_date"]?.ToString()!), + EndDate = DateTime.Parse(r["end_date"]?.ToString()!), + Location = r["location"] as string, + Query = r["query"] as string, + }); + } catch (Exception ex) { + logger.LogError("Failed to load query metadata from '{dbPath}'. Query: {query}. Exception {ex}", dbConfig.Path, dbConfig.Query, ex); + } + return Enumerable.Empty(); + } + + public IAsyncEnumerable LoadRowsAsync(string dataType, string venue, string? symbol, DateTime beginDate, DateTime endDate) { + venue = resolveVenue(venue); + var queryMetadatas = LoadQueryMetadata().ToList(); + var perDateQueries = queryMetadatas + .Where(m => m.DataType == dataType && m.Symbol == symbol && m.BeginDate < endDate && m.EndDate > beginDate) + .ToList(); + if (perDateQueries.Count == 0) { + return AsyncEnumerable.Empty(); + } + return perDateQueries.ToAsyncEnumerable().SelectMany(q => { + var location = q.Location ?? throw new InvalidOperationException($"'location' is null in metadata for {dataType} stream"); + var query = q.Query ?? throw new InvalidOperationException($"'query' is null in metadata for {dataType} stream"); + if (!Path.IsPathRooted(location)) { + var dir = Path.GetDirectoryName(dbConfig.Path) ?? Assembly.GetExecutingAssembly().Location; + location = Path.Combine(dir, location); + } + return new SQLiteEnumerable(location, query); + }); + } + + public IObservable>? LoadTablesObservable( + string dataType, + string venue, + string symbol, + DateTime beginDate, + DateTime endDate, + TimeSpan interval, + Func getTimestamp, + Func rowFactory + ) where T : ITimeSeriesRow { + return LoadRowsAsync(dataType, venue, symbol, beginDate, endDate) + .ToObservable() + .Scan(TableUpdate.mk_empty(r => r.Id), AddPage); + + TableUpdate AddPage(TableUpdate res, IBlueRowCollection blueRows) { + var bd = res.to_builder(); + foreach (IBlueRow blueRow in blueRows) { + var ts = getTimestamp(blueRow); + if (!ts.HasValue || ts < beginDate || ts >= endDate) { + continue; + } + var delta = ts - beginDate; + var groupKey = (int)(delta / interval); + if (!bd.TryGetValue(groupKey, out var row) || row.Timestamp < ts) { + row = rowFactory(blueRow, groupKey, ts.Value); + if (row is null) { + continue; + } + bd.add_or_update(row); + } + } + return bd.to_table(); + } + } + + static string resolveVenue(string venue) => + venue switch { + "CME" or "XCME" or "CBOT" or "XCBT" => "GLBX", + _ => venue + }; +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/ITimeSeriesRow.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/ITimeSeriesRow.cs new file mode 100644 index 000000000..90051de04 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/ITimeSeriesRow.cs @@ -0,0 +1,11 @@ +namespace bts.SilverPine; + +using BCG; + +using mbts.utils; + + +public interface ITimeSeriesRow : IBlueRow { + int Id => ExceptionAssert.of_type(this["id"]); + DateTime Timestamp { get; } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamFactory.cs new file mode 100644 index 000000000..c606c73de --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamFactory.cs @@ -0,0 +1,185 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.api.core; +using bts.SilverPine.StreamFactory.Logging; +using bts.SilverPine.Tables; + + +using Bts.DataStream; +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Spark.Core; + +using System.Reactive.Linq; +using System.Reflection; + +public class MbpStreamFactory : IDataStreamFactory { + readonly DatabaseConfig config; + readonly ILogger logger; + + public MbpStreamFactory(IOptions config, ILogger logger) + : this(config.Value, logger) { } + + public MbpStreamFactory(DatabaseConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.Mbp.IsMatch(streamId)) { + return create(args); + } + if (SupportedStreams.OrderImbalance.IsMatch(streamId)) { + return create(args); + } + return null; + } + + public static IObservable>? CreateTables( + DatabaseConfig config, string venue, string symbol, DateTime beginDate, DateTime endDate, TimeSpan interval, int numLevels) { + return loadPagesAsync(config, venue, symbol, beginDate, endDate) + .ToObservable() + .Scan(TableUpdate.mk_empty(r => r.Id), AddPage) + .Where(t => t.has_changes()); + + TableUpdate AddPage(TableUpdate res, IBlueRowCollection rows) { + var bd = res.to_builder(); + var refDate = new DateTime(2000, 1, 1); + foreach (IBlueRow r in rows) { + var tsObj = r["ts_event"]; + if (tsObj is not DateTime ts && !DateTime.TryParse(tsObj as string, out ts)) { + continue; + } + if (ts < refDate) { + continue; + } + var groupKey = (int)((ts - refDate) / interval); + if (!res.state.TryGetValue(groupKey, out var bboRow) || bboRow.Timestamp > ts) { + var mbpRow = new MbpVariableDepthRow(numLevels) { + Id = r.Id(), + Timestamp = r.get_datetime("ts_event"), + Venue = venue, + Symbol = symbol, + }; + for (int i = 0; i < numLevels; i++) { + mbpRow.BidQuantities[i] = r.GetDouble($"bid_sz_{i:00}") ?? 0; + mbpRow.BidPrices[i] = mbpRow.BidQuantities[i] > 0 ? r.GetDouble($"bid_px_{i:00}").assert_value() : double.NaN; + mbpRow.AskQuantities[i] = r.GetDouble($"ask_sz_{i:00}") ?? 0; + mbpRow.AskPrices[i] = mbpRow.AskQuantities[i] > 0 ? r.GetDouble($"ask_px_{i:00}").assert_value() : double.NaN; + } + bd.add_or_update(mbpRow); + } + } + return bd.to_table(); + } + } + + static string resolveVenue(string venue) => + venue switch { + "CME" or "XCME" or "CBOT" or "XCBT" => "GLBX", + _ => venue + }; + + static IEnumerable loadQueryMetadata(DatabaseConfig config, ILogger? logger = null) { + var dbPath = config.Path; + var query = config.Query; + if (dbPath is not null && query is not null) { + try { + return StreamQueryMetadata.Load(dbPath, query); + } catch (Exception ex) { + logger?.LogError("Failed to load query metadata from '{dbPath}'. Exception {ex}", dbPath, ex); + } + } + return Enumerable.Empty(); + } + + ViewStream? create(IEnumerable<(string? Name, int? Index, object? Value)> args) { + var argStore = new StreamFactoryArgStore(args); + var venue = argStore.GetVenueString(); + if (venue is null) { + logger.LogMissingParam("bbo", "venue"); + return null; + } + var symbol = argStore.GetSymbol(); + if (symbol is null) { + logger.LogMissingParam("bbo", "symbol"); + return null; + } + var begin = argStore.GetDateTime("begin"); + if (!begin.HasValue) { + logger.LogMissingParam("bbo", "begin"); + return null; + } + var end = argStore.GetDateTime("end"); + if (!end.HasValue) { + logger.LogMissingParam("bbo", "end"); + return null; + } + var interval = argStore.GetTimeSpan("interval"); + if (!interval.HasValue) { + logger.LogMissingParam("bbo", "interval"); + return null; + } + var numLevels = argStore.GetInt32("levels") ?? 1; + var schema = MbpVariableDepthRow.GetSchema(numLevels); + var vs = CreateTables(config, venue, symbol, begin.Value, end.Value, interval.Value, numLevels) + ?.AsDataPackets(schema) + ?.AsViewStream(); + return vs; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.Mbp; + } + + static IAsyncEnumerable loadPagesAsync(DatabaseConfig config, string venue, string? symbol, DateTime beginDate, DateTime endDate) { + const string dataType = "mbp10"; + venue = resolveVenue(venue); + var queryMetadatas = loadQueryMetadata(config).ToList(); + var perDateQueries = queryMetadatas + .Where(m => m.DataType == dataType && m.Symbol == symbol && m.BeginDate < endDate && m.EndDate > beginDate) + .ToList(); + if (perDateQueries.Count == 0) { + return AsyncEnumerable.Empty(); + } + return perDateQueries.ToAsyncEnumerable().SelectMany(q => { + var location = q.Location ?? throw new InvalidOperationException($"'location' is null in metadata for {dataType} stream"); + if (!Path.IsPathRooted(location)) { + var dir = Path.GetDirectoryName(config.Path) ?? Assembly.GetExecutingAssembly().Location; + location = Path.Combine(dir, location); + } + var query = q.Query ?? throw new InvalidOperationException($"'query' is null in metadata for {dataType} stream"); + var andFilters = new List(2); + if (beginDate > DateTime.MinValue) { + var unixTicks = beginDate.Ticks - DateTime.UnixEpoch.Ticks; + andFilters.Add($"ts_event_ns > {unixTicks * 100 - 1}"); + } + if (endDate < DateTime.MaxValue) { + var unixTicks = endDate.Ticks - DateTime.UnixEpoch.Ticks; + andFilters.Add($"ts_event_ns < {unixTicks * 100}"); + } + if (andFilters.Count > 0) { + if (query.EndsWith(';')) { + query = query.Substring(0, query.Length - 1); + } + query += " where " + string.Join(" and ", andFilters) + ';'; + } + return new SQLiteEnumerable(location, query); + }); + } + + + static class SupportedStreams { + public static readonly DataStreamMetadata Mbp = new() { + Name = "mbp" + }; + public static readonly DataStreamMetadata OrderImbalance = new() { + Name = OrderImbalanceRow.Schema.tableName + }; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/MbpVariableDepthRow.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpVariableDepthRow.cs new file mode 100644 index 000000000..b9e499184 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpVariableDepthRow.cs @@ -0,0 +1,109 @@ +namespace bts.SilverPine; + +using BCG; + +using Bts.DataStreamCore; + +public struct MbpVariableDepthRow : IBlueRow, IEquatable { + static List? schemaByNumLevels; + + public MbpVariableDepthRow(int numLevels) { + schema = GetSchema(numLevels); + Id = 0; + Venue = string.Empty; + Symbol = string.Empty; + BidPrices = new double[numLevels]; + BidQuantities = new double[numLevels]; + AskPrices = new double[numLevels]; + AskQuantities = new double[numLevels]; + while (numLevels-- > 0) { + BidPrices[numLevels] = double.NaN; + AskPrices[numLevels] = double.NaN; + } + } + + public int Id { get; init; } + public DateTime Timestamp { get; init; } + public string Venue { get; init; } + public string Symbol { get; init; } + public double[] BidPrices { get; } + public double[] BidQuantities { get; } + public double[] AskPrices { get; } + public double[] AskQuantities { get; } + public IBlueTableSchema schema { get; } + + public readonly object? this[int index] { + get { + switch (index) { + case 0: + return Id; + case 1: + return Timestamp; + case 2: + return Venue; + case 3: + return Symbol; + case < 0: + throw new IndexOutOfRangeException(); + } + index -= 4; + // All arrays are same length (we arbitrarily get it from BidPrices) + var numLevels = BidPrices.Length; + var (prices, quantities) = (BidPrices, BidQuantities); + if (index >= 2 * numLevels) { + (prices, quantities) = (AskPrices, AskQuantities); + index -= 2 * numLevels; + } + var vals = index % 2 == 0 ? prices : quantities; + index /= 2; + return vals[index]; + } + } + + public static IBlueTableSchema GetSchema(int numLevels) { + // Use local variable to avoid concurrency issues. + var temp = schemaByNumLevels ?? new(); + if (temp.Count > numLevels) { + return temp[numLevels]; + } + while (numLevels >= temp.Count) { + temp.Add(createSchema(numLevels)); + } + var rv = temp[numLevels]; + schemaByNumLevels = temp; + return rv; + + static IBlueTableSchema createSchema(int numLevels) { + var schema = new BlueTableSchemaBuilder($"mbp-{numLevels}"); + schema.Add("id", typeof(int)); + schema.Add("timestamp", typeof(DateTime)); + schema.Add("venue", typeof(string)); + schema.Add("symbol", typeof(string)); + for (int i = 0; i < numLevels; i++) { + schema.Add($"bid_p_{i}", typeof(double)); + schema.Add($"bid_q_{i}", typeof(double)); + } + for (int i = 0; i < numLevels; i++) { + schema.Add($"ask_p_{i}", typeof(double)); + schema.Add($"ask_q_{i}", typeof(double)); + } + return schema.ToSchema(); + } + } + + public bool Equals(MbpVariableDepthRow other) => + Id == other.Id + && Timestamp == other.Timestamp + && Venue == other.Venue + && Symbol == other.Symbol + && BidPrices.SequenceEqual(other.BidPrices) + && BidQuantities.SequenceEqual(other.BidQuantities) + && AskPrices.SequenceEqual(other.AskPrices) + && AskQuantities.SequenceEqual(other.AskQuantities); + + public override bool Equals(object? obj) => + obj is MbpVariableDepthRow other && Equals(other); + + public override int GetHashCode() => + Id; +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/OhlcvStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/OhlcvStreamFactory.cs new file mode 100644 index 000000000..8711974ac --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/OhlcvStreamFactory.cs @@ -0,0 +1,105 @@ +namespace bts.SilverPine; + +using BCG; + +using Bts.DataStream; + +using mbts.utils; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; + +public class OhlcvStreamFactory : IDataStreamFactory { + readonly DatabaseConfig config; + readonly ILogger logger; + + public OhlcvStreamFactory(IOptions config, ILogger logger) : this(config.Value, logger) { } + + public OhlcvStreamFactory(DatabaseConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.Ohlcv.IsMatch(streamId)) { + return createOhlcvStream(args); + } + return null; + } + + ViewStream? createOhlcvStream(IEnumerable<(string? Name, int? Index, object? Value)> args) { + var argStore = new StreamFactoryArgStore(args); + var barType = argStore.GetEnum() ?? OHLCVStream.OHLCVType.HISTORIC; + var barSize = argStore.GetEnum() ?? OHLCVStream.OHLCVBarSize.ONE_DAY; + var secType = argStore.GetSecurityType(); + var symbol = argStore.GetSymbol() ?? argStore.GetStrings().Where(s => s.Length > 1 && s.Length < 5).FirstOrDefault(); + var vs = createOhlcvStream(barType, barSize, symbol); + if (vs is null) { + logger.LogWarning("Failed to create OhlcvStream from args: {args}", + string.Join(", ", args.Select(a => $"({a.Name}, {a.Index}, {a.Value})"))); + } + return vs; + } + + ViewStream? createOhlcvStream(OHLCVStream.OHLCVType ohlcvType, OHLCVStream.OHLCVBarSize ohlcvBarSize, string? symbol) { + switch (ohlcvType) { + case OHLCVStream.OHLCVType.LIVE: + case OHLCVStream.OHLCVType.LIVE_AND_HISTORIC: + Log.assert(false); + break; + case OHLCVStream.OHLCVType.HISTORIC: { + var dataType = ohlcvBarSize switch { + OHLCVStream.OHLCVBarSize.ONE_SEC => "ohlcv1s", + OHLCVStream.OHLCVBarSize.ONE_MIN => "ohlcv1m", + OHLCVStream.OHLCVBarSize.ONE_HOUR => "ohlcv1h", + OHLCVStream.OHLCVBarSize.ONE_DAY => "ohlcv1d", + _ => null, + }; + ViewStream vs = new AppendStream( + loadQueryMetadata() + .Where(m => m.DataType == dataType && m.Symbol == symbol) + .Select(m => new SQLiteStream($"Data Source={m.Location};Version=3", m.Query + ?? throw new InvalidOperationException("'query' is null in metadata for bbo1m stream"))) + .ToArray()); + vs = new SchemaGuesserStream(vs); + if (ohlcvBarSize == OHLCVStream.OHLCVBarSize.ONE_DAY) { + vs = new DateStandardizationStream(vs, "ts_event", DateConversion.ChopTime); + } + vs = new NewProjectStream(vs, "symbol:contract"); + return vs; + } + } + return null; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.Ohlcv; + } + + IEnumerable loadQueryMetadata() { + var dbPath = config.Path; + var query = config.Query; + if (dbPath is not null && query is not null) { + try { + return StreamQueryMetadata.Load(dbPath, query); + } catch (Exception ex) { + logger.LogError("Failed to load query metadata from '{dbPath}'. Exception {ex}", dbPath, ex); + } + } + return Enumerable.Empty(); + } + + static class SupportedStreams { + public static readonly DataStreamMetadata Ohlcv = new() { + Name = "ohlcv", + AlternateNames = new() { "bars" } + }; + + public static string OhlcvName => ExceptionAssert.not_null(Ohlcv.Name); + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamFactory.cs new file mode 100644 index 000000000..4494f9bfe --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamFactory.cs @@ -0,0 +1,110 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.api.core; +using bts.SilverPine.StreamFactory.Logging; +using bts.SilverPine.Tables; + +using Bts.DataStream; + +using mbts.utils; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using System.Reactive.Linq; + +public class OrderImbalanceStreamFactory : IDataStreamFactory { + readonly DatabaseConfig config; + readonly ILogger logger; + + public OrderImbalanceStreamFactory(IOptions config, ILogger logger) + : this(config.Value, logger) { } + + public OrderImbalanceStreamFactory(DatabaseConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.OrderImbalance.IsMatch(streamId)) { + return create(args); + } + return null; + } + + ViewStream? create(IEnumerable<(string? Name, int? Index, object? Value)> args) { + var argStore = new StreamFactoryArgStore(args); + var venue = argStore.GetVenueString(); + if (venue is null) { + logger.LogMissingParam("orderImbalance", "venue"); + return null; + } + var symbol = argStore.GetSymbol(); + if (symbol is null) { + logger.LogMissingParam("orderImbalance", "symbol"); + return null; + } + var begin = argStore.GetDateTime("begin"); + if (!begin.HasValue) { + logger.LogMissingParam("orderImbalance", "begin"); + return null; + } + var end = argStore.GetDateTime("end"); + if (!end.HasValue) { + logger.LogMissingParam("orderImbalance", "end"); + return null; + } + var interval = argStore.GetTimeSpan("interval"); + if (!interval.HasValue) { + var totalDur = end.Value - begin.Value; + var targetInterval = totalDur / 100; + if (targetInterval.TotalDays > 5) { + interval = TimeSpan.FromDays(1); + } else if (targetInterval.TotalDays > 1) { + interval = TimeSpan.FromHours(1); + } else if (targetInterval.TotalMinutes > 60) { + interval = TimeSpan.FromMinutes(1); + } else if (targetInterval.TotalMinutes > 10) { + interval = TimeSpan.FromSeconds(1); + } else { + logger.LogWarning("OrderImbalanceStream interval wasn't specified. Defaulting to 1 second"); + interval = TimeSpan.FromSeconds(1); + } + // logger.LogMissingParam("orderImbalance", "interval"); + // return null; + } + var numLevels = argStore.GetInt32("levels") ?? 5; + var mbpSource = MbpStreamFactory.CreateTables(config, venue, symbol, begin.Value, end.Value, interval.Value, numLevels); + if (mbpSource is null) { + return null; + } + var imbalanceSource = mbpSource.SelectKeyValue( + r => new OrderImbalanceRow() { + Id = r.Id, + Timestamp = r.Timestamp, + Venue = r.Venue, + Symbol = r.Symbol, + // BidPrice = r.BidPrices.TakeWhile(p => !double.IsNaN(p)).LastOrDefault(double.NaN), + // BidQuantity = r.BidQuantities.TakeWhile(q => q > 0).Sum(), + // AskPrice = r.BidPrices.TakeWhile(p => !double.IsNaN(p)).LastOrDefault(double.NaN), + // AskQuantity = r.BidQuantities.TakeWhile(q => q > 0).Sum(), + }, + r => r.Id); + var vs = imbalanceSource + ?.AsDataPackets(MbpRow.Schema) + ?.AsViewStream(); + return vs; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.OrderImbalance; + } + + static class SupportedStreams { + public static readonly DataStreamMetadata OrderImbalance = new() { + Name = OrderImbalanceRow.Schema.tableName + }; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/SettlementStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/SettlementStreamFactory.cs new file mode 100644 index 000000000..216bfb0e7 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/SettlementStreamFactory.cs @@ -0,0 +1,70 @@ +namespace bts.SilverPine; + +using BCG; + +using Bts.DataStream; + +using mbts.utils; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; + +public class SettlementStreamFactory : IDataStreamFactory { + readonly OhlcvStreamConfig config; + readonly ILogger logger; + + public SettlementStreamFactory(IOptions config, ILogger logger) : this(config.Value, logger) { } + + public SettlementStreamFactory(OhlcvStreamConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.Settlement.IsMatch(streamId)) { + return createSettlementStream(args); + } + return null; + } + + ViewStream? createSettlementStream(IEnumerable<(string? Name, int? Index, object? Value)> args) { + var argList = args.Take(3).ToList(); + var vs = argList.Count switch { + 2 => Create(argList[0].Value as string, argList[1].Value as DateTime?), + 3 => Create(argList[0].Value as string, argList[1].Value as DateTime?, argList[2].Value as DateTime?), + _ => null, + }; + if (vs is null) { + logger.LogWarning("Failed to create SettlementStream from args: {args}", + string.Join(", ", args.Select(a => $"({a.Name}, {a.Index}, {a.Value})"))); + } + return vs; + + ViewStream? Create(string? symbol, DateTime? beginDate, DateTime? endDate = null) { + if (symbol is null || !beginDate.HasValue) { + return null; + } + return endDate.HasValue + ? new SettlementPriceStream(symbol, beginDate.Value, endDate.Value) + : new SettlementPriceStream(symbol, beginDate.Value); + } + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.Settlement; + } + + static class SupportedStreams { + public static readonly DataStreamMetadata Settlement = new() { + Name = "settlement", + AlternateNames = new() { "settlementPrice", "settle", "settlePrice" } + }; + + public static string SettlementName => ExceptionAssert.not_null(Settlement.Name); + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/StreamArgumentAttribute.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamArgumentAttribute.cs new file mode 100644 index 000000000..ccfbdbc4d --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamArgumentAttribute.cs @@ -0,0 +1,15 @@ +namespace bts.SilverPine; + +public class StreamArgumentAttribute : Attribute { + public StreamArgumentAttribute(params string[] names) => + Names = names; + + public StreamArgumentAttribute(WellKnownStreamArgumentType wellKnownType, params string[] names) { + WellKnownType = wellKnownType; + Names = names; + } + + public string[] Names { get; } + public WellKnownStreamArgumentType WellKnownType { get; } + public bool IsRequired { get; set; } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryArgStore.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryArgStore.cs new file mode 100644 index 000000000..8b8038507 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryArgStore.cs @@ -0,0 +1,243 @@ +namespace bts.SilverPine; + +using bts.api; + +using Microsoft.Extensions.Logging; + +using System.Diagnostics.CodeAnalysis; + +using System.Reflection; +using System.Text; + +class StreamFactoryArgStore { + readonly IEnumerable<(string? Name, int? Index, object? Value)> args; + + public StreamFactoryArgStore(IEnumerable<(string? Name, int? Index, object? Value)> args) => + this.args = args; + + public static bool TryGetParams( + IEnumerable<(string? Name, int? Index, object? Value)> args, + [MaybeNull][NotNullWhen(true)] out T result, + [NotNullWhen(false)] out string? errorMessage + ) => + new StreamFactoryArgStore(args).TryGetParams(out result, out errorMessage); + + public bool TryGetParams([MaybeNull][NotNullWhen(true)]out T result, [NotNullWhen(false)]out string? errorMessage) { + var props = typeof(T).GetProperties(); + var res = Activator.CreateInstance(); + if (res is null) { + errorMessage = $"Failed to instantiate {typeof(T)}"; + result = default; + return false; + } + foreach (var p in props) { + var att = p.GetCustomAttribute(); + if (TryGet(att?.Names ?? new[] { p.Name }, out var val)) { + p.SetValue(res, val); + } else if (att?.IsRequired ?? false) { + errorMessage = $"Missing required param '{p.Name}'"; + result = default; + return false; + } + } + errorMessage = null; + result = res; + return true; + } + + public Venue GetVenue() => + GetVenue(new[] { "venue", "exchange", "exch" }); + + public Venue GetVenue(string[] preferredNames) { + return preferredNames + .Select(n => Get(n)) + .Concat(args.Select(a => a.Value)) + .Select(ConvertObj) + .SelectWhereValues() + .FirstOrDefault(Venue.Unset); + + static Venue? ConvertObj(object? obj) { + if (obj is Venue res) { + return res; + } + var str = obj as string ?? obj?.ToString(); + if (tryParseVenue(str, out res)) { + return res; + } + return null; + } + } + + public string? GetVenueString() => + GetVenueString("venue", "exchange", "exch"); + + public string? GetVenueString(params string[] preferredNames) { + var obj = Get(preferredNames); + var str = obj as string ?? obj?.ToString(); + return str?.ToUpper()?.Replace("VENUE_", ""); + } + + public SecurityType GetSecurityType() => + GetSecurityType(new[] { "secType", "securityType", "type" }); + + public SecurityType GetSecurityType(params string[] preferredNames) { + return preferredNames + .Select(n => Get(n)) + .Concat(args.Select(a => a.Value)) + .Select(ConvertObj) + .SelectWhereValues() + .FirstOrDefault(SecurityType.Unset); + + static SecurityType? ConvertObj(object? obj) { + if (obj is SecurityType res) { + return res; + } + var str = obj as string ?? obj?.ToString(); + if (tryParseSecurityType(str, out res)) { + return res; + } + return null; + } + } + + public string? GetSymbol() => + Get("symbol", "product", "productCode", "asset")?.ToString(); + + public int? GetInt32(params string[] names) { + var obj = Get(names); + if (obj is int i) { + return i; + } + var s = obj as string ?? obj?.ToString(); + return int.TryParse(s, out i) ? i : null; + } + + public TimeSpan? GetTimeSpan(string name) => + GetTimeSpan(new[] { name }); + + public TimeSpan? GetTimeSpan(string[] preferredNames) => + preferredNames + .Select(n => BCG.BlueDataUtility.get_timespan(Get(n))) + .Where(ts => ts.HasValue) + .FirstOrDefault(); + + public IEnumerable GetStrings() => + args.Select(a => a.Value).OfType(); + + // First, parses the args associated with preferredNames and returns the first successful + // parse result. If none were successful, tries to parse every arg. + public T? GetEnum(params string[] preferredNames) where T : struct { + var candidateObjs = preferredNames.Select(n => Get(n)).Concat(args.Select(a => a.Value)); + foreach (var obj in candidateObjs) { + if (obj is T res) { + return res; + } + if (obj is string s && Enum.TryParse(s, true, out res)) { + return res; + } + } + return null; + } + + public IEnumerable GetDateTimes() => + args.Select(a => convertToDateTime(a.Value)).SelectWhereValues(); + + public DateTime? GetDateTime(params string[] names) => + names.Select(n => Get(n)).Select(convertToDateTime).Where(dt => dt.HasValue).FirstOrDefault(); + + public object? Get(params string[] names) => + TryGet(names, out var result) ? result : null; + + public bool TryGet(string name, out object? result) => + TryGet(new[] { name }, out result); + + public bool TryGet(string[] names, out object? result) { + foreach (var name in names.SelectMany(Variants)) { + if (args.TryGetFirst(a => a.Name == name, out var a)) { + result = a.Value; + return true; + } + } + result = null; + return false; + + IEnumerable Variants(string s) { + if (string.IsNullOrEmpty(s)) { + yield break; + } + yield return s; + // Convert to snake_case + var bd = new StringBuilder(char.ToLower(s[0])); + for (int i = 1; i < s.Length; i++) { + if (i > 1 && char.IsUpper(s[i]) && char.IsLower(s[i - 1])) { + bd.Append('_'); + } + bd.Append(char.ToLower(s[i])); + } + yield return bd.ToString(); + // Convert to camelCase + bd.Clear(); + for (int i = 0; i < s.Length; i++) { + int numUnderscores = 0; + while (s[i + numUnderscores] == '_' & (i + numUnderscores < s.Length)) { + numUnderscores++; + } + if (numUnderscores > 0) { + i += numUnderscores; + if (i >= s.Length) { + break; + } + bd.Append(char.ToUpper(s[i])); + } else { + bd.Append(s[i]); + } + } + yield return bd.ToString(); + } + } + + static bool tryParseVenue(string? str, out Venue v) { + if (str is null) { + v = Venue.Unset; + return false; + } + if (Enum.TryParse(str, true, out v) || Enum.TryParse("VENUE_" + str, true, out v)) { + return true; + } + v = str.ToUpper() switch { + "CME" or "XCME" or "CEC" or "XCEC" => Venue.Glbx, + "CBOT" or "CBT" or "XCBT" => Venue.Cboe, + _ => Venue.Unset, + }; + return v != Venue.Unset; + } + + static bool tryParseSecurityType(string? str, out SecurityType t) { + if (str is null) { + // bbo(symbol: "ZN", venue: "GLBX", begin: "2024-10-03T18:00:00-00:05", end: "2024-10-03T19:00:00-00:05", interval: "PT1S") + t = SecurityType.Unset; + return false; + } + return Enum.TryParse(str, true, out t) || Enum.TryParse("SECURITYTYPE_" + str, true, out t); + } + + static DateTime? convertToDateTime(object? obj) => + tryConvert(obj, out DateTime dt) ? dt : null; + + static bool tryConvert(object? obj, out DateTime res) { + if (obj is DateTime dt) { + res = dt; + return true; + } + if (obj is string s) { + if (DateTimeOffset.TryParse(s, out var resOff)) { + res = resOff.DateTime; + return true; + } + return DateTime.TryParse(s, out res); + } + res = default; + return false; + } + +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryLogs.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryLogs.cs new file mode 100644 index 000000000..fcceea235 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryLogs.cs @@ -0,0 +1,11 @@ +namespace bts.SilverPine.StreamFactory.Logging; + +using Microsoft.Extensions.Logging; + +internal static partial class StreamFactoryLogs { + [LoggerMessage(EventId = 1, Level = LogLevel.Warning, Message = "Unable to create '{StreamType}' stream. Missing required parameter '{Parameter}'.")] + public static partial void LogMissingParam(this ILogger logger, string streamType, string parameter); + + [LoggerMessage(EventId = 2, Level = LogLevel.Warning, Message = "Unable to create '{StreamType}' stream. {error}.")] + public static partial void LogInvalidParams(this ILogger logger, string streamType, string error); +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/WellKnownStreamArgumentType.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/WellKnownStreamArgumentType.cs new file mode 100644 index 000000000..aaa729292 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/WellKnownStreamArgumentType.cs @@ -0,0 +1,10 @@ +namespace bts.SilverPine; + +public enum WellKnownStreamArgumentType { + None, + Venue, + Symbol, + SecurityType, + StartDate, + StopDate, +} diff --git a/src/client/spark_api/data/SecurityCriteriaExtensions.cs b/src/client/spark_api/data/SecurityCriteriaExtensions.cs index d26c90792..99f788bb4 100644 --- a/src/client/spark_api/data/SecurityCriteriaExtensions.cs +++ b/src/client/spark_api/data/SecurityCriteriaExtensions.cs @@ -106,10 +106,19 @@ namespace mbts.spark { security_criteria.is_defined() && security_criteria.clauses.empty_if_null().Any(c => c.includes(security)); + public static bool includes( + this SecurityCriteria security_criteria, (Venue? Venue, SecurityType? Type, string? Symbol) tup) => + security_criteria.is_defined() && + security_criteria.clauses.empty_if_null().Any(c => c.includes(tup)); + public static bool includes( this SecurityCriteriaClause clause, SecurityImmutable security) => _to_predicates(clause).AnyAndAll(p => p(security)); + public static bool includes( + this SecurityCriteriaClause clause, (Venue? Venue, SecurityType? Type, string? Symbol) tup) => + _to_symbol_predicates(clause).AnyAndAll(p => p(tup)); + public static bool is_subset_of( this SecurityCriteriaClause clause, SecurityCriteriaClause other) => // exchanges @@ -176,6 +185,10 @@ namespace mbts.spark { ) ); + public static bool includes(this SecurityCriteria security_criteria, (Venue Venue, SecurityType Type, string? symbol) sec_tup) => + security_criteria.is_defined() && + security_criteria.clauses.empty_if_null().Any(c => c.includes(sec_tup)); + public static SecurityCriteriaClause with(this SecurityCriteriaClause clause, Venue[]? exchanges = null, SecurityType[]? security_types = null, @@ -470,5 +483,17 @@ namespace mbts.spark { yield return sec => clause.names.Contains(sec.ToString()); } } + + static IEnumerable> _to_symbol_predicates(SecurityCriteriaClause clause) { + if (!clause.exchanges.is_null_or_empty()) { + yield return tup => !tup.Venue.HasValue || clause.exchanges.Contains(tup.Venue.Value); + } + if (!clause.symbols.is_null_or_empty()) { + yield return tup => tup.Symbol is null || clause.symbols.Contains(tup.Symbol); + } + if (!clause.security_types.is_null_or_empty()) { + yield return tup => !tup.Type.HasValue || clause.security_types.Contains(tup.Type.Value); + } + } } } diff --git a/src/client/spark_core/Json/JsonMultiLineString.cs b/src/client/spark_core/Json/JsonMultiLineString.cs new file mode 100644 index 000000000..9c0321018 --- /dev/null +++ b/src/client/spark_core/Json/JsonMultiLineString.cs @@ -0,0 +1,9 @@ +#nullable enable +namespace Spark.Core.Json; + +using System.Text.Json.Serialization; + +[JsonConverter(typeof(JsonMultiLineStringConverter))] +public class JsonMultiLineString { + public string? Value { get; set; } +} diff --git a/src/client/spark_core/Json/JsonMultiLineStringConverter.cs b/src/client/spark_core/Json/JsonMultiLineStringConverter.cs new file mode 100644 index 000000000..fc786364f --- /dev/null +++ b/src/client/spark_core/Json/JsonMultiLineStringConverter.cs @@ -0,0 +1,39 @@ +#nullable enable +namespace Spark.Core.Json; + +using System; + +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +// Reads normal strings and arrays of lines +public class JsonMultiLineStringConverter : JsonConverter { + public override JsonMultiLineString? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.ReadString(out var str)) { + return new() { Value = str }; + } + if (reader.ReadArray(out var array)) { + return new() { Value = string.Join('\n', array.Select(v => v?.AsString())) }; + } + throw new JsonException("Failed to parse multi-line string"); + } + + public override void Write(Utf8JsonWriter writer, JsonMultiLineString value, JsonSerializerOptions options) => + writer.WriteStringValue(value.Value); +} + +public class JsonMultiLineStringValueConverter : JsonConverter { + public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.ReadString(out var str)) { + return str; + } + if (reader.ReadArray(out var array)) { + return str; + } + throw new JsonException("Failed to parse multi-line string"); + } + + public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => + writer.WriteStringValue(value); +} diff --git a/src/client/spark_core/Update/TableUpdate.cs b/src/client/spark_core/Update/TableUpdate.cs index 888ddcfda..ff6b971e1 100644 --- a/src/client/spark_core/Update/TableUpdate.cs +++ b/src/client/spark_core/Update/TableUpdate.cs @@ -3,6 +3,7 @@ namespace bts.api.core { using System; using System.Collections.Generic; using System.Collections.Immutable; + using System.Diagnostics.CodeAnalysis; using System.Linq; public class TableUpdate where TKey : IEquatable { @@ -277,6 +278,17 @@ namespace bts.api.core { _removed[kv.Key] = kv.Value; } + public bool TryGetValue(TKey key, [MaybeNullWhen(false)]out T result) { + if(_added.TryGetValue(key, out result)) { + return true; + } + if (_changes.TryGetValue(key, out var ch)) { + result = ch.New; + return true; + } + return !_removed.ContainsKey(key) && _state.state.TryGetValue(key, out result); + } + /// /// Creates a TableUpdate instance from the current state /// diff --git a/src/client/spark_websocket/Core/WebsocketDataSync.cs b/src/client/spark_websocket/Core/WebsocketDataSync.cs index 4b8db1351..e9ee1cb1f 100644 --- a/src/client/spark_websocket/Core/WebsocketDataSync.cs +++ b/src/client/spark_websocket/Core/WebsocketDataSync.cs @@ -16,6 +16,7 @@ namespace bts.websocket_core { using bts.api.position; using bts.api.synthetic; using Spark.Api.Services; + using Bts.DataStream; /// @@ -26,10 +27,12 @@ namespace bts.websocket_core { readonly AdminService adminService; readonly Spark.Api.IServiceProvider _service_ctx; readonly IUserDataPermissions userDataPermissions; + readonly IDataStreamFactory streamFactory; - public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IUserDataPermissions userDataPermissions) { + public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IUserDataPermissions userDataPermissions, IDataStreamFactory streamFactory) { _service_ctx = service_ctx; this.userDataPermissions = userDataPermissions; + this.streamFactory = streamFactory; adminService = service_ctx.GetService(); } @@ -330,6 +333,7 @@ namespace bts.websocket_core { req, Md_Client, Orders_Client, + streamFactory, obj => send_message(obj, req.client_id), out hdlr, out status)) { diff --git a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs index c80cbcaf6..b6021e299 100644 --- a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs +++ b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs @@ -198,7 +198,12 @@ namespace bts.websocket_core { return true; } - static bool TryGetDataStream(WebsocketRequest req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out string error) { + static bool TryGetDataStream( + WebsocketRequest req, + IDataStreamFactory streamFactory, + [NotNullWhen(true)] out ViewStream? stream, + [NotNullWhen(false)] out string error + ) { stream = null; error = ""; var query = (req.parameters?["query"] as JValue)?.Value(); @@ -213,7 +218,7 @@ namespace bts.websocket_core { } Exception? exception = null; try { - stream = exp.getView(new Dictionary(0)); + stream = exp.getView(new Dictionary(0), streamFactory); } catch (Exception ex) { exception = ex; } @@ -261,10 +266,14 @@ namespace bts.websocket_core { } public static bool TryCreate( - WebsocketRequest req, MarketDataClient md, OrdersClient oc, + WebsocketRequest req, + MarketDataClient md, + OrdersClient oc, + IDataStreamFactory streamFactory, Action sm, [NotNullWhen(true)] out WebsocketSubscriptionHandler? hdlr, - out string status) { + out string status + ) { hdlr = null; if ( @@ -327,7 +336,7 @@ namespace bts.websocket_core { } case SUBSCRIPTION_TYPE.DATA_STREAM: { // TODO: make this nullable - if (!TryGetDataStream(req, out var filter, out status)) { + if (!TryGetDataStream(req, streamFactory, out var filter, out status)) { return false; } sub = GetDataStreamSubscriber(filter, id.Value.Val);