diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs index 940fc56ba..554768d05 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs @@ -80,7 +80,7 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config .AddMasterWorker(config.GetSection("MasterWorker")) .AddViewStreamDatabase() .AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams")) - .AddOhlcvStreams(config.GetSection("OhlcvStream")) + .AddOhlcvStreams(config.GetSection("StreamDatabase")) .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream")) .AddCompositeDataStreamFactory() .AddPubSubClient(config.GetSection("PubSub")); diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json index 90675f1c3..c8c0535ee 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json @@ -32,74 +32,9 @@ "OHLCVStream" ] }, - "OhlcvStream": { - "Database": { - "Mbo": { - "Path": "/home/bts/.local/share/bts/silver_pine/Databento/mbo/${VENUE}/${SYMBOL}/store.db", - "Query": "select * from ${SYMBOL}_mbo_${DATE}", - "QueryParams": [ - "SYMBOL", - "DATE" - ] - }, - "Bbo1s": { - "Path": "/home/bts/.local/share/bts/silver_pine/Databento/bbo-1s/${VENUE}/${SYMBOL}/store.db", - "Query": "select * from ${SYMBOL}_bbo1s_${DATE}", - "QueryParams": [ - "SYMBOL", - "DATE" - ] - }, - "Bbo1m": { - "Path": "/home/bts/.local/share/bts/silver_pine/Databento/bbo-1m/${VENUE}/${SYMBOL}/store.db", - "Query": "select * from ${SYMBOL}_bbo1m_${DATE}", - "QueryParams": [ - "SYMBOL", - "DATE" - ] - }, - "Ohlcv1d": { - "Path": "/home/bts/.local/share/bts/silver_pine/database/stream_data.db", - "Table": "${SYMBOL}_ohlcv_1day", - "Query": "select * from ${SYMBOL}_ohlcv_1day", - "QueryParams": [ - "SYMBOL" - ] - } - }, - "Databases": [ - { - "Path": "/home/bts/.local/share/bts/silver_pine/database/stream_data.db", - "Venues": [ - "GLBX" - ], - "Types": [ - "Future" - ], - "Symbols": [ - "ES" - ], - "Datasets": { - "mbo": { - } - } - }, - { - "Path": "/home/bts/.local/share/bts/silver_pine/Databento/bbo/GLBX/ZN/store.db", - "Tables": [ - { - "DataType": "mbo", - "Venue": "GLBX", - "SecurityType": "Future", - "Symbol": "ZN", - "TableName": "ZN_bbo1m_20241001", - "TradingDates": [ - "2024-10-01" - ] - } - ] - } - ] + "StreamDatabase": { + "Path": "/home/bts/.local/share/bts/silver_pine/store.db", + "Query": "select * from metadata" }, "NaturalLanguageStream": { "prompts": [ @@ -164,7 +99,7 @@ "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ", "Depending on the wording of the request, some arguments may be omitted. ", "The begin, end, and interval args should be expressed in iso 8601 format. ", - "Interpret dates according to the trading schedule (rather than calendar date), whenever possible. ", + "Interpret dates according to the trading schedule rather than calendar date whenever possible. ", "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ", "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00." ], @@ -172,26 +107,49 @@ "examples": [ { "natural_language": "Get SP500 future bbo from Wednesday.", - "dsl": "Bbo(symbol: \"ES\", venue: \"GLBX\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")" + "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")" }, { "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.", - "dsl": "Bbo(symbol: \"ES\", venue: \"GLBX\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00\", interval: \"PT1S\", exclusive_end: true)" + "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", exclusive_end: true)" + }, + { + "natural_language": "Retrieve NVIDIA L1 books from Oct 3", + "dsl": "Bbo(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\")" }, { "natural_language": "Show me L1 data for Crude oil futures this week", - "dsl": "Bbo(symbol: \"CL\", venue: \"GLBX\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")" + "dsl": "Bbo(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\")" }, { "natural_language": "Fetch hourly bbo for Gold futures this week", - "dsl": "Bbo(symbol: \"GC\", venue: \"GLBX\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")" + "dsl": "Bbo(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")" }, { "natural_language": "What was the minute by minute bbo for Tesla yesterday?", - "dsl": "Bbo(symbol: \"TSLA\", venue: NASD)" + "dsl": "Bbo(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")" } ] }, + { + "category": "Order flow imbalance", + "MulitLineDescription": [ + "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ", + "It accepts the following parameters: ", + "symbol (a product code such as ZN for 10 year T-Note futures), ", + "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ", + "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ", + "interval (sample frequency of the data), ", + "begin and end (inclusive beginning and end of the time window to query, respectively), ", + "inclusive_end (a bool flag that, when true, makes the end time inclusive). ", + "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ", + "Depending on the wording of the request, some arguments may be omitted. ", + "The begin, end, and interval args should be expressed in iso 8601 format. ", + "Interpret dates according to the trading schedule rather than calendar date whenever possible. ", + "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ", + "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00." + ] + }, { "category": "VWAP", "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.", diff --git a/src/DataStreamSuite/SharedStreams/NL/DatabaseConfig.cs b/src/DataStreamSuite/SharedStreams/NL/DatabaseConfig.cs new file mode 100644 index 000000000..5de7a4c40 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/NL/DatabaseConfig.cs @@ -0,0 +1,6 @@ +namespace bts.SilverPine; + +public class DatabaseConfig { + public string? Path { get; set; } + public string? Query { get; set; } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/StreamQueryMetadata.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamQueryMetadata.cs new file mode 100644 index 000000000..c32d9c8b2 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamQueryMetadata.cs @@ -0,0 +1,33 @@ +namespace bts.SilverPine; + +using System.Data; +using System.Data.SQLite; + +public class StreamQueryMetadata { + public string? DataType { get; set; } + public string? Symbol { get; set; } + public DateTime BeginDate { get; set; } + public DateTime EndDate { get; set; } + public string? Location { get; set; } + public string? Query { get; set; } + + public static IEnumerable Load(string dbPath, string query) { + var connectionStr = $"Data Source={dbPath};Version=3"; + var dataTable = new DataTable(); + using var conn = new SQLiteConnection($"Data Source={dbPath};Version=3"); + conn.Open(); + using var sqlCmd = new SQLiteCommand(query, conn); + using var adapter = new SQLiteDataAdapter(sqlCmd); + adapter.Fill(dataTable); + + return dataTable.Rows.OfType() + .Select(r => new StreamQueryMetadata() { + Symbol = r["symbol"] as string, + DataType = r["data_type"] as string, + BeginDate = DateTime.Parse(r["begin_date"]?.ToString()!), + EndDate = DateTime.Parse(r["end_date"]?.ToString()!), + Location = r["location"] as string, + Query = r["query"] as string, + }); + } +}