From 0cf44fcc4083d389bd2b3bd9f8e0fd8fc93dc210 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 22 Sep 2025 01:43:37 -0400 Subject: [PATCH] feat: Add historic market data streams factory Loads historic md from Databento files in structured paths. It is designed to work with files downloaded via databento_client.py --- .../BlueData/BlueTableImmutable.cs | 72 +++++++ .../BlueData/BlueTableSchemaAdapter.cs | 15 ++ .../Historical/DatabentoBboAdapter.cs | 64 ++++++ .../DatabentoContinuousContractAdapter.cs | 0 .../Historical/DatabentoFileStore.cs | 179 ++++++++++++++++ .../Historical/DatabentoInstrumentAdapter.cs | 114 ++++++++++ .../Historical/HistoricBboParams.cs | 99 +++++++++ .../Historical/HistoricBboRowData.cs | 86 ++++++++ .../Historical/HistoricBboSchemaBuilder.cs | 58 +++++ .../Historical/HistoricBboStream.cs | 28 +++ .../MarketData/Historical/HistoricBboTable.cs | 160 ++++++++++++++ .../HistoricContinuousContractStream.cs | 43 ++++ .../HistoricContinuousContractTable.cs | 102 +++++++++ .../Historical/HistoricInstrumentParams.cs | 93 ++++++++ .../Historical/HistoricInstrumentRowData.cs | 48 +++++ .../HistoricInstrumentSchemaBuilder.cs | 82 +++++++ .../Historical/HistoricInstrumentStream.cs | 28 +++ .../Historical/HistoricInstrumentTable.cs | 167 +++++++++++++++ .../MarketData/Historical/InstrumentKey.cs | 11 + .../MarketData/Historical/TimeseriesIndex.cs | 168 +++++++++++++++ .../MarketData/Historical/TimeseriesKey.cs | 9 + .../Business/MarketData/BBOStream.cs | 2 +- .../Services/SharedStreamsServices.cs | 11 +- .../StreamFactories/BookStreamFactory.cs | 104 ++++++++- .../StreamFactories/HistoricDataParams.cs | 60 ++++++ .../HistoricParquetDataLoader.cs | 200 ++++++++++++++++++ .../InstrumentSchemaBuilder.cs | 78 ------- .../StreamFactories/InstrumentStreamParams.cs | 66 ------ .../ParquetReaderStreamFactory.cs | 35 +++ .../StreamFactories/WellKnownColumns.cs | 68 ++++-- .../spark_core/Collections/ArrayList.cs | 79 +++++++ .../Collections/Extensions/ListExtensions.cs | 10 + src/client/spark_core/utils/ArrayUtils.cs | 17 ++ src/client/spark_core/utils/DelegateUtils.cs | 22 ++ 34 files changed, 2205 insertions(+), 173 deletions(-) create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoBboAdapter.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoContinuousContractAdapter.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoFileStore.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoInstrumentAdapter.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboParams.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboRowData.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboSchemaBuilder.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboTable.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricContinuousContractStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricContinuousContractTable.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentParams.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentRowData.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentSchemaBuilder.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentTable.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/InstrumentKey.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/TimeseriesIndex.cs create mode 100644 src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/TimeseriesKey.cs create mode 100644 src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataParams.cs create mode 100644 src/DataStreamSuite/SharedStreams/StreamFactories/HistoricParquetDataLoader.cs delete mode 100644 src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentSchemaBuilder.cs delete mode 100644 src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamParams.cs create mode 100644 src/DataStreamSuite/SharedStreams/StreamFactories/ParquetReaderStreamFactory.cs create mode 100644 src/client/spark_core/Collections/ArrayList.cs create mode 100644 src/client/spark_core/utils/DelegateUtils.cs diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs index d776116b5..7820113bf 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs @@ -9,6 +9,7 @@ using bts.collections; using Spark.Core.Collections; using System; +using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -76,9 +77,43 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { } } + public Builder ToBuilder() => + new(this); + public DataPacketBuilder ToDataPacketBuilder() => new(this); + public class Builder : IEnumerable { + readonly IBlueTableSchema schema; + readonly ImmutableSortedSetDictionary.Builder rowMap; + + public Builder(BlueTableImmutable source) { + schema = source.schema; + rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder()); + } + + public void AddRange(IEnumerable> rows) => + rowMap.AddRange(rows); + + public void AddOrUpdate(IBlueRow row) => + AddOrUpdate(row.Id(), row); + + public void AddOrUpdate(int id, IBlueRow row) => + rowMap[id] = row; + + public bool Remove(int id) => + rowMap.Remove(id); + + public BlueTableImmutable ToImmutable() => + new(schema, rowMap.ToImmutable().Source); + + public IEnumerator GetEnumerator() => + rowMap.Values.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } + public class DataPacketBuilder { readonly IBlueTableSchema schema; readonly ImmutableSortedSet>.Builder table; @@ -354,9 +389,46 @@ public class BlueTableImmutableAdapter : IBlueRowCollection { } } + public Builder ToBuilder() => + new(this); + public DataPacketBuilder ToDataPacketBuilder() => new(this); + public class Builder : IEnumerable { + readonly BlueTableSchemaAdapter schemaAdapter; + readonly ImmutableSortedSetDictionary.Builder rowMap; + + public Builder(BlueTableImmutableAdapter source) { + schemaAdapter = source.schemaAdapter; + rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder()); + } + + public void AddRange(IEnumerable> rows) => + rowMap.AddRange(rows); + + public void AddOrUpdate(T row) => + AddOrUpdate((int)schemaAdapter["id"][row]!, row); + + public void AddOrUpdate(int id, T row) => + rowMap[id] = row; + + public bool Remove(int id) => + rowMap.Remove(id); + + public BlueTableImmutableAdapter ToImmutable() => + new(schemaAdapter, rowMap.ToImmutable().Source); + + public IEnumerator GetEnumerator() { + foreach (var r in rowMap.Values) { + yield return schemaAdapter.GetBlueRow(r); + } + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } + public class DataPacketBuilder { readonly BlueTableSchemaAdapter schemaAdapter; readonly ImmutableSortedSet>.Builder table = diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdapter.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdapter.cs index 2f1aec37d..f6a3fffae 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdapter.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdapter.cs @@ -73,6 +73,21 @@ public class BlueTableSchemaAdapter : IMutableBlueRowFactory, IBlueTableSc columnNames = null; } + public void AddColumn(string name, Func getValue) { + var dataType = typeof(TResult); + if (!ColumnInfo.isAllowedType(dataType)) { + throw new ArgumentException( + $"'{dataType}' is not a supported column type", nameof(dataType)); + } + // Invalidate cached schema (set it to null) + schema = null; + var adapter = new ColumnAdapter(new(name, columns.Count, dataType), r => getValue(r)); + columns.Add(adapter); + columnNameIndex[name] = adapter; + columnInfos = null; + columnNames = null; + } + public ColumnAdapter this[int columnIndex] => columns[columnIndex]; diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoBboAdapter.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoBboAdapter.cs new file mode 100644 index 000000000..0ab2d5121 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoBboAdapter.cs @@ -0,0 +1,64 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using Bts.DataStreamCore; + +public static class DatabentoBboAdapter { + static readonly string[] keyColumnNames = new[] { "ts_event", "bid_px_00", "bid_sz_00", "ask_px_00", "ask_sz_00" }; + + public static ColumnIndices ResolveColumns(ColumnInfo[] columns) => + new(columns.ResolveIndices(keyColumnNames)); + + public static Func GetRowDataFactory(Func, int> getId, Func getInstrument, ColumnInfo[]? columns = null) { + ColumnIndices? colIndicesBacking = null; + if (columns is not null) { + colIndicesBacking = ResolveColumns(columns); + } + return RowDataFactoryWrapper; + + HistoricBboRowData? RowDataFactoryWrapper(IBlueRow row) { + colIndicesBacking ??= ResolveColumns(row.schema.columnInfo); + return RowDataFactory(row, colIndicesBacking.Value); + } + + HistoricBboRowData? RowDataFactory(IBlueRow mbp1Row, in ColumnIndices colIndices) { + var inst = getInstrument(mbp1Row); + if (inst is null) { + return null; + } + var ts = mbp1Row.GetDateTime(colIndices.TimestampIndex) ?? default; + var bidPrice = mbp1Row.GetFloat64(colIndices.BidPriceIndex) ?? double.NaN; + var bidQuantity = mbp1Row.GetFloat64(colIndices.BidQuantityIndex) ?? 0; + var askPrice = mbp1Row.GetFloat64(colIndices.BidPriceIndex) ?? double.NaN; + var askQuantity = mbp1Row.GetFloat64(colIndices.AskQuantityIndex) ?? 0; + int id = getId(inst.Key); + var rd = new HistoricBboRowData(id, inst, ts, bidPrice, bidQuantity, askPrice, askQuantity); + return rd; + } + } + + public readonly struct ColumnIndices { + public readonly int TimestampIndex; + public readonly int BidPriceIndex; + public readonly int BidQuantityIndex; + public readonly int AskPriceIndex; + public readonly int AskQuantityIndex; + + public ColumnIndices(int[] keyColumnIndices) + : this( + keyColumnIndices[0], + keyColumnIndices[1], + keyColumnIndices[2], + keyColumnIndices[3], + keyColumnIndices[4]) { } + + public ColumnIndices(int timestampIndex, int bidPriceIndex, int bidQuantityIndex, int askPriceIndex, int askQuantityIndex) { + TimestampIndex = timestampIndex; + BidPriceIndex = bidPriceIndex; + BidQuantityIndex = bidQuantityIndex; + AskPriceIndex = askPriceIndex; + AskQuantityIndex = askQuantityIndex; + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoContinuousContractAdapter.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoContinuousContractAdapter.cs new file mode 100644 index 000000000..e69de29bb diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoFileStore.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoFileStore.cs new file mode 100644 index 000000000..a8551d4d2 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoFileStore.cs @@ -0,0 +1,179 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts; +using bts.api; +using bts.BlueOrb; +using bts.co2; + +using Bts.DataStream; +using Bts.DataStreamCore; + +using System.Diagnostics.CodeAnalysis; +using System.Text.RegularExpressions; +using System.Linq; +using Bts.DataStreamCore.StreamFactories; +using Spark.Core; + +public static class DatabentoFileStore { + const int defaultMaxRowCount = 1_500_000; + + public static IEnumerable<(string Path, DateTime Date)> GetDataFiles(string databentoSchema, string? continuousContract, SecurityType? secType, Venue? venue = null, DateTime? beginDate = null, DateTime? endDate = null) { + var localAppDataDir = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); + var mbp1Dir = Path.Combine(localAppDataDir, "bts", "databento", databentoSchema); + var mbp1ChildDirs = Directory.GetDirectories(mbp1Dir); + var preview = new List(); + if (continuousContract is not null) { + foreach (var d in mbp1ChildDirs) { + var p = Path.Combine(d, continuousContract); + if (Directory.Exists(p)) { + if (p.Contains("HG")) { + } + preview.Add(p); + } + } + } + // var preview = mbp1ChildDirs.Select(d => Path.Combine(d, continuousContract!)).Where(Directory.Exists).ToList(); + IEnumerable symDirs = continuousContract is null + ? mbp1ChildDirs.SelectMany(Directory.GetDirectories) + : mbp1ChildDirs.Select(d => Path.Combine(d, continuousContract)).Where(Directory.Exists); + var files = symDirs.SelectMany(Directory.GetFiles).Select(GetFile).SelectWhereValues(); + if (beginDate.HasValue) { + files = files.Where(f => f.Date > beginDate.Value - TimeSpan.FromDays(1)); + } + if (endDate.HasValue) { + files = files.Where(f => f.Date < endDate.Value); + } + return files; + + (string Path, DateTime Date)? GetFile(string f) { + if (f is null) { + return null; + } + var parts = f.Split(Path.DirectorySeparatorChar); + Log.assert(parts.Length >= 3); + var fileVenueStr = parts[^3]; + var sym = parts[^2]; + var name = Path.GetFileName(f); + var match = Regex.Match(name, $"{databentoSchema}_{fileVenueStr}_{sym}_(STOCK|SPOT|FUT|OPT)_(\\d{{8}})\\.parquet"); + if (!match.Success) { + return null; + } + if (secType.HasValue) { + var fileSecType = match.Groups[1].Value switch { + "STOCK" or "SPOT" => SecurityType.Spot, + "FUT" => SecurityType.Future, + "OPT" => SecurityType.Option, + _ => throw new InvalidOperationException("Failed to parse SecurityType from filename"), + }; + if (fileSecType != secType.Value) { + return null; + } + } + if (venue.HasValue) { + if (!MarketDataArgStore.TryParseVenue(fileVenueStr, out var fileVenue)) { + return null; + } + if (fileVenue != venue.Value) { + return null; + } + } + int dateInt = int.Parse(match.Groups[2].Value); + PeelPow10(ref dateInt, 10_000, out int year); + PeelPow10(ref dateInt, 100, out int month); + var date = new DateTime(year, month, dateInt); + return (f, date); + + void PeelPow10(ref int i, int pow10, out int peeled) { + peeled = i / pow10; + i -= peeled * pow10; + } + } + } + + public static IEnumerable>> GetDefinitionAsync(string? asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func? filter = null) { + filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter); + var files = GetDataFiles("definition", asset, secType, beginDate: beginDate, endDate: endDate); + return files.Select(f => new Func>(() => GetDefinitionAsync(f.Path, filter))); + } + + public static async Task GetDefinitionAsync(string path, Func? filter = null) { + // string columnMaskExpression = "bid_px_00:bid_price bid_sz_00:bid_size ask_px_00:ask_price ask_sz_00:ask_size symbol:contract"; + string? columnMaskExpression = null; + return await GetDataAsync(path, columnMaskExpression, filter); + } + + public static IEnumerable>> GetBboAsync(string asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func? filter = null) { + filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter); + var tables = GetDataFiles("mbp-1", asset, secType, beginDate: beginDate, endDate: endDate) + .Select(f => new Func>(() => GetBboAsync(f.Path, filter))); + return tables; + } + + public static async Task GetBboAsync(string path, Func? filter = null) { + string columnMaskExpression = "bid_px_00:bid_price bid_sz_00:bid_size ask_px_00:ask_price ask_sz_00:ask_size symbol:contract"; + return await GetDataAsync(path, columnMaskExpression, filter); + } + + public static IEnumerable>> GetTradesAsync(string asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func? filter = null) { + filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter); + var tables = GetDataFiles("trades", asset, secType, beginDate: beginDate, endDate: endDate) + .Select(f => new Func>(() => GetTradesAsync(f.Path, filter))); + return tables; + } + + public static async Task GetTradesAsync(string path, Func? filter = null) { + string columnMaskExpression = "id ts_event price:trade_price size:trade_size side:aggressor_side symbol:contract"; + var table = await GetDataAsync(path, columnMaskExpression, filter); + var schema = table.schema; + var rowFactory = ColumnSelectorRowFactory.Create(schema, ("aggressor_side", typeof(AggressorSide), ConvertSide)); + return new SelectorTable(table, rowFactory.OutputSchema, rowFactory.Create); + + static object? ConvertSide(object? s) { + char c = s as char? ?? ((s as string ?? "N")[0]); + return c switch { + 'b' or 'B' => AggressorSide.Buy, + 's' or 'S' or 'a' or 'A' => AggressorSide.Sell, + _ => AggressorSide.None, + }; + } + } + + public static Task GetDataAsync(string path, string? columnMaskExpression, Func? filter) => + GetDataAsync(path, columnMaskExpression, 0, defaultMaxRowCount, filter); + + public static async Task GetDataAsync(string path, string? columnMaskExpression, int? start, int? stop, Func? filter) { + var table = await ParquetTableReader.ReadAsync(path, start, stop); + if (columnMaskExpression is not null) { + table = ColumnMaskTable.Create(table, columnMaskExpression); + } + if (filter is not null) { + table = new FilterTable(table, filter); + } + return table; + } + + [return: NotNullIfNotNull(nameof(beginDate))] + [return: NotNullIfNotNull(nameof(endDate))] + static Func? getTimeFilter(DateTime? beginDate, DateTime? endDate) { + if (!beginDate.HasValue && !endDate.HasValue) { + return null; + } + return TimeFilter; + + bool TimeFilter(IBlueRow r) { + var ts = r.GetDateTime("ts_event"); + if (!ts.HasValue) { + return false; + } + if (beginDate.HasValue && ts.Value < beginDate) { + return false; + } + if (endDate.HasValue && ts.Value >= endDate) { + return false; + } + return true; + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoInstrumentAdapter.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoInstrumentAdapter.cs new file mode 100644 index 000000000..de4994fef --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/DatabentoInstrumentAdapter.cs @@ -0,0 +1,114 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.api; + +using Bts.DataStreamCore; + +public static class DatabentoInstrumentAdapter { + static readonly string[] keyColumnNames = new[] { "activation", "exchange", "asset", "instrument_class", "currency", "symbol", "instrument_id", "expiration", "unit_of_measure_qty" }; + + public static ColumnIndices ResolveColumns(ColumnInfo[] columns) => + new(columns.ResolveIndices(keyColumnNames)); + + public static Func GetRowDataFactory(Func, int> getId, ColumnInfo[]? columns = null) { + ColumnIndices? colIndicesBacking = null; + if (columns is not null) { + colIndicesBacking = ResolveColumns(columns); + } + return RowDataFactoryWrapper; + + HistoricInstrumentRowData RowDataFactoryWrapper(IBlueRow row) { + colIndicesBacking ??= ResolveColumns(row.schema.columnInfo); + return RowDataFactory(row, colIndicesBacking.Value); + } + + HistoricInstrumentRowData RowDataFactory(IBlueRow row, in ColumnIndices colIndices) { + var ts = row.GetDateTime(colIndices.ActivationIndex) ?? default; + var venue = row.GetString(colIndices.VenueIndex) switch { + "XCME" => Venue.Glbx, + _ => throw new NotImplementedException(), + }; + var secType = GetSecurityType(row.GetString(colIndices.InstrumentClassIndex)); + var asset = row.GetString(colIndices.AssetIndex) ?? string.Empty; + var currency = row.GetString(colIndices.CurrencyIndex) ?? string.Empty; + var name = row.GetString(colIndices.NameIndex) ?? string.Empty; + var exp = row.GetDateTime(colIndices.ExpirationIndex) ?? GetExpiration(row.GetInt64(colIndices.ExpirationIndex)); + var cpp = row.GetFloat64(colIndices.UnitOfMeasureQtyIndex) ?? double.NaN; + switch (secType) { + case SecurityType.Future: + case SecurityType.Option: + if (!exp.HasValue) { + throw new ArgumentException($"Expiration is null for security of type '{secType}'", nameof(row)); + } + break; + case SecurityType.Spread: + break; + default: + if (exp.HasValue) { + throw new ArgumentException($"Expiration is non-null for security of type '{secType}'", nameof(row)); + } + break; + } + int id = getId(new((venue, name), ts)); + return new(id, ts, venue, secType, asset, currency, name, exp, cpp); + } + + static SecurityType GetSecurityType(string? instrumentClass) { + // Translate Databento's instrument_class codes to BTS SecurityType + // https://databento.com/docs/schemas-and-data-formats/instrument-definitions + return instrumentClass switch { + "B" /*Bond*/ or "K" /*Stock*/ or "X" /*FX spot*/ or "Y" /*Commodity spot*/ => SecurityType.Spot, + "F" => SecurityType.Future, + "C" /*Call*/ or "P" /*Put*/ => SecurityType.Option, + "M" /*Mixed spread*/ or "S" /*Future spread*/ or "T" /*Option spread*/ => SecurityType.Spread, + _ => SecurityType.Unset, + }; + } + + static DateTime? GetExpiration(long? expirationTimestamp) { + if (!expirationTimestamp.HasValue) { + return null; + } + var ticks = expirationTimestamp.Value / 100; + return new(ticks); + } + } + + public readonly struct ColumnIndices { + public readonly int ActivationIndex; + public readonly int VenueIndex; + public readonly int AssetIndex; + public readonly int InstrumentClassIndex; + public readonly int CurrencyIndex; + public readonly int NameIndex; + public readonly int InstrumentIdIndex; + public readonly int ExpirationIndex; + public readonly int UnitOfMeasureQtyIndex; + + public ColumnIndices(int[] keyColumnIndices) + : this( + keyColumnIndices[0], + keyColumnIndices[1], + keyColumnIndices[2], + keyColumnIndices[3], + keyColumnIndices[4], + keyColumnIndices[5], + keyColumnIndices[6], + keyColumnIndices[7], + keyColumnIndices[8]) { } + + public ColumnIndices(int activationIndex, int venueIndex, int assetIndex, int instrumentClassIndex, int currencyIndex, int nameIndex, int instrumentIdIndex, int expirationIndex, int unitOfMeasureQtyIndex) { + ActivationIndex = activationIndex; + VenueIndex = venueIndex; + AssetIndex = assetIndex; + InstrumentClassIndex = instrumentClassIndex; + CurrencyIndex = currencyIndex; + NameIndex = nameIndex; + InstrumentIdIndex = instrumentIdIndex; + ExpirationIndex = expirationIndex; + UnitOfMeasureQtyIndex = unitOfMeasureQtyIndex; + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboParams.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboParams.cs new file mode 100644 index 000000000..2e327e3d4 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboParams.cs @@ -0,0 +1,99 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.api; + +using Bts.DataStreamCore.StreamFactories; + +public class HistoricBboParams { + public HistoricBboParams() { } + + public HistoricBboParams(string venueStr, string asset) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, null) { } + + public HistoricBboParams(string venueStr, string asset, string typeStr) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr)) { } + + public HistoricBboParams(string venueStr, string asset, string typeStr, DateTime date) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), date) { } + + public HistoricBboParams(string venueStr, string asset, string typeStr, DateTime beginDate, DateTime endDate) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), beginDate, endDate) { } + + public HistoricBboParams(string venueStr, string asset, DateTime date) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, date) { } + + public HistoricBboParams(Venue venue, string asset, DateTime date) + : this(venue, asset, null, date) { } + + public HistoricBboParams(Venue venue, string asset, DateTime beginDate, DateTime endDate) + : this(venue, asset, null, beginDate, endDate) { } + + public HistoricBboParams(Venue venue, string asset, SecurityType? type) { + Venue = venue; + Asset = asset; + Type = type; + } + + public HistoricBboParams(Venue venue, string asset, SecurityType? type, DateTime date) + : this(venue, asset, type, date, date.AddDays(1)) { } + + public HistoricBboParams(Venue venue, string asset, SecurityType? type, DateTime beginDate, DateTime endDate) { + Venue = venue; + Asset = asset; + Type = type; + BeginDate = beginDate; + EndDate = endDate; + } + + public Venue? Venue { get; set; } + public string? Asset { get; set; } + public SecurityType? Type { get; set; } + public string? Name { get; set; } + public bool ShowVenue { get; set; } = true; + public bool ShowAsset { get; set; } = true; + public bool ShowType { get; set; } = true; + public bool ShowName { get; set; } = true; + public bool ShowExpiration { get; set; } + public bool ShowCashPerPoint { get; set; } = true; + public bool ShowCurrency { get; set; } = true; + public bool ShowNotionalValue { get; set; } = true; + public DateTime? BeginDate { get; set; } + public DateTime? EndDate { get; set; } + + public HistoricInstrumentParams GetInstrumentParams() => + new() { + Venue = Venue, + Asset = Asset, + Type = Type, + Name = Name, + ShowVenue = ShowVenue, + ShowAsset = ShowAsset, + ShowType = ShowType, + ShowExpiration = ShowExpiration, + ShowCashPerPoint = ShowCashPerPoint, + BeginDate = BeginDate, + EndDate = EndDate, + }; + + public mbts.spark.SecurityCriteria GetSecurityCriteria() { + var clause = new mbts.spark.SecurityCriteriaClause( + exchanges: Venue.HasValue ? new[] { Venue.Value } : null, + security_types: Type.HasValue ? new[] { Type.Value } : null, + symbols: !string.IsNullOrEmpty(Asset) ? new[] { Asset } : null, + names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null); + return clause.is_defined ? new(new[] { clause }) : new(); + } + + public HistoricBboSchemaBuilder GetSchemaBuilder() => + new() { + ShowVenue = ShowVenue, + ShowAsset = ShowAsset, + ShowType = ShowType, + ShowName = ShowName, + ShowExpiration = ShowExpiration, + ShowCashPerPoint = ShowCashPerPoint, + ShowCurrency = ShowCurrency, + ShowNotionalValue = ShowNotionalValue, + }; +} + diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboRowData.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboRowData.cs new file mode 100644 index 000000000..164c80d8f --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboRowData.cs @@ -0,0 +1,86 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.api; + +public class HistoricBboRowData { + public HistoricBboRowData(int id, HistoricInstrumentRowData instrument, DateTime timestamp, double bidPrice, double bidQuantity, double askPrice, double askQuantity) + : this( + id, + timestamp, + instrument.Venue, + instrument.SecurityType, + instrument.Asset, + instrument.Currency, + instrument.Name, + instrument.Expiration, + instrument.CashPerPoint, + bidPrice, + bidQuantity, + askPrice, + askQuantity) { } + + public HistoricBboRowData( + int id, + DateTime timestamp, + Venue venue, + SecurityType securityType, + string asset, + string currency, + string name, + DateTime? expiration, + double cashPerPoint, + double bidPrice, + double bidQuantity, + double askPrice, + double askQuantity + ) { + Id = id; + Timestamp = timestamp; + Venue = venue; + SecurityType = securityType; + Asset = asset; + Currency = currency; + Name = name; + Expiration = expiration; + CashPerPoint = cashPerPoint; + BidPrice = bidPrice; + BidQuantity = bidQuantity; + AskPrice = askPrice; + AskQuantity = askQuantity; + } + + public int Id { get; } + public DateTime Timestamp { get; } + public Venue Venue { get; } + public SecurityType SecurityType { get; } + public TimeseriesKey Key => new((Venue, Name), Timestamp); + public string Asset { get; } + public string Currency { get; } + public string Name { get; } + public DateTime? Expiration { get; } + public double CashPerPoint { get; } + + public double BidPrice { get; } + public double BidQuantity { get; } + public double BidNotional => CashPerPoint * BidPrice * BidQuantity; + public double AskPrice { get; } + public double AskQuantity { get; } + public double AskNotional => CashPerPoint * AskPrice * AskQuantity; + + public HistoricBboRowData With( + double? bidPrice = null, double? bidQuantity = null, double? askPrice = null, double? askQuantity = null) => + new( + Id, + Timestamp, + Venue, + SecurityType, + Asset, + Currency, + Name, + Expiration, + CashPerPoint, + bidPrice ?? BidPrice, + bidQuantity ?? BidQuantity, + askPrice ?? AskPrice, + askQuantity ?? AskQuantity); +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboSchemaBuilder.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboSchemaBuilder.cs new file mode 100644 index 000000000..db0793a4e --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboSchemaBuilder.cs @@ -0,0 +1,58 @@ +namespace Bts.SharedStreams.MarketData; + +using Bts.DataStreamCore; +using Bts.SharedStreams.StreamFactories; + +public class HistoricBboSchemaBuilder { + public bool ShowVenue { get; set; } = true; + public bool ShowAsset { get; set; } = true; + public bool ShowType { get; set; } = true; + public bool ShowName { get; set; } = true; + public bool ShowExpiration { get; set; } = true; + public bool ShowCashPerPoint { get; set; } = true; + public bool ShowCurrency { get; set; } = true; + public bool ShowPrice { get; set; } = true; + public bool ShowQuantity { get; set; } = true; + public bool ShowNotionalValue { get; set; } = true; + public string TableName { get; set; } = "HistoricBbo"; + + public BlueTableSchemaAdapter DatabentoAdapter() { + var adapter = new BlueTableSchemaAdapter(TableName); + adapter.AddColumn("id", r => r.Id); + adapter.AddColumn(WellKnownColumnNames.Timestamp, r => r.Timestamp); + if (ShowVenue) { + adapter.AddColumn(WellKnownColumnNames.Venue, r => r.Venue); + } + if (ShowAsset) { + adapter.AddColumn(WellKnownColumnNames.Asset, r => r.Asset); + } + if (ShowType) { + adapter.AddColumn(WellKnownColumnNames.SecurityType, r => r.SecurityType); + } + if (ShowName) { + adapter.AddColumn(WellKnownColumnNames.SecurityName, r => r.Name); + } + if (ShowExpiration) { + adapter.AddColumn(WellKnownColumnNames.Expiration, r => r.Expiration); + } + if (ShowCurrency) { + adapter.AddColumn(WellKnownColumnNames.Currency, r => r.Currency); + } + if (ShowCashPerPoint) { + adapter.AddColumn(WellKnownColumnNames.CashPerPoint, r => r.Currency); + } + if (ShowNotionalValue) { + adapter.AddColumn(WellKnownColumnNames.BidNotional, r => r.BidNotional); + adapter.AddColumn(WellKnownColumnNames.AskNotional, r => r.AskNotional); + } + if (ShowQuantity) { + adapter.AddColumn(WellKnownColumnNames.BidQuantity, r => r.BidQuantity); + adapter.AddColumn(WellKnownColumnNames.AskQuantity, r => r.AskQuantity); + } + if (ShowPrice) { + adapter.AddColumn(WellKnownColumnNames.BidPrice, r => r.BidPrice); + adapter.AddColumn(WellKnownColumnNames.AskPrice, r => r.AskPrice); + } + return adapter; + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboStream.cs new file mode 100644 index 000000000..0203d9e7b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboStream.cs @@ -0,0 +1,28 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using Bts.DataStream.BlueOrb; +using Bts.PubSub; + +using Microsoft.Extensions.Logging; + +/// +/// Creates a time series of bbos for a date range. +/// +public class HistoricBboStream : ViewStream { + public HistoricBboStream(HistoricBboParams streamParams, IWorkScheduler workScheduler, ILogger logger) => + mySetRealSource(Create(streamParams, workScheduler, logger)); + + public static ViewStream Create(HistoricBboParams streamParams, IWorkScheduler workScheduler, ILogger logger) => + AsyncViewStream.Create(workScheduler, () => CreateAsync(streamParams, logger)); + + public static async Task CreateAsync(HistoricBboParams sp, ILogger logger) { + logger.LogInformation("Creating HistoricBboStream"); + var table = await HistoricBboTable.CreateAsync(sp, logger); + if (table is null) { + return null; + } + return new BlueRowCollectionStaticStream(table); + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboTable.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboTable.cs new file mode 100644 index 000000000..904c373b7 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricBboTable.cs @@ -0,0 +1,160 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts; +using bts.BlueOrb; +using bts.collections; + +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; + +using Spark.Core; +using Spark.Core.Collections; + +using System.Collections.Immutable; + +public class HistoricBboTable : IBlueRowCollection { + readonly BlueTableSchemaAdapter schemaAdapter; + readonly TimeseriesIndex bboIndex; + readonly ImmutableSortedSet> rowSet; + + public HistoricBboTable(BlueTableSchemaAdapter schemaAdapter) { + this.schemaAdapter = schemaAdapter; + bboIndex = new(); + rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer()); + } + + HistoricBboTable( + BlueTableSchemaAdapter schemaAdapter, + TimeseriesIndex bboIndex, + ImmutableSortedSet> rowSet) { + this.schemaAdapter = schemaAdapter; + this.bboIndex = bboIndex; + this.rowSet = rowSet; + } + + public IBlueRow this[int id] => + rowSet.TryGetValue(new(id, null!), out var res) + ? schemaAdapter.GetBlueRow(res.Value) + : throw new KeyNotFoundException(); + + public IBlueTableSchema schema => schemaAdapter.Schema; + + public int Count => rowSet.Count; + + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + + public static async Task CreateAsync(HistoricBboParams sp, ILogger logger, bool useContinuousContracts = true) { + logger.LogInformation("Creating HistoricBboTable"); + var beginDate = sp.BeginDate; + var endDate = sp.EndDate; + var instParams = sp.GetInstrumentParams(); + var instSchemaBd = instParams.GetSchemaBuilder(); + var instSchemaAdapter = instSchemaBd.DatabentoAdapter(); + var mbp1FileList = DatabentoFileStore.GetDataFiles("mbp-1", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToListSorted(f => f.Date); + if (!beginDate.HasValue && !endDate.HasValue) { + beginDate = mbp1FileList[^1].Date; + endDate = beginDate.Value + TimeSpan.FromDays(1); + } else { + beginDate ??= DateTime.MinValue; + endDate ??= DateTime.MaxValue; + } + mbp1FileList.RemoveAll(f => f.Date < beginDate.Value || f.Date >= endDate.Value); + if (mbp1FileList.Count == 0) { + return null; + } + var definitionFileList = DatabentoFileStore.GetDataFiles("definition", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToListSorted(f => f.Date); + if (!mbp1FileList.WrapSelector(f => f.Date, null).SequenceEqual(definitionFileList.WrapSelector(f => f.Date, null))) { + logger.LogInvalidParams("HistoricBboStream", "Missing one or more definition (instrument) files in date range"); + return null; + } + var tablePairs = new (IBlueRowCollection Mbp1, IBlueRowCollection Definition)[mbp1FileList.Count]; + Parallel.For(0, mbp1FileList.Count, LoadTables); + var instrumentIndex = new TimeseriesIndex(); + var instrumentIndexBd = instrumentIndex.ToBuilder(); + var bboSchemaBd = sp.GetSchemaBuilder(); + var bboSchemaAdapter = bboSchemaBd.DatabentoAdapter(); + var bboTable = new HistoricBboTable(bboSchemaAdapter); + foreach (var (mbp1, definition) in tablePairs) { + var filteredDefs = definition; + if (useContinuousContracts) { + filteredDefs = HistoricContinuousContractTable.Create(filteredDefs); + } + var definitionMap = filteredDefs.ToDictionary(r => r.GetInt64("instrument_id").assert_value()); + var instRowDataFactory = DatabentoInstrumentAdapter.GetRowDataFactory(instrumentIndexBd.GetOrAdd, filteredDefs.schema.columnInfo); + bboTable = bboTable.AddBbos(mbp1, GetInstrumentRowData); + + HistoricInstrumentRowData? GetInstrumentRowData(IBlueRow mbp1Row) { + var instId = mbp1Row.GetInt64("instrument_id"); + Log.notnull(instId); + if (!definitionMap.TryGetValue(instId.Value, out var defRow)) { + Log.assert(false); + } + var instRowData = instRowDataFactory(defRow!); + return instRowData; + } + } + return bboTable; + + void LoadTables(int index) { + var definitionPath = definitionFileList[index].Path; + var definition = ParquetTableReader.Read(definitionPath); + var mbp1Path = mbp1FileList[index].Path; + var mbp1 = ParquetTableReader.Read(mbp1Path); + tablePairs[index] = (mbp1, definition); + } + } + + /// + /// Add bbo records () to this table. + /// + /// The bbo records to add (columns should correspond to Databento 'mbp-1' schema) + /// Instruments table for the bbo records (i.e., for the same time period as bbo) + /// A new table (immutable) with the added rows + public HistoricBboTable AddBbos(IBlueRowCollection mbp1Rows, Func getInstrument) { + var bboIndex = this.bboIndex.ToBuilder(); + var rowSet = this.rowSet.ToBuilder(); + var rowDataFactory = DatabentoBboAdapter.GetRowDataFactory(bboIndex.GetOrAdd, getInstrument); + foreach (var mbp1Row in mbp1Rows) { + var rd = rowDataFactory(mbp1Row); + if (rd is null) { + continue; + } + rowSet.Add(new(rd.Id, rd)); + } + return new(schemaAdapter, bboIndex.ToImmutable(), rowSet.ToImmutable()); + } + + public bool containsID(int id) => + rowSet.Contains(new(id, null!)); + + public IBlueRow GetRow(int index) { + var rd = rowSet[index].Value; + var blueRow = schemaAdapter.GetBlueRow(rd); + return blueRow; + } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int idx = rowSet.IndexOf(new(id, null!)); + if (idx >= 0) { + var rd = rowSet[idx].Value; + var blueRow = schemaAdapter.GetBlueRow(rd); + if (BlueRowEqualityComparer.Equals(blueRow, item)) { + return idx; + } + } + return -1; + } + + public IEnumerator GetEnumerator() { + foreach (var kv in rowSet) { + yield return schemaAdapter.GetBlueRow(kv.Value); + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricContinuousContractStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricContinuousContractStream.cs new file mode 100644 index 000000000..743c7267b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricContinuousContractStream.cs @@ -0,0 +1,43 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.api; +using bts.BlueOrb; + +using Bts.DataStream; +using Bts.SharedStreams.StreamFactories; + +/// +/// Creates a time series of continuous contracts for a date range. This can be joined with +/// a timeseries of bbo data, for example, to get continuous bbo. +/// +public class HistoricContinuousContractStream : ViewStream { + public HistoricContinuousContractStream(ViewStream historicInstruments) => + mySetRealSource(Create(historicInstruments)); + + public static ViewStream? Create(ViewStream historicInstruments) { + var vs = historicInstruments.GroupByApply(new[] { WellKnownColumnNames.Venue, WellKnownColumnNames.SecurityType, WellKnownColumnNames.Asset }, Apply); + return vs; + + static IBlueRowCollection Apply(object?[] groupKeys, IBlueRowCollection inputRows) { + var secType = (SecurityType)groupKeys[1]!; + switch (secType) { + case SecurityType.Future: + case SecurityType.Option: + break; + default: + return inputRows; + } + var schema = inputRows.schema; + var minExpRow = inputRows.Select(r => (Row: r, Exp: r.GetDateTime(WellKnownColumnNames.Expiration))) + .Where(t => t.Exp.HasValue) + .MinElement(t => t.Exp!.Value) + .Row; + if (minExpRow is null) { + return inputRows; + } + return new SingleRowTable(minExpRow); + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricContinuousContractTable.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricContinuousContractTable.cs new file mode 100644 index 000000000..10bd92039 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricContinuousContractTable.cs @@ -0,0 +1,102 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts; +using bts.api; +using bts.BlueOrb; + +using Bts.DataStream; +using Bts.DataStreamCore; +using Bts.SharedStreams.StreamFactories; + +using Spark.Core; + +using System.Collections.Generic; + +/// +/// Creates a time series of continuous contracts for a date range. This can be joined with +/// a timeseries of bbo data, for example, to get continuous bbo. +/// +public class HistoricContinuousContractTable : IBlueRowCollection { + readonly IBlueRowCollection impl; + + public HistoricContinuousContractTable(IBlueRowCollection instruments) => + impl = Create(instruments); + + public IBlueTableSchema schema => impl.schema; + + public int Count => impl.Count; + + public IBlueRow this[int id] => impl[id]; + + IBlueRow IList.this[int index] { + get => ((IList)impl)[index]; + set => throw new NotSupportedException(); + } + + public static IBlueRowCollection Create(IBlueRowCollection instruments) { + var instGroups = instruments.GroupBy(GetKey) + .ToDictionary(g => g.Key, g => g.ToListSorted(r => GetExpiration(r) ?? DateTime.MaxValue)); + var bd = new BlueTableImmutable(instruments.schema).ToBuilder(); + foreach (var (cck, contracts) in instGroups) { + Log.assert(contracts.Count > 0); + switch (cck.Type) { + case SecurityType.Future: + bd.AddOrUpdate(contracts[0]); + break; + case SecurityType.Option: + throw new NotImplementedException("Options not implemented"); + case SecurityType.Spread: + foreach (var c in contracts) { + bd.AddOrUpdate(c); + } + break; + default: + Log.assert(contracts.Count == 1); + bd.AddOrUpdate(contracts[0]); + break; + } + bd.AddOrUpdate(contracts[0]); + } + var tableOut = bd.ToImmutable(); + return tableOut; + + static DateTime? GetExpiration(IBlueRow inst) { + var exp = inst.GetDateTime(WellKnownColumnNames.Expiration); + return exp; + } + + static ContinuousContractKey GetKey(IBlueRow inst) { + var venue = inst.GetEnum(WellKnownColumnNames.Venue).assert_value(); + var asset = inst.GetString(WellKnownColumnNames.Asset); + Log.notnull(asset); + var type = inst.GetEnum(WellKnownColumnNames.SecurityType).assert_value(); + return new(venue, asset, type); + } + } + + public bool containsID(int id) => + impl.containsID(id); + + public IEnumerator GetEnumerator() => + impl.GetEnumerator(); + + public IBlueRow GetRow(int index) => + impl.GetRow(index); + + public int IndexOf(IBlueRow item) => + impl.IndexOf(item); + + readonly struct ContinuousContractKey { + public readonly Venue Venue; + public readonly string Asset; + public readonly SecurityType Type; + + public ContinuousContractKey(Venue venue, string asset, SecurityType type) { + Venue = venue; + Asset = asset; + Type = type; + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentParams.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentParams.cs new file mode 100644 index 000000000..c2e17467b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentParams.cs @@ -0,0 +1,93 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.api; + +using Bts.DataStreamCore.StreamFactories; + +public class HistoricInstrumentParams { + public HistoricInstrumentParams() { } + + public HistoricInstrumentParams(string venueStr, string asset) + : this(MarketDataArgStore.ParseVenue(venueStr), asset) { } + + public HistoricInstrumentParams(string venueStr, string asset, string typeStr) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr)) { } + + public HistoricInstrumentParams(string venueStr, string asset, string typeStr, DateTime date) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), date) { } + + public HistoricInstrumentParams(string venueStr, string asset, string typeStr, DateTime beginDate, DateTime endDate) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), beginDate, endDate) { } + + public HistoricInstrumentParams(string venueStr, string asset, DateTime date) + : this(MarketDataArgStore.ParseVenue(venueStr), asset, date) { } + + public HistoricInstrumentParams(Venue venue, string asset) { + Venue = venue; + Asset = asset; + } + + public HistoricInstrumentParams(Venue venue, string asset, DateTime date) + : this(venue, asset, null, date) { } + + public HistoricInstrumentParams(Venue venue, string asset, DateTime beginDate, DateTime endDate) + : this(venue, asset, null, beginDate, endDate) { } + + public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type) { + Venue = venue; + Asset = asset; + Type = type; + } + + public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type, DateTime date) + : this(venue, asset, type, date, date.AddDays(1)) { } + + public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type, DateTime beginDate, DateTime endDate) { + Venue = venue; + Asset = asset; + Type = type; + BeginDate = beginDate; + EndDate = endDate; + } + + public Venue? Venue { get; set; } + public string? Asset { get; set; } + public SecurityType? Type { get; set; } + public string? Name { get; set; } + public int NumLegs { get; set; } = 5; + public bool ShowVenue { get; set; } = true; + public bool ShowType { get; set; } = true; + public bool ShowAsset { get; set; } = true; + public bool ShowExpiration { get; set; } + public bool ShowCashPerPoint { get; set; } = true; + public bool ShowLegVenue { get; set; } = true; + public bool ShowLegType { get; set; } = true; + public bool ShowLegAsset { get; set; } = true; + public bool ShowLegName { get; set; } = true; + public bool ShowLegExpiration { get; set; } + public bool ShowLegCashPerPoint { get; set; } = true; + public DateTime? BeginDate { get; set; } + public DateTime? EndDate { get; set; } + + public mbts.spark.SecurityCriteria GetSecurityCriteria() { + var clause = new mbts.spark.SecurityCriteriaClause( + exchanges: Venue.HasValue ? new[] { Venue.Value } : null, + security_types: Type.HasValue ? new[] { Type.Value } : null, + symbols: !string.IsNullOrEmpty(Asset) ? new[] { Asset } : null, + names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null); + return clause.is_defined ? new(new[] { clause }) : new(); + } + + public HistoricInstrumentSchemaBuilder GetSchemaBuilder() => + new() { + ShowVenue = ShowVenue, + ShowType = true, + ShowAsset = true, + ShowName = true, + ShowCashPerPoint = ShowCashPerPoint, + ShowLegVenue = ShowLegVenue, + ShowLegAsset = ShowLegAsset, + ShowLegType = ShowLegType, + ShowLegName = ShowLegVenue, + }; +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentRowData.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentRowData.cs new file mode 100644 index 000000000..11a12d41b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentRowData.cs @@ -0,0 +1,48 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.api; + +using System.Collections.Immutable; + +public class HistoricInstrumentRowData { + public HistoricInstrumentRowData( + int id, + DateTime timestamp, + Venue venue, + SecurityType securityType, + string asset, + string currency, + string name, + DateTime? expiration, + double cashPerPoint, + ImmutableList? legs = null + ) { + Id = id; + Timestamp = timestamp; + Venue = venue; + SecurityType = securityType; + Asset = asset; + Currency = currency; + Name = name; + Expiration = expiration; + CashPerPoint = cashPerPoint; + Legs = legs; + } + + public int Id { get; } + public DateTime Timestamp { get; } + public Venue Venue { get; } + public SecurityType SecurityType { get; } + public TimeseriesKey Key => new((Venue, Name), Timestamp); + public string Asset { get; } + public string Currency { get; } + public string Name { get; } + public DateTime? Expiration { get; } + public double CashPerPoint { get; } + public int NumLegs => Legs?.Count ?? 0; + // Some legs might not be resolved + public ImmutableList? Legs { get; } + + public HistoricInstrumentRowData AddLegs(IEnumerable legs) => + new(Id, Timestamp, Venue, SecurityType, Asset, Currency, Name, Expiration, CashPerPoint, ImmutableList.CreateRange(legs)); +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentSchemaBuilder.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentSchemaBuilder.cs new file mode 100644 index 000000000..34b706a8b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentSchemaBuilder.cs @@ -0,0 +1,82 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.collections; + +using Bts.DataStreamCore; +using Bts.SharedStreams.StreamFactories; + +public class HistoricInstrumentSchemaBuilder { + public bool ShowVenue { get; set; } = true; + public bool ShowAsset { get; set; } = true; + public bool ShowType { get; set; } = true; + public bool ShowName { get; set; } = true; + public bool ShowExpiration { get; set; } = true; + public bool ShowCashPerPoint { get; set; } = true; + public bool ShowCurrency { get; set; } = true; + public bool ShowLegVenue { get; set; } + public bool ShowLegAsset { get; set; } + public bool ShowLegType { get; set; } + public bool ShowLegName { get; set; } = true; + public bool ShowLegCashPerPoint { get; set; } + public int NumLegs { get; set; } = 2; + public string TableName { get; set; } = "HistoricInstrument"; + + public BlueTableSchemaAdapter DatabentoAdapter() { + var adapter = new BlueTableSchemaAdapter(TableName); + adapter.AddColumn("id", r => r.Id); + adapter.AddColumn(WellKnownColumnNames.Timestamp, r => r.Timestamp); + if (ShowVenue) { + adapter.AddColumn(WellKnownColumnNames.Venue, r => r.Venue); + } + if (ShowType) { + adapter.AddColumn(WellKnownColumnNames.SecurityType, r => r.SecurityType); + } + if (ShowAsset) { + adapter.AddColumn(WellKnownColumnNames.Asset, r => r.Asset); + } + if (ShowName) { + adapter.AddColumn(WellKnownColumnNames.SecurityName, r => r.Name); + } + if (ShowExpiration) { + adapter.AddColumn(WellKnownColumnNames.Expiration, r => r.Expiration); + } + if (ShowCurrency) { + adapter.AddColumn(WellKnownColumnNames.Currency, r => r.Currency); + } + if (ShowCashPerPoint) { + adapter.AddColumn(WellKnownColumnNames.CashPerPoint, r => r.Currency); + } + if (NumLegs > 0) { + adapter.AddColumn(WellKnownColumnNames.NumLegs, r => r.NumLegs); + for (int i = 0; i < NumLegs; i++) { + // Copy i to a local var (legIndex) so it can be referenced in column getters. + int legIndex = i; + if (ShowLegVenue) { + adapter.AddColumn(WellKnownColumnNames.LegVenue(i), Get(l => l.Venue)); + } + if (ShowLegType) { + adapter.AddColumn(WellKnownColumnNames.LegSecurityType(i), Get(l => l.SecurityType)); + } + if (ShowLegAsset) { + adapter.AddColumn(WellKnownColumnNames.LegAsset(i), Get(l => l.Asset)); + } + if (ShowLegName) { + adapter.AddColumn(WellKnownColumnNames.LegSecurityName(i), Get(l => l.Name)); + } + if (ShowLegCashPerPoint) { + adapter.AddColumn(WellKnownColumnNames.LegCashPerPoint(i), Get(l => l.CashPerPoint, double.NaN)); + } + + Func Get(Func getValue, T defaultValue = default!) => + s => { + var legs = s.Legs; + if (legs is null || !legs.TryGetAt(legIndex, out var leg)) { + return defaultValue; + } + return getValue(leg); + }; + } + } + return adapter; + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentStream.cs new file mode 100644 index 000000000..d3138b0e7 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentStream.cs @@ -0,0 +1,28 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using Bts.DataStream.BlueOrb; +using Bts.PubSub; + +using Microsoft.Extensions.Logging; + +/// +/// Creates a time series of instruments for a date range. +/// +public class HistoricInstrumentStream : ViewStream { + public HistoricInstrumentStream(HistoricInstrumentParams streamParams, IWorkScheduler workScheduler, ILogger logger) => + mySetRealSource(Create(streamParams, workScheduler, logger)); + + public static ViewStream Create(HistoricInstrumentParams streamParams, IWorkScheduler workScheduler, ILogger logger) => + AsyncViewStream.Create(workScheduler, () => CreateAsync(streamParams, logger)); + + public static async Task CreateAsync(HistoricInstrumentParams sp, ILogger logger) { + logger.LogInformation("Creating HistoricInstrumentStream"); + var table = await HistoricInstrumentTable.CreateAsync(sp, logger); + if (table is null) { + return null; + } + return new BlueRowCollectionStaticStream(table); + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentTable.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentTable.cs new file mode 100644 index 000000000..badca591b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricInstrumentTable.cs @@ -0,0 +1,167 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts; +using bts.api; +using bts.BlueOrb; +using bts.collections; + +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; + +using Spark.Core.Collections; + +using System.Collections.Immutable; + +public class HistoricInstrumentTable : IBlueRowCollection { + readonly BlueTableSchemaAdapter schemaAdapter; + readonly TimeseriesIndex instrumentIndex; + readonly ImmutableSortedSet> rowSet; + + public HistoricInstrumentTable(BlueTableSchemaAdapter schemaAdapter) { + this.schemaAdapter = schemaAdapter; + instrumentIndex = new(); + rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer()); + } + + public HistoricInstrumentTable(BlueTableSchemaAdapter schemaAdapter, TimeseriesIndex instrumentIndex) { + this.schemaAdapter = schemaAdapter; + this.instrumentIndex = instrumentIndex; + rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer()); + } + + HistoricInstrumentTable( + BlueTableSchemaAdapter schemaAdapter, + TimeseriesIndex instrumentIndex, + ImmutableSortedSet> rowSet) { + this.schemaAdapter = schemaAdapter; + this.instrumentIndex = instrumentIndex; + this.rowSet = rowSet; + } + + public IBlueRow this[int id] => + rowSet.TryGetValue(new(id, null!), out var res) + ? schemaAdapter.GetBlueRow(res.Value) + : throw new KeyNotFoundException(); + + public IBlueTableSchema schema => schemaAdapter.Schema; + + public int Count => rowSet.Count; + + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + + public static HistoricInstrumentTable Create(BlueTableSchemaAdapter schemaAdapter, IBlueRowCollection definitionRows) { + var table = new HistoricInstrumentTable(schemaAdapter); + table = table.AddInstruments(definitionRows); + return table; + } + + public static async Task CreateAsync(HistoricInstrumentParams sp, ILogger logger) { + logger.LogInformation("Creating HistoricInstrumentTable"); + var beginDate = sp.BeginDate; + var endDate = sp.EndDate; + var fileList = DatabentoFileStore.GetDataFiles("definition", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToList(); + if (!beginDate.HasValue && !endDate.HasValue) { + beginDate = fileList[^1].Date; + endDate = beginDate.Value + TimeSpan.FromDays(1); + } else { + beginDate ??= DateTime.MinValue; + endDate ??= DateTime.MaxValue; + } + fileList.RemoveAll(f => f.Date < beginDate.Value || f.Date >= endDate.Value); + if (fileList.Count == 0) { + return null; + } + var rowMapBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); + var rowSetBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); + var dateTables = fileList.ParallelSelect(f => ParquetTableReader.Read(f.Path)); + var tableRowMap = ImmutableDictionary.CreateBuilder(); + var schemaBd = sp.GetSchemaBuilder(); + var schemaAdapter = schemaBd.DatabentoAdapter(); + var nameToId = new Dictionary<(Venue, string), int>(); + var idToName = new Dictionary(); + var table = new HistoricInstrumentTable(schemaAdapter); + foreach (var dateTable in dateTables) { + table = table.AddInstruments(dateTable); + } + return table; + } + + /// + /// Add instruments () to this table. If there are spreads, the legs' contracts + /// should be included in as well. + /// + /// A table with columns corresponding to the Databento 'definition' schema + /// + /// Glbx spread does not have the correct number of legs (2) + /// Spread handler code not implemented at a particular venue + public HistoricInstrumentTable AddInstruments(IBlueRowCollection definitionRows) { + var instrumentIndex = this.instrumentIndex.ToBuilder(); + var rowSet = this.rowSet.ToBuilder(); + var spreadRows = new List(); + var rowDataFactory = DatabentoInstrumentAdapter.GetRowDataFactory(instrumentIndex.GetOrAdd, definitionRows.schema.columnInfo); + foreach (var r in definitionRows) { + var rd = rowDataFactory(r); + rowSet.Add(new(rd.Id, rd)); + if (rd.SecurityType == SecurityType.Spread) { + spreadRows.Add(rd); + } + } + // Resolve spread legs + for (int i = 0; i < spreadRows.Count; i++) { + var rd = spreadRows[i]; + switch (rd.Venue) { + case Venue.Glbx: + var legSymbols = rd.Name.Split('-'); + if (legSymbols.Length != 2) { + throw new ArgumentException($"Glbx spread has {legSymbols.Length} legs (expected 2)", nameof(definitionRows)); + } + var legs = legSymbols.Select(legSym => { + var legId = instrumentIndex.GetLatest((rd.Venue, legSym)); + bool foundLeg = rowSet.TryGetValue(new(legId, null!), out var leg); + Log.assert(foundLeg); + return leg.Value; + }); + rowSet.Remove(new(rd.Id, rd)); + rowSet.Add(new(rd.Id, rd.AddLegs(legs))); + break; + default: + throw new NotImplementedException($"Spreads not implemented at venue '{rd.Venue}'"); + } + } + return new(schemaAdapter, instrumentIndex.ToImmutable(), rowSet.ToImmutable()); + } + + public bool containsID(int id) => + instrumentIndex.ContainsId(id); + + public IBlueRow GetRow(int index) { + var rd = rowSet[index].Value; + var blueRow = schemaAdapter.GetBlueRow(rd); + return blueRow; + } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int idx = rowSet.IndexOf(new(id, null!)); + if (idx >= 0) { + var rd = rowSet[idx].Value; + var blueRow = schemaAdapter.GetBlueRow(rd); + if (BlueRowEqualityComparer.Equals(blueRow, item)) { + return idx; + } + } + return -1; + } + + public IEnumerator GetEnumerator() { + foreach (var kv in rowSet) { + yield return schemaAdapter.GetBlueRow(kv.Value); + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/InstrumentKey.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/InstrumentKey.cs new file mode 100644 index 000000000..4a9162a2c --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/InstrumentKey.cs @@ -0,0 +1,11 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.api; + +public record InstrumentKey(Venue Venue, string Name) { + public static implicit operator (Venue Venue, string Name)(InstrumentKey key) => + (key.Venue, key.Name); + + public static implicit operator InstrumentKey((Venue Venue, string Name) tup) => + new(tup.Venue, tup.Name); +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/TimeseriesIndex.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/TimeseriesIndex.cs new file mode 100644 index 000000000..f2732be4b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/TimeseriesIndex.cs @@ -0,0 +1,168 @@ +namespace Bts.SharedStreams.MarketData; + +using Spark.Core.Collections; + +using System.Collections.Immutable; + +/// +/// Maps a timeseries of keys to unique int32 ids. +/// +public class TimeseriesIndex where T : notnull { + static readonly IComparer> idTimeseriesComp = + KeyValuePairComparer.KeyComparer(); + + readonly ImmutableDictionary>> nameToId; + readonly ImmutableDictionary> idToName; + readonly int idCounter; + + public TimeseriesIndex() + : this(ImmutableDictionary>>.Empty, ImmutableDictionary>.Empty, 0) { } + + TimeseriesIndex(ImmutableDictionary>> nameToId, ImmutableDictionary> idToName, int idCounter) { + this.nameToId = nameToId; + this.idToName = idToName; + this.idCounter = idCounter; + } + + public bool ContainsId(int id) => + idToName.ContainsKey(id); + + public int GetNearest(TimeseriesKey key) { + if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) { + throw new KeyNotFoundException(); + } + int idx = ids.BinarySearch(new(key.Timestamp, -1)); + if (idx < 0) { + idx = ~idx; + } + if (idx == 0) { + throw new KeyNotFoundException(); + } + return ids[idx].Value; + } + + public TimeseriesIndex GetNearestOrAdd(TimeseriesKey key, out int result) { + int idx; + if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) { + ids = ImmutableList>.Empty; + idx = 0; + } else { + idx = ids.BinarySearch(new(key.Timestamp, -1)); + if (idx < 0) { + idx = ~idx; + } + } + if (idx == 0) { + result = idCounter + 1; + ids = ids.Insert(0, new(key.Timestamp, result)); + return new( + nameToId.SetItem(key.Name, ids), + idToName.SetItem(result, key), + result); + } + result = ids[idx].Value; + return this; + } + + public TimeseriesIndex GetOrAdd(TimeseriesKey key, out int result) { + int idx; + if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) { + ids = ImmutableList>.Empty; + idx = 0; + } else { + idx = ids.BinarySearch(new(key.Timestamp, -1)); + if (idx < 0) { + idx = ~idx; + } else { + result = ids[idx].Value; + return this; + } + } + result = idCounter + 1; + ids = ids.Insert(idx, new(key.Timestamp, result)); + return new( + nameToId.SetItem(key.Name, ids), + idToName.SetItem(result, key), + result); + } + + public IList> GetIds(T name) => + nameToId[name]; + + public Builder ToBuilder() => + new(this); + + public class Builder { + readonly ImmutableDictionary>>.Builder nameToId; + readonly ImmutableDictionary>.Builder idToName; + int idCounter; + + public Builder(TimeseriesIndex source) + : this(source.nameToId, source.idToName, source.idCounter) { } + + public Builder(ImmutableDictionary>> nameToId, ImmutableDictionary> idToName, int idCounter) { + this.nameToId = nameToId.ToBuilder(); + this.idToName = idToName.ToBuilder(); + this.idCounter = idCounter; + } + + public int GetNearest(TimeseriesKey key) => + getOrAdd(key, false, false); + + public int GetNearestOrAdd(TimeseriesKey key) => + getOrAdd(key, false, true); + + public int GetOrAdd(TimeseriesKey key) => + getOrAdd(key, true, true); + + public int GetLatest(T key) => + nameToId.TryGetValue(key, out var ids) + ? ids[^1].Value + : throw new KeyNotFoundException(); + + public bool TryGetLatest(T key, out int result) { + if (nameToId.TryGetValue(key, out var ids)) { + result = ids[^1].Value; + return true; + } + result = 0; + return false; + } + + public ImmutableList> GetIds(T name) => + nameToId.TryGetValue(name, out var ids) + ? ids + : ImmutableList>.Empty; + + public TimeseriesIndex ToImmutable() => + new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter); + + int getOrAdd(TimeseriesKey key, bool requireExactTimestamp, bool canAdd) { + var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList>.Empty); + int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp); + int id; + if (idx < 0) { + // We didn't get an exact match. If requireExactTimestamp is false and a record with an + // earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp. + idx = ~idx; + if (requireExactTimestamp || idx == 0) { + if (!canAdd) { + throw new KeyNotFoundException(); + } + id = ++idCounter; + ids = ids.Insert(idx, new(key.Timestamp, id)); + nameToId[key.Name] = ids; + idToName.Add(id, key); + } else { + if (idx == ids.Count) { + --idx; + } + id = ids[idx].Value; + } + } else { + id = ids[idx].Value; + } + return id; + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/TimeseriesKey.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/TimeseriesKey.cs new file mode 100644 index 000000000..ae2f360e7 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/TimeseriesKey.cs @@ -0,0 +1,9 @@ +namespace Bts.SharedStreams.MarketData; + +public record TimeseriesKey(T Name, DateTime Timestamp) { + public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey key) => + (key.Name, key.Timestamp); + + public static implicit operator TimeseriesKey((T Name, DateTime Timestamp) tup) => + new(tup.Name, tup.Timestamp); +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs index 03ec5edb7..49e8b72b7 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs @@ -100,7 +100,7 @@ namespace bts.BlueOrb { } } - [DataStreamUserRespresentation("bbo")] + [DataStreamUserRespresentation(false)] public class BBOStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; diff --git a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs index b74cd18a5..c33deccbc 100644 --- a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs +++ b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs @@ -1,7 +1,5 @@ namespace Bts.SharedStreams.Services; -using bts.SilverPine; - using Bts.DataStreamCore; using Bts.SharedStreams.StreamFactories; @@ -17,10 +15,15 @@ public static class SharedStreamsServices { .addStreamFactory() .addStreamFactory(); + public static IServiceCollection AddHistoricMarketDataStreams(this IServiceCollection services) => + services.addStreamFactory(); + public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) => services.Configure(configurationSection) - .AddSingleton() - .AddSingleton(sp => sp.GetRequiredService()); + .addStreamFactory(); + + public static IServiceCollection AddParquetReaderStream(this IServiceCollection services) => + services.addStreamFactory(); static IServiceCollection addStreamFactory(this IServiceCollection services) where T : class, IDataStreamFactory => services.AddSingleton() diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs index 96a3db861..5e7e02f4b 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs @@ -2,21 +2,26 @@ namespace Bts.SharedStreams.StreamFactories; using BCG; -using bts.md; - using Bts.DataStream; using Bts.DataStreamCore; +using Bts.DataStreamCore.StreamFactories; using Microsoft.Extensions.Logging; using Spark.Api.MarketData; using System; +using System.Collections.Generic; + +public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory { + readonly LiveBboStreamFactory liveBboFactory; + IDataStreamFactory? compositeFactory = null; -public class BookStreamFactory : DataStreamFactory { public BookStreamFactory(IMarketDataClient? mdClient, ILogger logger) { this.Add("book", CreateBook, logger); - this.Add(new[] { "bbo", "LiveBBOStream" }, CreateBbo, logger); + this.Add(new[] { "bbo" }, CreateBbo, logger); + liveBboFactory = new LiveBboStreamFactory(mdClient, logger); + Add(liveBboFactory); ViewStream? CreateBook(BookStreamParams streamParams) => mdClient?.Books(streamParams.GetSecurityCriteria()) @@ -24,10 +29,97 @@ public class BookStreamFactory : DataStreamFactory { ?.AsViewStream() ?? throw new InvalidOperationException("Live md not available"); - ViewStream? CreateBbo(BookStreamParams streamParams) => - mdClient?.Books(streamParams.GetSecurityCriteria()) + ViewStream? CreateBbo(BookStreamParams streamParams) { + return mdClient?.Books(streamParams.GetSecurityCriteria()) ?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()) ?.AsViewStream() ?? throw new InvalidOperationException("Live md not available"); + } + } + + public void OnCompositeFactory(IDataStreamFactory compositeFactory) { + this.compositeFactory = compositeFactory; + liveBboFactory.OnCompositeFactory(compositeFactory); + } + + class LiveBboStreamFactory : IDataStreamFactory, IRecursiveDataStreamFactory { + static readonly IDataStreamMetadata metadata = new DataStreamMetadata() { + Name = "LiveBBOStream", + AlternateNames = new(), + Signatures = new() { + new (string?, Type?)[] { + ("symbol", typeof(string)) + }, + new (string?, Type?)[] { + ("secType", typeof(string)), + ("symbol", typeof(string)), + } + } + }; + readonly IMarketDataClient? mdClient; + readonly ILogger logger; + IDataStreamFactory? compositeFactory = null; + + public LiveBboStreamFactory(IMarketDataClient? mdClient, ILogger logger) { + this.mdClient = mdClient; + this.logger = logger; + } + + public void OnCompositeFactory(IDataStreamFactory compositeFactory) => + this.compositeFactory = compositeFactory; + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!metadata.IsMatch(streamId)) { + return null; + } + var argList = args.ToList(); + switch (argList.Count) { + case 1: { + var sym = argList[0].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args)); + if (sym == "ES" && compositeFactory is not null) { + return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, sym) }); + } + var streamParams = new BookStreamParams() { + Symbol = sym + }; + return mdClient?.Books(streamParams.GetSecurityCriteria()) + ?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()) + ?.AsViewStream() + ?? throw new InvalidOperationException("Live md not available"); + } + case 2: { + var secTypeStr = argList[0].Value?.ToString() ?? string.Empty; + if (!MarketDataArgStore.TryParseSecurityType(secTypeStr, out var secType)) { + throw new ArgumentException("Failed to parse SecurityType arg", nameof(args)); + } + var sym = argList[1].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args)); + if (sym == "ES" && secType == bts.api.SecurityType.Future && compositeFactory is not null) { + return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, sym) }); + } + var streamParams = new BookStreamParams() { + SecurityType = secType, + Symbol = sym, + }; + return mdClient?.Books(streamParams.GetSecurityCriteria()) + ?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()) + ?.AsViewStream() + ?? throw new InvalidOperationException("Live md not available"); + } + default: + return null; + } + } + + public IEnumerable GetSupportedStreams() { + yield return metadata; + } + } + + public class LiveBboStreamParams { + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex }, Position = 0, IsRequired = false)] + public string? SecurityType { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex }, Position = 1, IsRequired = false)] + public string? ContinuousContract { get; set; } } } diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataParams.cs new file mode 100644 index 000000000..9ffed869a --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataParams.cs @@ -0,0 +1,60 @@ +namespace Bts.SharedStreams.StreamFactories; + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +public interface IHistoricDataParams { + string? DataType { get; } + string? Venue { get; } + string? Symbol { get; } + DateTime BeginDate { get; } + DateTime EndDate { get; } + TimeSpan? Interval { get; } + string? TsNanosCol { get; } +} + +public class HistoricDataParams : IHistoricDataParams { + public string? DataType { get; set; } + public string? Venue { get; set; } + public string? Symbol { get; set; } + public DateTime BeginDate { get; set; } + public DateTime EndDate { get; set; } + public TimeSpan? Interval { get; set; } + public string? TsNanosCol { get; set; } + + [MemberNotNull(nameof(DataType), nameof(Venue), nameof(Symbol), nameof(TsNanosCol))] + public HistoricDataParams Validate() { + validateNotNull(DataType); + validateNotNull(Venue); + validateNotNull(Symbol); + validateNotNull(TsNanosCol); + return this; + } + + [MemberNotNull(nameof(Interval))] + public HistoricDataParams ValidateInterval() { + validateNotNull(Interval); + return this; + } + + static void validateNotNull([NotNull]T arg, [CallerArgumentExpression(nameof(arg))]string argName = "") { + if (arg is null) { + throw new InvalidOperationException($"Invalid HistoricDataParams: {argName} is null"); + } + } +} + +public static class HistoricDataParamsExtensions { + // public static void Validate(this IHistoricDataParams hdp) { + // validateNotNull(hdp.DataType); + // validateNotNull(hdp.Venue); + // validateNotNull(hdp.Symbol); + // validateNotNull(hdp.TsNanosCol); + // } + + // static void validateNotNull([NotNull]T arg, [CallerArgumentExpression(nameof(arg))]string argName = "") { + // if (arg is null) { + // throw new InvalidOperationException($"Invalid HistoricDataParams: {argName} is null"); + // } + // } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricParquetDataLoader.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricParquetDataLoader.cs new file mode 100644 index 000000000..8b0b8309e --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricParquetDataLoader.cs @@ -0,0 +1,200 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts; +using bts.api; +using bts.BlueOrb; +using bts.BlueOrb.Parquet; +using bts.collections; + +using Bts.DataStream; +using Bts.DataStream.BlueOrb; +using Bts.DataStreamCore; +using Bts.DataStreamCore.StreamFactories; +using Bts.SharedStreams.MarketData; + +using Microsoft.Extensions.Logging; + +using Spark.Core.Collections; + +using System.Collections; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; +using System.Reactive.Linq; +using System.Threading.Tasks; + +class HistoricParquetDataLoader : DataStreamFactory { + public HistoricParquetDataLoader(ViewStreamDatabase viewStreamDatabase, ILogger logger) { + this.Add("HistoricBbo", CreateHistoricBbo, logger); + this.Add("ReplayFutureLiveBBOStream", CreateReplayFutureLiveBbo, logger); + this.Add("ReplayFutureLiveTradeStream", CreateReplayFutureLiveTrade, logger); + this.Add("HistoricInstrument", sp => HistoricInstrumentStream.Create(sp, viewStreamDatabase, logger)); + this.Add("HistoricContinuousContract", sp => { + var historicInstruments = HistoricInstrumentStream.Create(sp, viewStreamDatabase, logger); + var vs = HistoricContinuousContractStream.Create(historicInstruments); + return vs; + }); + + static SecurityType? ParseSecTypeArg(string? secTypeStr) { + if (string.IsNullOrEmpty(secTypeStr)) { + return null; + } + if (MarketDataArgStore.TryParseSecurityType(secTypeStr, out var secType)) { + return secType; + } + throw new ArgumentException("Failed to parse SecurityType", nameof(secTypeStr)); + } + + // static Venue? ParseVenueArg(string? venueStr) { + // if (string.IsNullOrEmpty(venueStr)) { + // return null; + // } + // if (MarketDataArgStore.TryParseVenue(venueStr, out var venue)) { + // return venue; + // } + // throw new ArgumentException("Failed to parse Venue", nameof(venueStr)); + // } + + // bool TryGetHistoricInstrumentStreamParams(IEnumerable<(string? Name, int? Index, object? Value)> args, [NotNullWhen(true)] out HistoricInstrumentStreamParams? result, [NotNullWhen(false)] out string? errorMessage) { + // var argList = args.ToList(); + // result = null; + // errorMessage = null; + // switch (argList.Count) { + // case 2: { + // var venue = ParseVenueArg(argList[0].Value as string); + // var asset = argList[1].Value as string; + // if (venue.HasValue && asset is not null) { + // var secType = SecurityType.Future; + // if (DatabentoFileStore.GetDataFiles("definition", asset, secType).TryGetMaxElement(f => f.Date, out var latestFile)) { + // result = new(venue.Value, asset, latestFile.Date) { + // Type = secType, + // }; + // } + // } + // break; + // } + // case 3: { + // var venue = ParseVenueArg(argList[0].Value as string); + // var asset = argList[1].Value as string; + // break; + // } + // } + // if (result is not null) { + // return true; + // } + // errorMessage ??= "Failed to convert args to HistoricInstrumentStreamParams"; + // return false; + // } + + ViewStream? CreateReplayFutureLiveBbo(ReplayFutureLiveBboParams opts) { + if (opts.ContinuousContract is null) { + logger.LogMissingParam("ReplayFutureLiveBBOStream", nameof(opts.ContinuousContract)); + return null; + } + Func? filter = opts.ContinuousContract == "ES" ? ContractFilter : null; + var tables = DatabentoFileStore.GetBboAsync(opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), opts.BeginDate, opts.EndDate, filter); + // var continuousContracts = HistoricContinuousContractStream.Create(new() { Symbol = opts.ContinuousContract, SecurityType = SecurityType.Future, BeginDate = opts.BeginDate, EndDate = opts.EndDate }, viewStreamDatabase, logger); + return ReplayStream.Create(tables, "ts_event", opts.ReplaySpeed, viewStreamDatabase); + + bool ContractFilter(IBlueRow r) => + r.GetString("contract") == "ESU5"; + } + + ViewStream? CreateReplayFutureLiveTrade(ReplayFutureLiveTradeParams opts) { + if (opts.ContinuousContract is null) { + logger.LogMissingParam("ReplayFutureLiveTradeStream", nameof(opts.ContinuousContract)); + return null; + } + Func? filter = opts.ContinuousContract == "ES" ? ContractFilter : null; + var tables = DatabentoFileStore.GetTradesAsync(opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), opts.BeginDate, opts.EndDate, filter); + return ReplayStream.Create(tables, "ts_event", opts.ReplaySpeed, viewStreamDatabase); + + bool ContractFilter(IBlueRow r) => + r.GetString("contract") == "ESU5"; + } + + ViewStream? CreateHistoricBbo(HistoricBboParams opts) { + if (opts.ContinuousContract is null) { + logger.LogMissingParam("HistoricBbo", nameof(opts.ContinuousContract)); + return null; + } + var streams = DatabentoFileStore.GetDataFiles( + "mbp-1", opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), beginDate: opts.BeginDate, endDate: opts.EndDate) + .Select(f => new ParquetReaderStream(f.Path, viewStreamDatabase)) + .ToArray(); + return streams.Length switch { + 0 => null, + 1 => streams[0], + _ => new MutableAppendStream(streams), + }; + } + } + + static BlueTableImmutable dropIndex(IBlueTableSchema schema, IEnumerable rows) { + int idIndex = schema.ResolveIndices("id"); + var rowSetBd = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer()).ToBuilder(); + foreach (var r in rows) { + int idx = rowSetBd.Count + 1; + var inputRow = (r as IndexRow)?.InputRow ?? r; + rowSetBd.Add(new(idx, new IndexRow(inputRow, idx, idIndex, schema))); + } + return new(schema, rowSetBd.ToImmutable()); + } + + struct ReplayFutureLiveTradeState { + public IDataPacket? Data; + public ColumnSelectorRowFactory? RowFactory; + } + + class ReplayFutureLiveBboParams { + [StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)] + public string? ContinuousContract { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1)] + public string? SecurityType { get; set; } + + [StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)] + public DateTime? BeginDate { get; set; } + + [StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)] + public DateTime? EndDate { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ReplaySpeedRegex }, Position = 4)] + public double ReplaySpeed { get; set; } = 1; + } + + class ReplayFutureLiveTradeParams { + [StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)] + public string? ContinuousContract { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1)] + public string? SecurityType { get; set; } + + [StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)] + public DateTime? BeginDate { get; set; } + + [StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)] + public DateTime? EndDate { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ReplaySpeedRegex }, Position = 4)] + public double ReplaySpeed { get; set; } = 1; + } + + class HistoricBboParams { + [StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)] + public string? ContinuousContract { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1, IsRequired = false)] + public string? SecurityType { get; set; } + + [StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)] + public DateTime? BeginDate { get; set; } + + [StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)] + public DateTime? EndDate { get; set; } + + [StreamArgument(RegexNames = new[] { "speed" }, Position = 4)] + public double ReplaySpeed { get; set; } = 1; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentSchemaBuilder.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentSchemaBuilder.cs deleted file mode 100644 index 25d95f120..000000000 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentSchemaBuilder.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace Bts.SharedStreams.StreamFactories; - -using BCG; - -using bts.api; - -using Bts.DataStreamCore; - -public class InstrumentSchemaBuilder { - public bool ShowVenue { get; set; } = true; - public bool ShowAsset { get; set; } = true; - public bool ShowType { get; set; } = true; - public bool ShowName { get; set; } = true; - public bool ShowId { get; set; } = true; - public bool ShowCashPerPoint { get; set; } = true; - public bool ShowLegId { get; set; } = true; - public bool ShowLegVenue { get; set; } - public bool ShowLegAsset { get; set; } - public bool ShowLegType { get; set; } - public bool ShowLegName { get; set; } = true; - public bool ShowLegCashPerPoint { get; set; } - public int NumLegs { get; set; } = 2; - public string TableName { get; set; } = "Instrument"; - - public BlueTableSchemaAdapter SecurityAdapter() { - var adapter = new BlueTableSchemaAdapter(TableName); - adapter.AddColumn("id", r => (int)r.Id); - if (ShowVenue) { - adapter.AddColumn(WellKnownColumns.Venue.Name, b => b.Exchange.ToString()); - } - if (ShowType) { - adapter.AddColumn(WellKnownColumns.SecurityType.Name, b => b.security_type().ToString()); - } - if (ShowAsset) { - adapter.AddColumn(WellKnownColumns.Asset.Name, b => b.Symbol); - } - if (ShowName) { - adapter.AddColumn(WellKnownColumns.SecurityName.Name, b => b.ToString()); - } - if (ShowId) { - adapter.AddColumn(WellKnownColumns.SecurityId.Name, b => b.Id); - } - if (ShowCashPerPoint) { - adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.ContractSpec.CashPerPoint); - } - for (int i = 0; i < NumLegs; i++) { - if (ShowLegVenue) { - adapter.AddColumn(WellKnownColumns.LegVenue(i).Name, Get(l => l.Security.Exchange.ToString(), null)); - } - if (ShowLegType) { - adapter.AddColumn(WellKnownColumns.LegSecurityType(i).Name, Get(l => l.Security.security_type().ToString(), null)); - } - if (ShowLegAsset) { - adapter.AddColumn(WellKnownColumns.LegAsset(i).Name, Get(l => l.Security.Symbol, null)); - } - if (ShowLegName) { - adapter.AddColumn(WellKnownColumns.LegSecurityName(i).Name, Get(l => l.Security.ToString(), null)); - } - if (ShowLegId) { - adapter.AddColumn(WellKnownColumns.LegSecurityId(i).Name, Get(l => (int)l.Security.Id, 0)); - } - if (ShowLegCashPerPoint) { - adapter.AddColumn(WellKnownColumns.LegCashPerPoint(i).Name, Get(l => l.Security.ContractSpec.CashPerPoint, double.NaN)); - } - - Func Get(Func getValue, T defaultValue) => - s => { - ContractLegImmutable? leg = s.ContractCase switch { - Security.ContractOneofCase.Spread => s.Spread.Legs.Skip(i).TryGetFirst(out var l) ? l : null, - Security.ContractOneofCase.Synthetic => s.Synthetic.Legs.Skip(i).TryGetFirst(out var l) ? l : null, - _ => null, - }; - return leg.HasValue ? getValue(leg.Value) : defaultValue; - }; - } - return adapter; - } -} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamParams.cs deleted file mode 100644 index 45a851f9f..000000000 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamParams.cs +++ /dev/null @@ -1,66 +0,0 @@ -namespace Bts.SharedStreams.StreamFactories; - -using bts.api; - -using Bts.DataStreamCore; - -public class InstrumentStreamParams { - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.VenueRegex })] - public Venue? Venue { get; set; } - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex })] - public string? Symbol { get; set; } - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex })] - public SecurityType? Type { get; set; } - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityNameRegex })] - public string? Name { get; set; } - - [StreamArgument("legs", "nLegs", "numLegs")] - public int NumLegs { get; set; } = 5; - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowVenueRegex })] - public bool ShowVenue { get; set; } = true; - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowSecurityTypeRegex })] - public bool ShowType { get; set; } = true; - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowAssetRegex })] - public bool ShowAsset { get; set; } = true; - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowCashPerPointRegex })] - public bool ShowCashPerPoint { get; set; } = true; - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegVenueRegex })] - public bool ShowLegVenue { get; set; } = true; - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegSecurityTypeRegex })] - public bool ShowLegType { get; set; } = true; - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegSymbolRegex })] - public bool ShowLegSymbol { get; set; } = true; - - [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegCashPerPointRegex })] - public bool ShowLegCashPerPoint { get; set; } = true; - - public mbts.spark.SecurityCriteria GetSecurityCriteria() { - var clause = new mbts.spark.SecurityCriteriaClause( - exchanges: Venue.HasValue ? new[] { Venue.Value } : null, - security_types: Type.HasValue ? new[] { Type.Value } : null, - symbols: !string.IsNullOrEmpty(Symbol) ? new[] { Symbol } : null, - names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null); - return clause.is_defined ? new(new[] { clause }) : new(); - } - - public InstrumentSchemaBuilder GetSchemaBuilder() => - new() { - NumLegs = NumLegs, - ShowVenue = ShowVenue, - ShowType = ShowType, - ShowAsset = ShowAsset, - ShowName = true, - ShowId = true, - ShowCashPerPoint = ShowCashPerPoint, - }; -} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/ParquetReaderStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/ParquetReaderStreamFactory.cs new file mode 100644 index 000000000..71b41c3a1 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/ParquetReaderStreamFactory.cs @@ -0,0 +1,35 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts.BlueOrb.Parquet; + +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; + +class ParquetReaderStreamFactory : DataStreamFactory { + public ParquetReaderStreamFactory(ViewStreamDatabase viewStreamDatabase, ILogger logger) { + this.Add("parquetReader", CreateParquetReader, logger); + + ViewStream? CreateParquetReader(ParquetReaderStreamParams streamParams) { + if (streamParams.File is null) { + logger.LogMissingParam("ParquetReader", nameof(streamParams.File)); + return null; + } + var stream = new ParquetReaderStream(streamParams.File, streamParams.Start, streamParams.Stop, viewStreamDatabase); + return stream; + } + } + + class ParquetReaderStreamParams { + [StreamArgument(RegexNames = new[] { "file|path|filePath" }, Position = 0, IsRequired = true)] + public string? File { get; set; } + + [StreamArgument(RegexNames = new[] { "^start|first|begin$" }, Position = 1, IsRequired = false)] + public int? Start { get; set; } + + [StreamArgument(RegexNames = new[] { "^stop|last|end$" }, Position = 2, IsRequired = false)] + public int? Stop { get; set; } + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/WellKnownColumns.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/WellKnownColumns.cs index 29d731e47..a268ae8e9 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/WellKnownColumns.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/WellKnownColumns.cs @@ -4,17 +4,49 @@ using bts.api; using Spark.Api.Orders; +public static class WellKnownColumnNames { + public const string Price = "price"; + public const string Quantity = "qty"; + public const string Timestamp = "ts_event"; + public const string BidPrice = "bid_price"; + public const string BidQuantity = "bid_size"; + public const string BidNotional = "bid_notl"; + public const string AskPrice = "ask_price"; + public const string Currency = "currency"; + public const string AskQuantity = "ask_size"; + public const string AskNotional = "ask_notl"; + public const string SecurityType = "sec_type"; + public const string Asset = "asset"; + public const string SecurityName = "sec_name"; + public const string Expiration = "expiration"; + public const string Venue = "exchange"; + public const string TradingStatus = "trading_status"; + public const string NumLegs = "num_legs"; + public const string CashPerPoint = "cpp"; + + public static string DepthPrice(Side side, int level) => $"{side.BidAskName()}Price{level}"; + public static string DepthQuantity(Side side, int level) => $"{side.BidAskName()}Qty{level}"; + public static string DepthNotional(Side side, int level) => $"{side.BidAskName()}Notl{level}"; + + public static string LegSecurityId(int legNum) => $"Leg{legNum}SecId"; + public static string LegVenue(int legNum) => $"Leg{legNum}Venue"; + public static string LegSecurityType(int legNum) => $"Leg{legNum}SecType"; + public static string LegAsset(int legNum) => $"Leg{legNum}Asset"; + public static string LegSecurityName(int legNum) => $"Leg{legNum}Sec"; + public static string LegCashPerPoint(int legNum) => $"Leg{legNum}Cpp"; +} + public static class WellKnownColumns { public static (string Name, Type Type) SecurityId => ("SecId", typeof(uint)); - public static (string Name, Type Type) Venue => ("Venue", typeof(string)); - public static (string Name, Type Type) SecurityType => ("SecType", typeof(string)); - public static (string Name, Type Type) Asset => ("Asset", typeof(string)); - public static (string Name, Type Type) SecurityName => ("Sec", typeof(string)); - public static (string Name, Type Type) CashPerPoint => ("Cpp", typeof(double)); - public static (string Name, Type Type) Price => ("Price", typeof(double)); - public static (string Name, Type Type) Quantity => ("Qty", typeof(double)); - public static (string Name, Type Type) Timestamp => ("Ts", typeof(DateTime)); - public static (string Name, Type Type) TimestampNanos => ("TsNanos", typeof(long)); + public static (string Name, Type Type) Venue => (WellKnownColumnNames.Venue, typeof(string)); + public static (string Name, Type Type) SecurityType => (WellKnownColumnNames.SecurityType, typeof(string)); + public static (string Name, Type Type) Asset => (WellKnownColumnNames.Asset, typeof(string)); + public static (string Name, Type Type) SecurityName => (WellKnownColumnNames.SecurityName, typeof(string)); + public static (string Name, Type Type) CashPerPoint => (WellKnownColumnNames.CashPerPoint, typeof(double)); + public static (string Name, Type Type) Price => (WellKnownColumnNames.Price, typeof(double)); + public static (string Name, Type Type) Quantity => (WellKnownColumnNames.Quantity, typeof(double)); + public static (string Name, Type Type) Timestamp => (WellKnownColumnNames.Timestamp, typeof(DateTime)); + public static (string Name, Type Type) TimestampNanos => ("ts_event_ns", typeof(long)); public static (string Name, Type Type) BidPrice => ("BidPrice", typeof(double)); public static (string Name, Type Type) BidQuantity => ("BidQty", typeof(double)); @@ -23,14 +55,14 @@ public static class WellKnownColumns { public static (string Name, Type Type) AskQuantity => ("AskQty", typeof(double)); public static (string Name, Type Type) AskNotional => ("AskNotl", typeof(double)); - public static (string Name, Type Type) DepthPrice(Side side, int level) => ($"{side.BidAskName()}Price{level}", typeof(double)); - public static (string Name, Type Type) DepthQuantity(Side side, int level) => ($"{side.BidAskName()}Qty{level}", typeof(double)); - public static (string Name, Type Type) DepthNotional(Side side, int level) => ($"{side.BidAskName()}Notl{level}", typeof(double)); + public static (string Name, Type Type) DepthPrice(Side side, int level) => (WellKnownColumnNames.DepthPrice(side, level), typeof(double)); + public static (string Name, Type Type) DepthQuantity(Side side, int level) => (WellKnownColumnNames.DepthQuantity(side, level), typeof(double)); + public static (string Name, Type Type) DepthNotional(Side side, int level) => (WellKnownColumnNames.DepthNotional(side, level), typeof(double)); - public static (string Name, Type Type) LegSecurityId(int legNum) => ($"Leg{legNum}SecId", typeof(uint)); - public static (string Name, Type Type) LegVenue(int legNum) => ($"Leg{legNum}Venue", typeof(string)); - public static (string Name, Type Type) LegSecurityType(int legNum) => ($"Leg{legNum}SecType", typeof(string)); - public static (string Name, Type Type) LegAsset(int legNum) => ($"Leg{legNum}Asset", typeof(string)); - public static (string Name, Type Type) LegSecurityName(int legNum) => ($"Leg{legNum}Sec", typeof(string)); - public static (string Name, Type Type) LegCashPerPoint(int legNum) => ($"Leg{legNum}Cpp", typeof(double)); + public static (string Name, Type Type) LegSecurityId(int legNum) => (WellKnownColumnNames.LegSecurityId(legNum), typeof(uint)); + public static (string Name, Type Type) LegVenue(int legNum) => (WellKnownColumnNames.LegVenue(legNum), typeof(string)); + public static (string Name, Type Type) LegSecurityType(int legNum) => (WellKnownColumnNames.LegSecurityType(legNum), typeof(string)); + public static (string Name, Type Type) LegAsset(int legNum) => (WellKnownColumnNames.LegAsset(legNum), typeof(string)); + public static (string Name, Type Type) LegSecurityName(int legNum) => (WellKnownColumnNames.LegSecurityName(legNum), typeof(string)); + public static (string Name, Type Type) LegCashPerPoint(int legNum) => (WellKnownColumnNames.LegCashPerPoint(legNum), typeof(double)); } diff --git a/src/client/spark_core/Collections/ArrayList.cs b/src/client/spark_core/Collections/ArrayList.cs new file mode 100644 index 000000000..4449f0f50 --- /dev/null +++ b/src/client/spark_core/Collections/ArrayList.cs @@ -0,0 +1,79 @@ +namespace Spark.Core.Collections; + +using bts.utils; + +using System; +using System.Collections; +using System.Collections.Generic; + +public class ArrayList : IList { + T[] items; + int count; + + public ArrayList(T[] items) { + this.items = items; + count = items.Length; + } + + public ArrayList(T[] items, int count) { + ArgumentOutOfRangeException.ThrowIfNegative(count); + ArgumentOutOfRangeException.ThrowIfGreaterThan(count, items.Length); + this.items = items; + this.count = count; + } + + public T this[int index] { + get => items[index]; + set => items[index] = value; + } + + public int Count => count; + + public bool IsReadOnly => false; + + public void Add(T item) => + ArrayHelper.Add(ref items, item, ref count); + + public void Clear() => + count = 0; + + public bool Contains(T item) => + IndexOf(item) != -1; + + public void CopyTo(T[] array, int arrayIndex) => + items.CopyTo(new Span(array, arrayIndex, count)); + + public IEnumerator GetEnumerator() { + for (int i = 0; i < count; i++) { + yield return items[i]; + } + } + + public int IndexOf(T item) { + var comp = EqualityComparer.Default; + for (int i = 0; i < count; i++) { + if (comp.Equals(items[i], item)) { + return i; + } + } + return -1; + } + + public void Insert(int index, T item) => + ArrayHelper.Insert(ref items, index, item, ref count); + + public bool Remove(T item) { + int idx = IndexOf(item); + if (idx == -1) { + return false; + } + ArrayHelper.remove_at(ref items, ref count, idx); + return true; + } + + public void RemoveAt(int index) => + ArrayHelper.remove_at(ref items, ref count, index); + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} diff --git a/src/client/spark_core/Collections/Extensions/ListExtensions.cs b/src/client/spark_core/Collections/Extensions/ListExtensions.cs index f05701528..e76ab5919 100644 --- a/src/client/spark_core/Collections/Extensions/ListExtensions.cs +++ b/src/client/spark_core/Collections/Extensions/ListExtensions.cs @@ -10,6 +10,7 @@ namespace bts.collections { using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; + using System.Threading.Tasks; public static class ListExtensions { public static bool AddDistinct(this IList list, T value) => @@ -286,6 +287,15 @@ namespace bts.collections { return false; } + public static IList ParallelSelect(this IList source, Func selector) { + var results = new TResult[source.Count]; + Parallel.For(0, source.Count, SetResult); + void SetResult(int i) { + results[i] = selector(source[i]); + } + return new ArrayList(results); + } + public static void sync_with( this IList list, IEnumerable source, diff --git a/src/client/spark_core/utils/ArrayUtils.cs b/src/client/spark_core/utils/ArrayUtils.cs index 61beccf69..c7c3c4147 100644 --- a/src/client/spark_core/utils/ArrayUtils.cs +++ b/src/client/spark_core/utils/ArrayUtils.cs @@ -243,6 +243,23 @@ namespace bts.utils { return clone; } + public static T[] Add(ref T[] array, T item, ref int count) { + ArgumentOutOfRangeException.ThrowIfGreaterThan(count, array.Length); + ArgumentOutOfRangeException.ThrowIfNegative(count); + if (count == array.Length) { + if (array.Length == 0) { + array = new T[1]; + } else { + var temp = array; + array = new T[checked(array.Length * 2)]; + Array.Copy(temp, 0, array, 0, count); + } + } + array[count] = item; + count++; + return array; + } + public static void add_sorted(ref T[] array, ref int size, T value, IComparer comparer) => add_sorted(ref array, ref size, value, comparer, out int pos); diff --git a/src/client/spark_core/utils/DelegateUtils.cs b/src/client/spark_core/utils/DelegateUtils.cs new file mode 100644 index 000000000..8d672df36 --- /dev/null +++ b/src/client/spark_core/utils/DelegateUtils.cs @@ -0,0 +1,22 @@ +#nullable enable +namespace Spark.Core; + +using System; +using System.Diagnostics.CodeAnalysis; + +public static class DelegateUtils { + [return:NotNullIfNotNull(nameof(pred1))] + [return:NotNullIfNotNull(nameof(pred2))] + public static Func? CombinePredicate(Func? pred1, Func? pred2) { + if (pred1 is null) { + return pred2; + } + if (pred2 is null) { + return pred1; + } + return And; + + bool And(T t) => + pred1(t) && pred2(t); + } +}