feat: Add historic market data streams factory

Loads historic md from Databento files in structured paths. It is
designed to work with files downloaded via databento_client.py
This commit is contained in:
John Colvin committed 2025-11-15 12:45:04 -05:00
1 parent 2549e34a17
commit 0cf44fcc40
34 files changed
+2205 -173

No files matched your search

@@ -9,6 +9,7 @@ using bts.collections;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
@@ -76,9 +77,43 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
}
}
public Builder ToBuilder() =>
new(this);
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class Builder : IEnumerable<IBlueRow> {
readonly IBlueTableSchema schema;
readonly ImmutableSortedSetDictionary<int, IBlueRow>.Builder rowMap;
public Builder(BlueTableImmutable source) {
schema = source.schema;
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
}
public void AddRange(IEnumerable<KeyValuePair<int, IBlueRow>> rows) =>
rowMap.AddRange(rows);
public void AddOrUpdate(IBlueRow row) =>
AddOrUpdate(row.Id(), row);
public void AddOrUpdate(int id, IBlueRow row) =>
rowMap[id] = row;
public bool Remove(int id) =>
rowMap.Remove(id);
public BlueTableImmutable ToImmutable() =>
new(schema, rowMap.ToImmutable().Source);
public IEnumerator<IBlueRow> GetEnumerator() =>
rowMap.Values.GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
public class DataPacketBuilder {
readonly IBlueTableSchema schema;
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder table;
@@ -354,9 +389,46 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection {
}
}
public Builder ToBuilder() =>
new(this);
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class Builder : IEnumerable<IBlueRow> {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableSortedSetDictionary<int, T>.Builder rowMap;
public Builder(BlueTableImmutableAdapter<T> source) {
schemaAdapter = source.schemaAdapter;
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
}
public void AddRange(IEnumerable<KeyValuePair<int, T>> rows) =>
rowMap.AddRange(rows);
public void AddOrUpdate(T row) =>
AddOrUpdate((int)schemaAdapter["id"][row]!, row);
public void AddOrUpdate(int id, T row) =>
rowMap[id] = row;
public bool Remove(int id) =>
rowMap.Remove(id);
public BlueTableImmutableAdapter<T> ToImmutable() =>
new(schemaAdapter, rowMap.ToImmutable().Source);
public IEnumerator<IBlueRow> GetEnumerator() {
foreach (var r in rowMap.Values) {
yield return schemaAdapter.GetBlueRow(r);
}
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
public class DataPacketBuilder {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table =
@@ -73,6 +73,21 @@ public class BlueTableSchemaAdapter<T> : IMutableBlueRowFactory<T>, IBlueTableSc
columnNames = null;
}
public void AddColumn<TResult>(string name, Func<T, TResult?> getValue) {
var dataType = typeof(TResult);
if (!ColumnInfo.isAllowedType(dataType)) {
throw new ArgumentException(
$"'{dataType}' is not a supported column type", nameof(dataType));
}
// Invalidate cached schema (set it to null)
schema = null;
var adapter = new ColumnAdapter(new(name, columns.Count, dataType), r => getValue(r));
columns.Add(adapter);
columnNameIndex[name] = adapter;
columnInfos = null;
columnNames = null;
}
public ColumnAdapter this[int columnIndex] =>
columns[columnIndex];
@@ -0,0 +1,64 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using Bts.DataStreamCore;
public static class DatabentoBboAdapter {
static readonly string[] keyColumnNames = new[] { "ts_event", "bid_px_00", "bid_sz_00", "ask_px_00", "ask_sz_00" };
public static ColumnIndices ResolveColumns(ColumnInfo[] columns) =>
new(columns.ResolveIndices(keyColumnNames));
public static Func<IBlueRow, HistoricBboRowData?> GetRowDataFactory(Func<TimeseriesKey<InstrumentKey>, int> getId, Func<IBlueRow, HistoricInstrumentRowData?> getInstrument, ColumnInfo[]? columns = null) {
ColumnIndices? colIndicesBacking = null;
if (columns is not null) {
colIndicesBacking = ResolveColumns(columns);
}
return RowDataFactoryWrapper;
HistoricBboRowData? RowDataFactoryWrapper(IBlueRow row) {
colIndicesBacking ??= ResolveColumns(row.schema.columnInfo);
return RowDataFactory(row, colIndicesBacking.Value);
}
HistoricBboRowData? RowDataFactory(IBlueRow mbp1Row, in ColumnIndices colIndices) {
var inst = getInstrument(mbp1Row);
if (inst is null) {
return null;
}
var ts = mbp1Row.GetDateTime(colIndices.TimestampIndex) ?? default;
var bidPrice = mbp1Row.GetFloat64(colIndices.BidPriceIndex) ?? double.NaN;
var bidQuantity = mbp1Row.GetFloat64(colIndices.BidQuantityIndex) ?? 0;
var askPrice = mbp1Row.GetFloat64(colIndices.BidPriceIndex) ?? double.NaN;
var askQuantity = mbp1Row.GetFloat64(colIndices.AskQuantityIndex) ?? 0;
int id = getId(inst.Key);
var rd = new HistoricBboRowData(id, inst, ts, bidPrice, bidQuantity, askPrice, askQuantity);
return rd;
}
}
public readonly struct ColumnIndices {
public readonly int TimestampIndex;
public readonly int BidPriceIndex;
public readonly int BidQuantityIndex;
public readonly int AskPriceIndex;
public readonly int AskQuantityIndex;
public ColumnIndices(int[] keyColumnIndices)
: this(
keyColumnIndices[0],
keyColumnIndices[1],
keyColumnIndices[2],
keyColumnIndices[3],
keyColumnIndices[4]) { }
public ColumnIndices(int timestampIndex, int bidPriceIndex, int bidQuantityIndex, int askPriceIndex, int askQuantityIndex) {
TimestampIndex = timestampIndex;
BidPriceIndex = bidPriceIndex;
BidQuantityIndex = bidQuantityIndex;
AskPriceIndex = askPriceIndex;
AskQuantityIndex = askQuantityIndex;
}
}
}
@@ -0,0 +1,179 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using bts;
using bts.api;
using bts.BlueOrb;
using bts.co2;
using Bts.DataStream;
using Bts.DataStreamCore;
using System.Diagnostics.CodeAnalysis;
using System.Text.RegularExpressions;
using System.Linq;
using Bts.DataStreamCore.StreamFactories;
using Spark.Core;
public static class DatabentoFileStore {
const int defaultMaxRowCount = 1_500_000;
public static IEnumerable<(string Path, DateTime Date)> GetDataFiles(string databentoSchema, string? continuousContract, SecurityType? secType, Venue? venue = null, DateTime? beginDate = null, DateTime? endDate = null) {
var localAppDataDir = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
var mbp1Dir = Path.Combine(localAppDataDir, "bts", "databento", databentoSchema);
var mbp1ChildDirs = Directory.GetDirectories(mbp1Dir);
var preview = new List<string>();
if (continuousContract is not null) {
foreach (var d in mbp1ChildDirs) {
var p = Path.Combine(d, continuousContract);
if (Directory.Exists(p)) {
if (p.Contains("HG")) {
}
preview.Add(p);
}
}
}
// var preview = mbp1ChildDirs.Select(d => Path.Combine(d, continuousContract!)).Where(Directory.Exists).ToList();
IEnumerable<string> symDirs = continuousContract is null
? mbp1ChildDirs.SelectMany(Directory.GetDirectories)
: mbp1ChildDirs.Select(d => Path.Combine(d, continuousContract)).Where(Directory.Exists);
var files = symDirs.SelectMany(Directory.GetFiles).Select(GetFile).SelectWhereValues();
if (beginDate.HasValue) {
files = files.Where(f => f.Date > beginDate.Value - TimeSpan.FromDays(1));
}
if (endDate.HasValue) {
files = files.Where(f => f.Date < endDate.Value);
}
return files;
(string Path, DateTime Date)? GetFile(string f) {
if (f is null) {
return null;
}
var parts = f.Split(Path.DirectorySeparatorChar);
Log.assert(parts.Length >= 3);
var fileVenueStr = parts[^3];
var sym = parts[^2];
var name = Path.GetFileName(f);
var match = Regex.Match(name, $"{databentoSchema}_{fileVenueStr}_{sym}_(STOCK|SPOT|FUT|OPT)_(\\d{{8}})\\.parquet");
if (!match.Success) {
return null;
}
if (secType.HasValue) {
var fileSecType = match.Groups[1].Value switch {
"STOCK" or "SPOT" => SecurityType.Spot,
"FUT" => SecurityType.Future,
"OPT" => SecurityType.Option,
_ => throw new InvalidOperationException("Failed to parse SecurityType from filename"),
};
if (fileSecType != secType.Value) {
return null;
}
}
if (venue.HasValue) {
if (!MarketDataArgStore.TryParseVenue(fileVenueStr, out var fileVenue)) {
return null;
}
if (fileVenue != venue.Value) {
return null;
}
}
int dateInt = int.Parse(match.Groups[2].Value);
PeelPow10(ref dateInt, 10_000, out int year);
PeelPow10(ref dateInt, 100, out int month);
var date = new DateTime(year, month, dateInt);
return (f, date);
void PeelPow10(ref int i, int pow10, out int peeled) {
peeled = i / pow10;
i -= peeled * pow10;
}
}
}
public static IEnumerable<Func<Task<IBlueRowCollection>>> GetDefinitionAsync(string? asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func<IBlueRow, bool>? filter = null) {
filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter);
var files = GetDataFiles("definition", asset, secType, beginDate: beginDate, endDate: endDate);
return files.Select(f => new Func<Task<IBlueRowCollection>>(() => GetDefinitionAsync(f.Path, filter)));
}
public static async Task<IBlueRowCollection> GetDefinitionAsync(string path, Func<IBlueRow, bool>? filter = null) {
// string columnMaskExpression = "bid_px_00:bid_price bid_sz_00:bid_size ask_px_00:ask_price ask_sz_00:ask_size symbol:contract";
string? columnMaskExpression = null;
return await GetDataAsync(path, columnMaskExpression, filter);
}
public static IEnumerable<Func<Task<IBlueRowCollection>>> GetBboAsync(string asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func<IBlueRow, bool>? filter = null) {
filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter);
var tables = GetDataFiles("mbp-1", asset, secType, beginDate: beginDate, endDate: endDate)
.Select(f => new Func<Task<IBlueRowCollection>>(() => GetBboAsync(f.Path, filter)));
return tables;
}
public static async Task<IBlueRowCollection> GetBboAsync(string path, Func<IBlueRow, bool>? filter = null) {
string columnMaskExpression = "bid_px_00:bid_price bid_sz_00:bid_size ask_px_00:ask_price ask_sz_00:ask_size symbol:contract";
return await GetDataAsync(path, columnMaskExpression, filter);
}
public static IEnumerable<Func<Task<IBlueRowCollection>>> GetTradesAsync(string asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func<IBlueRow, bool>? filter = null) {
filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter);
var tables = GetDataFiles("trades", asset, secType, beginDate: beginDate, endDate: endDate)
.Select(f => new Func<Task<IBlueRowCollection>>(() => GetTradesAsync(f.Path, filter)));
return tables;
}
public static async Task<IBlueRowCollection> GetTradesAsync(string path, Func<IBlueRow, bool>? filter = null) {
string columnMaskExpression = "id ts_event price:trade_price size:trade_size side:aggressor_side symbol:contract";
var table = await GetDataAsync(path, columnMaskExpression, filter);
var schema = table.schema;
var rowFactory = ColumnSelectorRowFactory.Create(schema, ("aggressor_side", typeof(AggressorSide), ConvertSide));
return new SelectorTable(table, rowFactory.OutputSchema, rowFactory.Create);
static object? ConvertSide(object? s) {
char c = s as char? ?? ((s as string ?? "N")[0]);
return c switch {
'b' or 'B' => AggressorSide.Buy,
's' or 'S' or 'a' or 'A' => AggressorSide.Sell,
_ => AggressorSide.None,
};
}
}
public static Task<IBlueRowCollection> GetDataAsync(string path, string? columnMaskExpression, Func<IBlueRow, bool>? filter) =>
GetDataAsync(path, columnMaskExpression, 0, defaultMaxRowCount, filter);
public static async Task<IBlueRowCollection> GetDataAsync(string path, string? columnMaskExpression, int? start, int? stop, Func<IBlueRow, bool>? filter) {
var table = await ParquetTableReader.ReadAsync(path, start, stop);
if (columnMaskExpression is not null) {
table = ColumnMaskTable.Create(table, columnMaskExpression);
}
if (filter is not null) {
table = new FilterTable(table, filter);
}
return table;
}
[return: NotNullIfNotNull(nameof(beginDate))]
[return: NotNullIfNotNull(nameof(endDate))]
static Func<IBlueRow, bool>? getTimeFilter(DateTime? beginDate, DateTime? endDate) {
if (!beginDate.HasValue && !endDate.HasValue) {
return null;
}
return TimeFilter;
bool TimeFilter(IBlueRow r) {
var ts = r.GetDateTime("ts_event");
if (!ts.HasValue) {
return false;
}
if (beginDate.HasValue && ts.Value < beginDate) {
return false;
}
if (endDate.HasValue && ts.Value >= endDate) {
return false;
}
return true;
}
}
}
@@ -0,0 +1,114 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using bts.api;
using Bts.DataStreamCore;
public static class DatabentoInstrumentAdapter {
static readonly string[] keyColumnNames = new[] { "activation", "exchange", "asset", "instrument_class", "currency", "symbol", "instrument_id", "expiration", "unit_of_measure_qty" };
public static ColumnIndices ResolveColumns(ColumnInfo[] columns) =>
new(columns.ResolveIndices(keyColumnNames));
public static Func<IBlueRow, HistoricInstrumentRowData> GetRowDataFactory(Func<TimeseriesKey<InstrumentKey>, int> getId, ColumnInfo[]? columns = null) {
ColumnIndices? colIndicesBacking = null;
if (columns is not null) {
colIndicesBacking = ResolveColumns(columns);
}
return RowDataFactoryWrapper;
HistoricInstrumentRowData RowDataFactoryWrapper(IBlueRow row) {
colIndicesBacking ??= ResolveColumns(row.schema.columnInfo);
return RowDataFactory(row, colIndicesBacking.Value);
}
HistoricInstrumentRowData RowDataFactory(IBlueRow row, in ColumnIndices colIndices) {
var ts = row.GetDateTime(colIndices.ActivationIndex) ?? default;
var venue = row.GetString(colIndices.VenueIndex) switch {
"XCME" => Venue.Glbx,
_ => throw new NotImplementedException(),
};
var secType = GetSecurityType(row.GetString(colIndices.InstrumentClassIndex));
var asset = row.GetString(colIndices.AssetIndex) ?? string.Empty;
var currency = row.GetString(colIndices.CurrencyIndex) ?? string.Empty;
var name = row.GetString(colIndices.NameIndex) ?? string.Empty;
var exp = row.GetDateTime(colIndices.ExpirationIndex) ?? GetExpiration(row.GetInt64(colIndices.ExpirationIndex));
var cpp = row.GetFloat64(colIndices.UnitOfMeasureQtyIndex) ?? double.NaN;
switch (secType) {
case SecurityType.Future:
case SecurityType.Option:
if (!exp.HasValue) {
throw new ArgumentException($"Expiration is null for security of type '{secType}'", nameof(row));
}
break;
case SecurityType.Spread:
break;
default:
if (exp.HasValue) {
throw new ArgumentException($"Expiration is non-null for security of type '{secType}'", nameof(row));
}
break;
}
int id = getId(new((venue, name), ts));
return new(id, ts, venue, secType, asset, currency, name, exp, cpp);
}
static SecurityType GetSecurityType(string? instrumentClass) {
// Translate Databento's instrument_class codes to BTS SecurityType
// https://databento.com/docs/schemas-and-data-formats/instrument-definitions
return instrumentClass switch {
"B" /*Bond*/ or "K" /*Stock*/ or "X" /*FX spot*/ or "Y" /*Commodity spot*/ => SecurityType.Spot,
"F" => SecurityType.Future,
"C" /*Call*/ or "P" /*Put*/ => SecurityType.Option,
"M" /*Mixed spread*/ or "S" /*Future spread*/ or "T" /*Option spread*/ => SecurityType.Spread,
_ => SecurityType.Unset,
};
}
static DateTime? GetExpiration(long? expirationTimestamp) {
if (!expirationTimestamp.HasValue) {
return null;
}
var ticks = expirationTimestamp.Value / 100;
return new(ticks);
}
}
public readonly struct ColumnIndices {
public readonly int ActivationIndex;
public readonly int VenueIndex;
public readonly int AssetIndex;
public readonly int InstrumentClassIndex;
public readonly int CurrencyIndex;
public readonly int NameIndex;
public readonly int InstrumentIdIndex;
public readonly int ExpirationIndex;
public readonly int UnitOfMeasureQtyIndex;
public ColumnIndices(int[] keyColumnIndices)
: this(
keyColumnIndices[0],
keyColumnIndices[1],
keyColumnIndices[2],
keyColumnIndices[3],
keyColumnIndices[4],
keyColumnIndices[5],
keyColumnIndices[6],
keyColumnIndices[7],
keyColumnIndices[8]) { }
public ColumnIndices(int activationIndex, int venueIndex, int assetIndex, int instrumentClassIndex, int currencyIndex, int nameIndex, int instrumentIdIndex, int expirationIndex, int unitOfMeasureQtyIndex) {
ActivationIndex = activationIndex;
VenueIndex = venueIndex;
AssetIndex = assetIndex;
InstrumentClassIndex = instrumentClassIndex;
CurrencyIndex = currencyIndex;
NameIndex = nameIndex;
InstrumentIdIndex = instrumentIdIndex;
ExpirationIndex = expirationIndex;
UnitOfMeasureQtyIndex = unitOfMeasureQtyIndex;
}
}
}
@@ -0,0 +1,99 @@
namespace Bts.SharedStreams.MarketData;
using bts.api;
using Bts.DataStreamCore.StreamFactories;
public class HistoricBboParams {
public HistoricBboParams() { }
public HistoricBboParams(string venueStr, string asset)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, null) { }
public HistoricBboParams(string venueStr, string asset, string typeStr)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr)) { }
public HistoricBboParams(string venueStr, string asset, string typeStr, DateTime date)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), date) { }
public HistoricBboParams(string venueStr, string asset, string typeStr, DateTime beginDate, DateTime endDate)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), beginDate, endDate) { }
public HistoricBboParams(string venueStr, string asset, DateTime date)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, date) { }
public HistoricBboParams(Venue venue, string asset, DateTime date)
: this(venue, asset, null, date) { }
public HistoricBboParams(Venue venue, string asset, DateTime beginDate, DateTime endDate)
: this(venue, asset, null, beginDate, endDate) { }
public HistoricBboParams(Venue venue, string asset, SecurityType? type) {
Venue = venue;
Asset = asset;
Type = type;
}
public HistoricBboParams(Venue venue, string asset, SecurityType? type, DateTime date)
: this(venue, asset, type, date, date.AddDays(1)) { }
public HistoricBboParams(Venue venue, string asset, SecurityType? type, DateTime beginDate, DateTime endDate) {
Venue = venue;
Asset = asset;
Type = type;
BeginDate = beginDate;
EndDate = endDate;
}
public Venue? Venue { get; set; }
public string? Asset { get; set; }
public SecurityType? Type { get; set; }
public string? Name { get; set; }
public bool ShowVenue { get; set; } = true;
public bool ShowAsset { get; set; } = true;
public bool ShowType { get; set; } = true;
public bool ShowName { get; set; } = true;
public bool ShowExpiration { get; set; }
public bool ShowCashPerPoint { get; set; } = true;
public bool ShowCurrency { get; set; } = true;
public bool ShowNotionalValue { get; set; } = true;
public DateTime? BeginDate { get; set; }
public DateTime? EndDate { get; set; }
public HistoricInstrumentParams GetInstrumentParams() =>
new() {
Venue = Venue,
Asset = Asset,
Type = Type,
Name = Name,
ShowVenue = ShowVenue,
ShowAsset = ShowAsset,
ShowType = ShowType,
ShowExpiration = ShowExpiration,
ShowCashPerPoint = ShowCashPerPoint,
BeginDate = BeginDate,
EndDate = EndDate,
};
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
var clause = new mbts.spark.SecurityCriteriaClause(
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
security_types: Type.HasValue ? new[] { Type.Value } : null,
symbols: !string.IsNullOrEmpty(Asset) ? new[] { Asset } : null,
names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null);
return clause.is_defined ? new(new[] { clause }) : new();
}
public HistoricBboSchemaBuilder GetSchemaBuilder() =>
new() {
ShowVenue = ShowVenue,
ShowAsset = ShowAsset,
ShowType = ShowType,
ShowName = ShowName,
ShowExpiration = ShowExpiration,
ShowCashPerPoint = ShowCashPerPoint,
ShowCurrency = ShowCurrency,
ShowNotionalValue = ShowNotionalValue,
};
}
@@ -0,0 +1,86 @@
namespace Bts.SharedStreams.MarketData;
using bts.api;
public class HistoricBboRowData {
public HistoricBboRowData(int id, HistoricInstrumentRowData instrument, DateTime timestamp, double bidPrice, double bidQuantity, double askPrice, double askQuantity)
: this(
id,
timestamp,
instrument.Venue,
instrument.SecurityType,
instrument.Asset,
instrument.Currency,
instrument.Name,
instrument.Expiration,
instrument.CashPerPoint,
bidPrice,
bidQuantity,
askPrice,
askQuantity) { }
public HistoricBboRowData(
int id,
DateTime timestamp,
Venue venue,
SecurityType securityType,
string asset,
string currency,
string name,
DateTime? expiration,
double cashPerPoint,
double bidPrice,
double bidQuantity,
double askPrice,
double askQuantity
) {
Id = id;
Timestamp = timestamp;
Venue = venue;
SecurityType = securityType;
Asset = asset;
Currency = currency;
Name = name;
Expiration = expiration;
CashPerPoint = cashPerPoint;
BidPrice = bidPrice;
BidQuantity = bidQuantity;
AskPrice = askPrice;
AskQuantity = askQuantity;
}
public int Id { get; }
public DateTime Timestamp { get; }
public Venue Venue { get; }
public SecurityType SecurityType { get; }
public TimeseriesKey<InstrumentKey> Key => new((Venue, Name), Timestamp);
public string Asset { get; }
public string Currency { get; }
public string Name { get; }
public DateTime? Expiration { get; }
public double CashPerPoint { get; }
public double BidPrice { get; }
public double BidQuantity { get; }
public double BidNotional => CashPerPoint * BidPrice * BidQuantity;
public double AskPrice { get; }
public double AskQuantity { get; }
public double AskNotional => CashPerPoint * AskPrice * AskQuantity;
public HistoricBboRowData With(
double? bidPrice = null, double? bidQuantity = null, double? askPrice = null, double? askQuantity = null) =>
new(
Id,
Timestamp,
Venue,
SecurityType,
Asset,
Currency,
Name,
Expiration,
CashPerPoint,
bidPrice ?? BidPrice,
bidQuantity ?? BidQuantity,
askPrice ?? AskPrice,
askQuantity ?? AskQuantity);
}
@@ -0,0 +1,58 @@
namespace Bts.SharedStreams.MarketData;
using Bts.DataStreamCore;
using Bts.SharedStreams.StreamFactories;
public class HistoricBboSchemaBuilder {
public bool ShowVenue { get; set; } = true;
public bool ShowAsset { get; set; } = true;
public bool ShowType { get; set; } = true;
public bool ShowName { get; set; } = true;
public bool ShowExpiration { get; set; } = true;
public bool ShowCashPerPoint { get; set; } = true;
public bool ShowCurrency { get; set; } = true;
public bool ShowPrice { get; set; } = true;
public bool ShowQuantity { get; set; } = true;
public bool ShowNotionalValue { get; set; } = true;
public string TableName { get; set; } = "HistoricBbo";
public BlueTableSchemaAdapter<HistoricBboRowData> DatabentoAdapter() {
var adapter = new BlueTableSchemaAdapter<HistoricBboRowData>(TableName);
adapter.AddColumn("id", r => r.Id);
adapter.AddColumn(WellKnownColumnNames.Timestamp, r => r.Timestamp);
if (ShowVenue) {
adapter.AddColumn(WellKnownColumnNames.Venue, r => r.Venue);
}
if (ShowAsset) {
adapter.AddColumn(WellKnownColumnNames.Asset, r => r.Asset);
}
if (ShowType) {
adapter.AddColumn(WellKnownColumnNames.SecurityType, r => r.SecurityType);
}
if (ShowName) {
adapter.AddColumn(WellKnownColumnNames.SecurityName, r => r.Name);
}
if (ShowExpiration) {
adapter.AddColumn(WellKnownColumnNames.Expiration, r => r.Expiration);
}
if (ShowCurrency) {
adapter.AddColumn(WellKnownColumnNames.Currency, r => r.Currency);
}
if (ShowCashPerPoint) {
adapter.AddColumn(WellKnownColumnNames.CashPerPoint, r => r.Currency);
}
if (ShowNotionalValue) {
adapter.AddColumn(WellKnownColumnNames.BidNotional, r => r.BidNotional);
adapter.AddColumn(WellKnownColumnNames.AskNotional, r => r.AskNotional);
}
if (ShowQuantity) {
adapter.AddColumn(WellKnownColumnNames.BidQuantity, r => r.BidQuantity);
adapter.AddColumn(WellKnownColumnNames.AskQuantity, r => r.AskQuantity);
}
if (ShowPrice) {
adapter.AddColumn(WellKnownColumnNames.BidPrice, r => r.BidPrice);
adapter.AddColumn(WellKnownColumnNames.AskPrice, r => r.AskPrice);
}
return adapter;
}
}
@@ -0,0 +1,28 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using Bts.DataStream.BlueOrb;
using Bts.PubSub;
using Microsoft.Extensions.Logging;
/// <summary>
/// Creates a time series of bbos for a date range.
/// </summary>
public class HistoricBboStream : ViewStream {
public HistoricBboStream(HistoricBboParams streamParams, IWorkScheduler workScheduler, ILogger logger) =>
mySetRealSource(Create(streamParams, workScheduler, logger));
public static ViewStream Create(HistoricBboParams streamParams, IWorkScheduler workScheduler, ILogger logger) =>
AsyncViewStream.Create(workScheduler, () => CreateAsync(streamParams, logger));
public static async Task<ViewStream?> CreateAsync(HistoricBboParams sp, ILogger logger) {
logger.LogInformation("Creating HistoricBboStream");
var table = await HistoricBboTable.CreateAsync(sp, logger);
if (table is null) {
return null;
}
return new BlueRowCollectionStaticStream(table);
}
}
@@ -0,0 +1,160 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using bts;
using bts.BlueOrb;
using bts.collections;
using Bts.DataStreamCore;
using Microsoft.Extensions.Logging;
using Spark.Core;
using Spark.Core.Collections;
using System.Collections.Immutable;
public class HistoricBboTable : IBlueRowCollection {
readonly BlueTableSchemaAdapter<HistoricBboRowData> schemaAdapter;
readonly TimeseriesIndex<InstrumentKey> bboIndex;
readonly ImmutableSortedSet<KeyValuePair<int, HistoricBboRowData>> rowSet;
public HistoricBboTable(BlueTableSchemaAdapter<HistoricBboRowData> schemaAdapter) {
this.schemaAdapter = schemaAdapter;
bboIndex = new();
rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, HistoricBboRowData>());
}
HistoricBboTable(
BlueTableSchemaAdapter<HistoricBboRowData> schemaAdapter,
TimeseriesIndex<InstrumentKey> bboIndex,
ImmutableSortedSet<KeyValuePair<int, HistoricBboRowData>> rowSet) {
this.schemaAdapter = schemaAdapter;
this.bboIndex = bboIndex;
this.rowSet = rowSet;
}
public IBlueRow this[int id] =>
rowSet.TryGetValue(new(id, null!), out var res)
? schemaAdapter.GetBlueRow(res.Value)
: throw new KeyNotFoundException();
public IBlueTableSchema schema => schemaAdapter.Schema;
public int Count => rowSet.Count;
IBlueRow IList<IBlueRow>.this[int index] {
get => GetRow(index);
set => throw new NotSupportedException();
}
public static async Task<HistoricBboTable?> CreateAsync(HistoricBboParams sp, ILogger logger, bool useContinuousContracts = true) {
logger.LogInformation("Creating HistoricBboTable");
var beginDate = sp.BeginDate;
var endDate = sp.EndDate;
var instParams = sp.GetInstrumentParams();
var instSchemaBd = instParams.GetSchemaBuilder();
var instSchemaAdapter = instSchemaBd.DatabentoAdapter();
var mbp1FileList = DatabentoFileStore.GetDataFiles("mbp-1", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToListSorted(f => f.Date);
if (!beginDate.HasValue && !endDate.HasValue) {
beginDate = mbp1FileList[^1].Date;
endDate = beginDate.Value + TimeSpan.FromDays(1);
} else {
beginDate ??= DateTime.MinValue;
endDate ??= DateTime.MaxValue;
}
mbp1FileList.RemoveAll(f => f.Date < beginDate.Value || f.Date >= endDate.Value);
if (mbp1FileList.Count == 0) {
return null;
}
var definitionFileList = DatabentoFileStore.GetDataFiles("definition", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToListSorted(f => f.Date);
if (!mbp1FileList.WrapSelector(f => f.Date, null).SequenceEqual(definitionFileList.WrapSelector(f => f.Date, null))) {
logger.LogInvalidParams("HistoricBboStream", "Missing one or more definition (instrument) files in date range");
return null;
}
var tablePairs = new (IBlueRowCollection Mbp1, IBlueRowCollection Definition)[mbp1FileList.Count];
Parallel.For(0, mbp1FileList.Count, LoadTables);
var instrumentIndex = new TimeseriesIndex<InstrumentKey>();
var instrumentIndexBd = instrumentIndex.ToBuilder();
var bboSchemaBd = sp.GetSchemaBuilder();
var bboSchemaAdapter = bboSchemaBd.DatabentoAdapter();
var bboTable = new HistoricBboTable(bboSchemaAdapter);
foreach (var (mbp1, definition) in tablePairs) {
var filteredDefs = definition;
if (useContinuousContracts) {
filteredDefs = HistoricContinuousContractTable.Create(filteredDefs);
}
var definitionMap = filteredDefs.ToDictionary(r => r.GetInt64("instrument_id").assert_value());
var instRowDataFactory = DatabentoInstrumentAdapter.GetRowDataFactory(instrumentIndexBd.GetOrAdd, filteredDefs.schema.columnInfo);
bboTable = bboTable.AddBbos(mbp1, GetInstrumentRowData);
HistoricInstrumentRowData? GetInstrumentRowData(IBlueRow mbp1Row) {
var instId = mbp1Row.GetInt64("instrument_id");
Log.notnull(instId);
if (!definitionMap.TryGetValue(instId.Value, out var defRow)) {
Log.assert(false);
}
var instRowData = instRowDataFactory(defRow!);
return instRowData;
}
}
return bboTable;
void LoadTables(int index) {
var definitionPath = definitionFileList[index].Path;
var definition = ParquetTableReader.Read(definitionPath);
var mbp1Path = mbp1FileList[index].Path;
var mbp1 = ParquetTableReader.Read(mbp1Path);
tablePairs[index] = (mbp1, definition);
}
}
/// <summary>
/// Add bbo records (<see cref="rows"/>) to this table.
/// </summary>
/// <param name="mbp1Rows">The bbo records to add (columns should correspond to Databento 'mbp-1' schema)</param>
/// <param name="instrumentRows">Instruments table for the bbo records (i.e., for the same time period as bbo)</param>
/// <returns>A new table (immutable) with the added rows</returns>
public HistoricBboTable AddBbos(IBlueRowCollection mbp1Rows, Func<IBlueRow, HistoricInstrumentRowData?> getInstrument) {
var bboIndex = this.bboIndex.ToBuilder();
var rowSet = this.rowSet.ToBuilder();
var rowDataFactory = DatabentoBboAdapter.GetRowDataFactory(bboIndex.GetOrAdd, getInstrument);
foreach (var mbp1Row in mbp1Rows) {
var rd = rowDataFactory(mbp1Row);
if (rd is null) {
continue;
}
rowSet.Add(new(rd.Id, rd));
}
return new(schemaAdapter, bboIndex.ToImmutable(), rowSet.ToImmutable());
}
public bool containsID(int id) =>
rowSet.Contains(new(id, null!));
public IBlueRow GetRow(int index) {
var rd = rowSet[index].Value;
var blueRow = schemaAdapter.GetBlueRow(rd);
return blueRow;
}
public int IndexOf(IBlueRow item) {
int id = item.Id();
int idx = rowSet.IndexOf(new(id, null!));
if (idx >= 0) {
var rd = rowSet[idx].Value;
var blueRow = schemaAdapter.GetBlueRow(rd);
if (BlueRowEqualityComparer.Equals(blueRow, item)) {
return idx;
}
}
return -1;
}
public IEnumerator<IBlueRow> GetEnumerator() {
foreach (var kv in rowSet) {
yield return schemaAdapter.GetBlueRow(kv.Value);
}
}
}
@@ -0,0 +1,43 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using bts.api;
using bts.BlueOrb;
using Bts.DataStream;
using Bts.SharedStreams.StreamFactories;
/// <summary>
/// Creates a time series of continuous contracts for a date range. This can be joined with
/// a timeseries of bbo data, for example, to get continuous bbo.
/// </summary>
public class HistoricContinuousContractStream : ViewStream {
public HistoricContinuousContractStream(ViewStream historicInstruments) =>
mySetRealSource(Create(historicInstruments));
public static ViewStream? Create(ViewStream historicInstruments) {
var vs = historicInstruments.GroupByApply(new[] { WellKnownColumnNames.Venue, WellKnownColumnNames.SecurityType, WellKnownColumnNames.Asset }, Apply);
return vs;
static IBlueRowCollection Apply(object?[] groupKeys, IBlueRowCollection inputRows) {
var secType = (SecurityType)groupKeys[1]!;
switch (secType) {
case SecurityType.Future:
case SecurityType.Option:
break;
default:
return inputRows;
}
var schema = inputRows.schema;
var minExpRow = inputRows.Select(r => (Row: r, Exp: r.GetDateTime(WellKnownColumnNames.Expiration)))
.Where(t => t.Exp.HasValue)
.MinElement(t => t.Exp!.Value)
.Row;
if (minExpRow is null) {
return inputRows;
}
return new SingleRowTable(minExpRow);
}
}
}
@@ -0,0 +1,102 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using bts;
using bts.api;
using bts.BlueOrb;
using Bts.DataStream;
using Bts.DataStreamCore;
using Bts.SharedStreams.StreamFactories;
using Spark.Core;
using System.Collections.Generic;
/// <summary>
/// Creates a time series of continuous contracts for a date range. This can be joined with
/// a timeseries of bbo data, for example, to get continuous bbo.
/// </summary>
public class HistoricContinuousContractTable : IBlueRowCollection {
readonly IBlueRowCollection impl;
public HistoricContinuousContractTable(IBlueRowCollection instruments) =>
impl = Create(instruments);
public IBlueTableSchema schema => impl.schema;
public int Count => impl.Count;
public IBlueRow this[int id] => impl[id];
IBlueRow IList<IBlueRow>.this[int index] {
get => ((IList<IBlueRow>)impl)[index];
set => throw new NotSupportedException();
}
public static IBlueRowCollection Create(IBlueRowCollection instruments) {
var instGroups = instruments.GroupBy(GetKey)
.ToDictionary(g => g.Key, g => g.ToListSorted(r => GetExpiration(r) ?? DateTime.MaxValue));
var bd = new BlueTableImmutable(instruments.schema).ToBuilder();
foreach (var (cck, contracts) in instGroups) {
Log.assert(contracts.Count > 0);
switch (cck.Type) {
case SecurityType.Future:
bd.AddOrUpdate(contracts[0]);
break;
case SecurityType.Option:
throw new NotImplementedException("Options not implemented");
case SecurityType.Spread:
foreach (var c in contracts) {
bd.AddOrUpdate(c);
}
break;
default:
Log.assert(contracts.Count == 1);
bd.AddOrUpdate(contracts[0]);
break;
}
bd.AddOrUpdate(contracts[0]);
}
var tableOut = bd.ToImmutable();
return tableOut;
static DateTime? GetExpiration(IBlueRow inst) {
var exp = inst.GetDateTime(WellKnownColumnNames.Expiration);
return exp;
}
static ContinuousContractKey GetKey(IBlueRow inst) {
var venue = inst.GetEnum<Venue>(WellKnownColumnNames.Venue).assert_value();
var asset = inst.GetString(WellKnownColumnNames.Asset);
Log.notnull(asset);
var type = inst.GetEnum<SecurityType>(WellKnownColumnNames.SecurityType).assert_value();
return new(venue, asset, type);
}
}
public bool containsID(int id) =>
impl.containsID(id);
public IEnumerator<IBlueRow> GetEnumerator() =>
impl.GetEnumerator();
public IBlueRow GetRow(int index) =>
impl.GetRow(index);
public int IndexOf(IBlueRow item) =>
impl.IndexOf(item);
readonly struct ContinuousContractKey {
public readonly Venue Venue;
public readonly string Asset;
public readonly SecurityType Type;
public ContinuousContractKey(Venue venue, string asset, SecurityType type) {
Venue = venue;
Asset = asset;
Type = type;
}
}
}
@@ -0,0 +1,93 @@
namespace Bts.SharedStreams.MarketData;
using bts.api;
using Bts.DataStreamCore.StreamFactories;
public class HistoricInstrumentParams {
public HistoricInstrumentParams() { }
public HistoricInstrumentParams(string venueStr, string asset)
: this(MarketDataArgStore.ParseVenue(venueStr), asset) { }
public HistoricInstrumentParams(string venueStr, string asset, string typeStr)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr)) { }
public HistoricInstrumentParams(string venueStr, string asset, string typeStr, DateTime date)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), date) { }
public HistoricInstrumentParams(string venueStr, string asset, string typeStr, DateTime beginDate, DateTime endDate)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), beginDate, endDate) { }
public HistoricInstrumentParams(string venueStr, string asset, DateTime date)
: this(MarketDataArgStore.ParseVenue(venueStr), asset, date) { }
public HistoricInstrumentParams(Venue venue, string asset) {
Venue = venue;
Asset = asset;
}
public HistoricInstrumentParams(Venue venue, string asset, DateTime date)
: this(venue, asset, null, date) { }
public HistoricInstrumentParams(Venue venue, string asset, DateTime beginDate, DateTime endDate)
: this(venue, asset, null, beginDate, endDate) { }
public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type) {
Venue = venue;
Asset = asset;
Type = type;
}
public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type, DateTime date)
: this(venue, asset, type, date, date.AddDays(1)) { }
public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type, DateTime beginDate, DateTime endDate) {
Venue = venue;
Asset = asset;
Type = type;
BeginDate = beginDate;
EndDate = endDate;
}
public Venue? Venue { get; set; }
public string? Asset { get; set; }
public SecurityType? Type { get; set; }
public string? Name { get; set; }
public int NumLegs { get; set; } = 5;
public bool ShowVenue { get; set; } = true;
public bool ShowType { get; set; } = true;
public bool ShowAsset { get; set; } = true;
public bool ShowExpiration { get; set; }
public bool ShowCashPerPoint { get; set; } = true;
public bool ShowLegVenue { get; set; } = true;
public bool ShowLegType { get; set; } = true;
public bool ShowLegAsset { get; set; } = true;
public bool ShowLegName { get; set; } = true;
public bool ShowLegExpiration { get; set; }
public bool ShowLegCashPerPoint { get; set; } = true;
public DateTime? BeginDate { get; set; }
public DateTime? EndDate { get; set; }
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
var clause = new mbts.spark.SecurityCriteriaClause(
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
security_types: Type.HasValue ? new[] { Type.Value } : null,
symbols: !string.IsNullOrEmpty(Asset) ? new[] { Asset } : null,
names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null);
return clause.is_defined ? new(new[] { clause }) : new();
}
public HistoricInstrumentSchemaBuilder GetSchemaBuilder() =>
new() {
ShowVenue = ShowVenue,
ShowType = true,
ShowAsset = true,
ShowName = true,
ShowCashPerPoint = ShowCashPerPoint,
ShowLegVenue = ShowLegVenue,
ShowLegAsset = ShowLegAsset,
ShowLegType = ShowLegType,
ShowLegName = ShowLegVenue,
};
}
@@ -0,0 +1,48 @@
namespace Bts.SharedStreams.MarketData;
using bts.api;
using System.Collections.Immutable;
public class HistoricInstrumentRowData {
public HistoricInstrumentRowData(
int id,
DateTime timestamp,
Venue venue,
SecurityType securityType,
string asset,
string currency,
string name,
DateTime? expiration,
double cashPerPoint,
ImmutableList<HistoricInstrumentRowData>? legs = null
) {
Id = id;
Timestamp = timestamp;
Venue = venue;
SecurityType = securityType;
Asset = asset;
Currency = currency;
Name = name;
Expiration = expiration;
CashPerPoint = cashPerPoint;
Legs = legs;
}
public int Id { get; }
public DateTime Timestamp { get; }
public Venue Venue { get; }
public SecurityType SecurityType { get; }
public TimeseriesKey<InstrumentKey> Key => new((Venue, Name), Timestamp);
public string Asset { get; }
public string Currency { get; }
public string Name { get; }
public DateTime? Expiration { get; }
public double CashPerPoint { get; }
public int NumLegs => Legs?.Count ?? 0;
// Some legs might not be resolved
public ImmutableList<HistoricInstrumentRowData>? Legs { get; }
public HistoricInstrumentRowData AddLegs(IEnumerable<HistoricInstrumentRowData> legs) =>
new(Id, Timestamp, Venue, SecurityType, Asset, Currency, Name, Expiration, CashPerPoint, ImmutableList.CreateRange(legs));
}
@@ -0,0 +1,82 @@
namespace Bts.SharedStreams.MarketData;
using bts.collections;
using Bts.DataStreamCore;
using Bts.SharedStreams.StreamFactories;
public class HistoricInstrumentSchemaBuilder {
public bool ShowVenue { get; set; } = true;
public bool ShowAsset { get; set; } = true;
public bool ShowType { get; set; } = true;
public bool ShowName { get; set; } = true;
public bool ShowExpiration { get; set; } = true;
public bool ShowCashPerPoint { get; set; } = true;
public bool ShowCurrency { get; set; } = true;
public bool ShowLegVenue { get; set; }
public bool ShowLegAsset { get; set; }
public bool ShowLegType { get; set; }
public bool ShowLegName { get; set; } = true;
public bool ShowLegCashPerPoint { get; set; }
public int NumLegs { get; set; } = 2;
public string TableName { get; set; } = "HistoricInstrument";
public BlueTableSchemaAdapter<HistoricInstrumentRowData> DatabentoAdapter() {
var adapter = new BlueTableSchemaAdapter<HistoricInstrumentRowData>(TableName);
adapter.AddColumn("id", r => r.Id);
adapter.AddColumn(WellKnownColumnNames.Timestamp, r => r.Timestamp);
if (ShowVenue) {
adapter.AddColumn(WellKnownColumnNames.Venue, r => r.Venue);
}
if (ShowType) {
adapter.AddColumn(WellKnownColumnNames.SecurityType, r => r.SecurityType);
}
if (ShowAsset) {
adapter.AddColumn(WellKnownColumnNames.Asset, r => r.Asset);
}
if (ShowName) {
adapter.AddColumn(WellKnownColumnNames.SecurityName, r => r.Name);
}
if (ShowExpiration) {
adapter.AddColumn(WellKnownColumnNames.Expiration, r => r.Expiration);
}
if (ShowCurrency) {
adapter.AddColumn(WellKnownColumnNames.Currency, r => r.Currency);
}
if (ShowCashPerPoint) {
adapter.AddColumn(WellKnownColumnNames.CashPerPoint, r => r.Currency);
}
if (NumLegs > 0) {
adapter.AddColumn(WellKnownColumnNames.NumLegs, r => r.NumLegs);
for (int i = 0; i < NumLegs; i++) {
// Copy i to a local var (legIndex) so it can be referenced in column getters.
int legIndex = i;
if (ShowLegVenue) {
adapter.AddColumn(WellKnownColumnNames.LegVenue(i), Get(l => l.Venue));
}
if (ShowLegType) {
adapter.AddColumn(WellKnownColumnNames.LegSecurityType(i), Get(l => l.SecurityType));
}
if (ShowLegAsset) {
adapter.AddColumn(WellKnownColumnNames.LegAsset(i), Get(l => l.Asset));
}
if (ShowLegName) {
adapter.AddColumn(WellKnownColumnNames.LegSecurityName(i), Get(l => l.Name));
}
if (ShowLegCashPerPoint) {
adapter.AddColumn(WellKnownColumnNames.LegCashPerPoint(i), Get(l => l.CashPerPoint, double.NaN));
}
Func<HistoricInstrumentRowData, T> Get<T>(Func<HistoricInstrumentRowData, T> getValue, T defaultValue = default!) =>
s => {
var legs = s.Legs;
if (legs is null || !legs.TryGetAt(legIndex, out var leg)) {
return defaultValue;
}
return getValue(leg);
};
}
}
return adapter;
}
}
@@ -0,0 +1,28 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using Bts.DataStream.BlueOrb;
using Bts.PubSub;
using Microsoft.Extensions.Logging;
/// <summary>
/// Creates a time series of instruments for a date range.
/// </summary>
public class HistoricInstrumentStream : ViewStream {
public HistoricInstrumentStream(HistoricInstrumentParams streamParams, IWorkScheduler workScheduler, ILogger logger) =>
mySetRealSource(Create(streamParams, workScheduler, logger));
public static ViewStream Create(HistoricInstrumentParams streamParams, IWorkScheduler workScheduler, ILogger logger) =>
AsyncViewStream.Create(workScheduler, () => CreateAsync(streamParams, logger));
public static async Task<ViewStream?> CreateAsync(HistoricInstrumentParams sp, ILogger logger) {
logger.LogInformation("Creating HistoricInstrumentStream");
var table = await HistoricInstrumentTable.CreateAsync(sp, logger);
if (table is null) {
return null;
}
return new BlueRowCollectionStaticStream(table);
}
}
@@ -0,0 +1,167 @@
namespace Bts.SharedStreams.MarketData;
using BCG;
using bts;
using bts.api;
using bts.BlueOrb;
using bts.collections;
using Bts.DataStreamCore;
using Microsoft.Extensions.Logging;
using Spark.Core.Collections;
using System.Collections.Immutable;
public class HistoricInstrumentTable : IBlueRowCollection {
readonly BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter;
readonly TimeseriesIndex<InstrumentKey> instrumentIndex;
readonly ImmutableSortedSet<KeyValuePair<int, HistoricInstrumentRowData>> rowSet;
public HistoricInstrumentTable(BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter) {
this.schemaAdapter = schemaAdapter;
instrumentIndex = new();
rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, HistoricInstrumentRowData>());
}
public HistoricInstrumentTable(BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter, TimeseriesIndex<InstrumentKey> instrumentIndex) {
this.schemaAdapter = schemaAdapter;
this.instrumentIndex = instrumentIndex;
rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, HistoricInstrumentRowData>());
}
HistoricInstrumentTable(
BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter,
TimeseriesIndex<InstrumentKey> instrumentIndex,
ImmutableSortedSet<KeyValuePair<int, HistoricInstrumentRowData>> rowSet) {
this.schemaAdapter = schemaAdapter;
this.instrumentIndex = instrumentIndex;
this.rowSet = rowSet;
}
public IBlueRow this[int id] =>
rowSet.TryGetValue(new(id, null!), out var res)
? schemaAdapter.GetBlueRow(res.Value)
: throw new KeyNotFoundException();
public IBlueTableSchema schema => schemaAdapter.Schema;
public int Count => rowSet.Count;
IBlueRow IList<IBlueRow>.this[int index] {
get => GetRow(index);
set => throw new NotSupportedException();
}
public static HistoricInstrumentTable Create(BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter, IBlueRowCollection definitionRows) {
var table = new HistoricInstrumentTable(schemaAdapter);
table = table.AddInstruments(definitionRows);
return table;
}
public static async Task<HistoricInstrumentTable?> CreateAsync(HistoricInstrumentParams sp, ILogger logger) {
logger.LogInformation("Creating HistoricInstrumentTable");
var beginDate = sp.BeginDate;
var endDate = sp.EndDate;
var fileList = DatabentoFileStore.GetDataFiles("definition", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToList();
if (!beginDate.HasValue && !endDate.HasValue) {
beginDate = fileList[^1].Date;
endDate = beginDate.Value + TimeSpan.FromDays(1);
} else {
beginDate ??= DateTime.MinValue;
endDate ??= DateTime.MaxValue;
}
fileList.RemoveAll(f => f.Date < beginDate.Value || f.Date >= endDate.Value);
if (fileList.Count == 0) {
return null;
}
var rowMapBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer<int, HistoricInstrumentRowData>());
var rowSetBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer<int, IBlueRow>());
var dateTables = fileList.ParallelSelect(f => ParquetTableReader.Read(f.Path));
var tableRowMap = ImmutableDictionary.CreateBuilder<int, HistoricInstrumentRowData>();
var schemaBd = sp.GetSchemaBuilder();
var schemaAdapter = schemaBd.DatabentoAdapter();
var nameToId = new Dictionary<(Venue, string), int>();
var idToName = new Dictionary<int, (Venue, string)>();
var table = new HistoricInstrumentTable(schemaAdapter);
foreach (var dateTable in dateTables) {
table = table.AddInstruments(dateTable);
}
return table;
}
/// <summary>
/// Add instruments (<see cref="rows"/>) to this table. If there are spreads, the legs' contracts
/// should be included in <see cref="rows"/> as well.
/// </summary>
/// <param name="definitionRows">A table with columns corresponding to the Databento 'definition' schema</param>
/// <returns></returns>
/// <exception cref="ArgumentException">Glbx spread does not have the correct number of legs (2)</exception>
/// <exception cref="NotImplementedException">Spread handler code not implemented at a particular venue</exception>
public HistoricInstrumentTable AddInstruments(IBlueRowCollection definitionRows) {
var instrumentIndex = this.instrumentIndex.ToBuilder();
var rowSet = this.rowSet.ToBuilder();
var spreadRows = new List<HistoricInstrumentRowData>();
var rowDataFactory = DatabentoInstrumentAdapter.GetRowDataFactory(instrumentIndex.GetOrAdd, definitionRows.schema.columnInfo);
foreach (var r in definitionRows) {
var rd = rowDataFactory(r);
rowSet.Add(new(rd.Id, rd));
if (rd.SecurityType == SecurityType.Spread) {
spreadRows.Add(rd);
}
}
// Resolve spread legs
for (int i = 0; i < spreadRows.Count; i++) {
var rd = spreadRows[i];
switch (rd.Venue) {
case Venue.Glbx:
var legSymbols = rd.Name.Split('-');
if (legSymbols.Length != 2) {
throw new ArgumentException($"Glbx spread has {legSymbols.Length} legs (expected 2)", nameof(definitionRows));
}
var legs = legSymbols.Select(legSym => {
var legId = instrumentIndex.GetLatest((rd.Venue, legSym));
bool foundLeg = rowSet.TryGetValue(new(legId, null!), out var leg);
Log.assert(foundLeg);
return leg.Value;
});
rowSet.Remove(new(rd.Id, rd));
rowSet.Add(new(rd.Id, rd.AddLegs(legs)));
break;
default:
throw new NotImplementedException($"Spreads not implemented at venue '{rd.Venue}'");
}
}
return new(schemaAdapter, instrumentIndex.ToImmutable(), rowSet.ToImmutable());
}
public bool containsID(int id) =>
instrumentIndex.ContainsId(id);
public IBlueRow GetRow(int index) {
var rd = rowSet[index].Value;
var blueRow = schemaAdapter.GetBlueRow(rd);
return blueRow;
}
public int IndexOf(IBlueRow item) {
int id = item.Id();
int idx = rowSet.IndexOf(new(id, null!));
if (idx >= 0) {
var rd = rowSet[idx].Value;
var blueRow = schemaAdapter.GetBlueRow(rd);
if (BlueRowEqualityComparer.Equals(blueRow, item)) {
return idx;
}
}
return -1;
}
public IEnumerator<IBlueRow> GetEnumerator() {
foreach (var kv in rowSet) {
yield return schemaAdapter.GetBlueRow(kv.Value);
}
}
}
@@ -0,0 +1,11 @@
namespace Bts.SharedStreams.MarketData;
using bts.api;
public record InstrumentKey(Venue Venue, string Name) {
public static implicit operator (Venue Venue, string Name)(InstrumentKey key) =>
(key.Venue, key.Name);
public static implicit operator InstrumentKey((Venue Venue, string Name) tup) =>
new(tup.Venue, tup.Name);
}
@@ -0,0 +1,168 @@
namespace Bts.SharedStreams.MarketData;
using Spark.Core.Collections;
using System.Collections.Immutable;
/// <summary>
/// Maps a timeseries of keys to unique int32 ids.
/// </summary>
public class TimeseriesIndex<T> where T : notnull {
static readonly IComparer<KeyValuePair<DateTime, int>> idTimeseriesComp =
KeyValuePairComparer.KeyComparer<DateTime, int>();
readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId;
readonly ImmutableDictionary<int, TimeseriesKey<T>> idToName;
readonly int idCounter;
public TimeseriesIndex()
: this(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Empty, ImmutableDictionary<int, TimeseriesKey<T>>.Empty, 0) { }
TimeseriesIndex(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
this.nameToId = nameToId;
this.idToName = idToName;
this.idCounter = idCounter;
}
public bool ContainsId(int id) =>
idToName.ContainsKey(id);
public int GetNearest(TimeseriesKey<T> key) {
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
throw new KeyNotFoundException();
}
int idx = ids.BinarySearch(new(key.Timestamp, -1));
if (idx < 0) {
idx = ~idx;
}
if (idx == 0) {
throw new KeyNotFoundException();
}
return ids[idx].Value;
}
public TimeseriesIndex<T> GetNearestOrAdd(TimeseriesKey<T> key, out int result) {
int idx;
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
idx = 0;
} else {
idx = ids.BinarySearch(new(key.Timestamp, -1));
if (idx < 0) {
idx = ~idx;
}
}
if (idx == 0) {
result = idCounter + 1;
ids = ids.Insert(0, new(key.Timestamp, result));
return new(
nameToId.SetItem(key.Name, ids),
idToName.SetItem(result, key),
result);
}
result = ids[idx].Value;
return this;
}
public TimeseriesIndex<T> GetOrAdd(TimeseriesKey<T> key, out int result) {
int idx;
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
idx = 0;
} else {
idx = ids.BinarySearch(new(key.Timestamp, -1));
if (idx < 0) {
idx = ~idx;
} else {
result = ids[idx].Value;
return this;
}
}
result = idCounter + 1;
ids = ids.Insert(idx, new(key.Timestamp, result));
return new(
nameToId.SetItem(key.Name, ids),
idToName.SetItem(result, key),
result);
}
public IList<KeyValuePair<DateTime, int>> GetIds(T name) =>
nameToId[name];
public Builder ToBuilder() =>
new(this);
public class Builder {
readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Builder nameToId;
readonly ImmutableDictionary<int, TimeseriesKey<T>>.Builder idToName;
int idCounter;
public Builder(TimeseriesIndex<T> source)
: this(source.nameToId, source.idToName, source.idCounter) { }
public Builder(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
this.nameToId = nameToId.ToBuilder();
this.idToName = idToName.ToBuilder();
this.idCounter = idCounter;
}
public int GetNearest(TimeseriesKey<T> key) =>
getOrAdd(key, false, false);
public int GetNearestOrAdd(TimeseriesKey<T> key) =>
getOrAdd(key, false, true);
public int GetOrAdd(TimeseriesKey<T> key) =>
getOrAdd(key, true, true);
public int GetLatest(T key) =>
nameToId.TryGetValue(key, out var ids)
? ids[^1].Value
: throw new KeyNotFoundException();
public bool TryGetLatest(T key, out int result) {
if (nameToId.TryGetValue(key, out var ids)) {
result = ids[^1].Value;
return true;
}
result = 0;
return false;
}
public ImmutableList<KeyValuePair<DateTime, int>> GetIds(T name) =>
nameToId.TryGetValue(name, out var ids)
? ids
: ImmutableList<KeyValuePair<DateTime, int>>.Empty;
public TimeseriesIndex<T> ToImmutable() =>
new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter);
int getOrAdd(TimeseriesKey<T> key, bool requireExactTimestamp, bool canAdd) {
var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList<KeyValuePair<DateTime, int>>.Empty);
int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp);
int id;
if (idx < 0) {
// We didn't get an exact match. If requireExactTimestamp is false and a record with an
// earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp.
idx = ~idx;
if (requireExactTimestamp || idx == 0) {
if (!canAdd) {
throw new KeyNotFoundException();
}
id = ++idCounter;
ids = ids.Insert(idx, new(key.Timestamp, id));
nameToId[key.Name] = ids;
idToName.Add(id, key);
} else {
if (idx == ids.Count) {
--idx;
}
id = ids[idx].Value;
}
} else {
id = ids[idx].Value;
}
return id;
}
}
}
@@ -0,0 +1,9 @@
namespace Bts.SharedStreams.MarketData;
public record TimeseriesKey<T>(T Name, DateTime Timestamp) {
public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey<T> key) =>
(key.Name, key.Timestamp);
public static implicit operator TimeseriesKey<T>((T Name, DateTime Timestamp) tup) =>
new(tup.Name, tup.Timestamp);
}
@@ -100,7 +100,7 @@ namespace bts.BlueOrb {
}
}
[DataStreamUserRespresentation("bbo")]
[DataStreamUserRespresentation(false)]
public class BBOStream : ViewStream, IWorker {
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
private DateTime _last_poll_time = DateTime.MinValue;
@@ -1,7 +1,5 @@
namespace Bts.SharedStreams.Services;
using bts.SilverPine;
using Bts.DataStreamCore;
using Bts.SharedStreams.StreamFactories;
@@ -17,10 +15,15 @@ public static class SharedStreamsServices {
.addStreamFactory<SpatialFuturePairsStreamFactory>()
.addStreamFactory<SpatialArbStatsStreamFactory>();
public static IServiceCollection AddHistoricMarketDataStreams(this IServiceCollection services) =>
services.addStreamFactory<HistoricParquetDataLoader>();
public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) =>
services.Configure<NaturalLanguageStreamFactoryConfig>(configurationSection)
.AddSingleton<NaturalLanguageStreamFactory>()
.AddSingleton<IDataStreamFactory, NaturalLanguageStreamFactory>(sp => sp.GetRequiredService<NaturalLanguageStreamFactory>());
.addStreamFactory<NaturalLanguageStreamFactory>();
public static IServiceCollection AddParquetReaderStream(this IServiceCollection services) =>
services.addStreamFactory<ParquetReaderStreamFactory>();
static IServiceCollection addStreamFactory<T>(this IServiceCollection services) where T : class, IDataStreamFactory =>
services.AddSingleton<T>()
@@ -2,21 +2,26 @@ namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.md;
using Bts.DataStream;
using Bts.DataStreamCore;
using Bts.DataStreamCore.StreamFactories;
using Microsoft.Extensions.Logging;
using Spark.Api.MarketData;
using System;
using System.Collections.Generic;
public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory {
readonly LiveBboStreamFactory liveBboFactory;
IDataStreamFactory? compositeFactory = null;
public class BookStreamFactory : DataStreamFactory {
public BookStreamFactory(IMarketDataClient? mdClient, ILogger<BookStreamFactory> logger) {
this.Add<BookStreamParams>("book", CreateBook, logger);
this.Add<BookStreamParams>(new[] { "bbo", "LiveBBOStream" }, CreateBbo, logger);
this.Add<BookStreamParams>(new[] { "bbo" }, CreateBbo, logger);
liveBboFactory = new LiveBboStreamFactory(mdClient, logger);
Add(liveBboFactory);
ViewStream? CreateBook(BookStreamParams streamParams) =>
mdClient?.Books(streamParams.GetSecurityCriteria())
@@ -24,10 +29,97 @@ public class BookStreamFactory : DataStreamFactory {
?.AsViewStream()
?? throw new InvalidOperationException("Live md not available");
ViewStream? CreateBbo(BookStreamParams streamParams) =>
mdClient?.Books(streamParams.GetSecurityCriteria())
ViewStream? CreateBbo(BookStreamParams streamParams) {
return mdClient?.Books(streamParams.GetSecurityCriteria())
?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
?.AsViewStream()
?? throw new InvalidOperationException("Live md not available");
}
}
public void OnCompositeFactory(IDataStreamFactory compositeFactory) {
this.compositeFactory = compositeFactory;
liveBboFactory.OnCompositeFactory(compositeFactory);
}
class LiveBboStreamFactory : IDataStreamFactory, IRecursiveDataStreamFactory {
static readonly IDataStreamMetadata metadata = new DataStreamMetadata() {
Name = "LiveBBOStream",
AlternateNames = new(),
Signatures = new() {
new (string?, Type?)[] {
("symbol", typeof(string))
},
new (string?, Type?)[] {
("secType", typeof(string)),
("symbol", typeof(string)),
}
}
};
readonly IMarketDataClient? mdClient;
readonly ILogger logger;
IDataStreamFactory? compositeFactory = null;
public LiveBboStreamFactory(IMarketDataClient? mdClient, ILogger logger) {
this.mdClient = mdClient;
this.logger = logger;
}
public void OnCompositeFactory(IDataStreamFactory compositeFactory) =>
this.compositeFactory = compositeFactory;
public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
if (!metadata.IsMatch(streamId)) {
return null;
}
var argList = args.ToList();
switch (argList.Count) {
case 1: {
var sym = argList[0].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args));
if (sym == "ES" && compositeFactory is not null) {
return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, sym) });
}
var streamParams = new BookStreamParams() {
Symbol = sym
};
return mdClient?.Books(streamParams.GetSecurityCriteria())
?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
?.AsViewStream()
?? throw new InvalidOperationException("Live md not available");
}
case 2: {
var secTypeStr = argList[0].Value?.ToString() ?? string.Empty;
if (!MarketDataArgStore.TryParseSecurityType(secTypeStr, out var secType)) {
throw new ArgumentException("Failed to parse SecurityType arg", nameof(args));
}
var sym = argList[1].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args));
if (sym == "ES" && secType == bts.api.SecurityType.Future && compositeFactory is not null) {
return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, sym) });
}
var streamParams = new BookStreamParams() {
SecurityType = secType,
Symbol = sym,
};
return mdClient?.Books(streamParams.GetSecurityCriteria())
?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
?.AsViewStream()
?? throw new InvalidOperationException("Live md not available");
}
default:
return null;
}
}
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
yield return metadata;
}
}
public class LiveBboStreamParams {
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex }, Position = 0, IsRequired = false)]
public string? SecurityType { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex }, Position = 1, IsRequired = false)]
public string? ContinuousContract { get; set; }
}
}
@@ -0,0 +1,60 @@
namespace Bts.SharedStreams.StreamFactories;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
public interface IHistoricDataParams {
string? DataType { get; }
string? Venue { get; }
string? Symbol { get; }
DateTime BeginDate { get; }
DateTime EndDate { get; }
TimeSpan? Interval { get; }
string? TsNanosCol { get; }
}
public class HistoricDataParams : IHistoricDataParams {
public string? DataType { get; set; }
public string? Venue { get; set; }
public string? Symbol { get; set; }
public DateTime BeginDate { get; set; }
public DateTime EndDate { get; set; }
public TimeSpan? Interval { get; set; }
public string? TsNanosCol { get; set; }
[MemberNotNull(nameof(DataType), nameof(Venue), nameof(Symbol), nameof(TsNanosCol))]
public HistoricDataParams Validate() {
validateNotNull(DataType);
validateNotNull(Venue);
validateNotNull(Symbol);
validateNotNull(TsNanosCol);
return this;
}
[MemberNotNull(nameof(Interval))]
public HistoricDataParams ValidateInterval() {
validateNotNull(Interval);
return this;
}
static void validateNotNull<T>([NotNull]T arg, [CallerArgumentExpression(nameof(arg))]string argName = "") {
if (arg is null) {
throw new InvalidOperationException($"Invalid HistoricDataParams: {argName} is null");
}
}
}
public static class HistoricDataParamsExtensions {
// public static void Validate(this IHistoricDataParams hdp) {
// validateNotNull(hdp.DataType);
// validateNotNull(hdp.Venue);
// validateNotNull(hdp.Symbol);
// validateNotNull(hdp.TsNanosCol);
// }
// static void validateNotNull<T>([NotNull]T arg, [CallerArgumentExpression(nameof(arg))]string argName = "") {
// if (arg is null) {
// throw new InvalidOperationException($"Invalid HistoricDataParams: {argName} is null");
// }
// }
}
@@ -0,0 +1,200 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts;
using bts.api;
using bts.BlueOrb;
using bts.BlueOrb.Parquet;
using bts.collections;
using Bts.DataStream;
using Bts.DataStream.BlueOrb;
using Bts.DataStreamCore;
using Bts.DataStreamCore.StreamFactories;
using Bts.SharedStreams.MarketData;
using Microsoft.Extensions.Logging;
using Spark.Core.Collections;
using System.Collections;
using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
using System.Reactive.Linq;
using System.Threading.Tasks;
class HistoricParquetDataLoader : DataStreamFactory {
public HistoricParquetDataLoader(ViewStreamDatabase viewStreamDatabase, ILogger<HistoricParquetDataLoader> logger) {
this.Add<HistoricBboParams>("HistoricBbo", CreateHistoricBbo, logger);
this.Add<ReplayFutureLiveBboParams>("ReplayFutureLiveBBOStream", CreateReplayFutureLiveBbo, logger);
this.Add<ReplayFutureLiveTradeParams>("ReplayFutureLiveTradeStream", CreateReplayFutureLiveTrade, logger);
this.Add<HistoricInstrumentParams>("HistoricInstrument", sp => HistoricInstrumentStream.Create(sp, viewStreamDatabase, logger));
this.Add<HistoricInstrumentParams>("HistoricContinuousContract", sp => {
var historicInstruments = HistoricInstrumentStream.Create(sp, viewStreamDatabase, logger);
var vs = HistoricContinuousContractStream.Create(historicInstruments);
return vs;
});
static SecurityType? ParseSecTypeArg(string? secTypeStr) {
if (string.IsNullOrEmpty(secTypeStr)) {
return null;
}
if (MarketDataArgStore.TryParseSecurityType(secTypeStr, out var secType)) {
return secType;
}
throw new ArgumentException("Failed to parse SecurityType", nameof(secTypeStr));
}
// static Venue? ParseVenueArg(string? venueStr) {
// if (string.IsNullOrEmpty(venueStr)) {
// return null;
// }
// if (MarketDataArgStore.TryParseVenue(venueStr, out var venue)) {
// return venue;
// }
// throw new ArgumentException("Failed to parse Venue", nameof(venueStr));
// }
// bool TryGetHistoricInstrumentStreamParams(IEnumerable<(string? Name, int? Index, object? Value)> args, [NotNullWhen(true)] out HistoricInstrumentStreamParams? result, [NotNullWhen(false)] out string? errorMessage) {
// var argList = args.ToList();
// result = null;
// errorMessage = null;
// switch (argList.Count) {
// case 2: {
// var venue = ParseVenueArg(argList[0].Value as string);
// var asset = argList[1].Value as string;
// if (venue.HasValue && asset is not null) {
// var secType = SecurityType.Future;
// if (DatabentoFileStore.GetDataFiles("definition", asset, secType).TryGetMaxElement(f => f.Date, out var latestFile)) {
// result = new(venue.Value, asset, latestFile.Date) {
// Type = secType,
// };
// }
// }
// break;
// }
// case 3: {
// var venue = ParseVenueArg(argList[0].Value as string);
// var asset = argList[1].Value as string;
// break;
// }
// }
// if (result is not null) {
// return true;
// }
// errorMessage ??= "Failed to convert args to HistoricInstrumentStreamParams";
// return false;
// }
ViewStream? CreateReplayFutureLiveBbo(ReplayFutureLiveBboParams opts) {
if (opts.ContinuousContract is null) {
logger.LogMissingParam("ReplayFutureLiveBBOStream", nameof(opts.ContinuousContract));
return null;
}
Func<IBlueRow, bool>? filter = opts.ContinuousContract == "ES" ? ContractFilter : null;
var tables = DatabentoFileStore.GetBboAsync(opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), opts.BeginDate, opts.EndDate, filter);
// var continuousContracts = HistoricContinuousContractStream.Create(new() { Symbol = opts.ContinuousContract, SecurityType = SecurityType.Future, BeginDate = opts.BeginDate, EndDate = opts.EndDate }, viewStreamDatabase, logger);
return ReplayStream.Create(tables, "ts_event", opts.ReplaySpeed, viewStreamDatabase);
bool ContractFilter(IBlueRow r) =>
r.GetString("contract") == "ESU5";
}
ViewStream? CreateReplayFutureLiveTrade(ReplayFutureLiveTradeParams opts) {
if (opts.ContinuousContract is null) {
logger.LogMissingParam("ReplayFutureLiveTradeStream", nameof(opts.ContinuousContract));
return null;
}
Func<IBlueRow, bool>? filter = opts.ContinuousContract == "ES" ? ContractFilter : null;
var tables = DatabentoFileStore.GetTradesAsync(opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), opts.BeginDate, opts.EndDate, filter);
return ReplayStream.Create(tables, "ts_event", opts.ReplaySpeed, viewStreamDatabase);
bool ContractFilter(IBlueRow r) =>
r.GetString("contract") == "ESU5";
}
ViewStream? CreateHistoricBbo(HistoricBboParams opts) {
if (opts.ContinuousContract is null) {
logger.LogMissingParam("HistoricBbo", nameof(opts.ContinuousContract));
return null;
}
var streams = DatabentoFileStore.GetDataFiles(
"mbp-1", opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), beginDate: opts.BeginDate, endDate: opts.EndDate)
.Select(f => new ParquetReaderStream(f.Path, viewStreamDatabase))
.ToArray();
return streams.Length switch {
0 => null,
1 => streams[0],
_ => new MutableAppendStream(streams),
};
}
}
static BlueTableImmutable dropIndex(IBlueTableSchema schema, IEnumerable<IBlueRow> rows) {
int idIndex = schema.ResolveIndices("id");
var rowSetBd = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, IBlueRow>()).ToBuilder();
foreach (var r in rows) {
int idx = rowSetBd.Count + 1;
var inputRow = (r as IndexRow)?.InputRow ?? r;
rowSetBd.Add(new(idx, new IndexRow(inputRow, idx, idIndex, schema)));
}
return new(schema, rowSetBd.ToImmutable());
}
struct ReplayFutureLiveTradeState {
public IDataPacket? Data;
public ColumnSelectorRowFactory? RowFactory;
}
class ReplayFutureLiveBboParams {
[StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)]
public string? ContinuousContract { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1)]
public string? SecurityType { get; set; }
[StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)]
public DateTime? BeginDate { get; set; }
[StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)]
public DateTime? EndDate { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ReplaySpeedRegex }, Position = 4)]
public double ReplaySpeed { get; set; } = 1;
}
class ReplayFutureLiveTradeParams {
[StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)]
public string? ContinuousContract { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1)]
public string? SecurityType { get; set; }
[StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)]
public DateTime? BeginDate { get; set; }
[StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)]
public DateTime? EndDate { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ReplaySpeedRegex }, Position = 4)]
public double ReplaySpeed { get; set; } = 1;
}
class HistoricBboParams {
[StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)]
public string? ContinuousContract { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1, IsRequired = false)]
public string? SecurityType { get; set; }
[StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)]
public DateTime? BeginDate { get; set; }
[StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)]
public DateTime? EndDate { get; set; }
[StreamArgument(RegexNames = new[] { "speed" }, Position = 4)]
public double ReplaySpeed { get; set; } = 1;
}
}
@@ -1,78 +0,0 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.api;
using Bts.DataStreamCore;
public class InstrumentSchemaBuilder {
public bool ShowVenue { get; set; } = true;
public bool ShowAsset { get; set; } = true;
public bool ShowType { get; set; } = true;
public bool ShowName { get; set; } = true;
public bool ShowId { get; set; } = true;
public bool ShowCashPerPoint { get; set; } = true;
public bool ShowLegId { get; set; } = true;
public bool ShowLegVenue { get; set; }
public bool ShowLegAsset { get; set; }
public bool ShowLegType { get; set; }
public bool ShowLegName { get; set; } = true;
public bool ShowLegCashPerPoint { get; set; }
public int NumLegs { get; set; } = 2;
public string TableName { get; set; } = "Instrument";
public BlueTableSchemaAdapter<SecurityImmutable> SecurityAdapter() {
var adapter = new BlueTableSchemaAdapter<SecurityImmutable>(TableName);
adapter.AddColumn("id", r => (int)r.Id);
if (ShowVenue) {
adapter.AddColumn(WellKnownColumns.Venue.Name, b => b.Exchange.ToString());
}
if (ShowType) {
adapter.AddColumn(WellKnownColumns.SecurityType.Name, b => b.security_type().ToString());
}
if (ShowAsset) {
adapter.AddColumn(WellKnownColumns.Asset.Name, b => b.Symbol);
}
if (ShowName) {
adapter.AddColumn(WellKnownColumns.SecurityName.Name, b => b.ToString());
}
if (ShowId) {
adapter.AddColumn(WellKnownColumns.SecurityId.Name, b => b.Id);
}
if (ShowCashPerPoint) {
adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.ContractSpec.CashPerPoint);
}
for (int i = 0; i < NumLegs; i++) {
if (ShowLegVenue) {
adapter.AddColumn(WellKnownColumns.LegVenue(i).Name, Get(l => l.Security.Exchange.ToString(), null));
}
if (ShowLegType) {
adapter.AddColumn(WellKnownColumns.LegSecurityType(i).Name, Get(l => l.Security.security_type().ToString(), null));
}
if (ShowLegAsset) {
adapter.AddColumn(WellKnownColumns.LegAsset(i).Name, Get(l => l.Security.Symbol, null));
}
if (ShowLegName) {
adapter.AddColumn(WellKnownColumns.LegSecurityName(i).Name, Get(l => l.Security.ToString(), null));
}
if (ShowLegId) {
adapter.AddColumn(WellKnownColumns.LegSecurityId(i).Name, Get(l => (int)l.Security.Id, 0));
}
if (ShowLegCashPerPoint) {
adapter.AddColumn(WellKnownColumns.LegCashPerPoint(i).Name, Get(l => l.Security.ContractSpec.CashPerPoint, double.NaN));
}
Func<SecurityImmutable, T> Get<T>(Func<ContractLegImmutable, T> getValue, T defaultValue) =>
s => {
ContractLegImmutable? leg = s.ContractCase switch {
Security.ContractOneofCase.Spread => s.Spread.Legs.Skip(i).TryGetFirst(out var l) ? l : null,
Security.ContractOneofCase.Synthetic => s.Synthetic.Legs.Skip(i).TryGetFirst(out var l) ? l : null,
_ => null,
};
return leg.HasValue ? getValue(leg.Value) : defaultValue;
};
}
return adapter;
}
}
@@ -1,66 +0,0 @@
namespace Bts.SharedStreams.StreamFactories;
using bts.api;
using Bts.DataStreamCore;
public class InstrumentStreamParams {
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.VenueRegex })]
public Venue? Venue { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex })]
public string? Symbol { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex })]
public SecurityType? Type { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityNameRegex })]
public string? Name { get; set; }
[StreamArgument("legs", "nLegs", "numLegs")]
public int NumLegs { get; set; } = 5;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowVenueRegex })]
public bool ShowVenue { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowSecurityTypeRegex })]
public bool ShowType { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowAssetRegex })]
public bool ShowAsset { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowCashPerPointRegex })]
public bool ShowCashPerPoint { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegVenueRegex })]
public bool ShowLegVenue { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegSecurityTypeRegex })]
public bool ShowLegType { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegSymbolRegex })]
public bool ShowLegSymbol { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegCashPerPointRegex })]
public bool ShowLegCashPerPoint { get; set; } = true;
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
var clause = new mbts.spark.SecurityCriteriaClause(
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
security_types: Type.HasValue ? new[] { Type.Value } : null,
symbols: !string.IsNullOrEmpty(Symbol) ? new[] { Symbol } : null,
names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null);
return clause.is_defined ? new(new[] { clause }) : new();
}
public InstrumentSchemaBuilder GetSchemaBuilder() =>
new() {
NumLegs = NumLegs,
ShowVenue = ShowVenue,
ShowType = ShowType,
ShowAsset = ShowAsset,
ShowName = true,
ShowId = true,
ShowCashPerPoint = ShowCashPerPoint,
};
}
@@ -0,0 +1,35 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.BlueOrb.Parquet;
using Bts.DataStreamCore;
using Microsoft.Extensions.Logging;
class ParquetReaderStreamFactory : DataStreamFactory {
public ParquetReaderStreamFactory(ViewStreamDatabase viewStreamDatabase, ILogger<ParquetReaderStreamFactory> logger) {
this.Add<ParquetReaderStreamParams>("parquetReader", CreateParquetReader, logger);
ViewStream? CreateParquetReader(ParquetReaderStreamParams streamParams) {
if (streamParams.File is null) {
logger.LogMissingParam("ParquetReader", nameof(streamParams.File));
return null;
}
var stream = new ParquetReaderStream(streamParams.File, streamParams.Start, streamParams.Stop, viewStreamDatabase);
return stream;
}
}
class ParquetReaderStreamParams {
[StreamArgument(RegexNames = new[] { "file|path|filePath" }, Position = 0, IsRequired = true)]
public string? File { get; set; }
[StreamArgument(RegexNames = new[] { "^start|first|begin$" }, Position = 1, IsRequired = false)]
public int? Start { get; set; }
[StreamArgument(RegexNames = new[] { "^stop|last|end$" }, Position = 2, IsRequired = false)]
public int? Stop { get; set; }
}
}
@@ -4,17 +4,49 @@ using bts.api;
using Spark.Api.Orders;
public static class WellKnownColumnNames {
public const string Price = "price";
public const string Quantity = "qty";
public const string Timestamp = "ts_event";
public const string BidPrice = "bid_price";
public const string BidQuantity = "bid_size";
public const string BidNotional = "bid_notl";
public const string AskPrice = "ask_price";
public const string Currency = "currency";
public const string AskQuantity = "ask_size";
public const string AskNotional = "ask_notl";
public const string SecurityType = "sec_type";
public const string Asset = "asset";
public const string SecurityName = "sec_name";
public const string Expiration = "expiration";
public const string Venue = "exchange";
public const string TradingStatus = "trading_status";
public const string NumLegs = "num_legs";
public const string CashPerPoint = "cpp";
public static string DepthPrice(Side side, int level) => $"{side.BidAskName()}Price{level}";
public static string DepthQuantity(Side side, int level) => $"{side.BidAskName()}Qty{level}";
public static string DepthNotional(Side side, int level) => $"{side.BidAskName()}Notl{level}";
public static string LegSecurityId(int legNum) => $"Leg{legNum}SecId";
public static string LegVenue(int legNum) => $"Leg{legNum}Venue";
public static string LegSecurityType(int legNum) => $"Leg{legNum}SecType";
public static string LegAsset(int legNum) => $"Leg{legNum}Asset";
public static string LegSecurityName(int legNum) => $"Leg{legNum}Sec";
public static string LegCashPerPoint(int legNum) => $"Leg{legNum}Cpp";
}
public static class WellKnownColumns {
public static (string Name, Type Type) SecurityId => ("SecId", typeof(uint));
public static (string Name, Type Type) Venue => ("Venue", typeof(string));
public static (string Name, Type Type) SecurityType => ("SecType", typeof(string));
public static (string Name, Type Type) Asset => ("Asset", typeof(string));
public static (string Name, Type Type) SecurityName => ("Sec", typeof(string));
public static (string Name, Type Type) CashPerPoint => ("Cpp", typeof(double));
public static (string Name, Type Type) Price => ("Price", typeof(double));
public static (string Name, Type Type) Quantity => ("Qty", typeof(double));
public static (string Name, Type Type) Timestamp => ("Ts", typeof(DateTime));
public static (string Name, Type Type) TimestampNanos => ("TsNanos", typeof(long));
public static (string Name, Type Type) Venue => (WellKnownColumnNames.Venue, typeof(string));
public static (string Name, Type Type) SecurityType => (WellKnownColumnNames.SecurityType, typeof(string));
public static (string Name, Type Type) Asset => (WellKnownColumnNames.Asset, typeof(string));
public static (string Name, Type Type) SecurityName => (WellKnownColumnNames.SecurityName, typeof(string));
public static (string Name, Type Type) CashPerPoint => (WellKnownColumnNames.CashPerPoint, typeof(double));
public static (string Name, Type Type) Price => (WellKnownColumnNames.Price, typeof(double));
public static (string Name, Type Type) Quantity => (WellKnownColumnNames.Quantity, typeof(double));
public static (string Name, Type Type) Timestamp => (WellKnownColumnNames.Timestamp, typeof(DateTime));
public static (string Name, Type Type) TimestampNanos => ("ts_event_ns", typeof(long));
public static (string Name, Type Type) BidPrice => ("BidPrice", typeof(double));
public static (string Name, Type Type) BidQuantity => ("BidQty", typeof(double));
@@ -23,14 +55,14 @@ public static class WellKnownColumns {
public static (string Name, Type Type) AskQuantity => ("AskQty", typeof(double));
public static (string Name, Type Type) AskNotional => ("AskNotl", typeof(double));
public static (string Name, Type Type) DepthPrice(Side side, int level) => ($"{side.BidAskName()}Price{level}", typeof(double));
public static (string Name, Type Type) DepthQuantity(Side side, int level) => ($"{side.BidAskName()}Qty{level}", typeof(double));
public static (string Name, Type Type) DepthNotional(Side side, int level) => ($"{side.BidAskName()}Notl{level}", typeof(double));
public static (string Name, Type Type) DepthPrice(Side side, int level) => (WellKnownColumnNames.DepthPrice(side, level), typeof(double));
public static (string Name, Type Type) DepthQuantity(Side side, int level) => (WellKnownColumnNames.DepthQuantity(side, level), typeof(double));
public static (string Name, Type Type) DepthNotional(Side side, int level) => (WellKnownColumnNames.DepthNotional(side, level), typeof(double));
public static (string Name, Type Type) LegSecurityId(int legNum) => ($"Leg{legNum}SecId", typeof(uint));
public static (string Name, Type Type) LegVenue(int legNum) => ($"Leg{legNum}Venue", typeof(string));
public static (string Name, Type Type) LegSecurityType(int legNum) => ($"Leg{legNum}SecType", typeof(string));
public static (string Name, Type Type) LegAsset(int legNum) => ($"Leg{legNum}Asset", typeof(string));
public static (string Name, Type Type) LegSecurityName(int legNum) => ($"Leg{legNum}Sec", typeof(string));
public static (string Name, Type Type) LegCashPerPoint(int legNum) => ($"Leg{legNum}Cpp", typeof(double));
public static (string Name, Type Type) LegSecurityId(int legNum) => (WellKnownColumnNames.LegSecurityId(legNum), typeof(uint));
public static (string Name, Type Type) LegVenue(int legNum) => (WellKnownColumnNames.LegVenue(legNum), typeof(string));
public static (string Name, Type Type) LegSecurityType(int legNum) => (WellKnownColumnNames.LegSecurityType(legNum), typeof(string));
public static (string Name, Type Type) LegAsset(int legNum) => (WellKnownColumnNames.LegAsset(legNum), typeof(string));
public static (string Name, Type Type) LegSecurityName(int legNum) => (WellKnownColumnNames.LegSecurityName(legNum), typeof(string));
public static (string Name, Type Type) LegCashPerPoint(int legNum) => (WellKnownColumnNames.LegCashPerPoint(legNum), typeof(double));
}
@@ -0,0 +1,79 @@
namespace Spark.Core.Collections;
using bts.utils;
using System;
using System.Collections;
using System.Collections.Generic;
public class ArrayList<T> : IList<T> {
T[] items;
int count;
public ArrayList(T[] items) {
this.items = items;
count = items.Length;
}
public ArrayList(T[] items, int count) {
ArgumentOutOfRangeException.ThrowIfNegative(count);
ArgumentOutOfRangeException.ThrowIfGreaterThan(count, items.Length);
this.items = items;
this.count = count;
}
public T this[int index] {
get => items[index];
set => items[index] = value;
}
public int Count => count;
public bool IsReadOnly => false;
public void Add(T item) =>
ArrayHelper.Add(ref items, item, ref count);
public void Clear() =>
count = 0;
public bool Contains(T item) =>
IndexOf(item) != -1;
public void CopyTo(T[] array, int arrayIndex) =>
items.CopyTo(new Span<T>(array, arrayIndex, count));
public IEnumerator<T> GetEnumerator() {
for (int i = 0; i < count; i++) {
yield return items[i];
}
}
public int IndexOf(T item) {
var comp = EqualityComparer<T>.Default;
for (int i = 0; i < count; i++) {
if (comp.Equals(items[i], item)) {
return i;
}
}
return -1;
}
public void Insert(int index, T item) =>
ArrayHelper.Insert(ref items, index, item, ref count);
public bool Remove(T item) {
int idx = IndexOf(item);
if (idx == -1) {
return false;
}
ArrayHelper.remove_at(ref items, ref count, idx);
return true;
}
public void RemoveAt(int index) =>
ArrayHelper.remove_at(ref items, ref count, index);
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
@@ -10,6 +10,7 @@ namespace bts.collections {
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading.Tasks;
public static class ListExtensions {
public static bool AddDistinct<T>(this IList<T> list, T value) =>
@@ -286,6 +287,15 @@ namespace bts.collections {
return false;
}
public static IList<TResult> ParallelSelect<T, TResult>(this IList<T> source, Func<T, TResult> selector) {
var results = new TResult[source.Count];
Parallel.For(0, source.Count, SetResult);
void SetResult(int i) {
results[i] = selector(source[i]);
}
return new ArrayList<TResult>(results);
}
public static void sync_with<T>(
this IList<T> list,
IEnumerable<T> source,
+17
View File
@@ -243,6 +243,23 @@ namespace bts.utils {
return clone;
}
public static T[] Add<T>(ref T[] array, T item, ref int count) {
ArgumentOutOfRangeException.ThrowIfGreaterThan(count, array.Length);
ArgumentOutOfRangeException.ThrowIfNegative(count);
if (count == array.Length) {
if (array.Length == 0) {
array = new T[1];
} else {
var temp = array;
array = new T[checked(array.Length * 2)];
Array.Copy(temp, 0, array, 0, count);
}
}
array[count] = item;
count++;
return array;
}
public static void add_sorted<T>(ref T[] array, ref int size, T value, IComparer<T> comparer) =>
add_sorted(ref array, ref size, value, comparer, out int pos);
@@ -0,0 +1,22 @@
#nullable enable
namespace Spark.Core;
using System;
using System.Diagnostics.CodeAnalysis;
public static class DelegateUtils {
[return:NotNullIfNotNull(nameof(pred1))]
[return:NotNullIfNotNull(nameof(pred2))]
public static Func<T, bool>? CombinePredicate<T>(Func<T, bool>? pred1, Func<T, bool>? pred2) {
if (pred1 is null) {
return pred2;
}
if (pred2 is null) {
return pred1;
}
return And;
bool And(T t) =>
pred1(t) && pred2(t);
}
}