feat: Add historic market data streams factory
Loads historic md from Databento files in structured paths. It is designed to work with files downloaded via databento_client.py
This commit is contained in:
1 parent
2549e34a17
commit
0cf44fcc40
34 files changed
+2205
-173
No files matched your search
@@ -9,6 +9,7 @@ using bts.collections;
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
@@ -76,9 +77,43 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
|
||||
}
|
||||
}
|
||||
|
||||
public Builder ToBuilder() =>
|
||||
new(this);
|
||||
|
||||
public DataPacketBuilder ToDataPacketBuilder() =>
|
||||
new(this);
|
||||
|
||||
public class Builder : IEnumerable<IBlueRow> {
|
||||
readonly IBlueTableSchema schema;
|
||||
readonly ImmutableSortedSetDictionary<int, IBlueRow>.Builder rowMap;
|
||||
|
||||
public Builder(BlueTableImmutable source) {
|
||||
schema = source.schema;
|
||||
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
|
||||
}
|
||||
|
||||
public void AddRange(IEnumerable<KeyValuePair<int, IBlueRow>> rows) =>
|
||||
rowMap.AddRange(rows);
|
||||
|
||||
public void AddOrUpdate(IBlueRow row) =>
|
||||
AddOrUpdate(row.Id(), row);
|
||||
|
||||
public void AddOrUpdate(int id, IBlueRow row) =>
|
||||
rowMap[id] = row;
|
||||
|
||||
public bool Remove(int id) =>
|
||||
rowMap.Remove(id);
|
||||
|
||||
public BlueTableImmutable ToImmutable() =>
|
||||
new(schema, rowMap.ToImmutable().Source);
|
||||
|
||||
public IEnumerator<IBlueRow> GetEnumerator() =>
|
||||
rowMap.Values.GetEnumerator();
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator() =>
|
||||
GetEnumerator();
|
||||
}
|
||||
|
||||
public class DataPacketBuilder {
|
||||
readonly IBlueTableSchema schema;
|
||||
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder table;
|
||||
@@ -354,9 +389,46 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection {
|
||||
}
|
||||
}
|
||||
|
||||
public Builder ToBuilder() =>
|
||||
new(this);
|
||||
|
||||
public DataPacketBuilder ToDataPacketBuilder() =>
|
||||
new(this);
|
||||
|
||||
public class Builder : IEnumerable<IBlueRow> {
|
||||
readonly BlueTableSchemaAdapter<T> schemaAdapter;
|
||||
readonly ImmutableSortedSetDictionary<int, T>.Builder rowMap;
|
||||
|
||||
public Builder(BlueTableImmutableAdapter<T> source) {
|
||||
schemaAdapter = source.schemaAdapter;
|
||||
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
|
||||
}
|
||||
|
||||
public void AddRange(IEnumerable<KeyValuePair<int, T>> rows) =>
|
||||
rowMap.AddRange(rows);
|
||||
|
||||
public void AddOrUpdate(T row) =>
|
||||
AddOrUpdate((int)schemaAdapter["id"][row]!, row);
|
||||
|
||||
public void AddOrUpdate(int id, T row) =>
|
||||
rowMap[id] = row;
|
||||
|
||||
public bool Remove(int id) =>
|
||||
rowMap.Remove(id);
|
||||
|
||||
public BlueTableImmutableAdapter<T> ToImmutable() =>
|
||||
new(schemaAdapter, rowMap.ToImmutable().Source);
|
||||
|
||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||
foreach (var r in rowMap.Values) {
|
||||
yield return schemaAdapter.GetBlueRow(r);
|
||||
}
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator() =>
|
||||
GetEnumerator();
|
||||
}
|
||||
|
||||
public class DataPacketBuilder {
|
||||
readonly BlueTableSchemaAdapter<T> schemaAdapter;
|
||||
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table =
|
||||
|
||||
@@ -73,6 +73,21 @@ public class BlueTableSchemaAdapter<T> : IMutableBlueRowFactory<T>, IBlueTableSc
|
||||
columnNames = null;
|
||||
}
|
||||
|
||||
public void AddColumn<TResult>(string name, Func<T, TResult?> getValue) {
|
||||
var dataType = typeof(TResult);
|
||||
if (!ColumnInfo.isAllowedType(dataType)) {
|
||||
throw new ArgumentException(
|
||||
$"'{dataType}' is not a supported column type", nameof(dataType));
|
||||
}
|
||||
// Invalidate cached schema (set it to null)
|
||||
schema = null;
|
||||
var adapter = new ColumnAdapter(new(name, columns.Count, dataType), r => getValue(r));
|
||||
columns.Add(adapter);
|
||||
columnNameIndex[name] = adapter;
|
||||
columnInfos = null;
|
||||
columnNames = null;
|
||||
}
|
||||
|
||||
public ColumnAdapter this[int columnIndex] =>
|
||||
columns[columnIndex];
|
||||
|
||||
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
public static class DatabentoBboAdapter {
|
||||
static readonly string[] keyColumnNames = new[] { "ts_event", "bid_px_00", "bid_sz_00", "ask_px_00", "ask_sz_00" };
|
||||
|
||||
public static ColumnIndices ResolveColumns(ColumnInfo[] columns) =>
|
||||
new(columns.ResolveIndices(keyColumnNames));
|
||||
|
||||
public static Func<IBlueRow, HistoricBboRowData?> GetRowDataFactory(Func<TimeseriesKey<InstrumentKey>, int> getId, Func<IBlueRow, HistoricInstrumentRowData?> getInstrument, ColumnInfo[]? columns = null) {
|
||||
ColumnIndices? colIndicesBacking = null;
|
||||
if (columns is not null) {
|
||||
colIndicesBacking = ResolveColumns(columns);
|
||||
}
|
||||
return RowDataFactoryWrapper;
|
||||
|
||||
HistoricBboRowData? RowDataFactoryWrapper(IBlueRow row) {
|
||||
colIndicesBacking ??= ResolveColumns(row.schema.columnInfo);
|
||||
return RowDataFactory(row, colIndicesBacking.Value);
|
||||
}
|
||||
|
||||
HistoricBboRowData? RowDataFactory(IBlueRow mbp1Row, in ColumnIndices colIndices) {
|
||||
var inst = getInstrument(mbp1Row);
|
||||
if (inst is null) {
|
||||
return null;
|
||||
}
|
||||
var ts = mbp1Row.GetDateTime(colIndices.TimestampIndex) ?? default;
|
||||
var bidPrice = mbp1Row.GetFloat64(colIndices.BidPriceIndex) ?? double.NaN;
|
||||
var bidQuantity = mbp1Row.GetFloat64(colIndices.BidQuantityIndex) ?? 0;
|
||||
var askPrice = mbp1Row.GetFloat64(colIndices.BidPriceIndex) ?? double.NaN;
|
||||
var askQuantity = mbp1Row.GetFloat64(colIndices.AskQuantityIndex) ?? 0;
|
||||
int id = getId(inst.Key);
|
||||
var rd = new HistoricBboRowData(id, inst, ts, bidPrice, bidQuantity, askPrice, askQuantity);
|
||||
return rd;
|
||||
}
|
||||
}
|
||||
|
||||
public readonly struct ColumnIndices {
|
||||
public readonly int TimestampIndex;
|
||||
public readonly int BidPriceIndex;
|
||||
public readonly int BidQuantityIndex;
|
||||
public readonly int AskPriceIndex;
|
||||
public readonly int AskQuantityIndex;
|
||||
|
||||
public ColumnIndices(int[] keyColumnIndices)
|
||||
: this(
|
||||
keyColumnIndices[0],
|
||||
keyColumnIndices[1],
|
||||
keyColumnIndices[2],
|
||||
keyColumnIndices[3],
|
||||
keyColumnIndices[4]) { }
|
||||
|
||||
public ColumnIndices(int timestampIndex, int bidPriceIndex, int bidQuantityIndex, int askPriceIndex, int askQuantityIndex) {
|
||||
TimestampIndex = timestampIndex;
|
||||
BidPriceIndex = bidPriceIndex;
|
||||
BidQuantityIndex = bidQuantityIndex;
|
||||
AskPriceIndex = askPriceIndex;
|
||||
AskQuantityIndex = askQuantityIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
Whitespace-only changes.
+179
@@ -0,0 +1,179 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
using bts.api;
|
||||
using bts.BlueOrb;
|
||||
using bts.co2;
|
||||
|
||||
using Bts.DataStream;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Linq;
|
||||
using Bts.DataStreamCore.StreamFactories;
|
||||
using Spark.Core;
|
||||
|
||||
public static class DatabentoFileStore {
|
||||
const int defaultMaxRowCount = 1_500_000;
|
||||
|
||||
public static IEnumerable<(string Path, DateTime Date)> GetDataFiles(string databentoSchema, string? continuousContract, SecurityType? secType, Venue? venue = null, DateTime? beginDate = null, DateTime? endDate = null) {
|
||||
var localAppDataDir = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
|
||||
var mbp1Dir = Path.Combine(localAppDataDir, "bts", "databento", databentoSchema);
|
||||
var mbp1ChildDirs = Directory.GetDirectories(mbp1Dir);
|
||||
var preview = new List<string>();
|
||||
if (continuousContract is not null) {
|
||||
foreach (var d in mbp1ChildDirs) {
|
||||
var p = Path.Combine(d, continuousContract);
|
||||
if (Directory.Exists(p)) {
|
||||
if (p.Contains("HG")) {
|
||||
}
|
||||
preview.Add(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
// var preview = mbp1ChildDirs.Select(d => Path.Combine(d, continuousContract!)).Where(Directory.Exists).ToList();
|
||||
IEnumerable<string> symDirs = continuousContract is null
|
||||
? mbp1ChildDirs.SelectMany(Directory.GetDirectories)
|
||||
: mbp1ChildDirs.Select(d => Path.Combine(d, continuousContract)).Where(Directory.Exists);
|
||||
var files = symDirs.SelectMany(Directory.GetFiles).Select(GetFile).SelectWhereValues();
|
||||
if (beginDate.HasValue) {
|
||||
files = files.Where(f => f.Date > beginDate.Value - TimeSpan.FromDays(1));
|
||||
}
|
||||
if (endDate.HasValue) {
|
||||
files = files.Where(f => f.Date < endDate.Value);
|
||||
}
|
||||
return files;
|
||||
|
||||
(string Path, DateTime Date)? GetFile(string f) {
|
||||
if (f is null) {
|
||||
return null;
|
||||
}
|
||||
var parts = f.Split(Path.DirectorySeparatorChar);
|
||||
Log.assert(parts.Length >= 3);
|
||||
var fileVenueStr = parts[^3];
|
||||
var sym = parts[^2];
|
||||
var name = Path.GetFileName(f);
|
||||
var match = Regex.Match(name, $"{databentoSchema}_{fileVenueStr}_{sym}_(STOCK|SPOT|FUT|OPT)_(\\d{{8}})\\.parquet");
|
||||
if (!match.Success) {
|
||||
return null;
|
||||
}
|
||||
if (secType.HasValue) {
|
||||
var fileSecType = match.Groups[1].Value switch {
|
||||
"STOCK" or "SPOT" => SecurityType.Spot,
|
||||
"FUT" => SecurityType.Future,
|
||||
"OPT" => SecurityType.Option,
|
||||
_ => throw new InvalidOperationException("Failed to parse SecurityType from filename"),
|
||||
};
|
||||
if (fileSecType != secType.Value) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (venue.HasValue) {
|
||||
if (!MarketDataArgStore.TryParseVenue(fileVenueStr, out var fileVenue)) {
|
||||
return null;
|
||||
}
|
||||
if (fileVenue != venue.Value) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
int dateInt = int.Parse(match.Groups[2].Value);
|
||||
PeelPow10(ref dateInt, 10_000, out int year);
|
||||
PeelPow10(ref dateInt, 100, out int month);
|
||||
var date = new DateTime(year, month, dateInt);
|
||||
return (f, date);
|
||||
|
||||
void PeelPow10(ref int i, int pow10, out int peeled) {
|
||||
peeled = i / pow10;
|
||||
i -= peeled * pow10;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static IEnumerable<Func<Task<IBlueRowCollection>>> GetDefinitionAsync(string? asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func<IBlueRow, bool>? filter = null) {
|
||||
filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter);
|
||||
var files = GetDataFiles("definition", asset, secType, beginDate: beginDate, endDate: endDate);
|
||||
return files.Select(f => new Func<Task<IBlueRowCollection>>(() => GetDefinitionAsync(f.Path, filter)));
|
||||
}
|
||||
|
||||
public static async Task<IBlueRowCollection> GetDefinitionAsync(string path, Func<IBlueRow, bool>? filter = null) {
|
||||
// string columnMaskExpression = "bid_px_00:bid_price bid_sz_00:bid_size ask_px_00:ask_price ask_sz_00:ask_size symbol:contract";
|
||||
string? columnMaskExpression = null;
|
||||
return await GetDataAsync(path, columnMaskExpression, filter);
|
||||
}
|
||||
|
||||
public static IEnumerable<Func<Task<IBlueRowCollection>>> GetBboAsync(string asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func<IBlueRow, bool>? filter = null) {
|
||||
filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter);
|
||||
var tables = GetDataFiles("mbp-1", asset, secType, beginDate: beginDate, endDate: endDate)
|
||||
.Select(f => new Func<Task<IBlueRowCollection>>(() => GetBboAsync(f.Path, filter)));
|
||||
return tables;
|
||||
}
|
||||
|
||||
public static async Task<IBlueRowCollection> GetBboAsync(string path, Func<IBlueRow, bool>? filter = null) {
|
||||
string columnMaskExpression = "bid_px_00:bid_price bid_sz_00:bid_size ask_px_00:ask_price ask_sz_00:ask_size symbol:contract";
|
||||
return await GetDataAsync(path, columnMaskExpression, filter);
|
||||
}
|
||||
|
||||
public static IEnumerable<Func<Task<IBlueRowCollection>>> GetTradesAsync(string asset, SecurityType? secType, DateTime? beginDate = null, DateTime? endDate = null, Func<IBlueRow, bool>? filter = null) {
|
||||
filter = DelegateUtils.CombinePredicate(getTimeFilter(beginDate, endDate), filter);
|
||||
var tables = GetDataFiles("trades", asset, secType, beginDate: beginDate, endDate: endDate)
|
||||
.Select(f => new Func<Task<IBlueRowCollection>>(() => GetTradesAsync(f.Path, filter)));
|
||||
return tables;
|
||||
}
|
||||
|
||||
public static async Task<IBlueRowCollection> GetTradesAsync(string path, Func<IBlueRow, bool>? filter = null) {
|
||||
string columnMaskExpression = "id ts_event price:trade_price size:trade_size side:aggressor_side symbol:contract";
|
||||
var table = await GetDataAsync(path, columnMaskExpression, filter);
|
||||
var schema = table.schema;
|
||||
var rowFactory = ColumnSelectorRowFactory.Create(schema, ("aggressor_side", typeof(AggressorSide), ConvertSide));
|
||||
return new SelectorTable(table, rowFactory.OutputSchema, rowFactory.Create);
|
||||
|
||||
static object? ConvertSide(object? s) {
|
||||
char c = s as char? ?? ((s as string ?? "N")[0]);
|
||||
return c switch {
|
||||
'b' or 'B' => AggressorSide.Buy,
|
||||
's' or 'S' or 'a' or 'A' => AggressorSide.Sell,
|
||||
_ => AggressorSide.None,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public static Task<IBlueRowCollection> GetDataAsync(string path, string? columnMaskExpression, Func<IBlueRow, bool>? filter) =>
|
||||
GetDataAsync(path, columnMaskExpression, 0, defaultMaxRowCount, filter);
|
||||
|
||||
public static async Task<IBlueRowCollection> GetDataAsync(string path, string? columnMaskExpression, int? start, int? stop, Func<IBlueRow, bool>? filter) {
|
||||
var table = await ParquetTableReader.ReadAsync(path, start, stop);
|
||||
if (columnMaskExpression is not null) {
|
||||
table = ColumnMaskTable.Create(table, columnMaskExpression);
|
||||
}
|
||||
if (filter is not null) {
|
||||
table = new FilterTable(table, filter);
|
||||
}
|
||||
return table;
|
||||
}
|
||||
|
||||
[return: NotNullIfNotNull(nameof(beginDate))]
|
||||
[return: NotNullIfNotNull(nameof(endDate))]
|
||||
static Func<IBlueRow, bool>? getTimeFilter(DateTime? beginDate, DateTime? endDate) {
|
||||
if (!beginDate.HasValue && !endDate.HasValue) {
|
||||
return null;
|
||||
}
|
||||
return TimeFilter;
|
||||
|
||||
bool TimeFilter(IBlueRow r) {
|
||||
var ts = r.GetDateTime("ts_event");
|
||||
if (!ts.HasValue) {
|
||||
return false;
|
||||
}
|
||||
if (beginDate.HasValue && ts.Value < beginDate) {
|
||||
return false;
|
||||
}
|
||||
if (endDate.HasValue && ts.Value >= endDate) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.api;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
public static class DatabentoInstrumentAdapter {
|
||||
static readonly string[] keyColumnNames = new[] { "activation", "exchange", "asset", "instrument_class", "currency", "symbol", "instrument_id", "expiration", "unit_of_measure_qty" };
|
||||
|
||||
public static ColumnIndices ResolveColumns(ColumnInfo[] columns) =>
|
||||
new(columns.ResolveIndices(keyColumnNames));
|
||||
|
||||
public static Func<IBlueRow, HistoricInstrumentRowData> GetRowDataFactory(Func<TimeseriesKey<InstrumentKey>, int> getId, ColumnInfo[]? columns = null) {
|
||||
ColumnIndices? colIndicesBacking = null;
|
||||
if (columns is not null) {
|
||||
colIndicesBacking = ResolveColumns(columns);
|
||||
}
|
||||
return RowDataFactoryWrapper;
|
||||
|
||||
HistoricInstrumentRowData RowDataFactoryWrapper(IBlueRow row) {
|
||||
colIndicesBacking ??= ResolveColumns(row.schema.columnInfo);
|
||||
return RowDataFactory(row, colIndicesBacking.Value);
|
||||
}
|
||||
|
||||
HistoricInstrumentRowData RowDataFactory(IBlueRow row, in ColumnIndices colIndices) {
|
||||
var ts = row.GetDateTime(colIndices.ActivationIndex) ?? default;
|
||||
var venue = row.GetString(colIndices.VenueIndex) switch {
|
||||
"XCME" => Venue.Glbx,
|
||||
_ => throw new NotImplementedException(),
|
||||
};
|
||||
var secType = GetSecurityType(row.GetString(colIndices.InstrumentClassIndex));
|
||||
var asset = row.GetString(colIndices.AssetIndex) ?? string.Empty;
|
||||
var currency = row.GetString(colIndices.CurrencyIndex) ?? string.Empty;
|
||||
var name = row.GetString(colIndices.NameIndex) ?? string.Empty;
|
||||
var exp = row.GetDateTime(colIndices.ExpirationIndex) ?? GetExpiration(row.GetInt64(colIndices.ExpirationIndex));
|
||||
var cpp = row.GetFloat64(colIndices.UnitOfMeasureQtyIndex) ?? double.NaN;
|
||||
switch (secType) {
|
||||
case SecurityType.Future:
|
||||
case SecurityType.Option:
|
||||
if (!exp.HasValue) {
|
||||
throw new ArgumentException($"Expiration is null for security of type '{secType}'", nameof(row));
|
||||
}
|
||||
break;
|
||||
case SecurityType.Spread:
|
||||
break;
|
||||
default:
|
||||
if (exp.HasValue) {
|
||||
throw new ArgumentException($"Expiration is non-null for security of type '{secType}'", nameof(row));
|
||||
}
|
||||
break;
|
||||
}
|
||||
int id = getId(new((venue, name), ts));
|
||||
return new(id, ts, venue, secType, asset, currency, name, exp, cpp);
|
||||
}
|
||||
|
||||
static SecurityType GetSecurityType(string? instrumentClass) {
|
||||
// Translate Databento's instrument_class codes to BTS SecurityType
|
||||
// https://databento.com/docs/schemas-and-data-formats/instrument-definitions
|
||||
return instrumentClass switch {
|
||||
"B" /*Bond*/ or "K" /*Stock*/ or "X" /*FX spot*/ or "Y" /*Commodity spot*/ => SecurityType.Spot,
|
||||
"F" => SecurityType.Future,
|
||||
"C" /*Call*/ or "P" /*Put*/ => SecurityType.Option,
|
||||
"M" /*Mixed spread*/ or "S" /*Future spread*/ or "T" /*Option spread*/ => SecurityType.Spread,
|
||||
_ => SecurityType.Unset,
|
||||
};
|
||||
}
|
||||
|
||||
static DateTime? GetExpiration(long? expirationTimestamp) {
|
||||
if (!expirationTimestamp.HasValue) {
|
||||
return null;
|
||||
}
|
||||
var ticks = expirationTimestamp.Value / 100;
|
||||
return new(ticks);
|
||||
}
|
||||
}
|
||||
|
||||
public readonly struct ColumnIndices {
|
||||
public readonly int ActivationIndex;
|
||||
public readonly int VenueIndex;
|
||||
public readonly int AssetIndex;
|
||||
public readonly int InstrumentClassIndex;
|
||||
public readonly int CurrencyIndex;
|
||||
public readonly int NameIndex;
|
||||
public readonly int InstrumentIdIndex;
|
||||
public readonly int ExpirationIndex;
|
||||
public readonly int UnitOfMeasureQtyIndex;
|
||||
|
||||
public ColumnIndices(int[] keyColumnIndices)
|
||||
: this(
|
||||
keyColumnIndices[0],
|
||||
keyColumnIndices[1],
|
||||
keyColumnIndices[2],
|
||||
keyColumnIndices[3],
|
||||
keyColumnIndices[4],
|
||||
keyColumnIndices[5],
|
||||
keyColumnIndices[6],
|
||||
keyColumnIndices[7],
|
||||
keyColumnIndices[8]) { }
|
||||
|
||||
public ColumnIndices(int activationIndex, int venueIndex, int assetIndex, int instrumentClassIndex, int currencyIndex, int nameIndex, int instrumentIdIndex, int expirationIndex, int unitOfMeasureQtyIndex) {
|
||||
ActivationIndex = activationIndex;
|
||||
VenueIndex = venueIndex;
|
||||
AssetIndex = assetIndex;
|
||||
InstrumentClassIndex = instrumentClassIndex;
|
||||
CurrencyIndex = currencyIndex;
|
||||
NameIndex = nameIndex;
|
||||
InstrumentIdIndex = instrumentIdIndex;
|
||||
ExpirationIndex = expirationIndex;
|
||||
UnitOfMeasureQtyIndex = unitOfMeasureQtyIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using bts.api;
|
||||
|
||||
using Bts.DataStreamCore.StreamFactories;
|
||||
|
||||
public class HistoricBboParams {
|
||||
public HistoricBboParams() { }
|
||||
|
||||
public HistoricBboParams(string venueStr, string asset)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, null) { }
|
||||
|
||||
public HistoricBboParams(string venueStr, string asset, string typeStr)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr)) { }
|
||||
|
||||
public HistoricBboParams(string venueStr, string asset, string typeStr, DateTime date)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), date) { }
|
||||
|
||||
public HistoricBboParams(string venueStr, string asset, string typeStr, DateTime beginDate, DateTime endDate)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), beginDate, endDate) { }
|
||||
|
||||
public HistoricBboParams(string venueStr, string asset, DateTime date)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, date) { }
|
||||
|
||||
public HistoricBboParams(Venue venue, string asset, DateTime date)
|
||||
: this(venue, asset, null, date) { }
|
||||
|
||||
public HistoricBboParams(Venue venue, string asset, DateTime beginDate, DateTime endDate)
|
||||
: this(venue, asset, null, beginDate, endDate) { }
|
||||
|
||||
public HistoricBboParams(Venue venue, string asset, SecurityType? type) {
|
||||
Venue = venue;
|
||||
Asset = asset;
|
||||
Type = type;
|
||||
}
|
||||
|
||||
public HistoricBboParams(Venue venue, string asset, SecurityType? type, DateTime date)
|
||||
: this(venue, asset, type, date, date.AddDays(1)) { }
|
||||
|
||||
public HistoricBboParams(Venue venue, string asset, SecurityType? type, DateTime beginDate, DateTime endDate) {
|
||||
Venue = venue;
|
||||
Asset = asset;
|
||||
Type = type;
|
||||
BeginDate = beginDate;
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
public Venue? Venue { get; set; }
|
||||
public string? Asset { get; set; }
|
||||
public SecurityType? Type { get; set; }
|
||||
public string? Name { get; set; }
|
||||
public bool ShowVenue { get; set; } = true;
|
||||
public bool ShowAsset { get; set; } = true;
|
||||
public bool ShowType { get; set; } = true;
|
||||
public bool ShowName { get; set; } = true;
|
||||
public bool ShowExpiration { get; set; }
|
||||
public bool ShowCashPerPoint { get; set; } = true;
|
||||
public bool ShowCurrency { get; set; } = true;
|
||||
public bool ShowNotionalValue { get; set; } = true;
|
||||
public DateTime? BeginDate { get; set; }
|
||||
public DateTime? EndDate { get; set; }
|
||||
|
||||
public HistoricInstrumentParams GetInstrumentParams() =>
|
||||
new() {
|
||||
Venue = Venue,
|
||||
Asset = Asset,
|
||||
Type = Type,
|
||||
Name = Name,
|
||||
ShowVenue = ShowVenue,
|
||||
ShowAsset = ShowAsset,
|
||||
ShowType = ShowType,
|
||||
ShowExpiration = ShowExpiration,
|
||||
ShowCashPerPoint = ShowCashPerPoint,
|
||||
BeginDate = BeginDate,
|
||||
EndDate = EndDate,
|
||||
};
|
||||
|
||||
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
|
||||
var clause = new mbts.spark.SecurityCriteriaClause(
|
||||
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
|
||||
security_types: Type.HasValue ? new[] { Type.Value } : null,
|
||||
symbols: !string.IsNullOrEmpty(Asset) ? new[] { Asset } : null,
|
||||
names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null);
|
||||
return clause.is_defined ? new(new[] { clause }) : new();
|
||||
}
|
||||
|
||||
public HistoricBboSchemaBuilder GetSchemaBuilder() =>
|
||||
new() {
|
||||
ShowVenue = ShowVenue,
|
||||
ShowAsset = ShowAsset,
|
||||
ShowType = ShowType,
|
||||
ShowName = ShowName,
|
||||
ShowExpiration = ShowExpiration,
|
||||
ShowCashPerPoint = ShowCashPerPoint,
|
||||
ShowCurrency = ShowCurrency,
|
||||
ShowNotionalValue = ShowNotionalValue,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using bts.api;
|
||||
|
||||
public class HistoricBboRowData {
|
||||
public HistoricBboRowData(int id, HistoricInstrumentRowData instrument, DateTime timestamp, double bidPrice, double bidQuantity, double askPrice, double askQuantity)
|
||||
: this(
|
||||
id,
|
||||
timestamp,
|
||||
instrument.Venue,
|
||||
instrument.SecurityType,
|
||||
instrument.Asset,
|
||||
instrument.Currency,
|
||||
instrument.Name,
|
||||
instrument.Expiration,
|
||||
instrument.CashPerPoint,
|
||||
bidPrice,
|
||||
bidQuantity,
|
||||
askPrice,
|
||||
askQuantity) { }
|
||||
|
||||
public HistoricBboRowData(
|
||||
int id,
|
||||
DateTime timestamp,
|
||||
Venue venue,
|
||||
SecurityType securityType,
|
||||
string asset,
|
||||
string currency,
|
||||
string name,
|
||||
DateTime? expiration,
|
||||
double cashPerPoint,
|
||||
double bidPrice,
|
||||
double bidQuantity,
|
||||
double askPrice,
|
||||
double askQuantity
|
||||
) {
|
||||
Id = id;
|
||||
Timestamp = timestamp;
|
||||
Venue = venue;
|
||||
SecurityType = securityType;
|
||||
Asset = asset;
|
||||
Currency = currency;
|
||||
Name = name;
|
||||
Expiration = expiration;
|
||||
CashPerPoint = cashPerPoint;
|
||||
BidPrice = bidPrice;
|
||||
BidQuantity = bidQuantity;
|
||||
AskPrice = askPrice;
|
||||
AskQuantity = askQuantity;
|
||||
}
|
||||
|
||||
public int Id { get; }
|
||||
public DateTime Timestamp { get; }
|
||||
public Venue Venue { get; }
|
||||
public SecurityType SecurityType { get; }
|
||||
public TimeseriesKey<InstrumentKey> Key => new((Venue, Name), Timestamp);
|
||||
public string Asset { get; }
|
||||
public string Currency { get; }
|
||||
public string Name { get; }
|
||||
public DateTime? Expiration { get; }
|
||||
public double CashPerPoint { get; }
|
||||
|
||||
public double BidPrice { get; }
|
||||
public double BidQuantity { get; }
|
||||
public double BidNotional => CashPerPoint * BidPrice * BidQuantity;
|
||||
public double AskPrice { get; }
|
||||
public double AskQuantity { get; }
|
||||
public double AskNotional => CashPerPoint * AskPrice * AskQuantity;
|
||||
|
||||
public HistoricBboRowData With(
|
||||
double? bidPrice = null, double? bidQuantity = null, double? askPrice = null, double? askQuantity = null) =>
|
||||
new(
|
||||
Id,
|
||||
Timestamp,
|
||||
Venue,
|
||||
SecurityType,
|
||||
Asset,
|
||||
Currency,
|
||||
Name,
|
||||
Expiration,
|
||||
CashPerPoint,
|
||||
bidPrice ?? BidPrice,
|
||||
bidQuantity ?? BidQuantity,
|
||||
askPrice ?? AskPrice,
|
||||
askQuantity ?? AskQuantity);
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.SharedStreams.StreamFactories;
|
||||
|
||||
public class HistoricBboSchemaBuilder {
|
||||
public bool ShowVenue { get; set; } = true;
|
||||
public bool ShowAsset { get; set; } = true;
|
||||
public bool ShowType { get; set; } = true;
|
||||
public bool ShowName { get; set; } = true;
|
||||
public bool ShowExpiration { get; set; } = true;
|
||||
public bool ShowCashPerPoint { get; set; } = true;
|
||||
public bool ShowCurrency { get; set; } = true;
|
||||
public bool ShowPrice { get; set; } = true;
|
||||
public bool ShowQuantity { get; set; } = true;
|
||||
public bool ShowNotionalValue { get; set; } = true;
|
||||
public string TableName { get; set; } = "HistoricBbo";
|
||||
|
||||
public BlueTableSchemaAdapter<HistoricBboRowData> DatabentoAdapter() {
|
||||
var adapter = new BlueTableSchemaAdapter<HistoricBboRowData>(TableName);
|
||||
adapter.AddColumn("id", r => r.Id);
|
||||
adapter.AddColumn(WellKnownColumnNames.Timestamp, r => r.Timestamp);
|
||||
if (ShowVenue) {
|
||||
adapter.AddColumn(WellKnownColumnNames.Venue, r => r.Venue);
|
||||
}
|
||||
if (ShowAsset) {
|
||||
adapter.AddColumn(WellKnownColumnNames.Asset, r => r.Asset);
|
||||
}
|
||||
if (ShowType) {
|
||||
adapter.AddColumn(WellKnownColumnNames.SecurityType, r => r.SecurityType);
|
||||
}
|
||||
if (ShowName) {
|
||||
adapter.AddColumn(WellKnownColumnNames.SecurityName, r => r.Name);
|
||||
}
|
||||
if (ShowExpiration) {
|
||||
adapter.AddColumn(WellKnownColumnNames.Expiration, r => r.Expiration);
|
||||
}
|
||||
if (ShowCurrency) {
|
||||
adapter.AddColumn(WellKnownColumnNames.Currency, r => r.Currency);
|
||||
}
|
||||
if (ShowCashPerPoint) {
|
||||
adapter.AddColumn(WellKnownColumnNames.CashPerPoint, r => r.Currency);
|
||||
}
|
||||
if (ShowNotionalValue) {
|
||||
adapter.AddColumn(WellKnownColumnNames.BidNotional, r => r.BidNotional);
|
||||
adapter.AddColumn(WellKnownColumnNames.AskNotional, r => r.AskNotional);
|
||||
}
|
||||
if (ShowQuantity) {
|
||||
adapter.AddColumn(WellKnownColumnNames.BidQuantity, r => r.BidQuantity);
|
||||
adapter.AddColumn(WellKnownColumnNames.AskQuantity, r => r.AskQuantity);
|
||||
}
|
||||
if (ShowPrice) {
|
||||
adapter.AddColumn(WellKnownColumnNames.BidPrice, r => r.BidPrice);
|
||||
adapter.AddColumn(WellKnownColumnNames.AskPrice, r => r.AskPrice);
|
||||
}
|
||||
return adapter;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStream.BlueOrb;
|
||||
using Bts.PubSub;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a time series of bbos for a date range.
|
||||
/// </summary>
|
||||
public class HistoricBboStream : ViewStream {
|
||||
public HistoricBboStream(HistoricBboParams streamParams, IWorkScheduler workScheduler, ILogger logger) =>
|
||||
mySetRealSource(Create(streamParams, workScheduler, logger));
|
||||
|
||||
public static ViewStream Create(HistoricBboParams streamParams, IWorkScheduler workScheduler, ILogger logger) =>
|
||||
AsyncViewStream.Create(workScheduler, () => CreateAsync(streamParams, logger));
|
||||
|
||||
public static async Task<ViewStream?> CreateAsync(HistoricBboParams sp, ILogger logger) {
|
||||
logger.LogInformation("Creating HistoricBboStream");
|
||||
var table = await HistoricBboTable.CreateAsync(sp, logger);
|
||||
if (table is null) {
|
||||
return null;
|
||||
}
|
||||
return new BlueRowCollectionStaticStream(table);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
using bts.BlueOrb;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Spark.Core;
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System.Collections.Immutable;
|
||||
|
||||
public class HistoricBboTable : IBlueRowCollection {
|
||||
readonly BlueTableSchemaAdapter<HistoricBboRowData> schemaAdapter;
|
||||
readonly TimeseriesIndex<InstrumentKey> bboIndex;
|
||||
readonly ImmutableSortedSet<KeyValuePair<int, HistoricBboRowData>> rowSet;
|
||||
|
||||
public HistoricBboTable(BlueTableSchemaAdapter<HistoricBboRowData> schemaAdapter) {
|
||||
this.schemaAdapter = schemaAdapter;
|
||||
bboIndex = new();
|
||||
rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, HistoricBboRowData>());
|
||||
}
|
||||
|
||||
HistoricBboTable(
|
||||
BlueTableSchemaAdapter<HistoricBboRowData> schemaAdapter,
|
||||
TimeseriesIndex<InstrumentKey> bboIndex,
|
||||
ImmutableSortedSet<KeyValuePair<int, HistoricBboRowData>> rowSet) {
|
||||
this.schemaAdapter = schemaAdapter;
|
||||
this.bboIndex = bboIndex;
|
||||
this.rowSet = rowSet;
|
||||
}
|
||||
|
||||
public IBlueRow this[int id] =>
|
||||
rowSet.TryGetValue(new(id, null!), out var res)
|
||||
? schemaAdapter.GetBlueRow(res.Value)
|
||||
: throw new KeyNotFoundException();
|
||||
|
||||
public IBlueTableSchema schema => schemaAdapter.Schema;
|
||||
|
||||
public int Count => rowSet.Count;
|
||||
|
||||
IBlueRow IList<IBlueRow>.this[int index] {
|
||||
get => GetRow(index);
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public static async Task<HistoricBboTable?> CreateAsync(HistoricBboParams sp, ILogger logger, bool useContinuousContracts = true) {
|
||||
logger.LogInformation("Creating HistoricBboTable");
|
||||
var beginDate = sp.BeginDate;
|
||||
var endDate = sp.EndDate;
|
||||
var instParams = sp.GetInstrumentParams();
|
||||
var instSchemaBd = instParams.GetSchemaBuilder();
|
||||
var instSchemaAdapter = instSchemaBd.DatabentoAdapter();
|
||||
var mbp1FileList = DatabentoFileStore.GetDataFiles("mbp-1", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToListSorted(f => f.Date);
|
||||
if (!beginDate.HasValue && !endDate.HasValue) {
|
||||
beginDate = mbp1FileList[^1].Date;
|
||||
endDate = beginDate.Value + TimeSpan.FromDays(1);
|
||||
} else {
|
||||
beginDate ??= DateTime.MinValue;
|
||||
endDate ??= DateTime.MaxValue;
|
||||
}
|
||||
mbp1FileList.RemoveAll(f => f.Date < beginDate.Value || f.Date >= endDate.Value);
|
||||
if (mbp1FileList.Count == 0) {
|
||||
return null;
|
||||
}
|
||||
var definitionFileList = DatabentoFileStore.GetDataFiles("definition", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToListSorted(f => f.Date);
|
||||
if (!mbp1FileList.WrapSelector(f => f.Date, null).SequenceEqual(definitionFileList.WrapSelector(f => f.Date, null))) {
|
||||
logger.LogInvalidParams("HistoricBboStream", "Missing one or more definition (instrument) files in date range");
|
||||
return null;
|
||||
}
|
||||
var tablePairs = new (IBlueRowCollection Mbp1, IBlueRowCollection Definition)[mbp1FileList.Count];
|
||||
Parallel.For(0, mbp1FileList.Count, LoadTables);
|
||||
var instrumentIndex = new TimeseriesIndex<InstrumentKey>();
|
||||
var instrumentIndexBd = instrumentIndex.ToBuilder();
|
||||
var bboSchemaBd = sp.GetSchemaBuilder();
|
||||
var bboSchemaAdapter = bboSchemaBd.DatabentoAdapter();
|
||||
var bboTable = new HistoricBboTable(bboSchemaAdapter);
|
||||
foreach (var (mbp1, definition) in tablePairs) {
|
||||
var filteredDefs = definition;
|
||||
if (useContinuousContracts) {
|
||||
filteredDefs = HistoricContinuousContractTable.Create(filteredDefs);
|
||||
}
|
||||
var definitionMap = filteredDefs.ToDictionary(r => r.GetInt64("instrument_id").assert_value());
|
||||
var instRowDataFactory = DatabentoInstrumentAdapter.GetRowDataFactory(instrumentIndexBd.GetOrAdd, filteredDefs.schema.columnInfo);
|
||||
bboTable = bboTable.AddBbos(mbp1, GetInstrumentRowData);
|
||||
|
||||
HistoricInstrumentRowData? GetInstrumentRowData(IBlueRow mbp1Row) {
|
||||
var instId = mbp1Row.GetInt64("instrument_id");
|
||||
Log.notnull(instId);
|
||||
if (!definitionMap.TryGetValue(instId.Value, out var defRow)) {
|
||||
Log.assert(false);
|
||||
}
|
||||
var instRowData = instRowDataFactory(defRow!);
|
||||
return instRowData;
|
||||
}
|
||||
}
|
||||
return bboTable;
|
||||
|
||||
void LoadTables(int index) {
|
||||
var definitionPath = definitionFileList[index].Path;
|
||||
var definition = ParquetTableReader.Read(definitionPath);
|
||||
var mbp1Path = mbp1FileList[index].Path;
|
||||
var mbp1 = ParquetTableReader.Read(mbp1Path);
|
||||
tablePairs[index] = (mbp1, definition);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Add bbo records (<see cref="rows"/>) to this table.
|
||||
/// </summary>
|
||||
/// <param name="mbp1Rows">The bbo records to add (columns should correspond to Databento 'mbp-1' schema)</param>
|
||||
/// <param name="instrumentRows">Instruments table for the bbo records (i.e., for the same time period as bbo)</param>
|
||||
/// <returns>A new table (immutable) with the added rows</returns>
|
||||
public HistoricBboTable AddBbos(IBlueRowCollection mbp1Rows, Func<IBlueRow, HistoricInstrumentRowData?> getInstrument) {
|
||||
var bboIndex = this.bboIndex.ToBuilder();
|
||||
var rowSet = this.rowSet.ToBuilder();
|
||||
var rowDataFactory = DatabentoBboAdapter.GetRowDataFactory(bboIndex.GetOrAdd, getInstrument);
|
||||
foreach (var mbp1Row in mbp1Rows) {
|
||||
var rd = rowDataFactory(mbp1Row);
|
||||
if (rd is null) {
|
||||
continue;
|
||||
}
|
||||
rowSet.Add(new(rd.Id, rd));
|
||||
}
|
||||
return new(schemaAdapter, bboIndex.ToImmutable(), rowSet.ToImmutable());
|
||||
}
|
||||
|
||||
public bool containsID(int id) =>
|
||||
rowSet.Contains(new(id, null!));
|
||||
|
||||
public IBlueRow GetRow(int index) {
|
||||
var rd = rowSet[index].Value;
|
||||
var blueRow = schemaAdapter.GetBlueRow(rd);
|
||||
return blueRow;
|
||||
}
|
||||
|
||||
public int IndexOf(IBlueRow item) {
|
||||
int id = item.Id();
|
||||
int idx = rowSet.IndexOf(new(id, null!));
|
||||
if (idx >= 0) {
|
||||
var rd = rowSet[idx].Value;
|
||||
var blueRow = schemaAdapter.GetBlueRow(rd);
|
||||
if (BlueRowEqualityComparer.Equals(blueRow, item)) {
|
||||
return idx;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||
foreach (var kv in rowSet) {
|
||||
yield return schemaAdapter.GetBlueRow(kv.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.api;
|
||||
using bts.BlueOrb;
|
||||
|
||||
using Bts.DataStream;
|
||||
using Bts.SharedStreams.StreamFactories;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a time series of continuous contracts for a date range. This can be joined with
|
||||
/// a timeseries of bbo data, for example, to get continuous bbo.
|
||||
/// </summary>
|
||||
public class HistoricContinuousContractStream : ViewStream {
|
||||
public HistoricContinuousContractStream(ViewStream historicInstruments) =>
|
||||
mySetRealSource(Create(historicInstruments));
|
||||
|
||||
public static ViewStream? Create(ViewStream historicInstruments) {
|
||||
var vs = historicInstruments.GroupByApply(new[] { WellKnownColumnNames.Venue, WellKnownColumnNames.SecurityType, WellKnownColumnNames.Asset }, Apply);
|
||||
return vs;
|
||||
|
||||
static IBlueRowCollection Apply(object?[] groupKeys, IBlueRowCollection inputRows) {
|
||||
var secType = (SecurityType)groupKeys[1]!;
|
||||
switch (secType) {
|
||||
case SecurityType.Future:
|
||||
case SecurityType.Option:
|
||||
break;
|
||||
default:
|
||||
return inputRows;
|
||||
}
|
||||
var schema = inputRows.schema;
|
||||
var minExpRow = inputRows.Select(r => (Row: r, Exp: r.GetDateTime(WellKnownColumnNames.Expiration)))
|
||||
.Where(t => t.Exp.HasValue)
|
||||
.MinElement(t => t.Exp!.Value)
|
||||
.Row;
|
||||
if (minExpRow is null) {
|
||||
return inputRows;
|
||||
}
|
||||
return new SingleRowTable(minExpRow);
|
||||
}
|
||||
}
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
using bts.api;
|
||||
using bts.BlueOrb;
|
||||
|
||||
using Bts.DataStream;
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.SharedStreams.StreamFactories;
|
||||
|
||||
using Spark.Core;
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a time series of continuous contracts for a date range. This can be joined with
|
||||
/// a timeseries of bbo data, for example, to get continuous bbo.
|
||||
/// </summary>
|
||||
public class HistoricContinuousContractTable : IBlueRowCollection {
|
||||
readonly IBlueRowCollection impl;
|
||||
|
||||
public HistoricContinuousContractTable(IBlueRowCollection instruments) =>
|
||||
impl = Create(instruments);
|
||||
|
||||
public IBlueTableSchema schema => impl.schema;
|
||||
|
||||
public int Count => impl.Count;
|
||||
|
||||
public IBlueRow this[int id] => impl[id];
|
||||
|
||||
IBlueRow IList<IBlueRow>.this[int index] {
|
||||
get => ((IList<IBlueRow>)impl)[index];
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public static IBlueRowCollection Create(IBlueRowCollection instruments) {
|
||||
var instGroups = instruments.GroupBy(GetKey)
|
||||
.ToDictionary(g => g.Key, g => g.ToListSorted(r => GetExpiration(r) ?? DateTime.MaxValue));
|
||||
var bd = new BlueTableImmutable(instruments.schema).ToBuilder();
|
||||
foreach (var (cck, contracts) in instGroups) {
|
||||
Log.assert(contracts.Count > 0);
|
||||
switch (cck.Type) {
|
||||
case SecurityType.Future:
|
||||
bd.AddOrUpdate(contracts[0]);
|
||||
break;
|
||||
case SecurityType.Option:
|
||||
throw new NotImplementedException("Options not implemented");
|
||||
case SecurityType.Spread:
|
||||
foreach (var c in contracts) {
|
||||
bd.AddOrUpdate(c);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
Log.assert(contracts.Count == 1);
|
||||
bd.AddOrUpdate(contracts[0]);
|
||||
break;
|
||||
}
|
||||
bd.AddOrUpdate(contracts[0]);
|
||||
}
|
||||
var tableOut = bd.ToImmutable();
|
||||
return tableOut;
|
||||
|
||||
static DateTime? GetExpiration(IBlueRow inst) {
|
||||
var exp = inst.GetDateTime(WellKnownColumnNames.Expiration);
|
||||
return exp;
|
||||
}
|
||||
|
||||
static ContinuousContractKey GetKey(IBlueRow inst) {
|
||||
var venue = inst.GetEnum<Venue>(WellKnownColumnNames.Venue).assert_value();
|
||||
var asset = inst.GetString(WellKnownColumnNames.Asset);
|
||||
Log.notnull(asset);
|
||||
var type = inst.GetEnum<SecurityType>(WellKnownColumnNames.SecurityType).assert_value();
|
||||
return new(venue, asset, type);
|
||||
}
|
||||
}
|
||||
|
||||
public bool containsID(int id) =>
|
||||
impl.containsID(id);
|
||||
|
||||
public IEnumerator<IBlueRow> GetEnumerator() =>
|
||||
impl.GetEnumerator();
|
||||
|
||||
public IBlueRow GetRow(int index) =>
|
||||
impl.GetRow(index);
|
||||
|
||||
public int IndexOf(IBlueRow item) =>
|
||||
impl.IndexOf(item);
|
||||
|
||||
readonly struct ContinuousContractKey {
|
||||
public readonly Venue Venue;
|
||||
public readonly string Asset;
|
||||
public readonly SecurityType Type;
|
||||
|
||||
public ContinuousContractKey(Venue venue, string asset, SecurityType type) {
|
||||
Venue = venue;
|
||||
Asset = asset;
|
||||
Type = type;
|
||||
}
|
||||
}
|
||||
}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using bts.api;
|
||||
|
||||
using Bts.DataStreamCore.StreamFactories;
|
||||
|
||||
public class HistoricInstrumentParams {
|
||||
public HistoricInstrumentParams() { }
|
||||
|
||||
public HistoricInstrumentParams(string venueStr, string asset)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset) { }
|
||||
|
||||
public HistoricInstrumentParams(string venueStr, string asset, string typeStr)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr)) { }
|
||||
|
||||
public HistoricInstrumentParams(string venueStr, string asset, string typeStr, DateTime date)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), date) { }
|
||||
|
||||
public HistoricInstrumentParams(string venueStr, string asset, string typeStr, DateTime beginDate, DateTime endDate)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, MarketDataArgStore.ParseSecurityType(typeStr), beginDate, endDate) { }
|
||||
|
||||
public HistoricInstrumentParams(string venueStr, string asset, DateTime date)
|
||||
: this(MarketDataArgStore.ParseVenue(venueStr), asset, date) { }
|
||||
|
||||
public HistoricInstrumentParams(Venue venue, string asset) {
|
||||
Venue = venue;
|
||||
Asset = asset;
|
||||
}
|
||||
|
||||
public HistoricInstrumentParams(Venue venue, string asset, DateTime date)
|
||||
: this(venue, asset, null, date) { }
|
||||
|
||||
public HistoricInstrumentParams(Venue venue, string asset, DateTime beginDate, DateTime endDate)
|
||||
: this(venue, asset, null, beginDate, endDate) { }
|
||||
|
||||
public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type) {
|
||||
Venue = venue;
|
||||
Asset = asset;
|
||||
Type = type;
|
||||
}
|
||||
|
||||
public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type, DateTime date)
|
||||
: this(venue, asset, type, date, date.AddDays(1)) { }
|
||||
|
||||
public HistoricInstrumentParams(Venue venue, string asset, SecurityType? type, DateTime beginDate, DateTime endDate) {
|
||||
Venue = venue;
|
||||
Asset = asset;
|
||||
Type = type;
|
||||
BeginDate = beginDate;
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
public Venue? Venue { get; set; }
|
||||
public string? Asset { get; set; }
|
||||
public SecurityType? Type { get; set; }
|
||||
public string? Name { get; set; }
|
||||
public int NumLegs { get; set; } = 5;
|
||||
public bool ShowVenue { get; set; } = true;
|
||||
public bool ShowType { get; set; } = true;
|
||||
public bool ShowAsset { get; set; } = true;
|
||||
public bool ShowExpiration { get; set; }
|
||||
public bool ShowCashPerPoint { get; set; } = true;
|
||||
public bool ShowLegVenue { get; set; } = true;
|
||||
public bool ShowLegType { get; set; } = true;
|
||||
public bool ShowLegAsset { get; set; } = true;
|
||||
public bool ShowLegName { get; set; } = true;
|
||||
public bool ShowLegExpiration { get; set; }
|
||||
public bool ShowLegCashPerPoint { get; set; } = true;
|
||||
public DateTime? BeginDate { get; set; }
|
||||
public DateTime? EndDate { get; set; }
|
||||
|
||||
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
|
||||
var clause = new mbts.spark.SecurityCriteriaClause(
|
||||
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
|
||||
security_types: Type.HasValue ? new[] { Type.Value } : null,
|
||||
symbols: !string.IsNullOrEmpty(Asset) ? new[] { Asset } : null,
|
||||
names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null);
|
||||
return clause.is_defined ? new(new[] { clause }) : new();
|
||||
}
|
||||
|
||||
public HistoricInstrumentSchemaBuilder GetSchemaBuilder() =>
|
||||
new() {
|
||||
ShowVenue = ShowVenue,
|
||||
ShowType = true,
|
||||
ShowAsset = true,
|
||||
ShowName = true,
|
||||
ShowCashPerPoint = ShowCashPerPoint,
|
||||
ShowLegVenue = ShowLegVenue,
|
||||
ShowLegAsset = ShowLegAsset,
|
||||
ShowLegType = ShowLegType,
|
||||
ShowLegName = ShowLegVenue,
|
||||
};
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using bts.api;
|
||||
|
||||
using System.Collections.Immutable;
|
||||
|
||||
public class HistoricInstrumentRowData {
|
||||
public HistoricInstrumentRowData(
|
||||
int id,
|
||||
DateTime timestamp,
|
||||
Venue venue,
|
||||
SecurityType securityType,
|
||||
string asset,
|
||||
string currency,
|
||||
string name,
|
||||
DateTime? expiration,
|
||||
double cashPerPoint,
|
||||
ImmutableList<HistoricInstrumentRowData>? legs = null
|
||||
) {
|
||||
Id = id;
|
||||
Timestamp = timestamp;
|
||||
Venue = venue;
|
||||
SecurityType = securityType;
|
||||
Asset = asset;
|
||||
Currency = currency;
|
||||
Name = name;
|
||||
Expiration = expiration;
|
||||
CashPerPoint = cashPerPoint;
|
||||
Legs = legs;
|
||||
}
|
||||
|
||||
public int Id { get; }
|
||||
public DateTime Timestamp { get; }
|
||||
public Venue Venue { get; }
|
||||
public SecurityType SecurityType { get; }
|
||||
public TimeseriesKey<InstrumentKey> Key => new((Venue, Name), Timestamp);
|
||||
public string Asset { get; }
|
||||
public string Currency { get; }
|
||||
public string Name { get; }
|
||||
public DateTime? Expiration { get; }
|
||||
public double CashPerPoint { get; }
|
||||
public int NumLegs => Legs?.Count ?? 0;
|
||||
// Some legs might not be resolved
|
||||
public ImmutableList<HistoricInstrumentRowData>? Legs { get; }
|
||||
|
||||
public HistoricInstrumentRowData AddLegs(IEnumerable<HistoricInstrumentRowData> legs) =>
|
||||
new(Id, Timestamp, Venue, SecurityType, Asset, Currency, Name, Expiration, CashPerPoint, ImmutableList.CreateRange(legs));
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.SharedStreams.StreamFactories;
|
||||
|
||||
public class HistoricInstrumentSchemaBuilder {
|
||||
public bool ShowVenue { get; set; } = true;
|
||||
public bool ShowAsset { get; set; } = true;
|
||||
public bool ShowType { get; set; } = true;
|
||||
public bool ShowName { get; set; } = true;
|
||||
public bool ShowExpiration { get; set; } = true;
|
||||
public bool ShowCashPerPoint { get; set; } = true;
|
||||
public bool ShowCurrency { get; set; } = true;
|
||||
public bool ShowLegVenue { get; set; }
|
||||
public bool ShowLegAsset { get; set; }
|
||||
public bool ShowLegType { get; set; }
|
||||
public bool ShowLegName { get; set; } = true;
|
||||
public bool ShowLegCashPerPoint { get; set; }
|
||||
public int NumLegs { get; set; } = 2;
|
||||
public string TableName { get; set; } = "HistoricInstrument";
|
||||
|
||||
public BlueTableSchemaAdapter<HistoricInstrumentRowData> DatabentoAdapter() {
|
||||
var adapter = new BlueTableSchemaAdapter<HistoricInstrumentRowData>(TableName);
|
||||
adapter.AddColumn("id", r => r.Id);
|
||||
adapter.AddColumn(WellKnownColumnNames.Timestamp, r => r.Timestamp);
|
||||
if (ShowVenue) {
|
||||
adapter.AddColumn(WellKnownColumnNames.Venue, r => r.Venue);
|
||||
}
|
||||
if (ShowType) {
|
||||
adapter.AddColumn(WellKnownColumnNames.SecurityType, r => r.SecurityType);
|
||||
}
|
||||
if (ShowAsset) {
|
||||
adapter.AddColumn(WellKnownColumnNames.Asset, r => r.Asset);
|
||||
}
|
||||
if (ShowName) {
|
||||
adapter.AddColumn(WellKnownColumnNames.SecurityName, r => r.Name);
|
||||
}
|
||||
if (ShowExpiration) {
|
||||
adapter.AddColumn(WellKnownColumnNames.Expiration, r => r.Expiration);
|
||||
}
|
||||
if (ShowCurrency) {
|
||||
adapter.AddColumn(WellKnownColumnNames.Currency, r => r.Currency);
|
||||
}
|
||||
if (ShowCashPerPoint) {
|
||||
adapter.AddColumn(WellKnownColumnNames.CashPerPoint, r => r.Currency);
|
||||
}
|
||||
if (NumLegs > 0) {
|
||||
adapter.AddColumn(WellKnownColumnNames.NumLegs, r => r.NumLegs);
|
||||
for (int i = 0; i < NumLegs; i++) {
|
||||
// Copy i to a local var (legIndex) so it can be referenced in column getters.
|
||||
int legIndex = i;
|
||||
if (ShowLegVenue) {
|
||||
adapter.AddColumn(WellKnownColumnNames.LegVenue(i), Get(l => l.Venue));
|
||||
}
|
||||
if (ShowLegType) {
|
||||
adapter.AddColumn(WellKnownColumnNames.LegSecurityType(i), Get(l => l.SecurityType));
|
||||
}
|
||||
if (ShowLegAsset) {
|
||||
adapter.AddColumn(WellKnownColumnNames.LegAsset(i), Get(l => l.Asset));
|
||||
}
|
||||
if (ShowLegName) {
|
||||
adapter.AddColumn(WellKnownColumnNames.LegSecurityName(i), Get(l => l.Name));
|
||||
}
|
||||
if (ShowLegCashPerPoint) {
|
||||
adapter.AddColumn(WellKnownColumnNames.LegCashPerPoint(i), Get(l => l.CashPerPoint, double.NaN));
|
||||
}
|
||||
|
||||
Func<HistoricInstrumentRowData, T> Get<T>(Func<HistoricInstrumentRowData, T> getValue, T defaultValue = default!) =>
|
||||
s => {
|
||||
var legs = s.Legs;
|
||||
if (legs is null || !legs.TryGetAt(legIndex, out var leg)) {
|
||||
return defaultValue;
|
||||
}
|
||||
return getValue(leg);
|
||||
};
|
||||
}
|
||||
}
|
||||
return adapter;
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStream.BlueOrb;
|
||||
using Bts.PubSub;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a time series of instruments for a date range.
|
||||
/// </summary>
|
||||
public class HistoricInstrumentStream : ViewStream {
|
||||
public HistoricInstrumentStream(HistoricInstrumentParams streamParams, IWorkScheduler workScheduler, ILogger logger) =>
|
||||
mySetRealSource(Create(streamParams, workScheduler, logger));
|
||||
|
||||
public static ViewStream Create(HistoricInstrumentParams streamParams, IWorkScheduler workScheduler, ILogger logger) =>
|
||||
AsyncViewStream.Create(workScheduler, () => CreateAsync(streamParams, logger));
|
||||
|
||||
public static async Task<ViewStream?> CreateAsync(HistoricInstrumentParams sp, ILogger logger) {
|
||||
logger.LogInformation("Creating HistoricInstrumentStream");
|
||||
var table = await HistoricInstrumentTable.CreateAsync(sp, logger);
|
||||
if (table is null) {
|
||||
return null;
|
||||
}
|
||||
return new BlueRowCollectionStaticStream(table);
|
||||
}
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
using bts.api;
|
||||
using bts.BlueOrb;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System.Collections.Immutable;
|
||||
|
||||
public class HistoricInstrumentTable : IBlueRowCollection {
|
||||
readonly BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter;
|
||||
readonly TimeseriesIndex<InstrumentKey> instrumentIndex;
|
||||
readonly ImmutableSortedSet<KeyValuePair<int, HistoricInstrumentRowData>> rowSet;
|
||||
|
||||
public HistoricInstrumentTable(BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter) {
|
||||
this.schemaAdapter = schemaAdapter;
|
||||
instrumentIndex = new();
|
||||
rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, HistoricInstrumentRowData>());
|
||||
}
|
||||
|
||||
public HistoricInstrumentTable(BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter, TimeseriesIndex<InstrumentKey> instrumentIndex) {
|
||||
this.schemaAdapter = schemaAdapter;
|
||||
this.instrumentIndex = instrumentIndex;
|
||||
rowSet = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, HistoricInstrumentRowData>());
|
||||
}
|
||||
|
||||
HistoricInstrumentTable(
|
||||
BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter,
|
||||
TimeseriesIndex<InstrumentKey> instrumentIndex,
|
||||
ImmutableSortedSet<KeyValuePair<int, HistoricInstrumentRowData>> rowSet) {
|
||||
this.schemaAdapter = schemaAdapter;
|
||||
this.instrumentIndex = instrumentIndex;
|
||||
this.rowSet = rowSet;
|
||||
}
|
||||
|
||||
public IBlueRow this[int id] =>
|
||||
rowSet.TryGetValue(new(id, null!), out var res)
|
||||
? schemaAdapter.GetBlueRow(res.Value)
|
||||
: throw new KeyNotFoundException();
|
||||
|
||||
public IBlueTableSchema schema => schemaAdapter.Schema;
|
||||
|
||||
public int Count => rowSet.Count;
|
||||
|
||||
IBlueRow IList<IBlueRow>.this[int index] {
|
||||
get => GetRow(index);
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public static HistoricInstrumentTable Create(BlueTableSchemaAdapter<HistoricInstrumentRowData> schemaAdapter, IBlueRowCollection definitionRows) {
|
||||
var table = new HistoricInstrumentTable(schemaAdapter);
|
||||
table = table.AddInstruments(definitionRows);
|
||||
return table;
|
||||
}
|
||||
|
||||
public static async Task<HistoricInstrumentTable?> CreateAsync(HistoricInstrumentParams sp, ILogger logger) {
|
||||
logger.LogInformation("Creating HistoricInstrumentTable");
|
||||
var beginDate = sp.BeginDate;
|
||||
var endDate = sp.EndDate;
|
||||
var fileList = DatabentoFileStore.GetDataFiles("definition", sp.Asset, sp.Type, sp.Venue, beginDate, endDate).ToList();
|
||||
if (!beginDate.HasValue && !endDate.HasValue) {
|
||||
beginDate = fileList[^1].Date;
|
||||
endDate = beginDate.Value + TimeSpan.FromDays(1);
|
||||
} else {
|
||||
beginDate ??= DateTime.MinValue;
|
||||
endDate ??= DateTime.MaxValue;
|
||||
}
|
||||
fileList.RemoveAll(f => f.Date < beginDate.Value || f.Date >= endDate.Value);
|
||||
if (fileList.Count == 0) {
|
||||
return null;
|
||||
}
|
||||
var rowMapBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer<int, HistoricInstrumentRowData>());
|
||||
var rowSetBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer<int, IBlueRow>());
|
||||
var dateTables = fileList.ParallelSelect(f => ParquetTableReader.Read(f.Path));
|
||||
var tableRowMap = ImmutableDictionary.CreateBuilder<int, HistoricInstrumentRowData>();
|
||||
var schemaBd = sp.GetSchemaBuilder();
|
||||
var schemaAdapter = schemaBd.DatabentoAdapter();
|
||||
var nameToId = new Dictionary<(Venue, string), int>();
|
||||
var idToName = new Dictionary<int, (Venue, string)>();
|
||||
var table = new HistoricInstrumentTable(schemaAdapter);
|
||||
foreach (var dateTable in dateTables) {
|
||||
table = table.AddInstruments(dateTable);
|
||||
}
|
||||
return table;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Add instruments (<see cref="rows"/>) to this table. If there are spreads, the legs' contracts
|
||||
/// should be included in <see cref="rows"/> as well.
|
||||
/// </summary>
|
||||
/// <param name="definitionRows">A table with columns corresponding to the Databento 'definition' schema</param>
|
||||
/// <returns></returns>
|
||||
/// <exception cref="ArgumentException">Glbx spread does not have the correct number of legs (2)</exception>
|
||||
/// <exception cref="NotImplementedException">Spread handler code not implemented at a particular venue</exception>
|
||||
public HistoricInstrumentTable AddInstruments(IBlueRowCollection definitionRows) {
|
||||
var instrumentIndex = this.instrumentIndex.ToBuilder();
|
||||
var rowSet = this.rowSet.ToBuilder();
|
||||
var spreadRows = new List<HistoricInstrumentRowData>();
|
||||
var rowDataFactory = DatabentoInstrumentAdapter.GetRowDataFactory(instrumentIndex.GetOrAdd, definitionRows.schema.columnInfo);
|
||||
foreach (var r in definitionRows) {
|
||||
var rd = rowDataFactory(r);
|
||||
rowSet.Add(new(rd.Id, rd));
|
||||
if (rd.SecurityType == SecurityType.Spread) {
|
||||
spreadRows.Add(rd);
|
||||
}
|
||||
}
|
||||
// Resolve spread legs
|
||||
for (int i = 0; i < spreadRows.Count; i++) {
|
||||
var rd = spreadRows[i];
|
||||
switch (rd.Venue) {
|
||||
case Venue.Glbx:
|
||||
var legSymbols = rd.Name.Split('-');
|
||||
if (legSymbols.Length != 2) {
|
||||
throw new ArgumentException($"Glbx spread has {legSymbols.Length} legs (expected 2)", nameof(definitionRows));
|
||||
}
|
||||
var legs = legSymbols.Select(legSym => {
|
||||
var legId = instrumentIndex.GetLatest((rd.Venue, legSym));
|
||||
bool foundLeg = rowSet.TryGetValue(new(legId, null!), out var leg);
|
||||
Log.assert(foundLeg);
|
||||
return leg.Value;
|
||||
});
|
||||
rowSet.Remove(new(rd.Id, rd));
|
||||
rowSet.Add(new(rd.Id, rd.AddLegs(legs)));
|
||||
break;
|
||||
default:
|
||||
throw new NotImplementedException($"Spreads not implemented at venue '{rd.Venue}'");
|
||||
}
|
||||
}
|
||||
return new(schemaAdapter, instrumentIndex.ToImmutable(), rowSet.ToImmutable());
|
||||
}
|
||||
|
||||
public bool containsID(int id) =>
|
||||
instrumentIndex.ContainsId(id);
|
||||
|
||||
public IBlueRow GetRow(int index) {
|
||||
var rd = rowSet[index].Value;
|
||||
var blueRow = schemaAdapter.GetBlueRow(rd);
|
||||
return blueRow;
|
||||
}
|
||||
|
||||
public int IndexOf(IBlueRow item) {
|
||||
int id = item.Id();
|
||||
int idx = rowSet.IndexOf(new(id, null!));
|
||||
if (idx >= 0) {
|
||||
var rd = rowSet[idx].Value;
|
||||
var blueRow = schemaAdapter.GetBlueRow(rd);
|
||||
if (BlueRowEqualityComparer.Equals(blueRow, item)) {
|
||||
return idx;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||
foreach (var kv in rowSet) {
|
||||
yield return schemaAdapter.GetBlueRow(kv.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using bts.api;
|
||||
|
||||
public record InstrumentKey(Venue Venue, string Name) {
|
||||
public static implicit operator (Venue Venue, string Name)(InstrumentKey key) =>
|
||||
(key.Venue, key.Name);
|
||||
|
||||
public static implicit operator InstrumentKey((Venue Venue, string Name) tup) =>
|
||||
new(tup.Venue, tup.Name);
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System.Collections.Immutable;
|
||||
|
||||
/// <summary>
|
||||
/// Maps a timeseries of keys to unique int32 ids.
|
||||
/// </summary>
|
||||
public class TimeseriesIndex<T> where T : notnull {
|
||||
static readonly IComparer<KeyValuePair<DateTime, int>> idTimeseriesComp =
|
||||
KeyValuePairComparer.KeyComparer<DateTime, int>();
|
||||
|
||||
readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId;
|
||||
readonly ImmutableDictionary<int, TimeseriesKey<T>> idToName;
|
||||
readonly int idCounter;
|
||||
|
||||
public TimeseriesIndex()
|
||||
: this(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Empty, ImmutableDictionary<int, TimeseriesKey<T>>.Empty, 0) { }
|
||||
|
||||
TimeseriesIndex(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
|
||||
this.nameToId = nameToId;
|
||||
this.idToName = idToName;
|
||||
this.idCounter = idCounter;
|
||||
}
|
||||
|
||||
public bool ContainsId(int id) =>
|
||||
idToName.ContainsKey(id);
|
||||
|
||||
public int GetNearest(TimeseriesKey<T> key) {
|
||||
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
|
||||
throw new KeyNotFoundException();
|
||||
}
|
||||
int idx = ids.BinarySearch(new(key.Timestamp, -1));
|
||||
if (idx < 0) {
|
||||
idx = ~idx;
|
||||
}
|
||||
if (idx == 0) {
|
||||
throw new KeyNotFoundException();
|
||||
}
|
||||
return ids[idx].Value;
|
||||
}
|
||||
|
||||
public TimeseriesIndex<T> GetNearestOrAdd(TimeseriesKey<T> key, out int result) {
|
||||
int idx;
|
||||
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
|
||||
ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
|
||||
idx = 0;
|
||||
} else {
|
||||
idx = ids.BinarySearch(new(key.Timestamp, -1));
|
||||
if (idx < 0) {
|
||||
idx = ~idx;
|
||||
}
|
||||
}
|
||||
if (idx == 0) {
|
||||
result = idCounter + 1;
|
||||
ids = ids.Insert(0, new(key.Timestamp, result));
|
||||
return new(
|
||||
nameToId.SetItem(key.Name, ids),
|
||||
idToName.SetItem(result, key),
|
||||
result);
|
||||
}
|
||||
result = ids[idx].Value;
|
||||
return this;
|
||||
}
|
||||
|
||||
public TimeseriesIndex<T> GetOrAdd(TimeseriesKey<T> key, out int result) {
|
||||
int idx;
|
||||
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
|
||||
ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
|
||||
idx = 0;
|
||||
} else {
|
||||
idx = ids.BinarySearch(new(key.Timestamp, -1));
|
||||
if (idx < 0) {
|
||||
idx = ~idx;
|
||||
} else {
|
||||
result = ids[idx].Value;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
result = idCounter + 1;
|
||||
ids = ids.Insert(idx, new(key.Timestamp, result));
|
||||
return new(
|
||||
nameToId.SetItem(key.Name, ids),
|
||||
idToName.SetItem(result, key),
|
||||
result);
|
||||
}
|
||||
|
||||
public IList<KeyValuePair<DateTime, int>> GetIds(T name) =>
|
||||
nameToId[name];
|
||||
|
||||
public Builder ToBuilder() =>
|
||||
new(this);
|
||||
|
||||
public class Builder {
|
||||
readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Builder nameToId;
|
||||
readonly ImmutableDictionary<int, TimeseriesKey<T>>.Builder idToName;
|
||||
int idCounter;
|
||||
|
||||
public Builder(TimeseriesIndex<T> source)
|
||||
: this(source.nameToId, source.idToName, source.idCounter) { }
|
||||
|
||||
public Builder(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
|
||||
this.nameToId = nameToId.ToBuilder();
|
||||
this.idToName = idToName.ToBuilder();
|
||||
this.idCounter = idCounter;
|
||||
}
|
||||
|
||||
public int GetNearest(TimeseriesKey<T> key) =>
|
||||
getOrAdd(key, false, false);
|
||||
|
||||
public int GetNearestOrAdd(TimeseriesKey<T> key) =>
|
||||
getOrAdd(key, false, true);
|
||||
|
||||
public int GetOrAdd(TimeseriesKey<T> key) =>
|
||||
getOrAdd(key, true, true);
|
||||
|
||||
public int GetLatest(T key) =>
|
||||
nameToId.TryGetValue(key, out var ids)
|
||||
? ids[^1].Value
|
||||
: throw new KeyNotFoundException();
|
||||
|
||||
public bool TryGetLatest(T key, out int result) {
|
||||
if (nameToId.TryGetValue(key, out var ids)) {
|
||||
result = ids[^1].Value;
|
||||
return true;
|
||||
}
|
||||
result = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
public ImmutableList<KeyValuePair<DateTime, int>> GetIds(T name) =>
|
||||
nameToId.TryGetValue(name, out var ids)
|
||||
? ids
|
||||
: ImmutableList<KeyValuePair<DateTime, int>>.Empty;
|
||||
|
||||
public TimeseriesIndex<T> ToImmutable() =>
|
||||
new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter);
|
||||
|
||||
int getOrAdd(TimeseriesKey<T> key, bool requireExactTimestamp, bool canAdd) {
|
||||
var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList<KeyValuePair<DateTime, int>>.Empty);
|
||||
int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp);
|
||||
int id;
|
||||
if (idx < 0) {
|
||||
// We didn't get an exact match. If requireExactTimestamp is false and a record with an
|
||||
// earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp.
|
||||
idx = ~idx;
|
||||
if (requireExactTimestamp || idx == 0) {
|
||||
if (!canAdd) {
|
||||
throw new KeyNotFoundException();
|
||||
}
|
||||
id = ++idCounter;
|
||||
ids = ids.Insert(idx, new(key.Timestamp, id));
|
||||
nameToId[key.Name] = ids;
|
||||
idToName.Add(id, key);
|
||||
} else {
|
||||
if (idx == ids.Count) {
|
||||
--idx;
|
||||
}
|
||||
id = ids[idx].Value;
|
||||
}
|
||||
} else {
|
||||
id = ids[idx].Value;
|
||||
}
|
||||
return id;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace Bts.SharedStreams.MarketData;
|
||||
|
||||
public record TimeseriesKey<T>(T Name, DateTime Timestamp) {
|
||||
public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey<T> key) =>
|
||||
(key.Name, key.Timestamp);
|
||||
|
||||
public static implicit operator TimeseriesKey<T>((T Name, DateTime Timestamp) tup) =>
|
||||
new(tup.Name, tup.Timestamp);
|
||||
}
|
||||
@@ -100,7 +100,7 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
}
|
||||
|
||||
[DataStreamUserRespresentation("bbo")]
|
||||
[DataStreamUserRespresentation(false)]
|
||||
public class BBOStream : ViewStream, IWorker {
|
||||
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
|
||||
private DateTime _last_poll_time = DateTime.MinValue;
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
namespace Bts.SharedStreams.Services;
|
||||
|
||||
using bts.SilverPine;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.SharedStreams.StreamFactories;
|
||||
|
||||
@@ -17,10 +15,15 @@ public static class SharedStreamsServices {
|
||||
.addStreamFactory<SpatialFuturePairsStreamFactory>()
|
||||
.addStreamFactory<SpatialArbStatsStreamFactory>();
|
||||
|
||||
public static IServiceCollection AddHistoricMarketDataStreams(this IServiceCollection services) =>
|
||||
services.addStreamFactory<HistoricParquetDataLoader>();
|
||||
|
||||
public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) =>
|
||||
services.Configure<NaturalLanguageStreamFactoryConfig>(configurationSection)
|
||||
.AddSingleton<NaturalLanguageStreamFactory>()
|
||||
.AddSingleton<IDataStreamFactory, NaturalLanguageStreamFactory>(sp => sp.GetRequiredService<NaturalLanguageStreamFactory>());
|
||||
.addStreamFactory<NaturalLanguageStreamFactory>();
|
||||
|
||||
public static IServiceCollection AddParquetReaderStream(this IServiceCollection services) =>
|
||||
services.addStreamFactory<ParquetReaderStreamFactory>();
|
||||
|
||||
static IServiceCollection addStreamFactory<T>(this IServiceCollection services) where T : class, IDataStreamFactory =>
|
||||
services.AddSingleton<T>()
|
||||
|
||||
@@ -2,21 +2,26 @@ namespace Bts.SharedStreams.StreamFactories;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.md;
|
||||
|
||||
using Bts.DataStream;
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.DataStreamCore.StreamFactories;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Spark.Api.MarketData;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory {
|
||||
readonly LiveBboStreamFactory liveBboFactory;
|
||||
IDataStreamFactory? compositeFactory = null;
|
||||
|
||||
public class BookStreamFactory : DataStreamFactory {
|
||||
public BookStreamFactory(IMarketDataClient? mdClient, ILogger<BookStreamFactory> logger) {
|
||||
this.Add<BookStreamParams>("book", CreateBook, logger);
|
||||
this.Add<BookStreamParams>(new[] { "bbo", "LiveBBOStream" }, CreateBbo, logger);
|
||||
this.Add<BookStreamParams>(new[] { "bbo" }, CreateBbo, logger);
|
||||
liveBboFactory = new LiveBboStreamFactory(mdClient, logger);
|
||||
Add(liveBboFactory);
|
||||
|
||||
ViewStream? CreateBook(BookStreamParams streamParams) =>
|
||||
mdClient?.Books(streamParams.GetSecurityCriteria())
|
||||
@@ -24,10 +29,97 @@ public class BookStreamFactory : DataStreamFactory {
|
||||
?.AsViewStream()
|
||||
?? throw new InvalidOperationException("Live md not available");
|
||||
|
||||
ViewStream? CreateBbo(BookStreamParams streamParams) =>
|
||||
mdClient?.Books(streamParams.GetSecurityCriteria())
|
||||
ViewStream? CreateBbo(BookStreamParams streamParams) {
|
||||
return mdClient?.Books(streamParams.GetSecurityCriteria())
|
||||
?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
|
||||
?.AsViewStream()
|
||||
?? throw new InvalidOperationException("Live md not available");
|
||||
}
|
||||
}
|
||||
|
||||
public void OnCompositeFactory(IDataStreamFactory compositeFactory) {
|
||||
this.compositeFactory = compositeFactory;
|
||||
liveBboFactory.OnCompositeFactory(compositeFactory);
|
||||
}
|
||||
|
||||
class LiveBboStreamFactory : IDataStreamFactory, IRecursiveDataStreamFactory {
|
||||
static readonly IDataStreamMetadata metadata = new DataStreamMetadata() {
|
||||
Name = "LiveBBOStream",
|
||||
AlternateNames = new(),
|
||||
Signatures = new() {
|
||||
new (string?, Type?)[] {
|
||||
("symbol", typeof(string))
|
||||
},
|
||||
new (string?, Type?)[] {
|
||||
("secType", typeof(string)),
|
||||
("symbol", typeof(string)),
|
||||
}
|
||||
}
|
||||
};
|
||||
readonly IMarketDataClient? mdClient;
|
||||
readonly ILogger logger;
|
||||
IDataStreamFactory? compositeFactory = null;
|
||||
|
||||
public LiveBboStreamFactory(IMarketDataClient? mdClient, ILogger logger) {
|
||||
this.mdClient = mdClient;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public void OnCompositeFactory(IDataStreamFactory compositeFactory) =>
|
||||
this.compositeFactory = compositeFactory;
|
||||
|
||||
public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
|
||||
if (!metadata.IsMatch(streamId)) {
|
||||
return null;
|
||||
}
|
||||
var argList = args.ToList();
|
||||
switch (argList.Count) {
|
||||
case 1: {
|
||||
var sym = argList[0].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args));
|
||||
if (sym == "ES" && compositeFactory is not null) {
|
||||
return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, sym) });
|
||||
}
|
||||
var streamParams = new BookStreamParams() {
|
||||
Symbol = sym
|
||||
};
|
||||
return mdClient?.Books(streamParams.GetSecurityCriteria())
|
||||
?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
|
||||
?.AsViewStream()
|
||||
?? throw new InvalidOperationException("Live md not available");
|
||||
}
|
||||
case 2: {
|
||||
var secTypeStr = argList[0].Value?.ToString() ?? string.Empty;
|
||||
if (!MarketDataArgStore.TryParseSecurityType(secTypeStr, out var secType)) {
|
||||
throw new ArgumentException("Failed to parse SecurityType arg", nameof(args));
|
||||
}
|
||||
var sym = argList[1].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args));
|
||||
if (sym == "ES" && secType == bts.api.SecurityType.Future && compositeFactory is not null) {
|
||||
return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, sym) });
|
||||
}
|
||||
var streamParams = new BookStreamParams() {
|
||||
SecurityType = secType,
|
||||
Symbol = sym,
|
||||
};
|
||||
return mdClient?.Books(streamParams.GetSecurityCriteria())
|
||||
?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
|
||||
?.AsViewStream()
|
||||
?? throw new InvalidOperationException("Live md not available");
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
|
||||
yield return metadata;
|
||||
}
|
||||
}
|
||||
|
||||
public class LiveBboStreamParams {
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex }, Position = 0, IsRequired = false)]
|
||||
public string? SecurityType { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex }, Position = 1, IsRequired = false)]
|
||||
public string? ContinuousContract { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
namespace Bts.SharedStreams.StreamFactories;
|
||||
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
public interface IHistoricDataParams {
|
||||
string? DataType { get; }
|
||||
string? Venue { get; }
|
||||
string? Symbol { get; }
|
||||
DateTime BeginDate { get; }
|
||||
DateTime EndDate { get; }
|
||||
TimeSpan? Interval { get; }
|
||||
string? TsNanosCol { get; }
|
||||
}
|
||||
|
||||
public class HistoricDataParams : IHistoricDataParams {
|
||||
public string? DataType { get; set; }
|
||||
public string? Venue { get; set; }
|
||||
public string? Symbol { get; set; }
|
||||
public DateTime BeginDate { get; set; }
|
||||
public DateTime EndDate { get; set; }
|
||||
public TimeSpan? Interval { get; set; }
|
||||
public string? TsNanosCol { get; set; }
|
||||
|
||||
[MemberNotNull(nameof(DataType), nameof(Venue), nameof(Symbol), nameof(TsNanosCol))]
|
||||
public HistoricDataParams Validate() {
|
||||
validateNotNull(DataType);
|
||||
validateNotNull(Venue);
|
||||
validateNotNull(Symbol);
|
||||
validateNotNull(TsNanosCol);
|
||||
return this;
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(Interval))]
|
||||
public HistoricDataParams ValidateInterval() {
|
||||
validateNotNull(Interval);
|
||||
return this;
|
||||
}
|
||||
|
||||
static void validateNotNull<T>([NotNull]T arg, [CallerArgumentExpression(nameof(arg))]string argName = "") {
|
||||
if (arg is null) {
|
||||
throw new InvalidOperationException($"Invalid HistoricDataParams: {argName} is null");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class HistoricDataParamsExtensions {
|
||||
// public static void Validate(this IHistoricDataParams hdp) {
|
||||
// validateNotNull(hdp.DataType);
|
||||
// validateNotNull(hdp.Venue);
|
||||
// validateNotNull(hdp.Symbol);
|
||||
// validateNotNull(hdp.TsNanosCol);
|
||||
// }
|
||||
|
||||
// static void validateNotNull<T>([NotNull]T arg, [CallerArgumentExpression(nameof(arg))]string argName = "") {
|
||||
// if (arg is null) {
|
||||
// throw new InvalidOperationException($"Invalid HistoricDataParams: {argName} is null");
|
||||
// }
|
||||
// }
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
namespace Bts.SharedStreams.StreamFactories;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
using bts.api;
|
||||
using bts.BlueOrb;
|
||||
using bts.BlueOrb.Parquet;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStream;
|
||||
using Bts.DataStream.BlueOrb;
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.DataStreamCore.StreamFactories;
|
||||
using Bts.SharedStreams.MarketData;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System.Collections;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Reactive.Linq;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
class HistoricParquetDataLoader : DataStreamFactory {
|
||||
public HistoricParquetDataLoader(ViewStreamDatabase viewStreamDatabase, ILogger<HistoricParquetDataLoader> logger) {
|
||||
this.Add<HistoricBboParams>("HistoricBbo", CreateHistoricBbo, logger);
|
||||
this.Add<ReplayFutureLiveBboParams>("ReplayFutureLiveBBOStream", CreateReplayFutureLiveBbo, logger);
|
||||
this.Add<ReplayFutureLiveTradeParams>("ReplayFutureLiveTradeStream", CreateReplayFutureLiveTrade, logger);
|
||||
this.Add<HistoricInstrumentParams>("HistoricInstrument", sp => HistoricInstrumentStream.Create(sp, viewStreamDatabase, logger));
|
||||
this.Add<HistoricInstrumentParams>("HistoricContinuousContract", sp => {
|
||||
var historicInstruments = HistoricInstrumentStream.Create(sp, viewStreamDatabase, logger);
|
||||
var vs = HistoricContinuousContractStream.Create(historicInstruments);
|
||||
return vs;
|
||||
});
|
||||
|
||||
static SecurityType? ParseSecTypeArg(string? secTypeStr) {
|
||||
if (string.IsNullOrEmpty(secTypeStr)) {
|
||||
return null;
|
||||
}
|
||||
if (MarketDataArgStore.TryParseSecurityType(secTypeStr, out var secType)) {
|
||||
return secType;
|
||||
}
|
||||
throw new ArgumentException("Failed to parse SecurityType", nameof(secTypeStr));
|
||||
}
|
||||
|
||||
// static Venue? ParseVenueArg(string? venueStr) {
|
||||
// if (string.IsNullOrEmpty(venueStr)) {
|
||||
// return null;
|
||||
// }
|
||||
// if (MarketDataArgStore.TryParseVenue(venueStr, out var venue)) {
|
||||
// return venue;
|
||||
// }
|
||||
// throw new ArgumentException("Failed to parse Venue", nameof(venueStr));
|
||||
// }
|
||||
|
||||
// bool TryGetHistoricInstrumentStreamParams(IEnumerable<(string? Name, int? Index, object? Value)> args, [NotNullWhen(true)] out HistoricInstrumentStreamParams? result, [NotNullWhen(false)] out string? errorMessage) {
|
||||
// var argList = args.ToList();
|
||||
// result = null;
|
||||
// errorMessage = null;
|
||||
// switch (argList.Count) {
|
||||
// case 2: {
|
||||
// var venue = ParseVenueArg(argList[0].Value as string);
|
||||
// var asset = argList[1].Value as string;
|
||||
// if (venue.HasValue && asset is not null) {
|
||||
// var secType = SecurityType.Future;
|
||||
// if (DatabentoFileStore.GetDataFiles("definition", asset, secType).TryGetMaxElement(f => f.Date, out var latestFile)) {
|
||||
// result = new(venue.Value, asset, latestFile.Date) {
|
||||
// Type = secType,
|
||||
// };
|
||||
// }
|
||||
// }
|
||||
// break;
|
||||
// }
|
||||
// case 3: {
|
||||
// var venue = ParseVenueArg(argList[0].Value as string);
|
||||
// var asset = argList[1].Value as string;
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// if (result is not null) {
|
||||
// return true;
|
||||
// }
|
||||
// errorMessage ??= "Failed to convert args to HistoricInstrumentStreamParams";
|
||||
// return false;
|
||||
// }
|
||||
|
||||
ViewStream? CreateReplayFutureLiveBbo(ReplayFutureLiveBboParams opts) {
|
||||
if (opts.ContinuousContract is null) {
|
||||
logger.LogMissingParam("ReplayFutureLiveBBOStream", nameof(opts.ContinuousContract));
|
||||
return null;
|
||||
}
|
||||
Func<IBlueRow, bool>? filter = opts.ContinuousContract == "ES" ? ContractFilter : null;
|
||||
var tables = DatabentoFileStore.GetBboAsync(opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), opts.BeginDate, opts.EndDate, filter);
|
||||
// var continuousContracts = HistoricContinuousContractStream.Create(new() { Symbol = opts.ContinuousContract, SecurityType = SecurityType.Future, BeginDate = opts.BeginDate, EndDate = opts.EndDate }, viewStreamDatabase, logger);
|
||||
return ReplayStream.Create(tables, "ts_event", opts.ReplaySpeed, viewStreamDatabase);
|
||||
|
||||
bool ContractFilter(IBlueRow r) =>
|
||||
r.GetString("contract") == "ESU5";
|
||||
}
|
||||
|
||||
ViewStream? CreateReplayFutureLiveTrade(ReplayFutureLiveTradeParams opts) {
|
||||
if (opts.ContinuousContract is null) {
|
||||
logger.LogMissingParam("ReplayFutureLiveTradeStream", nameof(opts.ContinuousContract));
|
||||
return null;
|
||||
}
|
||||
Func<IBlueRow, bool>? filter = opts.ContinuousContract == "ES" ? ContractFilter : null;
|
||||
var tables = DatabentoFileStore.GetTradesAsync(opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), opts.BeginDate, opts.EndDate, filter);
|
||||
return ReplayStream.Create(tables, "ts_event", opts.ReplaySpeed, viewStreamDatabase);
|
||||
|
||||
bool ContractFilter(IBlueRow r) =>
|
||||
r.GetString("contract") == "ESU5";
|
||||
}
|
||||
|
||||
ViewStream? CreateHistoricBbo(HistoricBboParams opts) {
|
||||
if (opts.ContinuousContract is null) {
|
||||
logger.LogMissingParam("HistoricBbo", nameof(opts.ContinuousContract));
|
||||
return null;
|
||||
}
|
||||
var streams = DatabentoFileStore.GetDataFiles(
|
||||
"mbp-1", opts.ContinuousContract, ParseSecTypeArg(opts.SecurityType), beginDate: opts.BeginDate, endDate: opts.EndDate)
|
||||
.Select(f => new ParquetReaderStream(f.Path, viewStreamDatabase))
|
||||
.ToArray();
|
||||
return streams.Length switch {
|
||||
0 => null,
|
||||
1 => streams[0],
|
||||
_ => new MutableAppendStream(streams),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
static BlueTableImmutable dropIndex(IBlueTableSchema schema, IEnumerable<IBlueRow> rows) {
|
||||
int idIndex = schema.ResolveIndices("id");
|
||||
var rowSetBd = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, IBlueRow>()).ToBuilder();
|
||||
foreach (var r in rows) {
|
||||
int idx = rowSetBd.Count + 1;
|
||||
var inputRow = (r as IndexRow)?.InputRow ?? r;
|
||||
rowSetBd.Add(new(idx, new IndexRow(inputRow, idx, idIndex, schema)));
|
||||
}
|
||||
return new(schema, rowSetBd.ToImmutable());
|
||||
}
|
||||
|
||||
struct ReplayFutureLiveTradeState {
|
||||
public IDataPacket? Data;
|
||||
public ColumnSelectorRowFactory? RowFactory;
|
||||
}
|
||||
|
||||
class ReplayFutureLiveBboParams {
|
||||
[StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)]
|
||||
public string? ContinuousContract { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1)]
|
||||
public string? SecurityType { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)]
|
||||
public DateTime? BeginDate { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)]
|
||||
public DateTime? EndDate { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ReplaySpeedRegex }, Position = 4)]
|
||||
public double ReplaySpeed { get; set; } = 1;
|
||||
}
|
||||
|
||||
class ReplayFutureLiveTradeParams {
|
||||
[StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)]
|
||||
public string? ContinuousContract { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1)]
|
||||
public string? SecurityType { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)]
|
||||
public DateTime? BeginDate { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)]
|
||||
public DateTime? EndDate { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ReplaySpeedRegex }, Position = 4)]
|
||||
public double ReplaySpeed { get; set; } = 1;
|
||||
}
|
||||
|
||||
class HistoricBboParams {
|
||||
[StreamArgument(RegexNames = new[] { "^(continuous)?Contract$" }, Position = 0, IsRequired = true)]
|
||||
public string? ContinuousContract { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex, WellKnownStreamArgumentAliases.TypeRegex }, Position = 1, IsRequired = false)]
|
||||
public string? SecurityType { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "(begin|start)(date)?" }, Position = 2)]
|
||||
public DateTime? BeginDate { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "(end|stop)(date)?" }, Position = 3)]
|
||||
public DateTime? EndDate { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "speed" }, Position = 4)]
|
||||
public double ReplaySpeed { get; set; } = 1;
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
namespace Bts.SharedStreams.StreamFactories;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.api;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
public class InstrumentSchemaBuilder {
|
||||
public bool ShowVenue { get; set; } = true;
|
||||
public bool ShowAsset { get; set; } = true;
|
||||
public bool ShowType { get; set; } = true;
|
||||
public bool ShowName { get; set; } = true;
|
||||
public bool ShowId { get; set; } = true;
|
||||
public bool ShowCashPerPoint { get; set; } = true;
|
||||
public bool ShowLegId { get; set; } = true;
|
||||
public bool ShowLegVenue { get; set; }
|
||||
public bool ShowLegAsset { get; set; }
|
||||
public bool ShowLegType { get; set; }
|
||||
public bool ShowLegName { get; set; } = true;
|
||||
public bool ShowLegCashPerPoint { get; set; }
|
||||
public int NumLegs { get; set; } = 2;
|
||||
public string TableName { get; set; } = "Instrument";
|
||||
|
||||
public BlueTableSchemaAdapter<SecurityImmutable> SecurityAdapter() {
|
||||
var adapter = new BlueTableSchemaAdapter<SecurityImmutable>(TableName);
|
||||
adapter.AddColumn("id", r => (int)r.Id);
|
||||
if (ShowVenue) {
|
||||
adapter.AddColumn(WellKnownColumns.Venue.Name, b => b.Exchange.ToString());
|
||||
}
|
||||
if (ShowType) {
|
||||
adapter.AddColumn(WellKnownColumns.SecurityType.Name, b => b.security_type().ToString());
|
||||
}
|
||||
if (ShowAsset) {
|
||||
adapter.AddColumn(WellKnownColumns.Asset.Name, b => b.Symbol);
|
||||
}
|
||||
if (ShowName) {
|
||||
adapter.AddColumn(WellKnownColumns.SecurityName.Name, b => b.ToString());
|
||||
}
|
||||
if (ShowId) {
|
||||
adapter.AddColumn(WellKnownColumns.SecurityId.Name, b => b.Id);
|
||||
}
|
||||
if (ShowCashPerPoint) {
|
||||
adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.ContractSpec.CashPerPoint);
|
||||
}
|
||||
for (int i = 0; i < NumLegs; i++) {
|
||||
if (ShowLegVenue) {
|
||||
adapter.AddColumn(WellKnownColumns.LegVenue(i).Name, Get(l => l.Security.Exchange.ToString(), null));
|
||||
}
|
||||
if (ShowLegType) {
|
||||
adapter.AddColumn(WellKnownColumns.LegSecurityType(i).Name, Get(l => l.Security.security_type().ToString(), null));
|
||||
}
|
||||
if (ShowLegAsset) {
|
||||
adapter.AddColumn(WellKnownColumns.LegAsset(i).Name, Get(l => l.Security.Symbol, null));
|
||||
}
|
||||
if (ShowLegName) {
|
||||
adapter.AddColumn(WellKnownColumns.LegSecurityName(i).Name, Get(l => l.Security.ToString(), null));
|
||||
}
|
||||
if (ShowLegId) {
|
||||
adapter.AddColumn(WellKnownColumns.LegSecurityId(i).Name, Get(l => (int)l.Security.Id, 0));
|
||||
}
|
||||
if (ShowLegCashPerPoint) {
|
||||
adapter.AddColumn(WellKnownColumns.LegCashPerPoint(i).Name, Get(l => l.Security.ContractSpec.CashPerPoint, double.NaN));
|
||||
}
|
||||
|
||||
Func<SecurityImmutable, T> Get<T>(Func<ContractLegImmutable, T> getValue, T defaultValue) =>
|
||||
s => {
|
||||
ContractLegImmutable? leg = s.ContractCase switch {
|
||||
Security.ContractOneofCase.Spread => s.Spread.Legs.Skip(i).TryGetFirst(out var l) ? l : null,
|
||||
Security.ContractOneofCase.Synthetic => s.Synthetic.Legs.Skip(i).TryGetFirst(out var l) ? l : null,
|
||||
_ => null,
|
||||
};
|
||||
return leg.HasValue ? getValue(leg.Value) : defaultValue;
|
||||
};
|
||||
}
|
||||
return adapter;
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
namespace Bts.SharedStreams.StreamFactories;
|
||||
|
||||
using bts.api;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
public class InstrumentStreamParams {
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.VenueRegex })]
|
||||
public Venue? Venue { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex })]
|
||||
public string? Symbol { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex })]
|
||||
public SecurityType? Type { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityNameRegex })]
|
||||
public string? Name { get; set; }
|
||||
|
||||
[StreamArgument("legs", "nLegs", "numLegs")]
|
||||
public int NumLegs { get; set; } = 5;
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowVenueRegex })]
|
||||
public bool ShowVenue { get; set; } = true;
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowSecurityTypeRegex })]
|
||||
public bool ShowType { get; set; } = true;
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowAssetRegex })]
|
||||
public bool ShowAsset { get; set; } = true;
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowCashPerPointRegex })]
|
||||
public bool ShowCashPerPoint { get; set; } = true;
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegVenueRegex })]
|
||||
public bool ShowLegVenue { get; set; } = true;
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegSecurityTypeRegex })]
|
||||
public bool ShowLegType { get; set; } = true;
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegSymbolRegex })]
|
||||
public bool ShowLegSymbol { get; set; } = true;
|
||||
|
||||
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegCashPerPointRegex })]
|
||||
public bool ShowLegCashPerPoint { get; set; } = true;
|
||||
|
||||
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
|
||||
var clause = new mbts.spark.SecurityCriteriaClause(
|
||||
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
|
||||
security_types: Type.HasValue ? new[] { Type.Value } : null,
|
||||
symbols: !string.IsNullOrEmpty(Symbol) ? new[] { Symbol } : null,
|
||||
names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null);
|
||||
return clause.is_defined ? new(new[] { clause }) : new();
|
||||
}
|
||||
|
||||
public InstrumentSchemaBuilder GetSchemaBuilder() =>
|
||||
new() {
|
||||
NumLegs = NumLegs,
|
||||
ShowVenue = ShowVenue,
|
||||
ShowType = ShowType,
|
||||
ShowAsset = ShowAsset,
|
||||
ShowName = true,
|
||||
ShowId = true,
|
||||
ShowCashPerPoint = ShowCashPerPoint,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
namespace Bts.SharedStreams.StreamFactories;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.BlueOrb.Parquet;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
class ParquetReaderStreamFactory : DataStreamFactory {
|
||||
public ParquetReaderStreamFactory(ViewStreamDatabase viewStreamDatabase, ILogger<ParquetReaderStreamFactory> logger) {
|
||||
this.Add<ParquetReaderStreamParams>("parquetReader", CreateParquetReader, logger);
|
||||
|
||||
ViewStream? CreateParquetReader(ParquetReaderStreamParams streamParams) {
|
||||
if (streamParams.File is null) {
|
||||
logger.LogMissingParam("ParquetReader", nameof(streamParams.File));
|
||||
return null;
|
||||
}
|
||||
var stream = new ParquetReaderStream(streamParams.File, streamParams.Start, streamParams.Stop, viewStreamDatabase);
|
||||
return stream;
|
||||
}
|
||||
}
|
||||
|
||||
class ParquetReaderStreamParams {
|
||||
[StreamArgument(RegexNames = new[] { "file|path|filePath" }, Position = 0, IsRequired = true)]
|
||||
public string? File { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "^start|first|begin$" }, Position = 1, IsRequired = false)]
|
||||
public int? Start { get; set; }
|
||||
|
||||
[StreamArgument(RegexNames = new[] { "^stop|last|end$" }, Position = 2, IsRequired = false)]
|
||||
public int? Stop { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -4,17 +4,49 @@ using bts.api;
|
||||
|
||||
using Spark.Api.Orders;
|
||||
|
||||
public static class WellKnownColumnNames {
|
||||
public const string Price = "price";
|
||||
public const string Quantity = "qty";
|
||||
public const string Timestamp = "ts_event";
|
||||
public const string BidPrice = "bid_price";
|
||||
public const string BidQuantity = "bid_size";
|
||||
public const string BidNotional = "bid_notl";
|
||||
public const string AskPrice = "ask_price";
|
||||
public const string Currency = "currency";
|
||||
public const string AskQuantity = "ask_size";
|
||||
public const string AskNotional = "ask_notl";
|
||||
public const string SecurityType = "sec_type";
|
||||
public const string Asset = "asset";
|
||||
public const string SecurityName = "sec_name";
|
||||
public const string Expiration = "expiration";
|
||||
public const string Venue = "exchange";
|
||||
public const string TradingStatus = "trading_status";
|
||||
public const string NumLegs = "num_legs";
|
||||
public const string CashPerPoint = "cpp";
|
||||
|
||||
public static string DepthPrice(Side side, int level) => $"{side.BidAskName()}Price{level}";
|
||||
public static string DepthQuantity(Side side, int level) => $"{side.BidAskName()}Qty{level}";
|
||||
public static string DepthNotional(Side side, int level) => $"{side.BidAskName()}Notl{level}";
|
||||
|
||||
public static string LegSecurityId(int legNum) => $"Leg{legNum}SecId";
|
||||
public static string LegVenue(int legNum) => $"Leg{legNum}Venue";
|
||||
public static string LegSecurityType(int legNum) => $"Leg{legNum}SecType";
|
||||
public static string LegAsset(int legNum) => $"Leg{legNum}Asset";
|
||||
public static string LegSecurityName(int legNum) => $"Leg{legNum}Sec";
|
||||
public static string LegCashPerPoint(int legNum) => $"Leg{legNum}Cpp";
|
||||
}
|
||||
|
||||
public static class WellKnownColumns {
|
||||
public static (string Name, Type Type) SecurityId => ("SecId", typeof(uint));
|
||||
public static (string Name, Type Type) Venue => ("Venue", typeof(string));
|
||||
public static (string Name, Type Type) SecurityType => ("SecType", typeof(string));
|
||||
public static (string Name, Type Type) Asset => ("Asset", typeof(string));
|
||||
public static (string Name, Type Type) SecurityName => ("Sec", typeof(string));
|
||||
public static (string Name, Type Type) CashPerPoint => ("Cpp", typeof(double));
|
||||
public static (string Name, Type Type) Price => ("Price", typeof(double));
|
||||
public static (string Name, Type Type) Quantity => ("Qty", typeof(double));
|
||||
public static (string Name, Type Type) Timestamp => ("Ts", typeof(DateTime));
|
||||
public static (string Name, Type Type) TimestampNanos => ("TsNanos", typeof(long));
|
||||
public static (string Name, Type Type) Venue => (WellKnownColumnNames.Venue, typeof(string));
|
||||
public static (string Name, Type Type) SecurityType => (WellKnownColumnNames.SecurityType, typeof(string));
|
||||
public static (string Name, Type Type) Asset => (WellKnownColumnNames.Asset, typeof(string));
|
||||
public static (string Name, Type Type) SecurityName => (WellKnownColumnNames.SecurityName, typeof(string));
|
||||
public static (string Name, Type Type) CashPerPoint => (WellKnownColumnNames.CashPerPoint, typeof(double));
|
||||
public static (string Name, Type Type) Price => (WellKnownColumnNames.Price, typeof(double));
|
||||
public static (string Name, Type Type) Quantity => (WellKnownColumnNames.Quantity, typeof(double));
|
||||
public static (string Name, Type Type) Timestamp => (WellKnownColumnNames.Timestamp, typeof(DateTime));
|
||||
public static (string Name, Type Type) TimestampNanos => ("ts_event_ns", typeof(long));
|
||||
|
||||
public static (string Name, Type Type) BidPrice => ("BidPrice", typeof(double));
|
||||
public static (string Name, Type Type) BidQuantity => ("BidQty", typeof(double));
|
||||
@@ -23,14 +55,14 @@ public static class WellKnownColumns {
|
||||
public static (string Name, Type Type) AskQuantity => ("AskQty", typeof(double));
|
||||
public static (string Name, Type Type) AskNotional => ("AskNotl", typeof(double));
|
||||
|
||||
public static (string Name, Type Type) DepthPrice(Side side, int level) => ($"{side.BidAskName()}Price{level}", typeof(double));
|
||||
public static (string Name, Type Type) DepthQuantity(Side side, int level) => ($"{side.BidAskName()}Qty{level}", typeof(double));
|
||||
public static (string Name, Type Type) DepthNotional(Side side, int level) => ($"{side.BidAskName()}Notl{level}", typeof(double));
|
||||
public static (string Name, Type Type) DepthPrice(Side side, int level) => (WellKnownColumnNames.DepthPrice(side, level), typeof(double));
|
||||
public static (string Name, Type Type) DepthQuantity(Side side, int level) => (WellKnownColumnNames.DepthQuantity(side, level), typeof(double));
|
||||
public static (string Name, Type Type) DepthNotional(Side side, int level) => (WellKnownColumnNames.DepthNotional(side, level), typeof(double));
|
||||
|
||||
public static (string Name, Type Type) LegSecurityId(int legNum) => ($"Leg{legNum}SecId", typeof(uint));
|
||||
public static (string Name, Type Type) LegVenue(int legNum) => ($"Leg{legNum}Venue", typeof(string));
|
||||
public static (string Name, Type Type) LegSecurityType(int legNum) => ($"Leg{legNum}SecType", typeof(string));
|
||||
public static (string Name, Type Type) LegAsset(int legNum) => ($"Leg{legNum}Asset", typeof(string));
|
||||
public static (string Name, Type Type) LegSecurityName(int legNum) => ($"Leg{legNum}Sec", typeof(string));
|
||||
public static (string Name, Type Type) LegCashPerPoint(int legNum) => ($"Leg{legNum}Cpp", typeof(double));
|
||||
public static (string Name, Type Type) LegSecurityId(int legNum) => (WellKnownColumnNames.LegSecurityId(legNum), typeof(uint));
|
||||
public static (string Name, Type Type) LegVenue(int legNum) => (WellKnownColumnNames.LegVenue(legNum), typeof(string));
|
||||
public static (string Name, Type Type) LegSecurityType(int legNum) => (WellKnownColumnNames.LegSecurityType(legNum), typeof(string));
|
||||
public static (string Name, Type Type) LegAsset(int legNum) => (WellKnownColumnNames.LegAsset(legNum), typeof(string));
|
||||
public static (string Name, Type Type) LegSecurityName(int legNum) => (WellKnownColumnNames.LegSecurityName(legNum), typeof(string));
|
||||
public static (string Name, Type Type) LegCashPerPoint(int legNum) => (WellKnownColumnNames.LegCashPerPoint(legNum), typeof(double));
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
namespace Spark.Core.Collections;
|
||||
|
||||
using bts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
public class ArrayList<T> : IList<T> {
|
||||
T[] items;
|
||||
int count;
|
||||
|
||||
public ArrayList(T[] items) {
|
||||
this.items = items;
|
||||
count = items.Length;
|
||||
}
|
||||
|
||||
public ArrayList(T[] items, int count) {
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(count);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(count, items.Length);
|
||||
this.items = items;
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public T this[int index] {
|
||||
get => items[index];
|
||||
set => items[index] = value;
|
||||
}
|
||||
|
||||
public int Count => count;
|
||||
|
||||
public bool IsReadOnly => false;
|
||||
|
||||
public void Add(T item) =>
|
||||
ArrayHelper.Add(ref items, item, ref count);
|
||||
|
||||
public void Clear() =>
|
||||
count = 0;
|
||||
|
||||
public bool Contains(T item) =>
|
||||
IndexOf(item) != -1;
|
||||
|
||||
public void CopyTo(T[] array, int arrayIndex) =>
|
||||
items.CopyTo(new Span<T>(array, arrayIndex, count));
|
||||
|
||||
public IEnumerator<T> GetEnumerator() {
|
||||
for (int i = 0; i < count; i++) {
|
||||
yield return items[i];
|
||||
}
|
||||
}
|
||||
|
||||
public int IndexOf(T item) {
|
||||
var comp = EqualityComparer<T>.Default;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (comp.Equals(items[i], item)) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public void Insert(int index, T item) =>
|
||||
ArrayHelper.Insert(ref items, index, item, ref count);
|
||||
|
||||
public bool Remove(T item) {
|
||||
int idx = IndexOf(item);
|
||||
if (idx == -1) {
|
||||
return false;
|
||||
}
|
||||
ArrayHelper.remove_at(ref items, ref count, idx);
|
||||
return true;
|
||||
}
|
||||
|
||||
public void RemoveAt(int index) =>
|
||||
ArrayHelper.remove_at(ref items, ref count, index);
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator() =>
|
||||
GetEnumerator();
|
||||
}
|
||||
@@ -10,6 +10,7 @@ namespace bts.collections {
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
public static class ListExtensions {
|
||||
public static bool AddDistinct<T>(this IList<T> list, T value) =>
|
||||
@@ -286,6 +287,15 @@ namespace bts.collections {
|
||||
return false;
|
||||
}
|
||||
|
||||
public static IList<TResult> ParallelSelect<T, TResult>(this IList<T> source, Func<T, TResult> selector) {
|
||||
var results = new TResult[source.Count];
|
||||
Parallel.For(0, source.Count, SetResult);
|
||||
void SetResult(int i) {
|
||||
results[i] = selector(source[i]);
|
||||
}
|
||||
return new ArrayList<TResult>(results);
|
||||
}
|
||||
|
||||
public static void sync_with<T>(
|
||||
this IList<T> list,
|
||||
IEnumerable<T> source,
|
||||
|
||||
@@ -243,6 +243,23 @@ namespace bts.utils {
|
||||
return clone;
|
||||
}
|
||||
|
||||
public static T[] Add<T>(ref T[] array, T item, ref int count) {
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(count, array.Length);
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(count);
|
||||
if (count == array.Length) {
|
||||
if (array.Length == 0) {
|
||||
array = new T[1];
|
||||
} else {
|
||||
var temp = array;
|
||||
array = new T[checked(array.Length * 2)];
|
||||
Array.Copy(temp, 0, array, 0, count);
|
||||
}
|
||||
}
|
||||
array[count] = item;
|
||||
count++;
|
||||
return array;
|
||||
}
|
||||
|
||||
public static void add_sorted<T>(ref T[] array, ref int size, T value, IComparer<T> comparer) =>
|
||||
add_sorted(ref array, ref size, value, comparer, out int pos);
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
#nullable enable
|
||||
namespace Spark.Core;
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
public static class DelegateUtils {
|
||||
[return:NotNullIfNotNull(nameof(pred1))]
|
||||
[return:NotNullIfNotNull(nameof(pred2))]
|
||||
public static Func<T, bool>? CombinePredicate<T>(Func<T, bool>? pred1, Func<T, bool>? pred2) {
|
||||
if (pred1 is null) {
|
||||
return pred2;
|
||||
}
|
||||
if (pred2 is null) {
|
||||
return pred1;
|
||||
}
|
||||
return And;
|
||||
|
||||
bool And(T t) =>
|
||||
pred1(t) && pred2(t);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user