feat: Add historic market data streams factory

Loads historic md from Databento files in structured paths. It is
designed to work with files downloaded via databento_client.py
This commit is contained in:
John Colvin committed 2025-11-15 12:45:04 -05:00
1 parent 2549e34a17
commit 0cf44fcc40
34 files changed
+2205 -173

No files matched your search

@@ -9,6 +9,7 @@ using bts.collections;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
@@ -76,9 +77,43 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
}
}
public Builder ToBuilder() =>
new(this);
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class Builder : IEnumerable<IBlueRow> {
readonly IBlueTableSchema schema;
readonly ImmutableSortedSetDictionary<int, IBlueRow>.Builder rowMap;
public Builder(BlueTableImmutable source) {
schema = source.schema;
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
}
public void AddRange(IEnumerable<KeyValuePair<int, IBlueRow>> rows) =>
rowMap.AddRange(rows);
public void AddOrUpdate(IBlueRow row) =>
AddOrUpdate(row.Id(), row);
public void AddOrUpdate(int id, IBlueRow row) =>
rowMap[id] = row;
public bool Remove(int id) =>
rowMap.Remove(id);
public BlueTableImmutable ToImmutable() =>
new(schema, rowMap.ToImmutable().Source);
public IEnumerator<IBlueRow> GetEnumerator() =>
rowMap.Values.GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
public class DataPacketBuilder {
readonly IBlueTableSchema schema;
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder table;
@@ -354,9 +389,46 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection {
}
}
public Builder ToBuilder() =>
new(this);
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class Builder : IEnumerable<IBlueRow> {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableSortedSetDictionary<int, T>.Builder rowMap;
public Builder(BlueTableImmutableAdapter<T> source) {
schemaAdapter = source.schemaAdapter;
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
}
public void AddRange(IEnumerable<KeyValuePair<int, T>> rows) =>
rowMap.AddRange(rows);
public void AddOrUpdate(T row) =>
AddOrUpdate((int)schemaAdapter["id"][row]!, row);
public void AddOrUpdate(int id, T row) =>
rowMap[id] = row;
public bool Remove(int id) =>
rowMap.Remove(id);
public BlueTableImmutableAdapter<T> ToImmutable() =>
new(schemaAdapter, rowMap.ToImmutable().Source);
public IEnumerator<IBlueRow> GetEnumerator() {
foreach (var r in rowMap.Values) {
yield return schemaAdapter.GetBlueRow(r);
}
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
public class DataPacketBuilder {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table =
@@ -73,6 +73,21 @@ public class BlueTableSchemaAdapter<T> : IMutableBlueRowFactory<T>, IBlueTableSc
columnNames = null;
}
public void AddColumn<TResult>(string name, Func<T, TResult?> getValue) {
var dataType = typeof(TResult);
if (!ColumnInfo.isAllowedType(dataType)) {
throw new ArgumentException(
$"'{dataType}' is not a supported column type", nameof(dataType));
}
// Invalidate cached schema (set it to null)
schema = null;
var adapter = new ColumnAdapter(new(name, columns.Count, dataType), r => getValue(r));
columns.Add(adapter);
columnNameIndex[name] = adapter;
columnInfos = null;
columnNames = null;
}
public ColumnAdapter this[int columnIndex] =>
columns[columnIndex];