feat: Add historic market data streams factory

Loads historic md from Databento files in structured paths. It is
designed to work with files downloaded via databento_client.py
This commit is contained in:
John Colvin committed 2025-11-19 14:40:15 -05:00
1 parent 0ea884b505
commit 04200a330a
33 files changed
+2145 -167

No files matched your search

@@ -9,6 +9,7 @@ using bts.collections;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
@@ -82,9 +83,43 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
}
}
public Builder ToBuilder() =>
new(this);
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class Builder : IEnumerable<IBlueRow> {
readonly IBlueTableSchema schema;
readonly ImmutableSortedSetDictionary<int, IBlueRow>.Builder rowMap;
public Builder(BlueTableImmutable source) {
schema = source.schema;
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
}
public void AddRange(IEnumerable<KeyValuePair<int, IBlueRow>> rows) =>
rowMap.AddRange(rows);
public void AddOrUpdate(IBlueRow row) =>
AddOrUpdate(row.Id(), row);
public void AddOrUpdate(int id, IBlueRow row) =>
rowMap[id] = row;
public bool Remove(int id) =>
rowMap.Remove(id);
public BlueTableImmutable ToImmutable() =>
new(schema, rowMap.ToImmutable().Source);
public IEnumerator<IBlueRow> GetEnumerator() =>
rowMap.Values.GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
public class DataPacketBuilder {
readonly IBlueTableSchema schema;
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder table;
@@ -340,9 +375,46 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection {
}
}
public Builder ToBuilder() =>
new(this);
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class Builder : IEnumerable<IBlueRow> {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableSortedSetDictionary<int, T>.Builder rowMap;
public Builder(BlueTableImmutableAdapter<T> source) {
schemaAdapter = source.schemaAdapter;
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
}
public void AddRange(IEnumerable<KeyValuePair<int, T>> rows) =>
rowMap.AddRange(rows);
public void AddOrUpdate(T row) =>
AddOrUpdate((int)schemaAdapter["id"][row]!, row);
public void AddOrUpdate(int id, T row) =>
rowMap[id] = row;
public bool Remove(int id) =>
rowMap.Remove(id);
public BlueTableImmutableAdapter<T> ToImmutable() =>
new(schemaAdapter, rowMap.ToImmutable().Source);
public IEnumerator<IBlueRow> GetEnumerator() {
foreach (var r in rowMap.Values) {
yield return schemaAdapter.GetBlueRow(r);
}
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
public class DataPacketBuilder {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table =