feat: Add historic market data streams factory
Loads historic md from Databento files in structured paths. It is designed to work with files downloaded via databento_client.py
This commit is contained in:
1 parent
2549e34a17
commit
0cf44fcc40
34 files changed
+2205
-173
No files matched your search
@@ -9,6 +9,7 @@ using bts.collections;
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
@@ -76,9 +77,43 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
|
||||
}
|
||||
}
|
||||
|
||||
public Builder ToBuilder() =>
|
||||
new(this);
|
||||
|
||||
public DataPacketBuilder ToDataPacketBuilder() =>
|
||||
new(this);
|
||||
|
||||
public class Builder : IEnumerable<IBlueRow> {
|
||||
readonly IBlueTableSchema schema;
|
||||
readonly ImmutableSortedSetDictionary<int, IBlueRow>.Builder rowMap;
|
||||
|
||||
public Builder(BlueTableImmutable source) {
|
||||
schema = source.schema;
|
||||
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
|
||||
}
|
||||
|
||||
public void AddRange(IEnumerable<KeyValuePair<int, IBlueRow>> rows) =>
|
||||
rowMap.AddRange(rows);
|
||||
|
||||
public void AddOrUpdate(IBlueRow row) =>
|
||||
AddOrUpdate(row.Id(), row);
|
||||
|
||||
public void AddOrUpdate(int id, IBlueRow row) =>
|
||||
rowMap[id] = row;
|
||||
|
||||
public bool Remove(int id) =>
|
||||
rowMap.Remove(id);
|
||||
|
||||
public BlueTableImmutable ToImmutable() =>
|
||||
new(schema, rowMap.ToImmutable().Source);
|
||||
|
||||
public IEnumerator<IBlueRow> GetEnumerator() =>
|
||||
rowMap.Values.GetEnumerator();
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator() =>
|
||||
GetEnumerator();
|
||||
}
|
||||
|
||||
public class DataPacketBuilder {
|
||||
readonly IBlueTableSchema schema;
|
||||
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder table;
|
||||
@@ -354,9 +389,46 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection {
|
||||
}
|
||||
}
|
||||
|
||||
public Builder ToBuilder() =>
|
||||
new(this);
|
||||
|
||||
public DataPacketBuilder ToDataPacketBuilder() =>
|
||||
new(this);
|
||||
|
||||
public class Builder : IEnumerable<IBlueRow> {
|
||||
readonly BlueTableSchemaAdapter<T> schemaAdapter;
|
||||
readonly ImmutableSortedSetDictionary<int, T>.Builder rowMap;
|
||||
|
||||
public Builder(BlueTableImmutableAdapter<T> source) {
|
||||
schemaAdapter = source.schemaAdapter;
|
||||
rowMap = ImmutableSortedSetDictionary.CreateBuilder(source.rows.ToBuilder());
|
||||
}
|
||||
|
||||
public void AddRange(IEnumerable<KeyValuePair<int, T>> rows) =>
|
||||
rowMap.AddRange(rows);
|
||||
|
||||
public void AddOrUpdate(T row) =>
|
||||
AddOrUpdate((int)schemaAdapter["id"][row]!, row);
|
||||
|
||||
public void AddOrUpdate(int id, T row) =>
|
||||
rowMap[id] = row;
|
||||
|
||||
public bool Remove(int id) =>
|
||||
rowMap.Remove(id);
|
||||
|
||||
public BlueTableImmutableAdapter<T> ToImmutable() =>
|
||||
new(schemaAdapter, rowMap.ToImmutable().Source);
|
||||
|
||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
||||
foreach (var r in rowMap.Values) {
|
||||
yield return schemaAdapter.GetBlueRow(r);
|
||||
}
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator() =>
|
||||
GetEnumerator();
|
||||
}
|
||||
|
||||
public class DataPacketBuilder {
|
||||
readonly BlueTableSchemaAdapter<T> schemaAdapter;
|
||||
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table =
|
||||
|
||||
@@ -73,6 +73,21 @@ public class BlueTableSchemaAdapter<T> : IMutableBlueRowFactory<T>, IBlueTableSc
|
||||
columnNames = null;
|
||||
}
|
||||
|
||||
public void AddColumn<TResult>(string name, Func<T, TResult?> getValue) {
|
||||
var dataType = typeof(TResult);
|
||||
if (!ColumnInfo.isAllowedType(dataType)) {
|
||||
throw new ArgumentException(
|
||||
$"'{dataType}' is not a supported column type", nameof(dataType));
|
||||
}
|
||||
// Invalidate cached schema (set it to null)
|
||||
schema = null;
|
||||
var adapter = new ColumnAdapter(new(name, columns.Count, dataType), r => getValue(r));
|
||||
columns.Add(adapter);
|
||||
columnNameIndex[name] = adapter;
|
||||
columnInfos = null;
|
||||
columnNames = null;
|
||||
}
|
||||
|
||||
public ColumnAdapter this[int columnIndex] =>
|
||||
columns[columnIndex];
|
||||
|
||||
|
||||
Reference in new issue
Block a user