I might have fixed the dates
This commit is contained in:
1 parent
8bcffea686
commit
ab2e9fe6bd
13 files changed
+228
-140
No files matched your search
@@ -58,7 +58,7 @@ namespace bts.history
|
||||
_symbol = options.replaysymbol;
|
||||
_securitytype = (SecurityType)Enum.Parse(typeof(SecurityType), options.replaysecuritytype);
|
||||
_tradedate = DateTime.Parse(options.replaystart);
|
||||
var path = HistoricalDataDownloader.data_path(_tradedate.Date);
|
||||
var path = DataDownloader.data_path(_tradedate.Date);
|
||||
new FileInfo(path).Directory.Create();
|
||||
_fs = new StreamWriter(
|
||||
File.Open(path + $"/{_symbol}_{_securitytype}_{algo.GetType().Name}_{algo.bucket.LowerBound.ToString("0") }_trades.txt", FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite) );
|
||||
|
||||
+4
-4
@@ -10,13 +10,13 @@ using Entry = System.Tuple<bts.api.SecurityType, bts.history.MarketDataIncrement
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalDataReplayer
|
||||
class BackTester
|
||||
{
|
||||
readonly HistoricalDataStreamer _stream;
|
||||
readonly DataStreamer _stream;
|
||||
MarketDataIncrementalRefresh _last = new MarketDataIncrementalRefresh() { SendingTime = new Timestamp(), TransactTime = new Timestamp() } ;
|
||||
public HistoricalDataReplayer()
|
||||
public BackTester()
|
||||
{
|
||||
_stream = new HistoricalDataStreamer();
|
||||
_stream = new DataStreamer();
|
||||
}
|
||||
|
||||
async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
|
||||
+4
-4
@@ -10,12 +10,12 @@ using bts.api;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalDataDownloader
|
||||
class DataDownloader
|
||||
{
|
||||
private static AmazonS3Client _client;
|
||||
private const string _access_key = "AKIAJ4XUDWPQ7M42352A";
|
||||
private const string _secret_key = "f/3gghKXdfHRnEPMh7Xhl3jvlGYbL6zcriUtnq4d";
|
||||
static HistoricalDataDownloader()
|
||||
static DataDownloader()
|
||||
{
|
||||
AmazonS3Config config = new AmazonS3Config();
|
||||
//config.ServiceURL = "http://cme-bts-datamine.s3.us-east-1.amazonaws.com/";
|
||||
@@ -25,7 +25,7 @@ namespace bts.history
|
||||
|
||||
private string _path;
|
||||
private string _bucket_name; //http://cme-bts-datamine.s3.us-east-1.amazonaws.com/
|
||||
public HistoricalDataDownloader(string bucket_name = "cme-bts-datamine")
|
||||
public DataDownloader(string bucket_name = "cme-bts-datamine")
|
||||
{
|
||||
_path = mbts.config.Configuration.get<bts.config.System>().data_path;
|
||||
_bucket_name = bucket_name;
|
||||
@@ -98,7 +98,7 @@ namespace bts.history
|
||||
case CMEMics.Nymex:
|
||||
return "xnym";
|
||||
default:
|
||||
bts.Log.assert(false, "unknown mic map HistoricalDataDownloader");
|
||||
bts.Log.assert(false, "unknown mic map DataDownloader");
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -9,17 +9,17 @@ using bts.co2;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalDataSender
|
||||
class DataSender
|
||||
{
|
||||
readonly HistoricalDataStreamer _stream;
|
||||
readonly DataStreamer _stream;
|
||||
readonly bts.co2.MarketDataReceiverService _mdservice;
|
||||
|
||||
Google.Protobuf.WellKnownTypes.Timestamp _time;
|
||||
Dictionary<Product, co2.ChannelReply> _channels = new Dictionary<Product, co2.ChannelReply>();
|
||||
|
||||
public HistoricalDataSender(ServiceContext context)
|
||||
public DataSender(ServiceContext context)
|
||||
{
|
||||
_stream = new HistoricalDataStreamer();
|
||||
_stream = new DataStreamer();
|
||||
_mdservice = context.locate<bts.co2.MarketDataReceiverService>();
|
||||
}
|
||||
|
||||
+4
-4
@@ -5,10 +5,10 @@ using bts.api;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalDataStreamer
|
||||
class DataStreamer
|
||||
{
|
||||
readonly HistoricalDataDownloader _dl = new HistoricalDataDownloader();
|
||||
public HistoricalDataStreamer()
|
||||
readonly DataDownloader _dl = new DataDownloader();
|
||||
public DataStreamer()
|
||||
{
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ namespace bts.history
|
||||
}
|
||||
|
||||
|
||||
async public Task process_md_stream_by_line(MDQuery request, Action<SecurityType, string, string> action, int peek_field = 55)
|
||||
async public Task process_md_stream_by_line(MDQuery request, Action<SecurityType, string, string> action, int peek_field = 0)
|
||||
{
|
||||
DateTime d = request.Begin.ToDateTime();
|
||||
while (d < request.End.ToDateTime())
|
||||
+5
-5
@@ -13,7 +13,7 @@ using System.Net.Security;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalDatabaseGenerator : IDisposable
|
||||
class DatabaseGenerator : IDisposable
|
||||
{
|
||||
const string _certstring =
|
||||
"MIIEMDCCApigAwIBAgIDWCURMA0GCSqGSIb3DQEBDAUAMDoxODA2BgNVBAMMLzMzZTdhMTgzLTY0YmYtNGVjYi1hOWRiLTFjZjhjZmQyZmY3NSBQcm9qZWN0IENBMB4XDTE5MTEyOTE4MTQzNVoXDTI5MTEyNjE4MTQzNVowOjE4MDYGA1UEAwwvMzNlN2ExODMtNjRiZi00ZWNiLWE5ZGItMWNmOGNmZDJmZjc1IFByb2plY3QgQ0EwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDMZDOJWfSlSYd4Qim73HvT9MyxvOuDBA4HI9dI/4h7/cCFwdFOxygmpiB0naEyaUB/Xz3YK+Z/0sbTXemWXiKkQD/LTRSCGjzs+3iN3COf6QcFe7M/nyIKFW+BwEsF1i4r2gVlJDhC5QfEYH8Vj5VyY6sVavVoyxIiZfgo13JafVOP2D5YbeO1Q7lFcl4Jl+UJCoBVsrsvgG0nmPs3XDdJuQndlYqJGB5i8t6XybAT6PzeEG2GbKOfX3IIaeq0O4fkJMBO2psa7JsublBj6/RhSorU1j+QMC+MMZkr2UYOPdkycAHdIT3GpEJtKx/H3BSewsvEZkDBWlQ6YpdCJculFHuYZGntNxTtFDLnTMVLKH/g8GJZBxomlj1bvRDQW+9MqKldQq/Oe7Lo2IjbMuj/0a9CTEG2jOu6lL0NOFFtslO78PjKTfG6tzbPWKwlti78qG2jvhQjr+CKdqvKN2OehUMMaB0FTNpPAJAWHMexw4YCRIAeF2neVmgkeaunxxMCAwEAAaM/MD0wHQYDVR0OBBYEFMmBSdQd0LEmHanWuSwNlQmj6e9IMA8GA1UdEwQIMAYBAf8CAQAwCwYDVR0PBAQDAgEGMA0GCSqGSIb3DQEBDAUAA4IBgQBJtdspuw62weMuPiOTjm70vjrCn8GZB0BGJ6HK47zUJ2sefdKCjhdn/L359fmln08ykTq7OPuNUAo3YS+QnLKm8tazcF+g7AL1o8sr5inQO2d1kERaWCgYdTT0R3yYc+J/3yxSsxvkbnCK/prs2fzIfUfaybylsvUuiQJqFmpb8h4dO0sXXMMDKyXrTczrYsrNR6r8d1LDofv6CcRVb1I8Ei/51dJEjNahUlARWD44rBwgipwzxADeG8dLkaoq2CK8yvmQ5jtEhEkJKtRZEsAmm8NjXkWuwbXQ8pMVrZbZode39MC4yM1lIp/XW98/34BvxF0kkBbCiPRZrVYK1Ok/7/hDP4Ek9xb3vhQD37T58n5tA2ZROtmSJ/BFxc29lLcfslBPdCmJW4JdcJvsY9rxwi11cHa7Ya8K1M2WVtMX34N3CGpvtScbwiQjOhaqL3S1GO2uYeiNF/qP82zvb2g1r1d7KZC45rst0uHKMyRmw+3tifhjWxkdqFGu65Lg+3c=";
|
||||
@@ -24,7 +24,7 @@ namespace bts.history
|
||||
public bool commit_now;
|
||||
}
|
||||
|
||||
readonly HistoricalDataStreamer _stream;
|
||||
readonly DataStreamer _stream;
|
||||
|
||||
readonly Dictionary<string, co2.BookMessage> _books = new Dictionary<string, co2.BookMessage>();
|
||||
readonly Dictionary<string, co2.TradeMessage> _trades = new Dictionary<string, co2.TradeMessage>();
|
||||
@@ -36,12 +36,12 @@ namespace bts.history
|
||||
readonly Npgsql.NpgsqlConnection _sql;
|
||||
bool _use_timescale = true;
|
||||
|
||||
public HistoricalDatabaseGenerator(string post_process_db, string post_process_db_host, string post_process_db_port,
|
||||
string user = "postgres", string pass = "enter")
|
||||
public DatabaseGenerator(string post_process_db, string post_process_db_host, string post_process_db_port,
|
||||
string user = "postgres", string pass = "enter")
|
||||
{
|
||||
if(_use_timescale && pass == "enter")
|
||||
pass = "password";
|
||||
_stream = new HistoricalDataStreamer();
|
||||
_stream = new DataStreamer();
|
||||
var conn = $@"Host={post_process_db_host}; Port={post_process_db_port}; Username={user}; Password={pass}; sslmode=Prefer;";
|
||||
var sql = new Npgsql.NpgsqlConnection(conn);
|
||||
sql.ProvideClientCertificatesCallback = provide_cert;
|
||||
@@ -11,7 +11,7 @@ namespace bts.history
|
||||
{
|
||||
class HistoricalDataServiceServer : HistoricalDataServiceBase
|
||||
{
|
||||
readonly HistoricalDataStreamer _stream = new HistoricalDataStreamer();
|
||||
readonly DataStreamer _stream = new DataStreamer();
|
||||
public HistoricalDataServiceServer(ServiceContext context) : base(context, "")
|
||||
{
|
||||
}
|
||||
|
||||
@@ -69,13 +69,15 @@ namespace bts.history
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<Tuple<string, string>> next_line(int peek_field = 55 /*sym*/)
|
||||
public IEnumerable<Tuple<string, string>> next_line(int peek_field = 0)
|
||||
{
|
||||
while (_reader.Peek() != -1)
|
||||
{
|
||||
_line++;
|
||||
string line = _reader.ReadLine();
|
||||
string peek = ProtoFix.peek_field(line, peek_field);
|
||||
string peek = string.Empty;
|
||||
if(peek_field != 0)
|
||||
peek = ProtoFix.peek_field(line, peek_field);
|
||||
yield return Tuple.Create(peek, line);
|
||||
}
|
||||
}
|
||||
|
||||
+74
-95
@@ -22,15 +22,16 @@ using MongoDB.Bson.Serialization.Serializers;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalMongoGenerator : IDisposable
|
||||
class MongoGenerator : IDisposable
|
||||
{
|
||||
class Operation
|
||||
{
|
||||
public string security_desc;
|
||||
{
|
||||
public string line;
|
||||
public MarketDataIncrementalRefresh md;
|
||||
public MDEntry entry;
|
||||
}
|
||||
|
||||
readonly HistoricalDataStreamer _stream;
|
||||
readonly DataStreamer _stream;
|
||||
readonly ConcurrentDictionary<string, co2.BookMessage> _books = new ConcurrentDictionary<string, co2.BookMessage>();
|
||||
readonly ConcurrentDictionary<string, co2.TradeMessage> _trades = new ConcurrentDictionary<string, co2.TradeMessage>();
|
||||
|
||||
@@ -42,11 +43,11 @@ namespace bts.history
|
||||
readonly MongoClient _client;
|
||||
readonly IMongoDatabase _db;
|
||||
|
||||
public HistoricalMongoGenerator(string post_process_db, string post_process_db_host, string post_process_db_port,
|
||||
public MongoGenerator(string post_process_db, string post_process_db_host, string post_process_db_port,
|
||||
string user = "mongo", string pass = "enter")
|
||||
{
|
||||
Query.setup_bson();
|
||||
_stream = new HistoricalDataStreamer();
|
||||
_stream = new DataStreamer();
|
||||
var conn = $@"mongodb://{user}:{pass}@{post_process_db_host}:{post_process_db_port}/admin";
|
||||
_client = new MongoClient(conn);
|
||||
_db = _client.GetDatabase(post_process_db);
|
||||
@@ -71,15 +72,13 @@ namespace bts.history
|
||||
int count = 0; Stopwatch watch = new Stopwatch();
|
||||
#pragma warning disable CS4014
|
||||
watch.Start();
|
||||
await _stream.process_md_stream_by_line(query, (sectype, secdesc, line) =>
|
||||
|
||||
start_processor(string.Empty).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted);
|
||||
var queue = _operations[string.Empty] = new ConcurrentQueue<Operation>();
|
||||
|
||||
await _stream.process_md_stream_by_line(query, (sectype, _, line) =>
|
||||
{
|
||||
if(!_operations.TryGetValue(secdesc, out ConcurrentQueue<Operation> queue))
|
||||
{
|
||||
queue = new ConcurrentQueue<Operation>();
|
||||
_operations[secdesc] = queue;
|
||||
start_processor(secdesc).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted);
|
||||
}
|
||||
queue.Enqueue(new Operation { security_desc = secdesc, line = line } );
|
||||
queue.Enqueue(new Operation { line = line } );
|
||||
|
||||
if (++count % 10_000 == 1)
|
||||
{
|
||||
@@ -87,7 +86,8 @@ namespace bts.history
|
||||
Console.WriteLine($"preprocessed {count.ToString("N0") } lines, { qlen.ToString("N0") } write queue length");
|
||||
if (qlen > 1_000_000)
|
||||
{
|
||||
while (qlen > 500_000)
|
||||
var reps = 0;
|
||||
while (qlen > 500_000 && reps++ < 30)
|
||||
{
|
||||
System.Threading.Thread.Sleep(2000);
|
||||
qlen = _operations.Values.Select(x => x.Count).Sum();
|
||||
@@ -113,22 +113,24 @@ namespace bts.history
|
||||
if(!_operations.TryGetValue(secdesc, out ConcurrentQueue<Operation> operations))
|
||||
continue;
|
||||
int count = 0;
|
||||
session.StartTransaction();
|
||||
while(operations.TryDequeue(out Operation oper))
|
||||
//the empty secdesc queue parses fix and routes MDEntry via security desc
|
||||
//preserving the order of per secdesc operations in their respective queues
|
||||
if(secdesc == string.Empty)
|
||||
{
|
||||
try
|
||||
route_operation(operations);
|
||||
}
|
||||
else
|
||||
{
|
||||
session.StartTransaction();
|
||||
while(operations.TryDequeue(out Operation oper))
|
||||
{
|
||||
var md = ProtoFix.parse_fix<MarketDataIncrementalRefresh>(oper.line);
|
||||
parse_packet(md);
|
||||
Log.assert(secdesc == oper.entry?.security_desc());
|
||||
parse_entry(oper.md, oper.entry);
|
||||
if(count++ > 10_000)
|
||||
break;
|
||||
}
|
||||
catch(QuickFix.MessageParseError ex)
|
||||
{
|
||||
Log.FIX.warn("Message parse error: " + ex.ToString());
|
||||
}
|
||||
if(count++ > 10_000)
|
||||
break;
|
||||
session.CommitTransactionAsync();
|
||||
}
|
||||
session.CommitTransactionAsync();
|
||||
}
|
||||
}
|
||||
}, _cts.Token);
|
||||
@@ -136,33 +138,56 @@ namespace bts.history
|
||||
await t;
|
||||
}
|
||||
|
||||
private void parse_packet(MarketDataIncrementalRefresh md)
|
||||
private void route_operation(ConcurrentQueue<Operation> operations)
|
||||
{
|
||||
foreach (var entry in md.MdEntries)
|
||||
while (operations.TryDequeue(out Operation oper))
|
||||
{
|
||||
string tname = table_name(entry.security_desc());
|
||||
if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
|
||||
Log.assert(oper.line != null && oper.entry == null && oper.md == null);
|
||||
try
|
||||
{
|
||||
if (!_books.TryGetValue(entry.security_desc(), out co2.BookMessage book))
|
||||
var md = ProtoFix.parse_fix<MarketDataIncrementalRefresh>(oper.line);
|
||||
foreach (var entry in md.MdEntries)
|
||||
{
|
||||
book = new co2.BookMessage();
|
||||
OrderBook.init_book(book);
|
||||
_books[entry.security_desc()] = book;
|
||||
}
|
||||
OrderBook.keep_book(book, entry);
|
||||
write_book(tname, book, entry, md);
|
||||
if(!_operations.TryGetValue(entry.security_desc(), out ConcurrentQueue<Operation> q))
|
||||
{
|
||||
q = new ConcurrentQueue<Operation>();
|
||||
_operations[entry.security_desc()] = q;
|
||||
start_processor(entry.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
|
||||
}
|
||||
q.Enqueue(new Operation { md = md, entry = entry });
|
||||
}
|
||||
}
|
||||
else if (entry.MdEntryType == MDEntryType.Trade)
|
||||
catch (QuickFix.MessageParseError ex)
|
||||
{
|
||||
co2.TradeMessage tm = OrderBook.create_trade(md, entry);
|
||||
if (!_trades.ContainsKey(entry.security_desc()))
|
||||
_trades[entry.security_desc()] = tm;
|
||||
|
||||
write_trade(tname, tm, entry, md);
|
||||
Log.FIX.warn("Message parse error: " + ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void parse_entry(MarketDataIncrementalRefresh md, MDEntry entry)
|
||||
{
|
||||
string tname = table_name(entry.security_desc());
|
||||
if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
|
||||
{
|
||||
if (!_books.TryGetValue(entry.security_desc(), out co2.BookMessage book))
|
||||
{
|
||||
book = new co2.BookMessage();
|
||||
OrderBook.init_book(book);
|
||||
_books[entry.security_desc()] = book;
|
||||
}
|
||||
OrderBook.keep_book(book, entry);
|
||||
write_book(tname, book, entry, md);
|
||||
}
|
||||
else if (entry.MdEntryType == MDEntryType.Trade)
|
||||
{
|
||||
co2.TradeMessage tm = OrderBook.create_trade(md, entry);
|
||||
if (!_trades.ContainsKey(entry.security_desc()))
|
||||
_trades[entry.security_desc()] = tm;
|
||||
|
||||
write_trade(tname, tm, entry, md);
|
||||
}
|
||||
}
|
||||
|
||||
string table_name(string security_desc)
|
||||
{
|
||||
var tname = $"{security_desc}";
|
||||
@@ -171,7 +196,7 @@ namespace bts.history
|
||||
|
||||
void write_trade(string tname, TradeMessage tm, MDEntry entry, MarketDataIncrementalRefresh md)
|
||||
{
|
||||
HistoryEntry row = new HistoryEntry
|
||||
HistoryEntry hist = new HistoryEntry
|
||||
{
|
||||
TradeDate = md.TradeDate,
|
||||
MsgSeqNum = md.MsgSeqNum,
|
||||
@@ -181,14 +206,14 @@ namespace bts.history
|
||||
Trade = tm
|
||||
};
|
||||
var table = _db.GetCollection<HistoryEntry>(tname);
|
||||
table.InsertOne(row);
|
||||
table.InsertOne( hist );
|
||||
}
|
||||
|
||||
void write_book(string tname, BookMessage book, MDEntry entry, MarketDataIncrementalRefresh md)
|
||||
{
|
||||
if (book.Bids.FirstOrDefault()?.Size == 0 && book.Asks.FirstOrDefault()?.Size == 0)
|
||||
return;
|
||||
HistoryEntry row = new HistoryEntry
|
||||
HistoryEntry hist = new HistoryEntry
|
||||
{
|
||||
TradeDate = md.TradeDate,
|
||||
MsgSeqNum = md.MsgSeqNum,
|
||||
@@ -198,9 +223,9 @@ namespace bts.history
|
||||
WorkingBook = book,
|
||||
};
|
||||
if (OrderBook.quote_event_done(md))
|
||||
row.Book = book;
|
||||
hist.Book = book;
|
||||
var table = _db.GetCollection<HistoryEntry>(tname);
|
||||
table.InsertOne(row);
|
||||
table.InsertOne( hist );
|
||||
}
|
||||
|
||||
public async void Dispose()
|
||||
@@ -210,50 +235,4 @@ namespace bts.history
|
||||
await qtask;
|
||||
}
|
||||
}
|
||||
|
||||
class TimestampSerializer : IBsonSerializer
|
||||
{
|
||||
readonly DateTimeSerializer _dt = new DateTimeSerializer(DateTimeKind.Utc, BsonType.Document);
|
||||
public System.Type ValueType => typeof(Timestamp);
|
||||
public TimestampSerializer()
|
||||
{
|
||||
}
|
||||
public object Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args)
|
||||
{
|
||||
// Timestamp val = new Timestamp();
|
||||
// val.Seconds = context.Reader.ReadInt64(nameof(val.Seconds));
|
||||
// val.Nanos = context.Reader.ReadInt32(nameof(val.Nanos));
|
||||
// return val;
|
||||
DateTime dt = _dt.Deserialize(context, args);
|
||||
return Timestamp.FromDateTime(dt);
|
||||
}
|
||||
|
||||
public void Serialize(BsonSerializationContext context, BsonSerializationArgs args, object value)
|
||||
{
|
||||
// Timestamp val = value as Timestamp;
|
||||
// if (val == null)
|
||||
// context.Writer.WriteNull();
|
||||
// else
|
||||
// {
|
||||
// context.Writer.WriteStartDocument();
|
||||
// context.Writer.WriteInt64(nameof(val.Seconds), val.Seconds);
|
||||
// context.Writer.WriteInt32(nameof(val.Nanos), val.Nanos);
|
||||
// var dt = BsonUtils.ToMillisecondsSinceEpoch(val.ToDateTime().ToUniversalTime());
|
||||
// context.Writer.WriteDateTime("DateTime", dt);
|
||||
// context.Writer.WriteEndDocument();
|
||||
// }
|
||||
DateTime dt = ((Timestamp)value).ToDateTime();
|
||||
_dt.Serialize(context, args, dt);
|
||||
}
|
||||
}
|
||||
|
||||
class SerializationProvider : IBsonSerializationProvider
|
||||
{
|
||||
public IBsonSerializer GetSerializer(System.Type type)
|
||||
{
|
||||
if (type == typeof(Timestamp))
|
||||
return new TimestampSerializer();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,17 +25,14 @@ namespace bts
|
||||
|
||||
static string get_between(string source, string start, string end)
|
||||
{
|
||||
int Start, End;
|
||||
if (source.Contains(start) && source.Contains(end))
|
||||
{
|
||||
Start = source.IndexOf(start, 0) + start.Length;
|
||||
End = source.IndexOf(end, Start);
|
||||
return source.Substring(Start, End - Start);
|
||||
}
|
||||
else
|
||||
{
|
||||
return "";
|
||||
}
|
||||
int spos = source.IndexOf(start, 0);
|
||||
if(spos < 0)
|
||||
return string.Empty;
|
||||
spos += start.Length;
|
||||
int epos = source.IndexOf(end, spos);
|
||||
if(epos < 0)
|
||||
return string.Empty;
|
||||
return source.Substring(spos, epos - spos);
|
||||
}
|
||||
|
||||
public static Timestamp peek_timestamp(string fixstring, int field = 60)
|
||||
|
||||
@@ -7,14 +7,46 @@ using MongoDB.Driver.Linq;
|
||||
using bts.history;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using MongoDB.Bson.Serialization;
|
||||
using bts.co2;
|
||||
using MongoDB.Bson.Serialization.Serializers;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
public class HistoryRow
|
||||
{
|
||||
public DateTime TradeDate { get; set; }
|
||||
public uint MsgSeqNum { get; set; }
|
||||
public DateTime SendingTime { get; set; }
|
||||
public DateTime TransactTime { get; set; }
|
||||
public string SecurityDesc { get; set; }
|
||||
public BookMessage Book { get; set; }
|
||||
public BookMessage WorkingBook { get; set; }
|
||||
public TradeMessage Trade { get; set; }
|
||||
public MBOOrderMessage Order { get; set; }
|
||||
|
||||
public HistoryRow(HistoryEntry entry)
|
||||
{
|
||||
this.entry = entry;
|
||||
this.TradeDate = entry.TradeDate.ToDateTime();
|
||||
this.MsgSeqNum = entry.MsgSeqNum;
|
||||
this.SendingTime = entry.SendingTime.ToDateTime();
|
||||
this.TransactTime = entry.TransactTime.ToDateTime();
|
||||
this.SecurityDesc = entry.SecurityDesc;
|
||||
this.Book = entry.Book;
|
||||
this.WorkingBook = entry.WorkingBook;
|
||||
this.Trade = entry.Trade;
|
||||
this.Order = entry.Order;
|
||||
}
|
||||
public HistoryEntry entry { get; }
|
||||
public DateTime timestamp() => entry.timestamp().ToDateTime();
|
||||
}
|
||||
|
||||
public class Query
|
||||
{
|
||||
readonly IMongoClient _client;
|
||||
readonly IMongoDatabase _db;
|
||||
|
||||
|
||||
public static void setup_bson() { }
|
||||
static Query()
|
||||
{
|
||||
BsonClassMap.RegisterClassMap<co2.BookMessage>(m =>
|
||||
@@ -22,15 +54,16 @@ namespace bts.history
|
||||
m.MapProperty(x => x.Bids);
|
||||
m.MapProperty(x => x.Asks);
|
||||
});
|
||||
// BsonClassMap.RegisterClassMap<Timestamp>(m =>
|
||||
// BsonClassMap.RegisterClassMap<HistoryRow>(m =>
|
||||
// {
|
||||
// m.AutoMap();
|
||||
// m.MapMember(x => x.TradeDate).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
|
||||
// m.MapMember(x => x.SendingTime).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
|
||||
// m.MapMember(x => x.TransactTime).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
|
||||
// m.SetIgnoreExtraElements(true);
|
||||
// });
|
||||
//BsonSerializer.RegisterSerializationProvider(new SerializationProvider());
|
||||
}
|
||||
|
||||
public static void setup_bson() {}
|
||||
BsonSerializer.RegisterSerializationProvider(new SerializationProvider());
|
||||
}
|
||||
|
||||
public Query(string database, string host = "sparkvm", int port = 27017,
|
||||
string user = "mongo", string pass = "enter")
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
|
||||
|
||||
using System;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using MongoDB.Bson.Serialization;
|
||||
using MongoDB.Bson.Serialization.Serializers;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class TimestampSerializer : SerializerBase<Timestamp>, IBsonDocumentSerializer
|
||||
{
|
||||
readonly DateTimeSerializer _dt = new DateTimeSerializer(DateTimeKind.Utc, BsonType.Document);
|
||||
public TimestampSerializer()
|
||||
{
|
||||
}
|
||||
public override Timestamp Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args)
|
||||
{
|
||||
Timestamp val = new Timestamp();
|
||||
val.Seconds = context.Reader.ReadInt64(nameof(val.Seconds));
|
||||
val.Nanos = context.Reader.ReadInt32(nameof(val.Nanos));
|
||||
return val;
|
||||
// DateTime dt = _dt.Deserialize(context, args);
|
||||
// return Timestamp.FromDateTime(dt);
|
||||
}
|
||||
|
||||
public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, Timestamp val)
|
||||
{
|
||||
if (val == null)
|
||||
context.Writer.WriteNull();
|
||||
else
|
||||
{
|
||||
context.Writer.WriteStartDocument();
|
||||
context.Writer.WriteInt64(nameof(val.Seconds), val.Seconds);
|
||||
context.Writer.WriteInt32(nameof(val.Nanos), val.Nanos);
|
||||
var dt = BsonUtils.ToMillisecondsSinceEpoch(val.ToDateTime().ToUniversalTime());
|
||||
context.Writer.WriteDateTime("DateTime", dt);
|
||||
context.Writer.WriteEndDocument();
|
||||
}
|
||||
// DateTime dt = ((Timestamp)value).ToDateTime();
|
||||
// _dt.Serialize(context, args, dt);
|
||||
}
|
||||
|
||||
public bool TryGetMemberSerializationInfo(string memberName, out BsonSerializationInfo serializationInfo)
|
||||
{
|
||||
switch (memberName)
|
||||
{
|
||||
case "Id":
|
||||
serializationInfo = new BsonSerializationInfo("_id", new ObjectIdSerializer(), typeof(ObjectId));
|
||||
return true;
|
||||
case nameof(Timestamp.Seconds):
|
||||
serializationInfo = new BsonSerializationInfo(nameof(Timestamp.Seconds), new Int64Serializer(), typeof(long));
|
||||
return true;
|
||||
case nameof(Timestamp.Nanos):
|
||||
serializationInfo = new BsonSerializationInfo(nameof(Timestamp.Nanos), new Int32Serializer(), typeof(int));
|
||||
return true;
|
||||
case "DateTime":
|
||||
serializationInfo = new BsonSerializationInfo("DateTime", new DateTimeSerializer(), typeof(DateTime));
|
||||
return true;
|
||||
default:
|
||||
serializationInfo = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class SerializationProvider : IBsonSerializationProvider
|
||||
{
|
||||
public IBsonSerializer GetSerializer(System.Type type)
|
||||
{
|
||||
if (type == typeof(Timestamp))
|
||||
return new TimestampSerializer();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -38,13 +38,13 @@ namespace bts.history
|
||||
st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecuritytype, true);
|
||||
if(!string.IsNullOrEmpty(o.post_process_db))
|
||||
{
|
||||
var rep = new HistoricalMongoGenerator(o.post_process_db, o.post_process_db_host, o.post_process_db_port,
|
||||
var rep = new MongoGenerator(o.post_process_db, o.post_process_db_host, o.post_process_db_port,
|
||||
o.post_process_db_user, o.post_process_db_pass);
|
||||
rep.generate(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult();
|
||||
}
|
||||
else
|
||||
{
|
||||
var rep = new HistoricalDataReplayer();
|
||||
var rep = new BackTester();
|
||||
rep.replay(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult();
|
||||
}
|
||||
Console.ReadKey();
|
||||
@@ -53,7 +53,7 @@ namespace bts.history
|
||||
|
||||
if (!string.IsNullOrEmpty(o.s3key))
|
||||
{
|
||||
var dl = new HistoricalDataDownloader();
|
||||
var dl = new DataDownloader();
|
||||
var str = dl.download(o.s3key).GetAwaiter().GetResult();
|
||||
using (MBOParser parser = new MBOParser(str))
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user