finally figured out timestamps

This commit is contained in:
Kevin Darby committed 2019-12-03 23:47:48 -05:00
1 parent ab2e9fe6bd
commit 33f12edce5
6 files changed
+157 -477

No files matched your search

@@ -0,0 +1,16 @@
using System;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
public sealed partial class HistoryEntry
{
public DateTime trade_date { get { return this.TradeDate.ToDateTime(); } set { this.TradeDate = Timestamp.FromDateTime(value); } }
public DateTime sending_time { get { return this.SendingTime.ToDateTime(); } set { this.SendingTime = Timestamp.FromDateTime(value); } }
public DateTime transact_time { get { return this.TransactTime.ToDateTime(); } set { this.TransactTime = Timestamp.FromDateTime(value); } }
public DateTime timestamp()
{
return this.sending_time;
}
}
}
@@ -8,10 +8,6 @@ namespace bts.history
{
return md.SendingTime;
}
public static Timestamp timestamp(this HistoryEntry hist)
{
return hist.SendingTime;
}
public static string security_desc(this MDEntry md)
{
return md.Symbol;
@@ -2,125 +2,11 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
]
},
{
"data": {
"text/html": [
"Installing package Google.Protobuf, version 3.6.1...................done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package Google.Protobuf, version 3.6.1"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package Microsoft.CSharp, version 4.5.0.......done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package Microsoft.CSharp, version 4.5.0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package Newtonsoft.Json, version 4.5.0.......done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package Newtonsoft.Json, version 4.5.0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package System.Linq.Dynamic, version 1.0.7.......done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package System.Linq.Dynamic, version 1.0.7"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package MongoDB.Driver, version 2.9.3........done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package MongoDB.Driver, version 2.9.3"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package MongoDB.Bson, version 2.9.3.......done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package MongoDB.Bson, version 2.9.3"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n",
"#r \"nuget:Google.Protobuf, 3.6.1\"\n",
@@ -134,23 +20,19 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
]
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n",
"using bts.history;\n",
@@ -161,8 +43,10 @@
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var q = new Query(\"roll\");"
@@ -171,14 +55,18 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"\n",
@@ -188,48 +76,34 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"20975"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"display(s1.Count())\n",
"display(s.Count())\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"20975"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var s2 = s.Where( x => x.SendingTime.Seconds != 0);\n",
"display(s2.Count())"
"var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
"display(s2.Count());"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var st = q.timeseries(\"ZNZ9-ZNH0\", new DateTime(2019, 11,26, 14,0, 0), \n",
@@ -238,40 +112,11 @@
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"ename": "Unhandled Exception",
"evalue": "{document}{SendingTime}.ToDateTime() is not supported.",
"output_type": "error",
"traceback": [
"Unhandled Exception",
"{document}{SendingTime}.ToDateTime() is not supported.",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.GetFieldExpression(Expression expression)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateComparison(Expression variableExpression, ExpressionType operatorType, ConstantExpression constantExpression)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateComparison(BinaryExpression binaryExpression)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateAndAlso(BinaryExpression node)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node, IBsonSerializerRegistry serializerRegistry)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslateWhere(WhereExpression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslateGroupBy(GroupByExpression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslatePipeline(PipelineExpression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node, IBsonSerializerRegistry serializerRegistry, ExpressionTranslationOptions translationOptions)",
" at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Translate(Expression expression)",
" at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Execute(Expression expression)",
" at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Execute[TResult](Expression expression)",
" at System.Linq.Queryable.Count[TSource](IQueryable`1 source)",
" at Submission#11.<<Initialize>>d__0.MoveNext()",
"--- End of stack trace from previous location where exception was thrown ---",
" at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)"
]
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"display(st.Count());"
]
@@ -279,7 +124,9 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
}
@@ -2,125 +2,11 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
]
},
{
"data": {
"text/html": [
"Installing package Google.Protobuf, version 3.6.1...................done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package Google.Protobuf, version 3.6.1"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package Microsoft.CSharp, version 4.5.0.......done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package Microsoft.CSharp, version 4.5.0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package Newtonsoft.Json, version 4.5.0.......done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package Newtonsoft.Json, version 4.5.0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package System.Linq.Dynamic, version 1.0.7.......done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package System.Linq.Dynamic, version 1.0.7"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package MongoDB.Driver, version 2.9.3........done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package MongoDB.Driver, version 2.9.3"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Installing package MongoDB.Bson, version 2.9.3.......done!"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Successfully added reference to package MongoDB.Bson, version 2.9.3"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n",
"#r \"nuget:Google.Protobuf, 3.6.1\"\n",
@@ -134,23 +20,19 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
]
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n",
"using bts.history;\n",
@@ -161,8 +43,10 @@
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var q = new Query(\"roll\");"
@@ -171,14 +55,18 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"\n",
@@ -188,48 +76,34 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"20975"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"display(s1.Count())\n",
"display(s.Count())\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"20975"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var s2 = s.Where( x => x.SendingTime.Seconds != 0);\n",
"display(s2.Count())"
"var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
"display(s2.Count());"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var st = q.timeseries(\"ZNZ9-ZNH0\", new DateTime(2019, 11,26, 14,0, 0), \n",
@@ -238,40 +112,11 @@
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"ename": "Unhandled Exception",
"evalue": "{document}{SendingTime}.ToDateTime() is not supported.",
"output_type": "error",
"traceback": [
"Unhandled Exception",
"{document}{SendingTime}.ToDateTime() is not supported.",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.GetFieldExpression(Expression expression)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateComparison(Expression variableExpression, ExpressionType operatorType, ConstantExpression constantExpression)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateComparison(BinaryExpression binaryExpression)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateAndAlso(BinaryExpression node)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node, IBsonSerializerRegistry serializerRegistry)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslateWhere(WhereExpression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslateGroupBy(GroupByExpression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslatePipeline(PipelineExpression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)",
" at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node, IBsonSerializerRegistry serializerRegistry, ExpressionTranslationOptions translationOptions)",
" at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Translate(Expression expression)",
" at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Execute(Expression expression)",
" at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Execute[TResult](Expression expression)",
" at System.Linq.Queryable.Count[TSource](IQueryable`1 source)",
" at Submission#11.<<Initialize>>d__0.MoveNext()",
"--- End of stack trace from previous location where exception was thrown ---",
" at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)"
]
}
],
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"display(st.Count());"
]
@@ -279,7 +124,9 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
}
+39 -67
View File
@@ -12,34 +12,6 @@ using MongoDB.Bson.Serialization.Serializers;
namespace bts.history
{
public class HistoryRow
{
public DateTime TradeDate { get; set; }
public uint MsgSeqNum { get; set; }
public DateTime SendingTime { get; set; }
public DateTime TransactTime { get; set; }
public string SecurityDesc { get; set; }
public BookMessage Book { get; set; }
public BookMessage WorkingBook { get; set; }
public TradeMessage Trade { get; set; }
public MBOOrderMessage Order { get; set; }
public HistoryRow(HistoryEntry entry)
{
this.entry = entry;
this.TradeDate = entry.TradeDate.ToDateTime();
this.MsgSeqNum = entry.MsgSeqNum;
this.SendingTime = entry.SendingTime.ToDateTime();
this.TransactTime = entry.TransactTime.ToDateTime();
this.SecurityDesc = entry.SecurityDesc;
this.Book = entry.Book;
this.WorkingBook = entry.WorkingBook;
this.Trade = entry.Trade;
this.Order = entry.Order;
}
public HistoryEntry entry { get; }
public DateTime timestamp() => entry.timestamp().ToDateTime();
}
public class Query
{
@@ -54,15 +26,18 @@ namespace bts.history
m.MapProperty(x => x.Bids);
m.MapProperty(x => x.Asks);
});
// BsonClassMap.RegisterClassMap<HistoryRow>(m =>
// {
// m.AutoMap();
// m.MapMember(x => x.TradeDate).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
// m.MapMember(x => x.SendingTime).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
// m.MapMember(x => x.TransactTime).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
// m.SetIgnoreExtraElements(true);
// });
BsonSerializer.RegisterSerializationProvider(new SerializationProvider());
BsonClassMap.RegisterClassMap<HistoryEntry>(m =>
{
m.AutoMap();
m.MapMember(x => x.trade_date).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
m.MapMember(x => x.sending_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
m.MapMember(x => x.transact_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
m.UnmapMember(x => x.TradeDate);
m.UnmapMember(x => x.SendingTime);
m.UnmapMember(x => x.TransactTime);
m.SetIgnoreExtraElements(true);
});
//BsonSerializer.RegisterSerializationProvider(new SerializationProvider());
}
public Query(string database, string host = "sparkvm", int port = 27017,
@@ -73,49 +48,46 @@ namespace bts.history
_db = _client.GetDatabase(database);
}
public IMongoQueryable<T> series<T>(string collection, string where = null, string orderby = null)
public IMongoQueryable<T> series<T>(string collection)
{
var coll = _db.GetCollection<T>(collection);
IMongoQueryable<T> res = coll.AsQueryable();
if(where != null)
{
var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda<T, bool>(where);
res = res.Where(expr);
}
if(orderby != null)
{
var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda<T, bool>(orderby);
res = res.OrderBy(expr);
}
return res;
}
public IMongoQueryable<HistoryEntry> timeseries(string collection, DateTime begin, DateTime end,
string where = null, string orderby = null)
public IMongoQueryable<HistoryEntry> timeseries(string collection, DateTime begin, DateTime end, bool use_sending_time = true)
{
var coll = _db.GetCollection<HistoryEntry>(collection);
IMongoQueryable<HistoryEntry> res = coll.AsQueryable();
Expression<Func<HistoryEntry, bool>> timeexpr = (HistoryEntry h) =>
h.SendingTime.ToDateTime() >= begin && h.SendingTime.ToDateTime() <= end;
Expression<Func<HistoryEntry, bool>> timeexpr = null;
if(use_sending_time)
timeexpr = (HistoryEntry h) => h.sending_time >= begin && h.sending_time <= end;
else
timeexpr = (HistoryEntry h) => h.transact_time >= begin && h.transact_time <= end;
res = res.Where( timeexpr );
if (where != null)
{
var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda<HistoryEntry, bool>(where);
res = res.Where(expr);
}
if (orderby != null)
{
var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda<HistoryEntry, bool>(orderby);
res = res.OrderBy(expr);
}
return res;
}
static public DateTime ToDateTime(Timestamp ts)
}
public static class QueryExtensions
{
static public IMongoQueryable<T> where<T>(this IMongoQueryable<T> q, string where)
{
var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda<T, bool>(where);
var res = q.Where(expr);
return res;
}
static public IMongoQueryable<T> order_by<T>(this IMongoQueryable<T> q, string orderby)
{
var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda<T, bool>(orderby);
var res = q.OrderBy(expr);
return res;
}
static public DateTime to_datetime(this Timestamp ts)
{
return ts.ToDateTime();
}
}
}
}
@@ -34,8 +34,10 @@ namespace bts.history
context.Writer.WriteStartDocument();
context.Writer.WriteInt64(nameof(val.Seconds), val.Seconds);
context.Writer.WriteInt32(nameof(val.Nanos), val.Nanos);
var dt = BsonUtils.ToMillisecondsSinceEpoch(val.ToDateTime().ToUniversalTime());
var dtime = val.ToDateTime().ToUniversalTime();
var dt = BsonUtils.ToMillisecondsSinceEpoch(dtime);
context.Writer.WriteDateTime("DateTime", dt);
context.Writer.WriteDateTime("Ticks", dtime.Ticks);
context.Writer.WriteEndDocument();
}
// DateTime dt = ((Timestamp)value).ToDateTime();
@@ -56,7 +58,7 @@ namespace bts.history
serializationInfo = new BsonSerializationInfo(nameof(Timestamp.Nanos), new Int32Serializer(), typeof(int));
return true;
case "DateTime":
serializationInfo = new BsonSerializationInfo("DateTime", new DateTimeSerializer(), typeof(DateTime));
serializationInfo = new BsonSerializationInfo("DateTime", new DateTimeSerializer(DateTimeKind.Utc, BsonType.Document), typeof(DateTime));
return true;
default:
serializationInfo = null;