From 33f12edce53111a66cc89c8bf86ba48e244fc845 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Tue, 3 Dec 2019 23:47:48 -0500 Subject: [PATCH] finally figured out timestamps --- src/client/spark_api/history/HistoryEntry.cs | 16 ++ .../spark_historical/HistoricalExtensions.cs | 4 - .../query_sample-checkpoint.ipynb | 251 ++++-------------- .../Notebooks/query_sample.ipynb | 251 ++++-------------- src/client/spark_historical/Query.cs | 106 +++----- .../spark_historical/TimestampSerializer.cs | 6 +- 6 files changed, 157 insertions(+), 477 deletions(-) create mode 100644 src/client/spark_api/history/HistoryEntry.cs diff --git a/src/client/spark_api/history/HistoryEntry.cs b/src/client/spark_api/history/HistoryEntry.cs new file mode 100644 index 000000000..809739582 --- /dev/null +++ b/src/client/spark_api/history/HistoryEntry.cs @@ -0,0 +1,16 @@ +using System; +using Google.Protobuf.WellKnownTypes; + +namespace bts.history +{ + public sealed partial class HistoryEntry + { + public DateTime trade_date { get { return this.TradeDate.ToDateTime(); } set { this.TradeDate = Timestamp.FromDateTime(value); } } + public DateTime sending_time { get { return this.SendingTime.ToDateTime(); } set { this.SendingTime = Timestamp.FromDateTime(value); } } + public DateTime transact_time { get { return this.TransactTime.ToDateTime(); } set { this.TransactTime = Timestamp.FromDateTime(value); } } + public DateTime timestamp() + { + return this.sending_time; + } + } +} \ No newline at end of file diff --git a/src/client/spark_historical/HistoricalExtensions.cs b/src/client/spark_historical/HistoricalExtensions.cs index 6f2db0ef2..c5a8fa5b2 100644 --- a/src/client/spark_historical/HistoricalExtensions.cs +++ b/src/client/spark_historical/HistoricalExtensions.cs @@ -8,10 +8,6 @@ namespace bts.history { return md.SendingTime; } - public static Timestamp timestamp(this HistoryEntry hist) - { - return hist.SendingTime; - } public static string security_desc(this MDEntry md) { return md.Symbol; diff --git a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/query_sample-checkpoint.ipynb b/src/client/spark_historical/Notebooks/.ipynb_checkpoints/query_sample-checkpoint.ipynb index 4d7fa12f9..d245c10ec 100644 --- a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/query_sample-checkpoint.ipynb +++ b/src/client/spark_historical/Notebooks/.ipynb_checkpoints/query_sample-checkpoint.ipynb @@ -2,125 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" - ] - }, - { - "data": { - "text/html": [ - "Installing package Google.Protobuf, version 3.6.1...................done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package Google.Protobuf, version 3.6.1" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package Microsoft.CSharp, version 4.5.0.......done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package Microsoft.CSharp, version 4.5.0" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package Newtonsoft.Json, version 4.5.0.......done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package Newtonsoft.Json, version 4.5.0" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package System.Linq.Dynamic, version 1.0.7.......done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package System.Linq.Dynamic, version 1.0.7" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package MongoDB.Driver, version 2.9.3........done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package MongoDB.Driver, version 2.9.3" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package MongoDB.Bson, version 2.9.3.......done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package MongoDB.Bson, version 2.9.3" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n", "#r \"nuget:Google.Protobuf, 3.6.1\"\n", @@ -134,23 +20,19 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" - ] - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n", "using bts.history;\n", @@ -161,8 +43,10 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, + "execution_count": null, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "var q = new Query(\"roll\");" @@ -171,14 +55,18 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 5, - "metadata": {}, + "execution_count": null, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "\n", @@ -188,48 +76,34 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "20975" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ - "display(s1.Count())\n", + "display(s.Count())\n", " " ] }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "20975" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ - "var s2 = s.Where( x => x.SendingTime.Seconds != 0);\n", - "display(s2.Count())" + "var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n", + "display(s2.Count());" ] }, { "cell_type": "code", - "execution_count": 10, - "metadata": {}, + "execution_count": null, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "var st = q.timeseries(\"ZNZ9-ZNH0\", new DateTime(2019, 11,26, 14,0, 0), \n", @@ -238,40 +112,11 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "ename": "Unhandled Exception", - "evalue": "{document}{SendingTime}.ToDateTime() is not supported.", - "output_type": "error", - "traceback": [ - "Unhandled Exception", - "{document}{SendingTime}.ToDateTime() is not supported.", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.GetFieldExpression(Expression expression)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateComparison(Expression variableExpression, ExpressionType operatorType, ConstantExpression constantExpression)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateComparison(BinaryExpression binaryExpression)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateAndAlso(BinaryExpression node)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node, IBsonSerializerRegistry serializerRegistry)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslateWhere(WhereExpression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslateGroupBy(GroupByExpression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslatePipeline(PipelineExpression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node, IBsonSerializerRegistry serializerRegistry, ExpressionTranslationOptions translationOptions)", - " at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Translate(Expression expression)", - " at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Execute(Expression expression)", - " at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Execute[TResult](Expression expression)", - " at System.Linq.Queryable.Count[TSource](IQueryable`1 source)", - " at Submission#11.<>d__0.MoveNext()", - "--- End of stack trace from previous location where exception was thrown ---", - " at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)" - ] - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "display(st.Count());" ] @@ -279,7 +124,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [] } diff --git a/src/client/spark_historical/Notebooks/query_sample.ipynb b/src/client/spark_historical/Notebooks/query_sample.ipynb index 4d7fa12f9..d245c10ec 100644 --- a/src/client/spark_historical/Notebooks/query_sample.ipynb +++ b/src/client/spark_historical/Notebooks/query_sample.ipynb @@ -2,125 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" - ] - }, - { - "data": { - "text/html": [ - "Installing package Google.Protobuf, version 3.6.1...................done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package Google.Protobuf, version 3.6.1" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package Microsoft.CSharp, version 4.5.0.......done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package Microsoft.CSharp, version 4.5.0" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package Newtonsoft.Json, version 4.5.0.......done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package Newtonsoft.Json, version 4.5.0" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package System.Linq.Dynamic, version 1.0.7.......done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package System.Linq.Dynamic, version 1.0.7" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package MongoDB.Driver, version 2.9.3........done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package MongoDB.Driver, version 2.9.3" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Installing package MongoDB.Bson, version 2.9.3.......done!" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Successfully added reference to package MongoDB.Bson, version 2.9.3" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n", "#r \"nuget:Google.Protobuf, 3.6.1\"\n", @@ -134,23 +20,19 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" - ] - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n", "using bts.history;\n", @@ -161,8 +43,10 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, + "execution_count": null, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "var q = new Query(\"roll\");" @@ -171,14 +55,18 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 5, - "metadata": {}, + "execution_count": null, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "\n", @@ -188,48 +76,34 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "20975" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ - "display(s1.Count())\n", + "display(s.Count())\n", " " ] }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "20975" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ - "var s2 = s.Where( x => x.SendingTime.Seconds != 0);\n", - "display(s2.Count())" + "var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n", + "display(s2.Count());" ] }, { "cell_type": "code", - "execution_count": 10, - "metadata": {}, + "execution_count": null, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "var st = q.timeseries(\"ZNZ9-ZNH0\", new DateTime(2019, 11,26, 14,0, 0), \n", @@ -238,40 +112,11 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "ename": "Unhandled Exception", - "evalue": "{document}{SendingTime}.ToDateTime() is not supported.", - "output_type": "error", - "traceback": [ - "Unhandled Exception", - "{document}{SendingTime}.ToDateTime() is not supported.", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.GetFieldExpression(Expression expression)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateComparison(Expression variableExpression, ExpressionType operatorType, ConstantExpression constantExpression)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateComparison(BinaryExpression binaryExpression)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.TranslateAndAlso(BinaryExpression node)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.PredicateTranslator.Translate(Expression node, IBsonSerializerRegistry serializerRegistry)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslateWhere(WhereExpression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslateGroupBy(GroupByExpression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.TranslatePipeline(PipelineExpression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node)", - " at MongoDB.Driver.Linq.Translators.QueryableTranslator.Translate(Expression node, IBsonSerializerRegistry serializerRegistry, ExpressionTranslationOptions translationOptions)", - " at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Translate(Expression expression)", - " at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Execute(Expression expression)", - " at MongoDB.Driver.Linq.MongoQueryProviderImpl`1.Execute[TResult](Expression expression)", - " at System.Linq.Queryable.Count[TSource](IQueryable`1 source)", - " at Submission#11.<>d__0.MoveNext()", - "--- End of stack trace from previous location where exception was thrown ---", - " at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)" - ] - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "display(st.Count());" ] @@ -279,7 +124,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [] } diff --git a/src/client/spark_historical/Query.cs b/src/client/spark_historical/Query.cs index 28d5ebd5a..7100b0c6c 100644 --- a/src/client/spark_historical/Query.cs +++ b/src/client/spark_historical/Query.cs @@ -12,34 +12,6 @@ using MongoDB.Bson.Serialization.Serializers; namespace bts.history { - public class HistoryRow - { - public DateTime TradeDate { get; set; } - public uint MsgSeqNum { get; set; } - public DateTime SendingTime { get; set; } - public DateTime TransactTime { get; set; } - public string SecurityDesc { get; set; } - public BookMessage Book { get; set; } - public BookMessage WorkingBook { get; set; } - public TradeMessage Trade { get; set; } - public MBOOrderMessage Order { get; set; } - - public HistoryRow(HistoryEntry entry) - { - this.entry = entry; - this.TradeDate = entry.TradeDate.ToDateTime(); - this.MsgSeqNum = entry.MsgSeqNum; - this.SendingTime = entry.SendingTime.ToDateTime(); - this.TransactTime = entry.TransactTime.ToDateTime(); - this.SecurityDesc = entry.SecurityDesc; - this.Book = entry.Book; - this.WorkingBook = entry.WorkingBook; - this.Trade = entry.Trade; - this.Order = entry.Order; - } - public HistoryEntry entry { get; } - public DateTime timestamp() => entry.timestamp().ToDateTime(); - } public class Query { @@ -54,15 +26,18 @@ namespace bts.history m.MapProperty(x => x.Bids); m.MapProperty(x => x.Asks); }); - // BsonClassMap.RegisterClassMap(m => - // { - // m.AutoMap(); - // m.MapMember(x => x.TradeDate).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); - // m.MapMember(x => x.SendingTime).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); - // m.MapMember(x => x.TransactTime).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); - // m.SetIgnoreExtraElements(true); - // }); - BsonSerializer.RegisterSerializationProvider(new SerializationProvider()); + BsonClassMap.RegisterClassMap(m => + { + m.AutoMap(); + m.MapMember(x => x.trade_date).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); + m.MapMember(x => x.sending_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); + m.MapMember(x => x.transact_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); + m.UnmapMember(x => x.TradeDate); + m.UnmapMember(x => x.SendingTime); + m.UnmapMember(x => x.TransactTime); + m.SetIgnoreExtraElements(true); + }); + //BsonSerializer.RegisterSerializationProvider(new SerializationProvider()); } public Query(string database, string host = "sparkvm", int port = 27017, @@ -73,49 +48,46 @@ namespace bts.history _db = _client.GetDatabase(database); } - public IMongoQueryable series(string collection, string where = null, string orderby = null) + public IMongoQueryable series(string collection) { var coll = _db.GetCollection(collection); IMongoQueryable res = coll.AsQueryable(); - if(where != null) - { - var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda(where); - res = res.Where(expr); - } - if(orderby != null) - { - var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda(orderby); - res = res.OrderBy(expr); - } return res; } - - - public IMongoQueryable timeseries(string collection, DateTime begin, DateTime end, - string where = null, string orderby = null) + public IMongoQueryable timeseries(string collection, DateTime begin, DateTime end, bool use_sending_time = true) { var coll = _db.GetCollection(collection); IMongoQueryable res = coll.AsQueryable(); - Expression> timeexpr = (HistoryEntry h) => - h.SendingTime.ToDateTime() >= begin && h.SendingTime.ToDateTime() <= end; + Expression> timeexpr = null; + if(use_sending_time) + timeexpr = (HistoryEntry h) => h.sending_time >= begin && h.sending_time <= end; + else + timeexpr = (HistoryEntry h) => h.transact_time >= begin && h.transact_time <= end; res = res.Where( timeexpr ); - if (where != null) - { - var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda(where); - res = res.Where(expr); - } - if (orderby != null) - { - var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda(orderby); - res = res.OrderBy(expr); - } return res; } - - static public DateTime ToDateTime(Timestamp ts) + } + + public static class QueryExtensions + { + static public IMongoQueryable where(this IMongoQueryable q, string where) + { + var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda(where); + var res = q.Where(expr); + return res; + } + + static public IMongoQueryable order_by(this IMongoQueryable q, string orderby) + { + var expr = System.Linq.Dynamic.DynamicExpression.ParseLambda(orderby); + var res = q.OrderBy(expr); + return res; + } + + static public DateTime to_datetime(this Timestamp ts) { return ts.ToDateTime(); - } + } } } \ No newline at end of file diff --git a/src/client/spark_historical/TimestampSerializer.cs b/src/client/spark_historical/TimestampSerializer.cs index 171ba40ee..a91696638 100644 --- a/src/client/spark_historical/TimestampSerializer.cs +++ b/src/client/spark_historical/TimestampSerializer.cs @@ -34,8 +34,10 @@ namespace bts.history context.Writer.WriteStartDocument(); context.Writer.WriteInt64(nameof(val.Seconds), val.Seconds); context.Writer.WriteInt32(nameof(val.Nanos), val.Nanos); - var dt = BsonUtils.ToMillisecondsSinceEpoch(val.ToDateTime().ToUniversalTime()); + var dtime = val.ToDateTime().ToUniversalTime(); + var dt = BsonUtils.ToMillisecondsSinceEpoch(dtime); context.Writer.WriteDateTime("DateTime", dt); + context.Writer.WriteDateTime("Ticks", dtime.Ticks); context.Writer.WriteEndDocument(); } // DateTime dt = ((Timestamp)value).ToDateTime(); @@ -56,7 +58,7 @@ namespace bts.history serializationInfo = new BsonSerializationInfo(nameof(Timestamp.Nanos), new Int32Serializer(), typeof(int)); return true; case "DateTime": - serializationInfo = new BsonSerializationInfo("DateTime", new DateTimeSerializer(), typeof(DateTime)); + serializationInfo = new BsonSerializationInfo("DateTime", new DateTimeSerializer(DateTimeKind.Utc, BsonType.Document), typeof(DateTime)); return true; default: serializationInfo = null;