fix parse bug for scientific notation

This commit is contained in:
Kevin Darby committed 2020-01-05 21:24:48 -05:00
1 parent 2cf9974b73
commit d1764997d3
3 files changed
+658 -3

No files matched your search

@@ -0,0 +1,287 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/protobuf-net/2.3.2/lib/netstandard1.3/protobuf-net.dll"
]
}
],
"source": [
"//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
"#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
"//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
"//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\" \n",
"//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
"#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
"//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
"#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
"//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
"#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\" \n",
"#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
"//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
"#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"\n",
"//#r \"nuget:protobuf-net, 2.3.2.0\"\n",
"#r \"C:/Users/darby/.nuget/packages/protobuf-net/2.3.2/lib/netstandard1.3/protobuf-net.dll\" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
]
}
],
"source": [
"#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n",
"using bts.history;\n",
"using MongoDB.Driver;\n",
"using MongoDB.Driver.Linq;\n",
"using Google.Protobuf.WellKnownTypes;\n",
"using static bts.history.Query;\n",
"using MathNet.Numerics.Statistics;\n",
"using mbts.co2;\n",
"\n",
"bts.BTS.init();\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var q1 = new Query(\"ES_201911_W4\", \"bigblue\");\n",
"var q2 = new Query(\"MES_201911_W4\", \"bigblue\");\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"var s1 = q1.collection<HistoryRow>(\"ESZ9\");\n",
"var s2 = q2.collection<HistoryRow>(\"MESZ9\");\n",
"\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var begin = DateTime.SpecifyKind(new DateTime(2019, 11, 25, 14, 0, 0), DateTimeKind.Utc); //9am east\n",
"var end = DateTime.SpecifyKind(new DateTime(2019, 11, 25, 22, 0, 0), DateTimeKind.Utc); //6pm east\n",
"\n",
"var s1t = q1.timeseries_cached<HistoryRow>(s1, begin, end);\n",
"var s2t = q2.timeseries_cached<HistoryRow>(s2, begin, end);\n",
"\n",
"await Task.WhenAll(new Task[] { s1t, s2t } );\n",
"\n",
"//local copy so we don't beat up the database\n",
"var s1f = await s1t;\n",
"var s2f = await s2t;\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"\n",
"\n",
"display(s1f.Count()); display(s2f.Count()); "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"//quotes the last leg\n",
"\n",
"var latency_in = 50_000u;\n",
"var latency_out = 50_000u;\n",
"var latency_iceberg = 60_000u;\n",
"var incoming_latency = new Bucket(latency_in, latency_in+1);\n",
"var outgoing_latency = new Bucket(latency_out, latency_out+1);\n",
"var sink = new LatencyOrderSink(incoming_latency, outgoing_latency);\n",
"var fill_model = new FIFOFillModel(outgoing_latency);\n",
"\n",
"var security_list = new [] {\"ESZ9\",\"MESZ9\"};\n",
"var data = new[] { s1f, s2f };\n",
"var algo_params = \"tick_size=1.0,1.0;hedge_factors=1,-10;price_factors=1,-1;profit_ticks=1.0;lean_factor=1\";\n",
"var qty = 1u;\n",
"var algo_results = algo_results</*Stack*/Spreader, HistoryRow>(sink, fill_model, algo_params, \n",
" bts.api.Side.Undefined, 0.0, qty, \n",
" security_list, //must match below\n",
" data, latency_iceberg);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"var algo_results_cached = algo_results.ToArray().OrderBy(x => x.order?.TransactTime ?? x.fill.TransactTime, \n",
" HistoricalExtensions.timestamp_comparer);\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var cpp = new Dictionary<string,double> { {\"ESZ9\", 10}, {\"MESZ9\", 1} };\n",
"var pnl = algo_pnl(algo_results_cached, cpp);\n",
"display(pnl);\n",
"display(algo_results_cached.Where(x => x.fill != null).Select(x => new Display(x)).Take(1000));"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"display(algo_results_cached.Select(x => new Display(x)).Take(2000));"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"//var exchange_orders = exchange_orders<HistoryRow>(h0f);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"//display(exchange_orders.Skip(20000).Take(100));\n",
"//display(exchange_orders.Where(x => x.OrderId == 846784119686).Select(x => new Display.MBODisplay(x)));"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".NET (C#)",
"language": "C#",
"name": ".net-csharp"
},
"language_info": {
"file_extension": ".cs",
"mimetype": "text/x-csharp",
"name": "C#",
"pygments_lexer": "csharp",
"version": "8.0"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
@@ -0,0 +1,354 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/protobuf-net/2.3.2/lib/netstandard1.3/protobuf-net.dll"
]
}
],
"source": [
"//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
"#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
"//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
"//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\" \n",
"//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
"#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
"//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
"#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
"//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
"#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\" \n",
"#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
"//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
"#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"\n",
"//#r \"nuget:protobuf-net, 2.3.2.0\"\n",
"#r \"C:/Users/darby/.nuget/packages/protobuf-net/2.3.2/lib/netstandard1.3/protobuf-net.dll\" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
]
}
],
"source": [
"#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n",
"using bts.history;\n",
"using MongoDB.Driver;\n",
"using MongoDB.Driver.Linq;\n",
"using Google.Protobuf.WellKnownTypes;\n",
"using static bts.history.Query;\n",
"using MathNet.Numerics.Statistics;\n",
"using mbts.co2;\n",
"\n",
"bts.BTS.init();\n"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var q1 = new Query(\"ES_201911_W4\", \"bigblue\");\n",
"var q2 = new Query(\"MES_201911_W4\", \"bigblue\");\n"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"var s1 = q1.collection<HistoryRow>(\"ESZ9\");\n",
"var s2 = q2.collection<HistoryRow>(\"MESZ9\");\n",
"\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"var begin = DateTime.SpecifyKind(new DateTime(2019, 11, 25, 14, 0, 0), DateTimeKind.Utc); //9am east\n",
"var end = DateTime.SpecifyKind(new DateTime(2019, 11, 25, 22, 0, 0), DateTimeKind.Utc); //6pm east\n",
"\n",
"var s1t = q1.timeseries_cached<HistoryRow>(s1, begin, end);\n",
"var s2t = q2.timeseries_cached<HistoryRow>(s2, begin, end);\n",
"\n",
"await Task.WhenAll(new Task[] { s1t, s2t } );\n",
"\n",
"//local copy so we don't beat up the database\n",
"var s1f = await s1t;\n",
"var s2f = await s2t;\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"331656"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"\n",
"\n",
"display(s1f.Count()); display(s2f.Count()); "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"//quotes the last leg\n",
"\n",
"var latency_in = 50_000u;\n",
"var latency_out = 50_000u;\n",
"var latency_iceberg = 60_000u;\n",
"var incoming_latency = new Bucket(latency_in, latency_in+1);\n",
"var outgoing_latency = new Bucket(latency_out, latency_out+1);\n",
"var sink = new LatencyOrderSink(incoming_latency, outgoing_latency);\n",
"var fill_model = new FIFOFillModel(outgoing_latency);\n",
"\n",
"var security_list = new [] {\"ESZ9\",\"MESZ9\"};\n",
"var data = new[] { s1f, s2f };\n",
"var algo_params = \"tick_size=1.0,1.0;hedge_factors=1,-10;price_factors=1,-1;profit_ticks=1.0;lean_factor=1\";\n",
"var qty = 1u;\n",
"var algo_results = algo_results</*Stack*/Spreader, HistoryRow>(sink, fill_model, algo_params, \n",
" bts.api.Side.Undefined, 0.0, qty, \n",
" security_list, //must match below\n",
" data, latency_iceberg);"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"ename": "Unhandled Exception",
"evalue": "Object reference not set to an instance of an object.",
"output_type": "error",
"traceback": [
"Unhandled Exception",
"Object reference not set to an instance of an object.",
" at bts.history.Query.<>c__12`2.<stream_data>b__12_0(T x, T y) in c:\\code\\bts\\spark\\src\\client\\spark_historical\\Query\\Query.cs:line 114",
" at System.Collections.Generic.ComparisonComparer`1.Compare(T x, T y)",
" at bts.history.HistoricalExtensions.IndexOfMin[T](IList`1 self, IComparer`1 comparer) in c:\\code\\bts\\spark\\src\\client\\spark_historical\\HistoricalExtensions.cs:line 63",
" at bts.history.Query.stream_data[T,U](IEnumerable`1[] datastream, Func`3 function, Func`1 end_of_stream)+MoveNext() in c:\\code\\bts\\spark\\src\\client\\spark_historical\\Query\\Query.cs:line 117",
" at System.Collections.Generic.LargeArrayBuilder`1.AddRange(IEnumerable`1 items)",
" at System.Collections.Generic.EnumerableHelpers.ToArray[T](IEnumerable`1 source)",
" at Submission#21.<<Initialize>>d__0.MoveNext()",
"--- End of stack trace from previous location where exception was thrown ---",
" at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)"
]
}
],
"source": [
"var algo_results_cached = algo_results.ToArray().OrderBy(x => x.order?.TransactTime ?? x.fill.TransactTime, \n",
" HistoricalExtensions.timestamp_comparer);\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"scrolled": true
},
"outputs": [
{
"ename": "Unhandled Exception",
"evalue": "Object reference not set to an instance of an object.",
"output_type": "error",
"traceback": [
"Unhandled Exception",
"Object reference not set to an instance of an object.",
" at bts.history.Query.algo_pnl(IEnumerable`1 results, Dictionary`2 cpp_mults) in c:\\code\\bts\\spark\\src\\client\\spark_historical\\Query\\Query.cs:line 212",
" at Submission#16.<<Initialize>>d__0.MoveNext()",
"--- End of stack trace from previous location where exception was thrown ---",
" at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)"
]
}
],
"source": [
"var cpp = new Dictionary<string,double> { {\"ESZ9\", 10}, {\"MESZ9\", 1} };\n",
"var pnl = algo_pnl(algo_results_cached, cpp);\n",
"display(pnl);\n",
"display(algo_results_cached.Where(x => x.fill != null).Select(x => new Display(x)).Take(1000));"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"ename": "Unhandled Exception",
"evalue": "Value cannot be null. (Parameter 'source')",
"output_type": "error",
"traceback": [
"Unhandled Exception",
"Value cannot be null. (Parameter 'source')",
" at System.Linq.ThrowHelper.ThrowArgumentNullException(ExceptionArgument argument)",
" at System.Linq.Enumerable.Select[TSource,TResult](IEnumerable`1 source, Func`2 selector)",
" at Submission#17.<<Initialize>>d__0.MoveNext()",
"--- End of stack trace from previous location where exception was thrown ---",
" at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)"
]
}
],
"source": [
"display(algo_results_cached.Select(x => new Display(x)).Take(2000));"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"//var exchange_orders = exchange_orders<HistoryRow>(h0f);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"//display(exchange_orders.Skip(20000).Take(100));\n",
"//display(exchange_orders.Where(x => x.OrderId == 846784119686).Select(x => new Display.MBODisplay(x)));"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".NET (C#)",
"language": "C#",
"name": ".net-csharp"
},
"language_info": {
"file_extension": ".cs",
"mimetype": "text/x-csharp",
"name": "C#",
"pygments_lexer": "csharp",
"version": "8.0"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
+17 -3
View File
@@ -130,9 +130,23 @@ namespace bts
case FieldType.SInt64:
case FieldType.UInt64:
{
var data = fix.GetDecimal(fd.FieldNumber);
object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
fd.Accessor.SetValue(msg, conv);
try
{
var data = fix.GetDecimal(fd.FieldNumber);
object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
fd.Accessor.SetValue(msg, conv);
}
catch(Exception ex) when (ex is QuickFix.FieldConvertError || ex is FormatException)
{
try
{
var snot = fix.GetString(fd.FieldNumber); //scientific notation apparently appears in the files
var data = Double.Parse(snot, System.Globalization.NumberStyles.Float);
object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
fd.Accessor.SetValue(msg, conv);
}
catch {}
}
}
break;
case FieldType.Enum: