From fed7ca1ff349d3db88840596b01ca1706da3cee3 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 5 Dec 2019 18:20:28 -0500 Subject: [PATCH] honing in --- .vscode/launch.json | 2 - .../spark_api/Co2/NativePriceHelpers.cs | 5 +- .../Untitled-Copy1-checkpoint.ipynb | 71 -------- .../Untitled-checkpoint.ipynb | 6 - .../query_sample-checkpoint.ipynb | 160 ++++++++++------ .../Notebooks/Untitled-Copy1.ipynb | 71 -------- .../spark_historical/Notebooks/Untitled.ipynb | 71 -------- .../Notebooks/query_sample.ipynb | 172 +++++++++++------- .../spark_historical/Query/MongoGenerator.cs | 21 ++- src/client/spark_historical/Query/Query.cs | 71 +++++++- src/client/spark_historical/setup_notes.txt | 14 +- 11 files changed, 317 insertions(+), 347 deletions(-) delete mode 100644 src/client/spark_historical/Notebooks/.ipynb_checkpoints/Untitled-Copy1-checkpoint.ipynb delete mode 100644 src/client/spark_historical/Notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb delete mode 100644 src/client/spark_historical/Notebooks/Untitled-Copy1.ipynb delete mode 100644 src/client/spark_historical/Notebooks/Untitled.ipynb diff --git a/.vscode/launch.json b/.vscode/launch.json index 5667f1343..d6621013a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -111,8 +111,6 @@ "--rsecuritytype", "Synthetic", "--post_process_db", "roll2", "--post_process_db_host", "sparkvm", - "--post_process_db_port", "27017", - "--post_process_db_user", "mongo", ], "cwd": "${workspaceRoot}", "stopAtEntry": false, diff --git a/src/client/spark_api/Co2/NativePriceHelpers.cs b/src/client/spark_api/Co2/NativePriceHelpers.cs index a49ab893e..c74553713 100644 --- a/src/client/spark_api/Co2/NativePriceHelpers.cs +++ b/src/client/spark_api/Co2/NativePriceHelpers.cs @@ -75,7 +75,10 @@ namespace mbts.co2 { var lev = book.Asks[level]; return lev.Size; } - + public static double price(this bts.co2.PriceSizeMessage p) + { + return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent); + } public static IList all_bids(this bts.co2.BookMessage book) { diff --git a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/Untitled-Copy1-checkpoint.ipynb b/src/client/spark_historical/Notebooks/.ipynb_checkpoints/Untitled-Copy1-checkpoint.ipynb deleted file mode 100644 index a44208e54..000000000 --- a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/Untitled-Copy1-checkpoint.ipynb +++ /dev/null @@ -1,71 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" - ] - } - ], - "source": [ - "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"\n", - "\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "ename": "Unhandled Exception", - "evalue": "(1,13): error CS7036: There is no argument given that corresponds to the required formal parameter 'database' of 'Query.Query(string, string, int, string, string)'", - "output_type": "error", - "traceback": [ - "(1,13): error CS7036: There is no argument given that corresponds to the required formal parameter 'database' of 'Query.Query(string, string, int, string, string)'" - ] - } - ], - "source": [ - "var q = new bts.history.Query();\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".NET (C#)", - "language": "C#", - "name": ".net-csharp" - }, - "language_info": { - "file_extension": ".cs", - "mimetype": "text/x-csharp", - "name": "C#", - "pygments_lexer": "csharp", - "version": "8.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/src/client/spark_historical/Notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb deleted file mode 100644 index 2fd64429b..000000000 --- a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/query_sample-checkpoint.ipynb b/src/client/spark_historical/Notebooks/.ipynb_checkpoints/query_sample-checkpoint.ipynb index 3e2801785..597c25e07 100644 --- a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/query_sample-checkpoint.ipynb +++ b/src/client/spark_historical/Notebooks/.ipynb_checkpoints/query_sample-checkpoint.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "scrolled": true }, @@ -11,12 +11,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll" + "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll" ] } ], "source": [ - "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n", "//#r \"nuget:Google.Protobuf, 3.6.1\"\n", "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n", "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n", @@ -43,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": { "scrolled": true }, @@ -52,21 +51,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" + "Invalid: Command: #r c:/code/bts/spark/build/Release/bin/netstandard2.0/spark_historical.dll" ] } ], "source": [ - "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n", + "#r \"c:/code/bts/spark/build/Release/bin/netstandard2.0/spark_historical.dll\" \n", "using bts.history;\n", "using MongoDB.Driver;\n", "using MongoDB.Driver.Linq;\n", - "using Google.Protobuf.WellKnownTypes;\n" + "using Google.Protobuf.WellKnownTypes;\n", + "using static bts.history.Query;\n" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 3, "metadata": { "scrolled": true }, @@ -75,6 +75,13 @@ "var q = new Query(\"roll\", \"bigblue\");" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -86,20 +93,7 @@ }, { "cell_type": "code", - "execution_count": 15, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "\n", - "var s = q.series(\"ZNZ9-ZNH0\").Where( x=> x.Book != null );\n", - "var s1 = s.Where( x => x.Book.Bids[0].Size > 0);" - ] - }, - { - "cell_type": "code", - "execution_count": 16, + "execution_count": 4, "metadata": { "scrolled": true }, @@ -107,7 +101,7 @@ { "data": { "text/html": [ - "106013" + "1982025" ] }, "metadata": {}, @@ -115,8 +109,65 @@ } ], "source": [ + "var s = q.series(\"ZNZ9-ZNH0\").Where( x=> x.Book != null );\n", + "var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n", "display(s.Count())\n", - " " + "var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n", + "display(s2.Count());" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "var z9 = q.timeseries(\"ZNZ9\", new DateTime(2019, 11, 25, 0,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));\n", + "var h0 = q.timeseries(\"ZNH0\", new DateTime(2019, 11, 25, 0,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));\n", + "var spread = q.timeseries(\"ZNZ9-ZNH0\", new DateTime(2019, 11, 25, 0,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "3499301" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "1900644" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "1562688" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display(z9.Count());\n", + "display(h0.Count());\n", + "display(spread.Count());\n" ] }, { @@ -125,61 +176,58 @@ "metadata": { "scrolled": true }, - "outputs": [ - { - "data": { - "text/html": [ - "89186" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n", - "display(s2.Count());" + "var z9t = z9.Where(x => x.TopBook != null).AsEnumerable().Select(x => x.TopBook);\n", + "var h0t = h0.Where(x => x.TopBook != null).AsEnumerable().Select(x => x.TopBook);\n", + "var spreadt = spread.Where(x => x.TopBook != null).AsEnumerable().Select(x => x.TopBook);\n", + "\n" ] }, { "cell_type": "code", "execution_count": 18, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ - "var st = q.timeseries(\"ZNZ9-ZNH0\", new DateTime(2019, 11,26, 14,0, 0), \n", - " new DateTime(2019, 11, 26, 22, 0, 0));" + "var synth = create_synthetic_top( new []{z9t, h0t, spreadt}, new[] {1.0, -1.0, 1.0} );" ] }, { "cell_type": "code", "execution_count": 19, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "0" - ] - }, - "metadata": {}, - "output_type": "display_data" + "ename": "Unhandled Exception", + "evalue": "Object reference not set to an instance of an object.", + "output_type": "error", + "traceback": [ + "Unhandled Exception", + "Object reference not set to an instance of an object.", + " at bts.history.Query.create_synthetic_top(IEnumerable`1[] datastream, Double[] ratio, Double[] price_mult)+MoveNext() in c:\\code\\bts\\spark\\src\\client\\spark_historical\\Query\\Query.cs:line 116", + " at System.Linq.Enumerable.EnumerablePartition`1.MoveNext()", + " at Microsoft.DotNet.Interactive.Rendering.HtmlFormatter`1.<>c__DisplayClass7_0.b__4(T instance, TextWriter writer)", + " at Microsoft.DotNet.Interactive.Rendering.HtmlFormatter`1.Format(T instance, TextWriter writer)", + " at Microsoft.DotNet.Interactive.Rendering.TypeFormatter`1.Microsoft.DotNet.Interactive.Rendering.ITypeFormatter.Format(Object instance, TextWriter writer)", + " at Microsoft.DotNet.Interactive.Rendering.Formatter`1.FormatTo(T obj, TextWriter writer, String mimeType)", + " at Microsoft.DotNet.Interactive.Rendering.Formatter.FormatTo[T](T obj, TextWriter writer, String mimeType)", + " at Microsoft.DotNet.Interactive.Rendering.Formatter.ToDisplayString(Object obj, String mimeType)", + " at Microsoft.DotNet.Interactive.Kernel.display(Object value, String mimeType)", + " at Submission#20.<>d__0.MoveNext()", + "--- End of stack trace from previous location where exception was thrown ---", + " at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)" + ] } ], "source": [ - "display(st.Count());" + "display(synth.Take(10));" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [] } diff --git a/src/client/spark_historical/Notebooks/Untitled-Copy1.ipynb b/src/client/spark_historical/Notebooks/Untitled-Copy1.ipynb deleted file mode 100644 index a44208e54..000000000 --- a/src/client/spark_historical/Notebooks/Untitled-Copy1.ipynb +++ /dev/null @@ -1,71 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" - ] - } - ], - "source": [ - "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"\n", - "\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "ename": "Unhandled Exception", - "evalue": "(1,13): error CS7036: There is no argument given that corresponds to the required formal parameter 'database' of 'Query.Query(string, string, int, string, string)'", - "output_type": "error", - "traceback": [ - "(1,13): error CS7036: There is no argument given that corresponds to the required formal parameter 'database' of 'Query.Query(string, string, int, string, string)'" - ] - } - ], - "source": [ - "var q = new bts.history.Query();\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".NET (C#)", - "language": "C#", - "name": ".net-csharp" - }, - "language_info": { - "file_extension": ".cs", - "mimetype": "text/x-csharp", - "name": "C#", - "pygments_lexer": "csharp", - "version": "8.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/client/spark_historical/Notebooks/Untitled.ipynb b/src/client/spark_historical/Notebooks/Untitled.ipynb deleted file mode 100644 index a44208e54..000000000 --- a/src/client/spark_historical/Notebooks/Untitled.ipynb +++ /dev/null @@ -1,71 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" - ] - } - ], - "source": [ - "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"\n", - "\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "ename": "Unhandled Exception", - "evalue": "(1,13): error CS7036: There is no argument given that corresponds to the required formal parameter 'database' of 'Query.Query(string, string, int, string, string)'", - "output_type": "error", - "traceback": [ - "(1,13): error CS7036: There is no argument given that corresponds to the required formal parameter 'database' of 'Query.Query(string, string, int, string, string)'" - ] - } - ], - "source": [ - "var q = new bts.history.Query();\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".NET (C#)", - "language": "C#", - "name": ".net-csharp" - }, - "language_info": { - "file_extension": ".cs", - "mimetype": "text/x-csharp", - "name": "C#", - "pygments_lexer": "csharp", - "version": "8.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/client/spark_historical/Notebooks/query_sample.ipynb b/src/client/spark_historical/Notebooks/query_sample.ipynb index 3e2801785..e572145b2 100644 --- a/src/client/spark_historical/Notebooks/query_sample.ipynb +++ b/src/client/spark_historical/Notebooks/query_sample.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "scrolled": true }, @@ -11,12 +11,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll" + "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll" ] } ], "source": [ - "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n", "//#r \"nuget:Google.Protobuf, 3.6.1\"\n", "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n", "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n", @@ -43,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "scrolled": true }, @@ -52,21 +51,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll" + "Invalid: Command: #r c:/code/bts/spark/build/Release/bin/netstandard2.0/spark_historical.dll" ] } ], "source": [ - "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\" \n", + "#r \"c:/code/bts/spark/build/Release/bin/netstandard2.0/spark_historical.dll\" \n", "using bts.history;\n", "using MongoDB.Driver;\n", "using MongoDB.Driver.Linq;\n", - "using Google.Protobuf.WellKnownTypes;\n" + "using Google.Protobuf.WellKnownTypes;\n", + "using static bts.history.Query;\n" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 3, "metadata": { "scrolled": true }, @@ -75,6 +75,13 @@ "var q = new Query(\"roll\", \"bigblue\");" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -86,76 +93,45 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 4, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "1982025" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "\n", "var s = q.series(\"ZNZ9-ZNH0\").Where( x=> x.Book != null );\n", - "var s1 = s.Where( x => x.Book.Bids[0].Size > 0);" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "106013" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ + "var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n", "display(s.Count())\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "89186" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ "var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n", "display(s2.Count());" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 4, "metadata": { "scrolled": true }, "outputs": [], "source": [ - "var st = q.timeseries(\"ZNZ9-ZNH0\", new DateTime(2019, 11,26, 14,0, 0), \n", - " new DateTime(2019, 11, 26, 22, 0, 0));" + "var z9 = q.timeseries(\"ZNZ9\", new DateTime(2019, 11, 25, 0,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));\n", + "var h0 = q.timeseries(\"ZNH0\", new DateTime(2019, 11, 25, 0,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));\n", + "var spread = q.timeseries(\"ZNZ9-ZNH0\", new DateTime(2019, 11, 25, 0,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 15, "metadata": { "scrolled": true }, @@ -163,7 +139,25 @@ { "data": { "text/html": [ - "0" + "3499301" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "1900644" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "1562688" ] }, "metadata": {}, @@ -171,15 +165,69 @@ } ], "source": [ - "display(st.Count());" + "display(z9.Count());\n", + "display(h0.Count());\n", + "display(spread.Count());\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "var z9t = z9.Where(x => x.TopBook != null).AsEnumerable().Select(x => x.TopBook);\n", + "var h0t = h0.Where(x => x.TopBook != null).AsEnumerable().Select(x => x.TopBook);\n", + "var spreadt = spread.Where(x => x.TopBook != null).AsEnumerable().Select(x => x.TopBook);\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "var synth = create_synthetic_top( new []{z9t, h0t, spreadt}, new[] {1.0, -1.0, 1.0}, new[] {1.0, -1.0, 1.0} );" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "ename": "Unhandled Exception", + "evalue": "Object reference not set to an instance of an object.", + "output_type": "error", + "traceback": [ + "Unhandled Exception", + "Object reference not set to an instance of an object.", + " at bts.history.Query.create_synthetic_top(IEnumerable`1[] datastream, Double[] ratio, Double[] price_mult)+MoveNext() in c:\\code\\bts\\spark\\src\\client\\spark_historical\\Query\\Query.cs:line 116", + " at System.Linq.Enumerable.EnumerablePartition`1.MoveNext()", + " at Microsoft.DotNet.Interactive.Rendering.HtmlFormatter`1.<>c__DisplayClass7_0.b__4(T instance, TextWriter writer)", + " at Microsoft.DotNet.Interactive.Rendering.HtmlFormatter`1.Format(T instance, TextWriter writer)", + " at Microsoft.DotNet.Interactive.Rendering.TypeFormatter`1.Microsoft.DotNet.Interactive.Rendering.ITypeFormatter.Format(Object instance, TextWriter writer)", + " at Microsoft.DotNet.Interactive.Rendering.Formatter`1.FormatTo(T obj, TextWriter writer, String mimeType)", + " at Microsoft.DotNet.Interactive.Rendering.Formatter.FormatTo[T](T obj, TextWriter writer, String mimeType)", + " at Microsoft.DotNet.Interactive.Rendering.Formatter.ToDisplayString(Object obj, String mimeType)", + " at Microsoft.DotNet.Interactive.Kernel.display(Object value, String mimeType)", + " at Submission#9.<>d__0.MoveNext()", + "--- End of stack trace from previous location where exception was thrown ---", + " at Microsoft.CodeAnalysis.Scripting.ScriptExecutionState.RunSubmissionsAsync[TResult](ImmutableArray`1 precedingExecutors, Func`2 currentExecutor, StrongBox`1 exceptionHolderOpt, Func`2 catchExceptionOpt, CancellationToken cancellationToken)" + ] + } + ], + "source": [ + "display(synth.Take(10));" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [] } diff --git a/src/client/spark_historical/Query/MongoGenerator.cs b/src/client/spark_historical/Query/MongoGenerator.cs index 3554afc24..a2d936d02 100644 --- a/src/client/spark_historical/Query/MongoGenerator.cs +++ b/src/client/spark_historical/Query/MongoGenerator.cs @@ -93,9 +93,24 @@ namespace bts.history } }); _done_reading = true; + Console.WriteLine("waiting for tasks to finish " + get_qlen() + " operations in q"); foreach (var qtask in _qtask) await qtask; - Console.WriteLine(get_qlen() +" operations in q"); + Console.WriteLine(get_qlen() +" operations in q"); + foreach(var secdesc in _books.Keys) + { + var tname = table_name(secdesc); + Console.WriteLine("creating indices " + tname); + var table = _db.GetCollection(tname); + + var builder = Builders.IndexKeys; + var index1 = new CreateIndexModel(builder.Ascending(x => x.sending_time)); + await table.Indexes.CreateOneAsync(index1).ConfigureAwait(false); + var index2 = new CreateIndexModel(builder.Ascending(x => x.transact_time)); + await table.Indexes.CreateOneAsync(index2).ConfigureAwait(false); + var index3 = new CreateIndexModel(builder.Ascending(x => x.trade_date)); + await table.Indexes.CreateOneAsync(index3).ConfigureAwait(false); + } watch.Stop(); Console.WriteLine("operation took " + watch.Elapsed.ToString() + " press any key"); Console.ReadKey(); @@ -236,9 +251,9 @@ namespace bts.history } } - string table_name(string v) + public static string table_name(string secdesc) { - return v; + return secdesc; } void write_trade(TradeMessage tm, MDEntry entry, MarketDataIncrementalRefresh md) diff --git a/src/client/spark_historical/Query/Query.cs b/src/client/spark_historical/Query/Query.cs index 6e325ce28..88b9f7b63 100644 --- a/src/client/spark_historical/Query/Query.cs +++ b/src/client/spark_historical/Query/Query.cs @@ -11,6 +11,7 @@ using MongoDB.Bson.Serialization; using bts.co2; using MongoDB.Bson.Serialization.Serializers; using System.Collections.Generic; +using mbts.co2; namespace bts.history { @@ -76,15 +77,81 @@ namespace bts.history IMongoQueryable res = coll.AsQueryable(); Expression> timeexpr = null; if(use_sending_time) - timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end; + timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end; else timeexpr = (H h) => h.transact_time >= begin && h.transact_time <= end; res = res.Where( timeexpr ); return res; } + //so I can get into a local linq through the jupyter nb + public static IEnumerable local(IMongoQueryable q) + { + return q.AsEnumerable(); + } + + public static IEnumerable create_synthetic_top(IEnumerable[] datastream, + double[] ratio, double[] price_mult = null) + { + if(price_mult == null) + price_mult = ratio; + + TopBookMessage[] valcache = new TopBookMessage[datastream.Length]; + bool go = false; bool once = false; + do + { + for(int i = 0; i < datastream.Length; i++) + { + go |= datastream[i].GetEnumerator().MoveNext(); + if(go) + { + valcache[i] = datastream[i].GetEnumerator().Current; + if(once) //we have a value for all legs + { + double bid = 0.0, ask = 0.0; uint bsize = uint.MaxValue, asize = uint.MaxValue; + for(int z = 0; z < valcache.Length; z++) + { + double r = ratio[z]; + double p = price_mult[z]; + var b = valcache[z].Bid; + var a = valcache[z].Ask; + if(r > 0.0) + { + //you receive the bid price for each buy leg if you sell + bid += b.price() * p; + bsize = Math.Min(bsize, (uint)(b.Size / r)); + + //you pay the ask price for each buy leg if you buy + ask += a.price() * p; + asize = Math.Min(asize, (uint)(a.Size / r)); + } + else //sell this leg, look at bid for bid + { + //you pay ask price for each sell leg if you sell + bid += a.price() * p; + bsize = Math.Min(bsize, (uint)(a.Size / r)); + + //you receive bid price for each sell leg if you buy + ask += b.price() * p; + asize = Math.Min(asize, (uint)(b.Size / r)); + } + } + var bi = IntPrice.from_double(bid, 7); + var ai = IntPrice.from_double(ask, 7); + yield return new TopBookMessage { + Bid = new PriceSizeMessage { Size = bsize, Mantissa = bi.mantissa, Exponent = bi.exponent }, + Ask = new PriceSizeMessage { Size = asize, Mantissa = ai.mantissa, Exponent = ai.exponent } + }; + } + } + } + once = true; + } + while(go); + } + public IEnumerable run_algo(IAlgo algo, IFillModel fill_model, uint latency, - IEnumerable datastream) + Dictionary< string, IEnumerable > datastream) { return null; } diff --git a/src/client/spark_historical/setup_notes.txt b/src/client/spark_historical/setup_notes.txt index 2c0fad197..68935ec7c 100644 --- a/src/client/spark_historical/setup_notes.txt +++ b/src/client/spark_historical/setup_notes.txt @@ -23,10 +23,20 @@ db.createUser( ) -#install python for jupyter notebooks -install python 3.x +#installation instructions +#install python https://www.python.org/downloads/ + +#for 3.x pip3 install future setuptools numpy protobuf typing hypothesis wheel jupyter graphviz pydot matplotlib +#for 2.x +pip install future protobuf jupyter + +#now install the windows x64 .net core 3 sdk here: https://dotnet.microsoft.com/download/dotnet-core/3.0 +#(https://dotnet.microsoft.com/download/dotnet-core/thank-you/sdk-3.0.101-windows-x64-installer) +#you may need to upgrade your visual studio (cuz msbuild) to at least 15.9 in order to build the system now +#this studio upgrade will take an hour + #jupyter .net dotnet tool install --global dotnet-try #see if jupyter is installed