From c6c0dc21e2b4e1bbeccdbef66728ad2bcb8b4804 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Sat, 14 Nov 2020 12:42:44 -0500 Subject: [PATCH] Added better logging to simulator --- src/data_files/files/spartan/sim_db.sql | 26 +- .../lining_up_books-checkpoint.ipynb | 1234 ++++++++++++++ .../spartan/analysis/lining_up_books.ipynb | 1448 +++++++++++++++++ .../sparkd/cmds/spartan/analysis/spartan.py | 212 +++ .../mdp3/back_testing/algos/PolyObserver.cpp | 76 +- .../md/mdp3/back_testing/algos/Sniper.cpp | 9 + .../orchestrator/SimOrchestrator.cpp | 19 +- .../orchestrator/SimSessionOptions.proto | 18 +- .../md/mdp3/back_testing/sim_db/SimDB.cpp | 4 + .../back_testing/simulator/ExchangeSim.cpp | 56 +- .../simulator/agents/GatewayAgent.cpp | 96 +- 11 files changed, 3091 insertions(+), 107 deletions(-) create mode 100644 src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints/lining_up_books-checkpoint.ipynb create mode 100644 src/server/sparkd/cmds/spartan/analysis/lining_up_books.ipynb create mode 100644 src/server/sparkd/cmds/spartan/analysis/spartan.py diff --git a/src/data_files/files/spartan/sim_db.sql b/src/data_files/files/spartan/sim_db.sql index 06441424b..ba39b2d86 100644 --- a/src/data_files/files/spartan/sim_db.sql +++ b/src/data_files/files/spartan/sim_db.sql @@ -59,13 +59,29 @@ create table log( val10 TEXT, foreign key(run_id) references run(run_id) ); -CREATE INDEX log_idx_0 ON log(run_id,entry_type); -CREATE INDEX log_idx_1 ON log(val1); -CREATE INDEX log_idx_2 ON log(val2); -CREATE INDEX log_idx_3 ON log(val3); +CREATE INDEX log_idx_0 ON log(run_id); + +/* Gateway log */ +drop view if exists gateway_log; +create view gateway_log as +select + log_entry_order, + t_time,h_t_time,entry_type, + cast (val1 as numeric) eid1, + cast (val2 as numeric) eid2, + val3,val4,val5,val6,val7, + val8,val9 +from + log +where + entry_type='gw_order' or + entry_type='gw_algo_order' or + entry_type='gw_fill' or + entry_type='gw_algo_fill' +order by + log_entry_order; /* sniper specific views */ - drop view if exists shot_beginnings; create view shot_beginnings as select diff --git a/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints/lining_up_books-checkpoint.ipynb b/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints/lining_up_books-checkpoint.ipynb new file mode 100644 index 000000000..43d3730c3 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints/lining_up_books-checkpoint.ipynb @@ -0,0 +1,1234 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import spartan \n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "# Log entire log from DB (this can easily be 10GB, so it might take a while)\n", + "log = spartan.qsim(\"select * from log\")\n", + "log.to_parquet(\"/tmp/log.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "log = pd.read_parquet(\"/tmp/log.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract and clean up book entries\n", + "books = log[log.entry_type=='gw_book']\n", + "books = spartan.clean_sim_books(books)\n", + "\n", + "# Create an event timeline for the books that form our main poligon\n", + "lined_up_books = spartan.get_310_lined_prices(books)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Grab shots\n", + "shots = spartan.get_shot_outcomes()" + ] + }, + { + "cell_type": "code", + "execution_count": 394, + "metadata": {}, + "outputs": [], + "source": [ + "r_books = lined_up_books.loc[20309598:20309692]" + ] + }, + { + "cell_type": "code", + "execution_count": 395, + "metadata": {}, + "outputs": [], + "source": [ + "r_shots = shots.loc[2918]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eid1eid2symbolbidbid_qtyaskask_qty
780714550ESM1300000.02.0343175.01.0
780814550ESU0-ESU1-6395.01.0-2805.01.0
780914550ESH1330850.01.0343000.01.0
781014550ESU0-ESZ0-1055.0812.0-1050.0247.0
781114550ESU0334525.02.0334550.07.0
........................
11679363879070ESZ0333475.09.0333500.02.0
11679393879080ESZ0333475.09.0333500.02.0
11679423879090ESH1332550.02.0332775.01.0
11679493879100ESZ0333475.09.0333500.01.0
11679523879110ESZ0333475.09.0333525.012.0
\n", + "

284111 rows × 7 columns

\n", + "
" + ], + "text/plain": [ + " eid1 eid2 symbol bid bid_qty ask ask_qty\n", + "7807 1455 0 ESM1 300000.0 2.0 343175.0 1.0\n", + "7808 1455 0 ESU0-ESU1 -6395.0 1.0 -2805.0 1.0\n", + "7809 1455 0 ESH1 330850.0 1.0 343000.0 1.0\n", + "7810 1455 0 ESU0-ESZ0 -1055.0 812.0 -1050.0 247.0\n", + "7811 1455 0 ESU0 334525.0 2.0 334550.0 7.0\n", + "... ... ... ... ... ... ... ...\n", + "1167936 387907 0 ESZ0 333475.0 9.0 333500.0 2.0\n", + "1167939 387908 0 ESZ0 333475.0 9.0 333500.0 2.0\n", + "1167942 387909 0 ESH1 332550.0 2.0 332775.0 1.0\n", + "1167949 387910 0 ESZ0 333475.0 9.0 333500.0 1.0\n", + "1167952 387911 0 ESZ0 333475.0 9.0 333525.0 12.0\n", + "\n", + "[284111 rows x 7 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "books" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1455" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "books.eid1.min()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "387911" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "books.eid1.max()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "range(1455, 387912)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "idx = pd.DataFrame({'eid1':range(books.eid1.min(),books.eid1.max()+1)})\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eid1
01455
11456
21457
31458
41459
......
386452387907
386453387908
386454387909
386455387910
386456387911
\n", + "

386457 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " eid1\n", + "0 1455\n", + "1 1456\n", + "2 1457\n", + "3 1458\n", + "4 1459\n", + "... ...\n", + "386452 387907\n", + "386453 387908\n", + "386454 387909\n", + "386455 387910\n", + "386456 387911\n", + "\n", + "[386457 rows x 1 columns]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "idx" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eid1eid2
014550
114560
214570
314580
414590
.........
3864523879070
3864533879080
3864543879090
3864553879100
3864563879110
\n", + "

386457 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " eid1 eid2\n", + "0 1455 0\n", + "1 1456 0\n", + "2 1457 0\n", + "3 1458 0\n", + "4 1459 0\n", + "... ... ...\n", + "386452 387907 0\n", + "386453 387908 0\n", + "386454 387909 0\n", + "386455 387910 0\n", + "386456 387911 0\n", + "\n", + "[386457 rows x 2 columns]" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "idx['eid2'] = 0\n", + "idx" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eid1eid2
014550
114560
214570
314580
414590
.........
3864523879070
3864533879080
3864543879090
3864553879100
3864563879110
\n", + "

386457 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " eid1 eid2\n", + "0 1455 0\n", + "1 1456 0\n", + "2 1457 0\n", + "3 1458 0\n", + "4 1459 0\n", + "... ... ...\n", + "386452 387907 0\n", + "386453 387908 0\n", + "386454 387909 0\n", + "386455 387910 0\n", + "386456 387911 0\n", + "\n", + "[386457 rows x 2 columns]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "idx" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "idx = idx.set_index(['eid1','eid2'],drop=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eid1eid2
eid2
014590
\n", + "
" + ], + "text/plain": [ + " eid1 eid2\n", + "eid2 \n", + "0 1459 0" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "idx.loc[1459]" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eid1eid2bid_ubid_qty_uask_uask_qty_ubid_zbid_qty_zask_zask_qty_zbid_u_zbid_qty_u_zask_u_zask_qty_u_zbuy_p_valuesell_p_value
eid1eid2
1455014550334525.02.0334550.07.0333475.02.0333525.01.0-1055.0812.0-1050.0247.0-27.5-12.5
1456014560334525.02.0334550.07.0333475.02.0333525.01.0-1055.0812.0-1050.0247.0-27.5-12.5
1457014570334525.02.0334550.07.0333475.02.0333525.01.0-1055.0812.0-1050.0247.0-27.5-12.5
1458014580334525.02.0334550.07.0333375.02.0333525.01.0-1055.0812.0-1050.0247.0-27.5-62.5
1459014590334525.02.0334550.07.0333375.02.0333525.01.0-1055.0812.0-1050.0247.0-27.5-62.5
......................................................
38790703879070334525.07.0334575.011.0333475.09.0333500.02.0-1065.0955.0-1060.01784.0-20.0-20.0
38790803879080334525.07.0334575.011.0333475.09.0333500.02.0-1065.0955.0-1060.01784.0-20.0-20.0
38790903879090334525.07.0334575.011.0333475.09.0333500.02.0-1065.0955.0-1060.01784.0-20.0-20.0
38791003879100334525.07.0334575.011.0333475.09.0333500.01.0-1065.0955.0-1060.01784.0-20.0-20.0
38791103879110334525.07.0334575.011.0333475.09.0333525.012.0-1065.0955.0-1060.01784.0-32.5-20.0
\n", + "

386457 rows × 16 columns

\n", + "
" + ], + "text/plain": [ + " eid1 eid2 bid_u bid_qty_u ask_u ask_qty_u bid_z \\\n", + "eid1 eid2 \n", + "1455 0 1455 0 334525.0 2.0 334550.0 7.0 333475.0 \n", + "1456 0 1456 0 334525.0 2.0 334550.0 7.0 333475.0 \n", + "1457 0 1457 0 334525.0 2.0 334550.0 7.0 333475.0 \n", + "1458 0 1458 0 334525.0 2.0 334550.0 7.0 333375.0 \n", + "1459 0 1459 0 334525.0 2.0 334550.0 7.0 333375.0 \n", + "... ... ... ... ... ... ... ... \n", + "387907 0 387907 0 334525.0 7.0 334575.0 11.0 333475.0 \n", + "387908 0 387908 0 334525.0 7.0 334575.0 11.0 333475.0 \n", + "387909 0 387909 0 334525.0 7.0 334575.0 11.0 333475.0 \n", + "387910 0 387910 0 334525.0 7.0 334575.0 11.0 333475.0 \n", + "387911 0 387911 0 334525.0 7.0 334575.0 11.0 333475.0 \n", + "\n", + " bid_qty_z ask_z ask_qty_z bid_u_z bid_qty_u_z ask_u_z \\\n", + "eid1 eid2 \n", + "1455 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n", + "1456 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n", + "1457 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n", + "1458 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n", + "1459 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n", + "... ... ... ... ... ... ... \n", + "387907 0 9.0 333500.0 2.0 -1065.0 955.0 -1060.0 \n", + "387908 0 9.0 333500.0 2.0 -1065.0 955.0 -1060.0 \n", + "387909 0 9.0 333500.0 2.0 -1065.0 955.0 -1060.0 \n", + "387910 0 9.0 333500.0 1.0 -1065.0 955.0 -1060.0 \n", + "387911 0 9.0 333525.0 12.0 -1065.0 955.0 -1060.0 \n", + "\n", + " ask_qty_u_z buy_p_value sell_p_value \n", + "eid1 eid2 \n", + "1455 0 247.0 -27.5 -12.5 \n", + "1456 0 247.0 -27.5 -12.5 \n", + "1457 0 247.0 -27.5 -12.5 \n", + "1458 0 247.0 -27.5 -62.5 \n", + "1459 0 247.0 -27.5 -62.5 \n", + "... ... ... ... \n", + "387907 0 1784.0 -20.0 -20.0 \n", + "387908 0 1784.0 -20.0 -20.0 \n", + "387909 0 1784.0 -20.0 -20.0 \n", + "387910 0 1784.0 -20.0 -20.0 \n", + "387911 0 1784.0 -32.5 -20.0 \n", + "\n", + "[386457 rows x 16 columns]" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lined_up_books = spartan.get_310_lined_prices(books)\n", + "lined_up_books" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eid1eid2bid_ubid_qty_uask_uask_qty_ubid_zbid_qty_zask_zask_qty_zbid_u_zbid_qty_u_zask_u_zask_qty_u_zbuy_p_valuesell_p_value
eid2
0100000334750.013.0334775.024.0333700.06.0333750.05.0-1055.0317.0-1050.0586.0-27.5-12.5
\n", + "
" + ], + "text/plain": [ + " eid1 eid2 bid_u bid_qty_u ask_u ask_qty_u bid_z \\\n", + "eid2 \n", + "0 10000 0 334750.0 13.0 334775.0 24.0 333700.0 \n", + "\n", + " bid_qty_z ask_z ask_qty_z bid_u_z bid_qty_u_z ask_u_z \\\n", + "eid2 \n", + "0 6.0 333750.0 5.0 -1055.0 317.0 -1050.0 \n", + "\n", + " ask_qty_u_z buy_p_value sell_p_value \n", + "eid2 \n", + "0 586.0 -27.5 -12.5 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lined_up_books.loc[10000]" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "250.0" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(lined_up_books.ask_u - lined_up_books.bid_u).max()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/server/sparkd/cmds/spartan/analysis/lining_up_books.ipynb b/src/server/sparkd/cmds/spartan/analysis/lining_up_books.ipynb new file mode 100644 index 000000000..caff1953d --- /dev/null +++ b/src/server/sparkd/cmds/spartan/analysis/lining_up_books.ipynb @@ -0,0 +1,1448 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import spartan \n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "# Log entire log from DB (this can easily be 10GB, so it might take a while)\n", + "log = spartan.qsim(\"select * from log\")\n", + "log.to_parquet(\"/tmp/log.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "log = pd.read_parquet(\"/tmp/log.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract and clean up book entries\n", + "books = log[log.entry_type=='gw_book']\n", + "books = spartan.clean_sim_books(books)\n", + "\n", + "# Create an event timeline for the books that form our main poligon\n", + "lined_up_books = spartan.get_310_lined_prices(books)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "# Grab shots\n", + "shots = spartan.get_shot_outcomes()" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "# Grab gateway logs\n", + "gw_logs = spartan.qsim(\"select * from gateway_log\")\n", + "gw_logs = gw_logs.set_index(['eid1','eid2'],drop=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
streak_idlen
0291847
1138043
2118838
3139837
4293135
.........
302726471
30285981
302926431
30305941
303120451
\n", + "

3032 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " streak_id len\n", + "0 2918 47\n", + "1 1380 43\n", + "2 1188 38\n", + "3 1398 37\n", + "4 2931 35\n", + "... ... ...\n", + "3027 2647 1\n", + "3028 598 1\n", + "3029 2643 1\n", + "3030 594 1\n", + "3031 2045 1\n", + "\n", + "[3032 rows x 2 columns]" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# find longest streaks of shots\n", + "streaks = pd.DataFrame(shots.streak_id.value_counts()).reset_index().rename({'index':'streak_id','streak_id':'len'},axis=1)\n", + "streaks" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
run_idt_timepolygono_eido_eid_2e_eide_eid_2targetrealizedstreak_id
559711599854398601848154+ESU0 +(ESU0-ESZ0) -ESZ020309598020309647317.517.52918
559811599854398601867165+ESU0 +(ESU0-ESZ0) -ESZ020309647320309667317.517.52918
559911599854398601879165+ESU0 +(ESU0-ESZ0) -ESZ020309667320309667617.517.52918
560011599854398601970880+ESU0 +(ESU0-ESZ0) -ESZ020309667620309668317.517.52918
560111599854398602128013+ESU0 +(ESU0-ESZ0) -ESZ020309668320309670317.517.52918
560211599854398602143173+ESU0 +(ESU0-ESZ0) -ESZ020309670320309677317.517.52918
560311599854398602246463+ESU0 +(ESU0-ESZ0) -ESZ020309677320309678317.517.52918
560411599854398602351943+ESU0 +(ESU0-ESZ0) -ESZ020309678320309689317.517.52918
560511599854398602428572+ESU0 +(ESU0-ESZ0) -ESZ020309689320309690317.517.52918
560611599854398602440572+ESU0 +(ESU0-ESZ0) -ESZ020309690320309690617.517.52918
560711599854398602452572+ESU0 +(ESU0-ESZ0) -ESZ020309690620309690917.517.52918
560811599854398602464572+ESU0 +(ESU0-ESZ0) -ESZ0203096909203096901217.517.52918
560911599854398602476572+ESU0 +(ESU0-ESZ0) -ESZ02030969012203096901517.517.52918
561011599854398602488572+ESU0 +(ESU0-ESZ0) -ESZ02030969015203096901817.517.52918
561111599854398602500572+ESU0 +(ESU0-ESZ0) -ESZ02030969018203096902117.517.52918
561211599854398602512572+ESU0 +(ESU0-ESZ0) -ESZ02030969021203096902417.517.52918
561311599854398602524572+ESU0 +(ESU0-ESZ0) -ESZ02030969024203096902717.517.52918
561411599854398602536572+ESU0 +(ESU0-ESZ0) -ESZ02030969027203096903017.517.52918
561511599854398602548572+ESU0 +(ESU0-ESZ0) -ESZ02030969030203096903317.517.52918
561611599854398602560572+ESU0 +(ESU0-ESZ0) -ESZ02030969033203096903617.517.52918
561711599854398602572572+ESU0 +(ESU0-ESZ0) -ESZ02030969036203096903917.517.52918
561811599854398602584572+ESU0 +(ESU0-ESZ0) -ESZ02030969039203096904217.517.52918
561911599854398602596572+ESU0 +(ESU0-ESZ0) -ESZ02030969042203096904517.517.52918
562011599854398602676203+ESU0 +(ESU0-ESZ0) -ESZ0203096904520309691317.517.52918
562111599854398602944102+ESU0 +(ESU0-ESZ0) -ESZ020309691320309692317.517.52918
562211599854398602956102+ESU0 +(ESU0-ESZ0) -ESZ020309692320309692617.517.52918
562311599854398602968102+ESU0 +(ESU0-ESZ0) -ESZ020309692620309692917.517.52918
562411599854398602980102+ESU0 +(ESU0-ESZ0) -ESZ0203096929203096921217.517.52918
562511599854398602992102+ESU0 +(ESU0-ESZ0) -ESZ02030969212203096921517.517.52918
562611599854398603004102+ESU0 +(ESU0-ESZ0) -ESZ02030969215203096921817.517.52918
562711599854398603016102+ESU0 +(ESU0-ESZ0) -ESZ02030969218203096922117.517.52918
562811599854398603028102+ESU0 +(ESU0-ESZ0) -ESZ02030969221203096922417.517.52918
562911599854398603040102+ESU0 +(ESU0-ESZ0) -ESZ02030969224203096922717.517.52918
563011599854398603052102+ESU0 +(ESU0-ESZ0) -ESZ02030969227203096923017.517.52918
563111599854398603064102+ESU0 +(ESU0-ESZ0) -ESZ02030969230203096923317.517.52918
563211599854398603076102+ESU0 +(ESU0-ESZ0) -ESZ02030969233203096923617.517.52918
563311599854398603088102+ESU0 +(ESU0-ESZ0) -ESZ02030969236203096923917.517.52918
563411599854398603100102+ESU0 +(ESU0-ESZ0) -ESZ02030969239203096924217.517.52918
563511599854398603112102+ESU0 +(ESU0-ESZ0) -ESZ02030969242203096924517.517.52918
563611599854398603124102+ESU0 +(ESU0-ESZ0) -ESZ02030969245203096924817.517.52918
563711599854398603136102+ESU0 +(ESU0-ESZ0) -ESZ02030969248203096925117.517.52918
563811599854398603148102+ESU0 +(ESU0-ESZ0) -ESZ02030969251203096925417.517.52918
563911599854398603160102+ESU0 +(ESU0-ESZ0) -ESZ02030969254203096925717.517.52918
564011599854398603172102+ESU0 +(ESU0-ESZ0) -ESZ02030969257203096926017.517.52918
564111599854398603184102+ESU0 +(ESU0-ESZ0) -ESZ02030969260203096926317.517.52918
564211599854398603196102+ESU0 +(ESU0-ESZ0) -ESZ02030969263203096926617.517.52918
564311599854398603208102+ESU0 +(ESU0-ESZ0) -ESZ02030969266203096926917.517.52918
\n", + "
" + ], + "text/plain": [ + " run_id t_time polygon o_eid \\\n", + "5597 1 1599854398601848154 +ESU0 +(ESU0-ESZ0) -ESZ0 20309598 \n", + "5598 1 1599854398601867165 +ESU0 +(ESU0-ESZ0) -ESZ0 20309647 \n", + "5599 1 1599854398601879165 +ESU0 +(ESU0-ESZ0) -ESZ0 20309667 \n", + "5600 1 1599854398601970880 +ESU0 +(ESU0-ESZ0) -ESZ0 20309667 \n", + "5601 1 1599854398602128013 +ESU0 +(ESU0-ESZ0) -ESZ0 20309668 \n", + "5602 1 1599854398602143173 +ESU0 +(ESU0-ESZ0) -ESZ0 20309670 \n", + "5603 1 1599854398602246463 +ESU0 +(ESU0-ESZ0) -ESZ0 20309677 \n", + "5604 1 1599854398602351943 +ESU0 +(ESU0-ESZ0) -ESZ0 20309678 \n", + "5605 1 1599854398602428572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309689 \n", + "5606 1 1599854398602440572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5607 1 1599854398602452572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5608 1 1599854398602464572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5609 1 1599854398602476572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5610 1 1599854398602488572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5611 1 1599854398602500572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5612 1 1599854398602512572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5613 1 1599854398602524572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5614 1 1599854398602536572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5615 1 1599854398602548572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5616 1 1599854398602560572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5617 1 1599854398602572572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5618 1 1599854398602584572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5619 1 1599854398602596572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5620 1 1599854398602676203 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n", + "5621 1 1599854398602944102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309691 \n", + "5622 1 1599854398602956102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5623 1 1599854398602968102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5624 1 1599854398602980102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5625 1 1599854398602992102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5626 1 1599854398603004102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5627 1 1599854398603016102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5628 1 1599854398603028102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5629 1 1599854398603040102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5630 1 1599854398603052102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5631 1 1599854398603064102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5632 1 1599854398603076102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5633 1 1599854398603088102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5634 1 1599854398603100102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5635 1 1599854398603112102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5636 1 1599854398603124102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5637 1 1599854398603136102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5638 1 1599854398603148102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5639 1 1599854398603160102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5640 1 1599854398603172102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5641 1 1599854398603184102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5642 1 1599854398603196102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "5643 1 1599854398603208102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n", + "\n", + " o_eid_2 e_eid e_eid_2 target realized streak_id \n", + "5597 0 20309647 3 17.5 17.5 2918 \n", + "5598 3 20309667 3 17.5 17.5 2918 \n", + "5599 3 20309667 6 17.5 17.5 2918 \n", + "5600 6 20309668 3 17.5 17.5 2918 \n", + "5601 3 20309670 3 17.5 17.5 2918 \n", + "5602 3 20309677 3 17.5 17.5 2918 \n", + "5603 3 20309678 3 17.5 17.5 2918 \n", + "5604 3 20309689 3 17.5 17.5 2918 \n", + "5605 3 20309690 3 17.5 17.5 2918 \n", + "5606 3 20309690 6 17.5 17.5 2918 \n", + "5607 6 20309690 9 17.5 17.5 2918 \n", + "5608 9 20309690 12 17.5 17.5 2918 \n", + "5609 12 20309690 15 17.5 17.5 2918 \n", + "5610 15 20309690 18 17.5 17.5 2918 \n", + "5611 18 20309690 21 17.5 17.5 2918 \n", + "5612 21 20309690 24 17.5 17.5 2918 \n", + "5613 24 20309690 27 17.5 17.5 2918 \n", + "5614 27 20309690 30 17.5 17.5 2918 \n", + "5615 30 20309690 33 17.5 17.5 2918 \n", + "5616 33 20309690 36 17.5 17.5 2918 \n", + "5617 36 20309690 39 17.5 17.5 2918 \n", + "5618 39 20309690 42 17.5 17.5 2918 \n", + "5619 42 20309690 45 17.5 17.5 2918 \n", + "5620 45 20309691 3 17.5 17.5 2918 \n", + "5621 3 20309692 3 17.5 17.5 2918 \n", + "5622 3 20309692 6 17.5 17.5 2918 \n", + "5623 6 20309692 9 17.5 17.5 2918 \n", + "5624 9 20309692 12 17.5 17.5 2918 \n", + "5625 12 20309692 15 17.5 17.5 2918 \n", + "5626 15 20309692 18 17.5 17.5 2918 \n", + "5627 18 20309692 21 17.5 17.5 2918 \n", + "5628 21 20309692 24 17.5 17.5 2918 \n", + "5629 24 20309692 27 17.5 17.5 2918 \n", + "5630 27 20309692 30 17.5 17.5 2918 \n", + "5631 30 20309692 33 17.5 17.5 2918 \n", + "5632 33 20309692 36 17.5 17.5 2918 \n", + "5633 36 20309692 39 17.5 17.5 2918 \n", + "5634 39 20309692 42 17.5 17.5 2918 \n", + "5635 42 20309692 45 17.5 17.5 2918 \n", + "5636 45 20309692 48 17.5 17.5 2918 \n", + "5637 48 20309692 51 17.5 17.5 2918 \n", + "5638 51 20309692 54 17.5 17.5 2918 \n", + "5639 54 20309692 57 17.5 17.5 2918 \n", + "5640 57 20309692 60 17.5 17.5 2918 \n", + "5641 60 20309692 63 17.5 17.5 2918 \n", + "5642 63 20309692 66 17.5 17.5 2918 \n", + "5643 66 20309692 69 17.5 17.5 2918 " + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get shots participating in the longest streak\n", + "longest_streak = streaks.streak_id.iloc[0]\n", + "first_event = shots[shots.streak_id==longest_streak].o_eid.min()\n", + "last_event = shots[shots.streak_id==longest_streak].o_eid.max()\n", + "shots = shots[shots.streak_id==longest_streak]\n", + "shots" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
t_timeh_t_timeentry_typeeid1_xeid2_xval3val4val5val6val7...bid_zbid_qty_zask_zask_qty_zbid_u_zbid_qty_u_zask_u_zask_qty_u_zbuy_p_valuesell_p_value
eid1eid2
203085980159985439847304592319:59:58.473045923gw_order2030859801599854398472684897updatedESU0-ESZ0-1080.0000001...333000.0155.0333025.013.0-1040.09473.0-1035.05116.0-20.0-7.5
203085990159985439847305375219:59:58.473053752gw_order2030859901599854398472684897updatedESU0334100.0000001...333000.0155.0333025.013.0-1040.09473.0-1035.05116.0-20.0-7.5
203086000159985439847356303019:59:58.473563030gw_order2030860001599854398472697817removedESZ0333025.0000000...333025.08.0333050.0158.0-1040.09473.0-1035.05116.0-32.55.0
0159985439847356303019:59:58.473563030gw_order2030860001599854398472697817removedESZ0333025.0000000...333025.08.0333050.0158.0-1040.09473.0-1035.05116.0-32.55.0
0159985439847356303019:59:58.473563030gw_order2030860001599854398472697817removedESZ0333025.0000000...333025.08.0333050.0158.0-1040.09473.0-1035.05116.0-32.55.0
.....................................................................
2030969269159985439860320810219:59:58.603208102gw_fill20309692691599854398603208102partialSIDE_SELLESU0-ESZ0-1035.000000...332975.0151.0333025.0153.0-1040.09759.0-1035.05063.0-45.05.0
69159985439860320810219:59:58.603208102gw_fill20309692691599854398603208102filledSIDE_BUYESU0-ESZ0-1035.000000...332975.0151.0333025.0153.0-1040.09759.0-1035.05063.0-45.05.0
203096930159985439860382410319:59:58.603824103gw_order2030969301599854398603739477addedESU0334000.0000001...332975.0151.0333025.0153.0-1040.09759.0-1035.05063.0-45.05.0
0159985439860382410319:59:58.603824103gw_fill2030969301599854398603739477partialSIDE_SELLESU0334000.000000...332975.0151.0333025.0153.0-1040.09759.0-1035.05063.0-45.05.0
0159985439860382410319:59:58.603824103gw_fill2030969301599854398603739477filledSIDE_BUYESU0334000.000000...332975.0151.0333025.0153.0-1040.09759.0-1035.05063.0-45.05.0
\n", + "

2094 rows × 28 columns

\n", + "
" + ], + "text/plain": [ + " t_time h_t_time entry_type eid1_x \\\n", + "eid1 eid2 \n", + "20308598 0 1599854398473045923 19:59:58.473045923 gw_order 20308598 \n", + "20308599 0 1599854398473053752 19:59:58.473053752 gw_order 20308599 \n", + "20308600 0 1599854398473563030 19:59:58.473563030 gw_order 20308600 \n", + " 0 1599854398473563030 19:59:58.473563030 gw_order 20308600 \n", + " 0 1599854398473563030 19:59:58.473563030 gw_order 20308600 \n", + "... ... ... ... ... \n", + "20309692 69 1599854398603208102 19:59:58.603208102 gw_fill 20309692 \n", + " 69 1599854398603208102 19:59:58.603208102 gw_fill 20309692 \n", + "20309693 0 1599854398603824103 19:59:58.603824103 gw_order 20309693 \n", + " 0 1599854398603824103 19:59:58.603824103 gw_fill 20309693 \n", + " 0 1599854398603824103 19:59:58.603824103 gw_fill 20309693 \n", + "\n", + " eid2_x val3 val4 val5 val6 \\\n", + "eid1 eid2 \n", + "20308598 0 0 1599854398472684897 updated ESU0-ESZ0 -1080.000000 \n", + "20308599 0 0 1599854398472684897 updated ESU0 334100.000000 \n", + "20308600 0 0 1599854398472697817 removed ESZ0 333025.000000 \n", + " 0 0 1599854398472697817 removed ESZ0 333025.000000 \n", + " 0 0 1599854398472697817 removed ESZ0 333025.000000 \n", + "... ... ... ... ... ... \n", + "20309692 69 69 1599854398603208102 partial SIDE_SELL ESU0-ESZ0 \n", + " 69 69 1599854398603208102 filled SIDE_BUY ESU0-ESZ0 \n", + "20309693 0 0 1599854398603739477 added ESU0 334000.000000 \n", + " 0 0 1599854398603739477 partial SIDE_SELL ESU0 \n", + " 0 0 1599854398603739477 filled SIDE_BUY ESU0 \n", + "\n", + " val7 ... bid_z bid_qty_z ask_z ask_qty_z \\\n", + "eid1 eid2 ... \n", + "20308598 0 1 ... 333000.0 155.0 333025.0 13.0 \n", + "20308599 0 1 ... 333000.0 155.0 333025.0 13.0 \n", + "20308600 0 0 ... 333025.0 8.0 333050.0 158.0 \n", + " 0 0 ... 333025.0 8.0 333050.0 158.0 \n", + " 0 0 ... 333025.0 8.0 333050.0 158.0 \n", + "... ... ... ... ... ... ... \n", + "20309692 69 -1035.000000 ... 332975.0 151.0 333025.0 153.0 \n", + " 69 -1035.000000 ... 332975.0 151.0 333025.0 153.0 \n", + "20309693 0 1 ... 332975.0 151.0 333025.0 153.0 \n", + " 0 334000.000000 ... 332975.0 151.0 333025.0 153.0 \n", + " 0 334000.000000 ... 332975.0 151.0 333025.0 153.0 \n", + "\n", + " bid_u_z bid_qty_u_z ask_u_z ask_qty_u_z buy_p_value \\\n", + "eid1 eid2 \n", + "20308598 0 -1040.0 9473.0 -1035.0 5116.0 -20.0 \n", + "20308599 0 -1040.0 9473.0 -1035.0 5116.0 -20.0 \n", + "20308600 0 -1040.0 9473.0 -1035.0 5116.0 -32.5 \n", + " 0 -1040.0 9473.0 -1035.0 5116.0 -32.5 \n", + " 0 -1040.0 9473.0 -1035.0 5116.0 -32.5 \n", + "... ... ... ... ... ... \n", + "20309692 69 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n", + " 69 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n", + "20309693 0 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n", + " 0 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n", + " 0 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n", + "\n", + " sell_p_value \n", + "eid1 eid2 \n", + "20308598 0 -7.5 \n", + "20308599 0 -7.5 \n", + "20308600 0 5.0 \n", + " 0 5.0 \n", + " 0 5.0 \n", + "... ... \n", + "20309692 69 5.0 \n", + " 69 5.0 \n", + "20309693 0 5.0 \n", + " 0 5.0 \n", + " 0 5.0 \n", + "\n", + "[2094 rows x 28 columns]" + ] + }, + "execution_count": 92, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "first_event = shots.o_eid.min() - 1000\n", + "last_event = shots.o_eid.max() + 1\n", + "\n", + "r_books = lined_up_books.loc[first_event:last_event]\n", + "\n", + "r_logs = gw_logs.loc[first_event:last_event]\n", + "long_streak_log = r_logs.merge(r_books,how='outer',left_index=True,right_index=True)\n", + "long_streak_log.to_csv(\"/home/ppinto/long_streak.csv\")\n", + "long_streak_log" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(20308598, 20309693)" + ] + }, + "execution_count": 93, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(first_event,last_event)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/server/sparkd/cmds/spartan/analysis/spartan.py b/src/server/sparkd/cmds/spartan/analysis/spartan.py new file mode 100644 index 000000000..6d5f5694e --- /dev/null +++ b/src/server/sparkd/cmds/spartan/analysis/spartan.py @@ -0,0 +1,212 @@ +#!/usr/bin/python3 + +import pandas as pd +import os +import sqlite3 +import pathlib + +store_root = pathlib.Path.home() / pathlib.Path("sparkdata/data") + +def set_pcap_store(store): + store_root = store + +def get_db_path(channel, date): + return store_root / pathlib.Path(channel) / pathlib.Path(date+"_"+channel+"_24h.sqlite") + +def get_pcap_db(channel,date): + path = get_db_path(channel,date) + assert os.path.exists(path) + return sqlite3.connect(path) + +def get_sim_db(): + from pathlib import Path + path = store_root / Path("databases") / Path("sim_db") /Path("sim_db.sqlite") + assert os.path.exists(path) + return sqlite3.connect(path) + +def qsim(query): + with get_sim_db() as conn: + return pd.read_sql_query(query,conn) + +def get_db(channel,store): + def querier(query): + with get_pcap_db(channel,store) as conn: + return pd.read_sql_query(query,conn) + return querier + +def clean_sim_books(books): + books = books.rename(columns={'val1':'eid1', 'val2':'eid2','val4':'symbol', + 'val5':'bid','val6':'bid_qty', 'val7':'ask','val8':'ask_qty'}) + books = books.drop(['log_entry_order','t_time','h_t_time', + 'run_id','val3','entry_type','val9','val10'],axis=1) + + books = books.astype({'eid1':'int32','eid2':'int32', + 'bid':'float64','bid_qty':'float64', + 'ask':'float64','ask_qty':'float64'}) + return books + +def get_310_lined_prices(books): + # create an idx with all events in the range (this is so that we + # can have a row for each exchange state even if nothing visibly + # changed in those events) + idx = pd.DataFrame({'eid1': range(books.eid1.min(),books.eid1.max()+1)}) + idx['eid2'] = 0 + + # Filter for the symbols we care + u=books[books.symbol=='ESU0'].drop(['symbol'],axis=1) + z=books[books.symbol=='ESZ0'].drop(['symbol'],axis=1) + u_z=books[books.symbol=='ESU0-ESZ0'].drop(['symbol'],axis=1) + + # Line up symbols so we have an event time line that includes all symbols + prices = idx + + prices = prices.merge(u, + how='outer', + left_on=['eid1','eid2'], + right_on=['eid1','eid2']) + prices = prices.rename({'bid':'bid_u','ask':'ask_u', + 'bid_qty':'bid_qty_u','ask_qty':'ask_qty_u'},axis=1) + + prices = prices.merge(z, + how='outer', + left_on=['eid1','eid2'], + right_on=['eid1','eid2']) + prices = prices.rename({'bid':'bid_z','ask':'ask_z', + 'bid_qty':'bid_qty_z','ask_qty':'ask_qty_z'},axis=1) + + prices = prices.merge(u_z, + how='outer', + left_on=['eid1','eid2'], + right_on=['eid1','eid2']) + prices = prices.rename({'bid':'bid_u_z','ask':'ask_u_z', + 'bid_qty':'bid_qty_u_z','ask_qty':'ask_qty_u_z'},axis=1) + + # set eid1, eid2 as indexes and then sort by them so that ffill + # below will propagate the last price in event order + prices = prices.set_index(['eid1','eid2'],drop=False) + prices = prices.sort_index() + + # Fill holes (every event only updates one symbol) with previous value of book + prices = prices.ffill() + + # Add columns to calculate polygon values for buy and sell + prices['buy_p_value'] = (-prices.ask_z+prices.bid_u+prices.bid_u_z)/2 + prices['sell_p_value'] = (prices.bid_z-prices.ask_u-prices.ask_u_z)/2 + return prices + +def get_shot_outcomes(run_id=1): + with get_sim_db() as conn: + return pd.read_sql_query("select * from shot_outcome where run_id="+str(run_id),conn) + + +def get_opp(*dates): + result = pd.DataFrame() + + for date in dates: + with sqlite3.connect("/home/ppinto/310/"+date+"_310_24h.sqlite") as conn: + result = result.append(pd.read_sql_query(("select '{}' date,\"sum(merged_qty*merged_value)\" " + "opportunity from total_opportunity").format(date),conn)) + return result + + +book_cache = None; +def books(): + def books_from_db(): + with get_md_conn() as conn: + df = pd.read_sql_query("select top_of_books.*,events.t_time " + "from top_of_books, events " + "where top_of_books.eid=events.eid",conn) + df = df.fillna(0) + df = df.astype({'eid':'int32','sid':'int32', + 't_time':'uint64', + 'bid':'int32','bid_qty':'int32', + 'ask':'int32','ask_qty':'int32'}) + df = df.set_index('eid') + df = df.sort_index() + return df + + global book_cache + if not book_cache is None: + return book_cache + elif os.path.exists("/tmp/books.parquet"): + book_cache = pd.read_parquet("/tmp/books.parquet") + return book_cache; + else: + book_cache = books_from_db() + book_cache.to_parquet("/tmp/books.parquet") + return book_cache + +def events_from_db(): + with get_md_conn() as conn: + df = pd.read_sql_query("select * from events",conn) + df = df.fillna(0) + df = df.astype({'eid':'int32','t_time':'uint64', + 'first_sn':'uint64','last_sn':'uint64', + 'first_st':'uint64','last_st':'uint64'}) + return df + +def securities(): + with get_md_conn() as conn: + return pd.read_sql_query("select * from securities",conn) + +def get_event_range(): + with get_md_conn() as conn: + with conn.cursor() as cursor: + cursor.execute("select min(eid),max(eid) from events") + min,max =cursor.fetchall()[0] + return int(min),int(max) + +def events_from_parquet(): + return pd.read_parquet("/home/ppinto/events.parquet") + +def get_sec_book(symbol): + b = books() + s = b[b.symbol==symbol] + s = s.reindex(range(0,b.shape[0]+1),method='ffill') + s = s.fillna(0) + s = s.astype({'bid':'int32','bid_qty':'int32','ask_qty':'int32', + 'ask':'int32','t_time':'uint64'}) + s = s.drop(columns=['symbol','sid']) + return s + +def line_up_books(*symbols): + result = None + for s in symbols: + df = get_sec_book(s) + df = df.add_suffix('_'+s) + if result is None: + result = df + else: + result = result.join(df) + return result + + +def upload_lined_books(df): + book_f = "/tmp/lined_up_books.csv" + df.to_csv(book_f); + with get_md_conn() as conn: + with conn.cursor() as cursor: + with open(book_f) as f: + cursor.copy_from(f,"lined_up_books",sep=",") + +def price_series(symbol): + b = books() + s=b[b.symbol==symbol] + s = s[(s.shift(1).bid != s.bid) | (s.shift(1).ask != s.ask)] + s=s.copy() + s['delta'] = s.t_time - s.shift(1).t_time + return s + +def get_polygon_valuation(): + lined_up = line_up_books('ESU0','ESZ0','ESU0-ESZ0') + lined_up['buy']=-lined_up.ask_ESZ0+lined_up.bid_ESU0+lined_up['bid_ESU0-ESZ0'] + lined_up['sell']=lined_up.bid_ESZ0-lined_up.ask_ESU0-lined_up['ask_ESU0-ESZ0'] + return lined_up + +def filter_for_opportunities(p): + p = p[ ((p.buy>0) | (p.shift(1).buy>0) | (p.shift(-1).buy>0)) | + ((p.sell>0) | (p.shift(1).sell>0) | (p.shift(-1).sell>0)) ] + return p + + + diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp index 729fbd01c..14fae2fc2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp @@ -35,19 +35,17 @@ struct PolyObserver::Opportunity { struct PolyObserver::PImpl { ISimServices *sim = nullptr; - std::unordered_map books; + std::unordered_map books; std::vector> targets; AlgoEventID event_id; MDP3TimeStamp transact_time; MDP3TimeStamp observable_time; - size_t max_events; }; PolyObserver::PolyObserver() : me(new PImpl()) {} PolyObserver::~PolyObserver() {} - -void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) { +void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &) { me->sim = &s; me->books.clear(); me->targets.clear(); @@ -55,8 +53,6 @@ void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) { me->event_id = {0, 0}; me->transact_time = MDP3TimeStamp{0}; me->observable_time = MDP3TimeStamp{0}; - me->max_events = - param_or_default(options, "MaxEvents", std::numeric_limits::max()); for (auto &p : s.polygons()) { auto buy_t = @@ -73,25 +69,21 @@ void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) { } void PolyObserver::on_notification(AlgoNotification const &n) { - if (n.event_id.first >= me->max_events) - me->sim->terminate_simulation(); - else { - if (me->event_id.first) { - for (auto i = me->event_id.first+1;i < n.event_id.first;i++) { - me->event_id.first = i; - me->event_id.second = 0; - me->transact_time = n.transact_time; - me->observable_time = n.observable_time; - eval(); - rank(); - log(); - } + if (me->event_id.first) { + for (auto i = me->event_id.first + 1; i < n.event_id.first; i++) { + me->event_id.first = i; + me->event_id.second = 0; + me->transact_time = n.transact_time; + me->observable_time = n.observable_time; + eval(); + rank(); + log(); } - me->event_id = n.event_id; - me->transact_time = n.transact_time; - me->observable_time = n.observable_time; - std::visit(*this, n.event); } + me->event_id = n.event_id; + me->transact_time = n.transact_time; + me->observable_time = n.observable_time; + std::visit(*this, n.event); } void PolyObserver::eval() { @@ -134,16 +126,17 @@ void PolyObserver::rank() { void PolyObserver::log() { for (auto const &t : me->targets) if (t->has_value()) - me->sim->log("opportunity", me->event_id.first, me->event_id.second, - t->descriptor, t->value, t->quantity.value()); + me->sim->log("poly_opportunity", me->event_id.first, + me->event_id.second, t->descriptor, t->value, + t->quantity.value()); } -std::pair available(Book const&b) { - std::pair result ; +std::pair available(Book const &b) { + std::pair result; result.first = md::IntPrice{}; - result.second= MDP3Quantity{0}; - - for (size_t i = 0; i < b.size();i++) { + result.second = MDP3Quantity{0}; + + for (size_t i = 0; i < b.size(); i++) { if (b[i].quantity.value() == 0) break; else { @@ -154,22 +147,15 @@ std::pair available(Book const&b) { return result; } - void PolyObserver::operator()(const MBPUpdate &mbp) { - - auto avb = available(mbp.mbp_buy); - auto avs = available(mbp.mbp_sell); - - me->sim->log("book", me->sim->get_sec(mbp.security_id).symbol(), - me->event_id.first, - mbp.mbp_buy[0].price.price(), - mbp.mbp_buy[0].quantity.value(), - mbp.mbp_sell[0].price.price(), - mbp.mbp_sell[0].quantity.value(), - avb.first.price(), - avb.second.value(), - avs.first.price(), - avs.second.value()); + me->sim->log( + "poly_book", + me->event_id.first, + me->event_id.second, + me->transact_time.value(), + me->sim->get_sec(mbp.security_id).symbol(), + mbp.mbp_buy[0].price.price(), mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), mbp.mbp_sell[0].quantity.value()); me->books[mbp.security_id] = mbp; eval(); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp index 7adbb21bb..2523b81d7 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp @@ -324,6 +324,15 @@ void Sniper::operator()(const MBPUpdate &mbp) { if (it == me->books.end()) return; + // me->sim->log("book", + // me->current_event.first, + // me->current_event.second, + // me->sim->get_sec(mbp.security_id).symbol(), + // mbp.mbp_buy[0].price.price(), + // mbp.mbp_buy[0].quantity.value(), + // mbp.mbp_sell[0].price.price(), + // mbp.mbp_sell[0].quantity.value()); + auto &book = it->second; book.book = mbp; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp index 1ac262ec1..ebd5d0b2b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp @@ -1,12 +1,12 @@ #include "SimOrchestrator.h" #include "../simulator/SimInput.h" #include "lib/utils/PBUtils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h" -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h" #include @@ -20,8 +20,9 @@ class SimOrchestrator::SimInputManager { if (auto ptr = maybe_input.lock()) return ptr; else { - LOG(Spartan,Warn, "Loading input from {}", path); - auto new_ptr = std::make_shared(make_sim_input_from_proto(path)); + LOG(Spartan, Warn, "Loading input from {}", path); + auto new_ptr = + std::make_shared(make_sim_input_from_proto(path)); maybe_input = new_ptr; return new_ptr; } @@ -123,7 +124,8 @@ SimOrchestrator::SimOrchestrator(protos::SimSessionOptions const &options) auto &these_inputs = this_run.inputs; these_options.set_algo(options.algo()); these_options.set_market_date(day); - + these_options.set_log_all_transactions(options.log_all_transactions()); + these_options.set_max_events(options.max_events()); // Set inputs for (auto const &[channel, sim_input] : day_inputs) { *these_options.add_channels() = channel; @@ -170,7 +172,7 @@ std::unique_ptr make_algo(std::string const &algo) { return std::make_unique(); if (algo == "PolyObserver") return std::make_unique(); - + bts_assert(false); return {}; } @@ -189,8 +191,8 @@ void SimOrchestrator::run() { auto algo = make_algo(r->options.algo()); auto rid = db.begin_run(sid, r->options); int rr = remaining_runs; - LOG(Spartan, Warn, "Starting run {} ({} remaining) {}", rid, - rr, utils::pb::to_string(r->options)); + LOG(Spartan, Warn, "Starting run {} ({} remaining) {}", rid, rr, + utils::pb::to_string(r->options)); auto logger = [rid, &db](auto ts, auto const &entry_type, auto const &args) { db.log(rid, ts, entry_type, args); @@ -219,8 +221,7 @@ void SimOrchestrator::run() { for (size_t i = 0; i < me->runs.size(); i++) workers[i % workers.size()].assigned_runs.push_back(&me->runs[i]); - auto session_id = - me->db.create_session(me->options.algo()); + auto session_id = me->db.create_session(me->options.algo()); std::atomic run_count = me->runs.size(); for (auto &w : workers) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto index e9ef8555a..57419f262 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto @@ -3,7 +3,6 @@ package bts.spartan.protos; /* algo: "Sniper" -session_title: "A memorable title" n_cores: 4 file: "/home/ppinto/360/2020-09-07_360_24h.sim" @@ -31,6 +30,17 @@ message SimSessionOptions { // The algo we are testing string algo = 1; + // Wheter the simulator will record every order, order report and + // price update. This produces fantastic ammounts of information so + // it is off by default. + bool log_all_transactions = 9; + + // The maximum number of events we will process before we terminate + // the simulation. This is usefull to speed initial prototyping of + // algos or analysis tools. Use 0 (the default) to signifiy all + // events. + uint64 max_events = 11; + // The sim input files we are using. File names should end with // "__24h.sim". Files for the same day will // grouped and used as a unit for each simulation run. @@ -54,7 +64,7 @@ message SimSessionOptions { // An estimate for the average transaction latency at the // exchange. We fallback on this value when we cannot use the send - // time of a packet to estimate for that purpose. + // time of a packet to estimate transaction duration. repeated uint64 transaction_latency_ns = 8; // Algo specific parameters used for this session. For each named @@ -63,12 +73,14 @@ message SimSessionOptions { message ParameterDomain { repeated string item = 1; } - map parameter = 9; + map parameter = 10; } // The parameters for a single simulation run. message SimRunOptions { string algo = 1; + bool log_all_transactions = 10; + uint64 max_events = 11; uint64 base_algo_latency_ns = 2; uint64 order_update_delay_ns = 3; uint64 mbp_update_delay_ns = 4; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp index 25e2fdbbf..215b0d1f6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp @@ -48,6 +48,10 @@ SimDB::SimDB(std::string const &loc) { db.exec(str); } me = std::make_unique(loc); + SQLite::Statement sync_off(me->db,"PRAGMA synchronous = OFF"); + sync_off.executeStep(); + SQLite::Statement journal_mem(me->db,"PRAGMA journal_mode = MEMORY"); + journal_mem.executeStep(); me->logger_thread = std::thread{[this]() { do_work(); }}; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp index 5737368e6..74dfd8d85 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp @@ -80,7 +80,8 @@ void ExchangeSim::prepare_run(std::vector const &algos, // Init gateways, add them to the simulator for (auto &i : me->inputs) { - auto &gw = me->run_state->gateways.emplace_back(me->logger,i->instruments, options); + auto &gw = me->run_state->gateways.emplace_back(me->logger, + i->instruments, options); me->run_state->simulator.add_agent(gw); } @@ -133,38 +134,45 @@ bool ExchangeSim::has_work() const { void ExchangeSim::do_work() { bts_assert(me->run_state); me->run_state->simulator.run_once(); - auto now = Timer::now(); - const auto log_period_s = 10; - if (ms_diff(now, me->run_state->last_time_log) > 1000 * log_period_s) { + if (me->run_state->options.max_events() && + me->run_state->last_completed_count >= + me->run_state->options.max_events()) { + me->run_state->simulator.end_simulation(); + } else { + auto now = Timer::now(); + const auto log_period_s = 10; + if (ms_diff(now, me->run_state->last_time_log) > 1000 * log_period_s) { - auto sim_time = me->run_state->simulator.simulation_time(); + auto sim_time = me->run_state->simulator.simulation_time(); - size_t total_events = me->run_state->starting_event_count; + size_t total_events = me->run_state->starting_event_count; - size_t remaining_events = me->run_state->simulator.pending_events_count(); + size_t remaining_events = + me->run_state->simulator.pending_events_count(); - size_t completed_events = total_events > remaining_events - ? total_events - remaining_events - : total_events; + size_t completed_events = total_events > remaining_events + ? total_events - remaining_events + : total_events; - size_t p_complete = (double)completed_events / total_events * 100; + size_t p_complete = (double)completed_events / total_events * 100; - size_t completed_this_cycle = - completed_events - me->run_state->last_completed_count; + size_t completed_this_cycle = + completed_events - me->run_state->last_completed_count; - size_t elapsed_secs = sec_diff(now, me->run_state->run_start); - auto eta = (remaining_events * elapsed_secs / completed_events); - auto TPS = completed_this_cycle / log_period_s; + size_t elapsed_secs = sec_diff(now, me->run_state->run_start); + auto eta = (remaining_events * elapsed_secs / completed_events); + auto TPS = completed_this_cycle / log_period_s; - me->run_state->last_time_log = now; - me->run_state->last_sim_time = sim_time; - me->run_state->last_completed_count = completed_events; + me->run_state->last_time_log = now; + me->run_state->last_sim_time = sim_time; + me->run_state->last_completed_count = completed_events; - LOG(Spartan, Diag, - "Run {} -> Time: {} TPS: {} Complete: {}% ETA: {} " - "secs", - me->run_id, to_time_string(MDP3TimeStamp{sim_time}), TPS, p_complete, - eta); + LOG(Spartan, Diag, + "Run {} -> Time: {} TPS: {} Complete: {}% ETA: {} " + "secs", + me->run_id, to_time_string(MDP3TimeStamp{sim_time}), TPS, + p_complete, eta); + } } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp index 81be7b08a..71e9ad174 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp @@ -117,6 +117,16 @@ void GatewayAgent::_handle(ExchangeTransactionEvent const &e) { else forward_clock_by(me->options.transaction_latency_ns()); + if (me->options.log_all_transactions()) + for (auto const &change : ev->order_changes) { + log(me->logger, time(), "gw_order", me->current_event.first, + me->current_event.second, ev->transact_time.value(), + change_name(change.change), + get_sec(change.order.security).symbol(), + change.order.price.price(), change.order.quantity.value(), + api::Side_Name(change.order.side), change.order.key.value()); + } + // Now match the recorded orders and send out their effects as // algo notification events std::vector effects; @@ -126,12 +136,34 @@ void GatewayAgent::_handle(ExchangeTransactionEvent const &e) { std::visit( overloaded{ [&](MBPUpdate &mbp) { + if (me->options.log_all_transactions()) { + log(me->logger, time(), + "gw_book", + me->current_event.first, + me->current_event.second, + ev->transact_time.value(), + get_sec(mbp.security_id).symbol(), + mbp.mbp_buy[0].price.price(), + mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), + mbp.mbp_sell[0].quantity.value()); + } emit_delayed_event( me->options.mbp_update_delay_ns(), AlgoNotification{me->current_event, ev->transact_time, MDP3TimeStamp{time()}, mbp}); }, [&](OrderUpdate const &order_update) { + if (me->options.log_all_transactions()) { + log(me->logger, time(), "gw_fill", + me->current_event.first, me->current_event.second, + ev->transact_time.value(), order_update.status_str(), + api::Side_Name(order_update.side), + get_sec(order_update.security_id).symbol(), + order_update.last_price.price(), + order_update.last_fills.value(), + order_update.key.value()); + } emit_delayed_event( me->options.order_update_delay_ns(), AlgoNotification{me->current_event, ev->transact_time, @@ -148,33 +180,55 @@ void GatewayAgent::_handle(AlgoActionEvent const &e) { auto const &algo_action = e.ev; auto const &change = algo_action.change; - - log(me->logger, time(), "gw_algo_order", algo_action.transact_time.value(), - change_name(change.change), get_sec(change.order.security).symbol(), - change.order.price.price(), change.order.quantity.value(), - api::Side_Name(change.order.side), change.order.key.value(), - change.order.security.value(), me->current_event.first, - me->current_event.second); + if (me->options.log_all_transactions()) { + log(me->logger, time(), "gw_algo_order", me->current_event.first, + me->current_event.second, algo_action.transact_time.value(), + change_name(change.change), get_sec(change.order.security).symbol(), + change.order.price.price(), change.order.quantity.value(), + api::Side_Name(change.order.side), change.order.key.value()); + } forward_clock_by(me->options.transaction_latency_ns()); std::vector effects; me->matcher.match(Matcher::ALGO, {algo_action.change}, &effects); for (auto &ef : effects) std::visit( - overloaded{[&](MBPUpdate &mbp) { - emit_delayed_event( - me->options.mbp_update_delay_ns(), - AlgoNotification{me->current_event, - algo_action.transact_time, - MDP3TimeStamp{time()}, mbp}); - }, - [&](OrderUpdate const &order_update) { - emit_delayed_event( - me->options.order_update_delay_ns(), - AlgoNotification{ - me->current_event, algo_action.transact_time, - MDP3TimeStamp{time()}, order_update}); - }}, + overloaded{ + [&](MBPUpdate &mbp) { + if (me->options.log_all_transactions()) { + log(me->logger, time(), "gw_book", me->current_event.first, + me->current_event.second, + algo_action.transact_time.value(), + get_sec(mbp.security_id).symbol(), + mbp.mbp_buy[0].price.price(), + mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), + mbp.mbp_sell[0].quantity.value()); + } + emit_delayed_event( + me->options.mbp_update_delay_ns(), + AlgoNotification{me->current_event, + algo_action.transact_time, + MDP3TimeStamp{time()}, mbp}); + }, + [&](OrderUpdate const &order_update) { + if (me->options.log_all_transactions()) { + log(me->logger, time(), "gw_algo_fill", me->current_event.first, + me->current_event.second, + algo_action.transact_time.value(), + order_update.status_str(), + api::Side_Name(order_update.side), + get_sec(order_update.security_id).symbol(), + order_update.last_price.price(), + order_update.last_fills.value(), + order_update.key.value()); + } + emit_delayed_event( + me->options.order_update_delay_ns(), + AlgoNotification{me->current_event, + algo_action.transact_time, + MDP3TimeStamp{time()}, order_update}); + }}, ef); }