diff --git a/src/data_files/files/spartan/sim_db.sql b/src/data_files/files/spartan/sim_db.sql
index 06441424b..ba39b2d86 100644
--- a/src/data_files/files/spartan/sim_db.sql
+++ b/src/data_files/files/spartan/sim_db.sql
@@ -59,13 +59,29 @@ create table log(
val10 TEXT,
foreign key(run_id) references run(run_id)
);
-CREATE INDEX log_idx_0 ON log(run_id,entry_type);
-CREATE INDEX log_idx_1 ON log(val1);
-CREATE INDEX log_idx_2 ON log(val2);
-CREATE INDEX log_idx_3 ON log(val3);
+CREATE INDEX log_idx_0 ON log(run_id);
+
+/* Gateway log */
+drop view if exists gateway_log;
+create view gateway_log as
+select
+ log_entry_order,
+ t_time,h_t_time,entry_type,
+ cast (val1 as numeric) eid1,
+ cast (val2 as numeric) eid2,
+ val3,val4,val5,val6,val7,
+ val8,val9
+from
+ log
+where
+ entry_type='gw_order' or
+ entry_type='gw_algo_order' or
+ entry_type='gw_fill' or
+ entry_type='gw_algo_fill'
+order by
+ log_entry_order;
/* sniper specific views */
-
drop view if exists shot_beginnings;
create view shot_beginnings as
select
diff --git a/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints/lining_up_books-checkpoint.ipynb b/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints/lining_up_books-checkpoint.ipynb
new file mode 100644
index 000000000..43d3730c3
--- /dev/null
+++ b/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints/lining_up_books-checkpoint.ipynb
@@ -0,0 +1,1234 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import spartan \n",
+ "import pandas as pd"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Log entire log from DB (this can easily be 10GB, so it might take a while)\n",
+ "log = spartan.qsim(\"select * from log\")\n",
+ "log.to_parquet(\"/tmp/log.parquet\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "log = pd.read_parquet(\"/tmp/log.parquet\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract and clean up book entries\n",
+ "books = log[log.entry_type=='gw_book']\n",
+ "books = spartan.clean_sim_books(books)\n",
+ "\n",
+ "# Create an event timeline for the books that form our main poligon\n",
+ "lined_up_books = spartan.get_310_lined_prices(books)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Grab shots\n",
+ "shots = spartan.get_shot_outcomes()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 394,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "r_books = lined_up_books.loc[20309598:20309692]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 395,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "r_shots = shots.loc[2918]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " eid1 | \n",
+ " eid2 | \n",
+ " symbol | \n",
+ " bid | \n",
+ " bid_qty | \n",
+ " ask | \n",
+ " ask_qty | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 7807 | \n",
+ " 1455 | \n",
+ " 0 | \n",
+ " ESM1 | \n",
+ " 300000.0 | \n",
+ " 2.0 | \n",
+ " 343175.0 | \n",
+ " 1.0 | \n",
+ "
\n",
+ " \n",
+ " | 7808 | \n",
+ " 1455 | \n",
+ " 0 | \n",
+ " ESU0-ESU1 | \n",
+ " -6395.0 | \n",
+ " 1.0 | \n",
+ " -2805.0 | \n",
+ " 1.0 | \n",
+ "
\n",
+ " \n",
+ " | 7809 | \n",
+ " 1455 | \n",
+ " 0 | \n",
+ " ESH1 | \n",
+ " 330850.0 | \n",
+ " 1.0 | \n",
+ " 343000.0 | \n",
+ " 1.0 | \n",
+ "
\n",
+ " \n",
+ " | 7810 | \n",
+ " 1455 | \n",
+ " 0 | \n",
+ " ESU0-ESZ0 | \n",
+ " -1055.0 | \n",
+ " 812.0 | \n",
+ " -1050.0 | \n",
+ " 247.0 | \n",
+ "
\n",
+ " \n",
+ " | 7811 | \n",
+ " 1455 | \n",
+ " 0 | \n",
+ " ESU0 | \n",
+ " 334525.0 | \n",
+ " 2.0 | \n",
+ " 334550.0 | \n",
+ " 7.0 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 1167936 | \n",
+ " 387907 | \n",
+ " 0 | \n",
+ " ESZ0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333500.0 | \n",
+ " 2.0 | \n",
+ "
\n",
+ " \n",
+ " | 1167939 | \n",
+ " 387908 | \n",
+ " 0 | \n",
+ " ESZ0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333500.0 | \n",
+ " 2.0 | \n",
+ "
\n",
+ " \n",
+ " | 1167942 | \n",
+ " 387909 | \n",
+ " 0 | \n",
+ " ESH1 | \n",
+ " 332550.0 | \n",
+ " 2.0 | \n",
+ " 332775.0 | \n",
+ " 1.0 | \n",
+ "
\n",
+ " \n",
+ " | 1167949 | \n",
+ " 387910 | \n",
+ " 0 | \n",
+ " ESZ0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333500.0 | \n",
+ " 1.0 | \n",
+ "
\n",
+ " \n",
+ " | 1167952 | \n",
+ " 387911 | \n",
+ " 0 | \n",
+ " ESZ0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333525.0 | \n",
+ " 12.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
284111 rows × 7 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " eid1 eid2 symbol bid bid_qty ask ask_qty\n",
+ "7807 1455 0 ESM1 300000.0 2.0 343175.0 1.0\n",
+ "7808 1455 0 ESU0-ESU1 -6395.0 1.0 -2805.0 1.0\n",
+ "7809 1455 0 ESH1 330850.0 1.0 343000.0 1.0\n",
+ "7810 1455 0 ESU0-ESZ0 -1055.0 812.0 -1050.0 247.0\n",
+ "7811 1455 0 ESU0 334525.0 2.0 334550.0 7.0\n",
+ "... ... ... ... ... ... ... ...\n",
+ "1167936 387907 0 ESZ0 333475.0 9.0 333500.0 2.0\n",
+ "1167939 387908 0 ESZ0 333475.0 9.0 333500.0 2.0\n",
+ "1167942 387909 0 ESH1 332550.0 2.0 332775.0 1.0\n",
+ "1167949 387910 0 ESZ0 333475.0 9.0 333500.0 1.0\n",
+ "1167952 387911 0 ESZ0 333475.0 9.0 333525.0 12.0\n",
+ "\n",
+ "[284111 rows x 7 columns]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "books"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1455"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "books.eid1.min()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "387911"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "books.eid1.max()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "range(1455, 387912)"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "idx = pd.DataFrame({'eid1':range(books.eid1.min(),books.eid1.max()+1)})\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " eid1 | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 1455 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 1456 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 1457 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 1458 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 1459 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 386452 | \n",
+ " 387907 | \n",
+ "
\n",
+ " \n",
+ " | 386453 | \n",
+ " 387908 | \n",
+ "
\n",
+ " \n",
+ " | 386454 | \n",
+ " 387909 | \n",
+ "
\n",
+ " \n",
+ " | 386455 | \n",
+ " 387910 | \n",
+ "
\n",
+ " \n",
+ " | 386456 | \n",
+ " 387911 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
386457 rows × 1 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " eid1\n",
+ "0 1455\n",
+ "1 1456\n",
+ "2 1457\n",
+ "3 1458\n",
+ "4 1459\n",
+ "... ...\n",
+ "386452 387907\n",
+ "386453 387908\n",
+ "386454 387909\n",
+ "386455 387910\n",
+ "386456 387911\n",
+ "\n",
+ "[386457 rows x 1 columns]"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "idx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " eid1 | \n",
+ " eid2 | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 1455 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 1456 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 1457 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 1458 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 1459 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 386452 | \n",
+ " 387907 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 386453 | \n",
+ " 387908 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 386454 | \n",
+ " 387909 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 386455 | \n",
+ " 387910 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 386456 | \n",
+ " 387911 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
386457 rows × 2 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " eid1 eid2\n",
+ "0 1455 0\n",
+ "1 1456 0\n",
+ "2 1457 0\n",
+ "3 1458 0\n",
+ "4 1459 0\n",
+ "... ... ...\n",
+ "386452 387907 0\n",
+ "386453 387908 0\n",
+ "386454 387909 0\n",
+ "386455 387910 0\n",
+ "386456 387911 0\n",
+ "\n",
+ "[386457 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "idx['eid2'] = 0\n",
+ "idx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " eid1 | \n",
+ " eid2 | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 1455 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 1456 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 1457 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 1458 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 1459 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 386452 | \n",
+ " 387907 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 386453 | \n",
+ " 387908 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 386454 | \n",
+ " 387909 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 386455 | \n",
+ " 387910 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 386456 | \n",
+ " 387911 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
386457 rows × 2 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " eid1 eid2\n",
+ "0 1455 0\n",
+ "1 1456 0\n",
+ "2 1457 0\n",
+ "3 1458 0\n",
+ "4 1459 0\n",
+ "... ... ...\n",
+ "386452 387907 0\n",
+ "386453 387908 0\n",
+ "386454 387909 0\n",
+ "386455 387910 0\n",
+ "386456 387911 0\n",
+ "\n",
+ "[386457 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "idx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "idx = idx.set_index(['eid1','eid2'],drop=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " eid1 | \n",
+ " eid2 | \n",
+ "
\n",
+ " \n",
+ " | eid2 | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 1459 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " eid1 eid2\n",
+ "eid2 \n",
+ "0 1459 0"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "idx.loc[1459]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " | \n",
+ " eid1 | \n",
+ " eid2 | \n",
+ " bid_u | \n",
+ " bid_qty_u | \n",
+ " ask_u | \n",
+ " ask_qty_u | \n",
+ " bid_z | \n",
+ " bid_qty_z | \n",
+ " ask_z | \n",
+ " ask_qty_z | \n",
+ " bid_u_z | \n",
+ " bid_qty_u_z | \n",
+ " ask_u_z | \n",
+ " ask_qty_u_z | \n",
+ " buy_p_value | \n",
+ " sell_p_value | \n",
+ "
\n",
+ " \n",
+ " | eid1 | \n",
+ " eid2 | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 1455 | \n",
+ " 0 | \n",
+ " 1455 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 2.0 | \n",
+ " 334550.0 | \n",
+ " 7.0 | \n",
+ " 333475.0 | \n",
+ " 2.0 | \n",
+ " 333525.0 | \n",
+ " 1.0 | \n",
+ " -1055.0 | \n",
+ " 812.0 | \n",
+ " -1050.0 | \n",
+ " 247.0 | \n",
+ " -27.5 | \n",
+ " -12.5 | \n",
+ "
\n",
+ " \n",
+ " | 1456 | \n",
+ " 0 | \n",
+ " 1456 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 2.0 | \n",
+ " 334550.0 | \n",
+ " 7.0 | \n",
+ " 333475.0 | \n",
+ " 2.0 | \n",
+ " 333525.0 | \n",
+ " 1.0 | \n",
+ " -1055.0 | \n",
+ " 812.0 | \n",
+ " -1050.0 | \n",
+ " 247.0 | \n",
+ " -27.5 | \n",
+ " -12.5 | \n",
+ "
\n",
+ " \n",
+ " | 1457 | \n",
+ " 0 | \n",
+ " 1457 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 2.0 | \n",
+ " 334550.0 | \n",
+ " 7.0 | \n",
+ " 333475.0 | \n",
+ " 2.0 | \n",
+ " 333525.0 | \n",
+ " 1.0 | \n",
+ " -1055.0 | \n",
+ " 812.0 | \n",
+ " -1050.0 | \n",
+ " 247.0 | \n",
+ " -27.5 | \n",
+ " -12.5 | \n",
+ "
\n",
+ " \n",
+ " | 1458 | \n",
+ " 0 | \n",
+ " 1458 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 2.0 | \n",
+ " 334550.0 | \n",
+ " 7.0 | \n",
+ " 333375.0 | \n",
+ " 2.0 | \n",
+ " 333525.0 | \n",
+ " 1.0 | \n",
+ " -1055.0 | \n",
+ " 812.0 | \n",
+ " -1050.0 | \n",
+ " 247.0 | \n",
+ " -27.5 | \n",
+ " -62.5 | \n",
+ "
\n",
+ " \n",
+ " | 1459 | \n",
+ " 0 | \n",
+ " 1459 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 2.0 | \n",
+ " 334550.0 | \n",
+ " 7.0 | \n",
+ " 333375.0 | \n",
+ " 2.0 | \n",
+ " 333525.0 | \n",
+ " 1.0 | \n",
+ " -1055.0 | \n",
+ " 812.0 | \n",
+ " -1050.0 | \n",
+ " 247.0 | \n",
+ " -27.5 | \n",
+ " -62.5 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 387907 | \n",
+ " 0 | \n",
+ " 387907 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 7.0 | \n",
+ " 334575.0 | \n",
+ " 11.0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333500.0 | \n",
+ " 2.0 | \n",
+ " -1065.0 | \n",
+ " 955.0 | \n",
+ " -1060.0 | \n",
+ " 1784.0 | \n",
+ " -20.0 | \n",
+ " -20.0 | \n",
+ "
\n",
+ " \n",
+ " | 387908 | \n",
+ " 0 | \n",
+ " 387908 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 7.0 | \n",
+ " 334575.0 | \n",
+ " 11.0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333500.0 | \n",
+ " 2.0 | \n",
+ " -1065.0 | \n",
+ " 955.0 | \n",
+ " -1060.0 | \n",
+ " 1784.0 | \n",
+ " -20.0 | \n",
+ " -20.0 | \n",
+ "
\n",
+ " \n",
+ " | 387909 | \n",
+ " 0 | \n",
+ " 387909 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 7.0 | \n",
+ " 334575.0 | \n",
+ " 11.0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333500.0 | \n",
+ " 2.0 | \n",
+ " -1065.0 | \n",
+ " 955.0 | \n",
+ " -1060.0 | \n",
+ " 1784.0 | \n",
+ " -20.0 | \n",
+ " -20.0 | \n",
+ "
\n",
+ " \n",
+ " | 387910 | \n",
+ " 0 | \n",
+ " 387910 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 7.0 | \n",
+ " 334575.0 | \n",
+ " 11.0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333500.0 | \n",
+ " 1.0 | \n",
+ " -1065.0 | \n",
+ " 955.0 | \n",
+ " -1060.0 | \n",
+ " 1784.0 | \n",
+ " -20.0 | \n",
+ " -20.0 | \n",
+ "
\n",
+ " \n",
+ " | 387911 | \n",
+ " 0 | \n",
+ " 387911 | \n",
+ " 0 | \n",
+ " 334525.0 | \n",
+ " 7.0 | \n",
+ " 334575.0 | \n",
+ " 11.0 | \n",
+ " 333475.0 | \n",
+ " 9.0 | \n",
+ " 333525.0 | \n",
+ " 12.0 | \n",
+ " -1065.0 | \n",
+ " 955.0 | \n",
+ " -1060.0 | \n",
+ " 1784.0 | \n",
+ " -32.5 | \n",
+ " -20.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
386457 rows × 16 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " eid1 eid2 bid_u bid_qty_u ask_u ask_qty_u bid_z \\\n",
+ "eid1 eid2 \n",
+ "1455 0 1455 0 334525.0 2.0 334550.0 7.0 333475.0 \n",
+ "1456 0 1456 0 334525.0 2.0 334550.0 7.0 333475.0 \n",
+ "1457 0 1457 0 334525.0 2.0 334550.0 7.0 333475.0 \n",
+ "1458 0 1458 0 334525.0 2.0 334550.0 7.0 333375.0 \n",
+ "1459 0 1459 0 334525.0 2.0 334550.0 7.0 333375.0 \n",
+ "... ... ... ... ... ... ... ... \n",
+ "387907 0 387907 0 334525.0 7.0 334575.0 11.0 333475.0 \n",
+ "387908 0 387908 0 334525.0 7.0 334575.0 11.0 333475.0 \n",
+ "387909 0 387909 0 334525.0 7.0 334575.0 11.0 333475.0 \n",
+ "387910 0 387910 0 334525.0 7.0 334575.0 11.0 333475.0 \n",
+ "387911 0 387911 0 334525.0 7.0 334575.0 11.0 333475.0 \n",
+ "\n",
+ " bid_qty_z ask_z ask_qty_z bid_u_z bid_qty_u_z ask_u_z \\\n",
+ "eid1 eid2 \n",
+ "1455 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n",
+ "1456 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n",
+ "1457 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n",
+ "1458 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n",
+ "1459 0 2.0 333525.0 1.0 -1055.0 812.0 -1050.0 \n",
+ "... ... ... ... ... ... ... \n",
+ "387907 0 9.0 333500.0 2.0 -1065.0 955.0 -1060.0 \n",
+ "387908 0 9.0 333500.0 2.0 -1065.0 955.0 -1060.0 \n",
+ "387909 0 9.0 333500.0 2.0 -1065.0 955.0 -1060.0 \n",
+ "387910 0 9.0 333500.0 1.0 -1065.0 955.0 -1060.0 \n",
+ "387911 0 9.0 333525.0 12.0 -1065.0 955.0 -1060.0 \n",
+ "\n",
+ " ask_qty_u_z buy_p_value sell_p_value \n",
+ "eid1 eid2 \n",
+ "1455 0 247.0 -27.5 -12.5 \n",
+ "1456 0 247.0 -27.5 -12.5 \n",
+ "1457 0 247.0 -27.5 -12.5 \n",
+ "1458 0 247.0 -27.5 -62.5 \n",
+ "1459 0 247.0 -27.5 -62.5 \n",
+ "... ... ... ... \n",
+ "387907 0 1784.0 -20.0 -20.0 \n",
+ "387908 0 1784.0 -20.0 -20.0 \n",
+ "387909 0 1784.0 -20.0 -20.0 \n",
+ "387910 0 1784.0 -20.0 -20.0 \n",
+ "387911 0 1784.0 -32.5 -20.0 \n",
+ "\n",
+ "[386457 rows x 16 columns]"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lined_up_books = spartan.get_310_lined_prices(books)\n",
+ "lined_up_books"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " eid1 | \n",
+ " eid2 | \n",
+ " bid_u | \n",
+ " bid_qty_u | \n",
+ " ask_u | \n",
+ " ask_qty_u | \n",
+ " bid_z | \n",
+ " bid_qty_z | \n",
+ " ask_z | \n",
+ " ask_qty_z | \n",
+ " bid_u_z | \n",
+ " bid_qty_u_z | \n",
+ " ask_u_z | \n",
+ " ask_qty_u_z | \n",
+ " buy_p_value | \n",
+ " sell_p_value | \n",
+ "
\n",
+ " \n",
+ " | eid2 | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 10000 | \n",
+ " 0 | \n",
+ " 334750.0 | \n",
+ " 13.0 | \n",
+ " 334775.0 | \n",
+ " 24.0 | \n",
+ " 333700.0 | \n",
+ " 6.0 | \n",
+ " 333750.0 | \n",
+ " 5.0 | \n",
+ " -1055.0 | \n",
+ " 317.0 | \n",
+ " -1050.0 | \n",
+ " 586.0 | \n",
+ " -27.5 | \n",
+ " -12.5 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " eid1 eid2 bid_u bid_qty_u ask_u ask_qty_u bid_z \\\n",
+ "eid2 \n",
+ "0 10000 0 334750.0 13.0 334775.0 24.0 333700.0 \n",
+ "\n",
+ " bid_qty_z ask_z ask_qty_z bid_u_z bid_qty_u_z ask_u_z \\\n",
+ "eid2 \n",
+ "0 6.0 333750.0 5.0 -1055.0 317.0 -1050.0 \n",
+ "\n",
+ " ask_qty_u_z buy_p_value sell_p_value \n",
+ "eid2 \n",
+ "0 586.0 -27.5 -12.5 "
+ ]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "lined_up_books.loc[10000]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "250.0"
+ ]
+ },
+ "execution_count": 37,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "(lined_up_books.ask_u - lined_up_books.bid_u).max()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/src/server/sparkd/cmds/spartan/analysis/lining_up_books.ipynb b/src/server/sparkd/cmds/spartan/analysis/lining_up_books.ipynb
new file mode 100644
index 000000000..caff1953d
--- /dev/null
+++ b/src/server/sparkd/cmds/spartan/analysis/lining_up_books.ipynb
@@ -0,0 +1,1448 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import spartan \n",
+ "import pandas as pd"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Log entire log from DB (this can easily be 10GB, so it might take a while)\n",
+ "log = spartan.qsim(\"select * from log\")\n",
+ "log.to_parquet(\"/tmp/log.parquet\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "log = pd.read_parquet(\"/tmp/log.parquet\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract and clean up book entries\n",
+ "books = log[log.entry_type=='gw_book']\n",
+ "books = spartan.clean_sim_books(books)\n",
+ "\n",
+ "# Create an event timeline for the books that form our main poligon\n",
+ "lined_up_books = spartan.get_310_lined_prices(books)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Grab shots\n",
+ "shots = spartan.get_shot_outcomes()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Grab gateway logs\n",
+ "gw_logs = spartan.qsim(\"select * from gateway_log\")\n",
+ "gw_logs = gw_logs.set_index(['eid1','eid2'],drop=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " streak_id | \n",
+ " len | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 2918 | \n",
+ " 47 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 1380 | \n",
+ " 43 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 1188 | \n",
+ " 38 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 1398 | \n",
+ " 37 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 2931 | \n",
+ " 35 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 3027 | \n",
+ " 2647 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | 3028 | \n",
+ " 598 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | 3029 | \n",
+ " 2643 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | 3030 | \n",
+ " 594 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | 3031 | \n",
+ " 2045 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
3032 rows × 2 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " streak_id len\n",
+ "0 2918 47\n",
+ "1 1380 43\n",
+ "2 1188 38\n",
+ "3 1398 37\n",
+ "4 2931 35\n",
+ "... ... ...\n",
+ "3027 2647 1\n",
+ "3028 598 1\n",
+ "3029 2643 1\n",
+ "3030 594 1\n",
+ "3031 2045 1\n",
+ "\n",
+ "[3032 rows x 2 columns]"
+ ]
+ },
+ "execution_count": 32,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# find longest streaks of shots\n",
+ "streaks = pd.DataFrame(shots.streak_id.value_counts()).reset_index().rename({'index':'streak_id','streak_id':'len'},axis=1)\n",
+ "streaks"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 63,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " run_id | \n",
+ " t_time | \n",
+ " polygon | \n",
+ " o_eid | \n",
+ " o_eid_2 | \n",
+ " e_eid | \n",
+ " e_eid_2 | \n",
+ " target | \n",
+ " realized | \n",
+ " streak_id | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 5597 | \n",
+ " 1 | \n",
+ " 1599854398601848154 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309598 | \n",
+ " 0 | \n",
+ " 20309647 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5598 | \n",
+ " 1 | \n",
+ " 1599854398601867165 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309647 | \n",
+ " 3 | \n",
+ " 20309667 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5599 | \n",
+ " 1 | \n",
+ " 1599854398601879165 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309667 | \n",
+ " 3 | \n",
+ " 20309667 | \n",
+ " 6 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5600 | \n",
+ " 1 | \n",
+ " 1599854398601970880 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309667 | \n",
+ " 6 | \n",
+ " 20309668 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5601 | \n",
+ " 1 | \n",
+ " 1599854398602128013 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309668 | \n",
+ " 3 | \n",
+ " 20309670 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5602 | \n",
+ " 1 | \n",
+ " 1599854398602143173 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309670 | \n",
+ " 3 | \n",
+ " 20309677 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5603 | \n",
+ " 1 | \n",
+ " 1599854398602246463 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309677 | \n",
+ " 3 | \n",
+ " 20309678 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5604 | \n",
+ " 1 | \n",
+ " 1599854398602351943 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309678 | \n",
+ " 3 | \n",
+ " 20309689 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5605 | \n",
+ " 1 | \n",
+ " 1599854398602428572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309689 | \n",
+ " 3 | \n",
+ " 20309690 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5606 | \n",
+ " 1 | \n",
+ " 1599854398602440572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 3 | \n",
+ " 20309690 | \n",
+ " 6 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5607 | \n",
+ " 1 | \n",
+ " 1599854398602452572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 6 | \n",
+ " 20309690 | \n",
+ " 9 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5608 | \n",
+ " 1 | \n",
+ " 1599854398602464572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 9 | \n",
+ " 20309690 | \n",
+ " 12 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5609 | \n",
+ " 1 | \n",
+ " 1599854398602476572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 12 | \n",
+ " 20309690 | \n",
+ " 15 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5610 | \n",
+ " 1 | \n",
+ " 1599854398602488572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 15 | \n",
+ " 20309690 | \n",
+ " 18 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5611 | \n",
+ " 1 | \n",
+ " 1599854398602500572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 18 | \n",
+ " 20309690 | \n",
+ " 21 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5612 | \n",
+ " 1 | \n",
+ " 1599854398602512572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 21 | \n",
+ " 20309690 | \n",
+ " 24 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5613 | \n",
+ " 1 | \n",
+ " 1599854398602524572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 24 | \n",
+ " 20309690 | \n",
+ " 27 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5614 | \n",
+ " 1 | \n",
+ " 1599854398602536572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 27 | \n",
+ " 20309690 | \n",
+ " 30 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5615 | \n",
+ " 1 | \n",
+ " 1599854398602548572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 30 | \n",
+ " 20309690 | \n",
+ " 33 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5616 | \n",
+ " 1 | \n",
+ " 1599854398602560572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 33 | \n",
+ " 20309690 | \n",
+ " 36 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5617 | \n",
+ " 1 | \n",
+ " 1599854398602572572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 36 | \n",
+ " 20309690 | \n",
+ " 39 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5618 | \n",
+ " 1 | \n",
+ " 1599854398602584572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 39 | \n",
+ " 20309690 | \n",
+ " 42 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5619 | \n",
+ " 1 | \n",
+ " 1599854398602596572 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 42 | \n",
+ " 20309690 | \n",
+ " 45 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5620 | \n",
+ " 1 | \n",
+ " 1599854398602676203 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309690 | \n",
+ " 45 | \n",
+ " 20309691 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5621 | \n",
+ " 1 | \n",
+ " 1599854398602944102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309691 | \n",
+ " 3 | \n",
+ " 20309692 | \n",
+ " 3 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5622 | \n",
+ " 1 | \n",
+ " 1599854398602956102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 3 | \n",
+ " 20309692 | \n",
+ " 6 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5623 | \n",
+ " 1 | \n",
+ " 1599854398602968102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 6 | \n",
+ " 20309692 | \n",
+ " 9 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5624 | \n",
+ " 1 | \n",
+ " 1599854398602980102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 9 | \n",
+ " 20309692 | \n",
+ " 12 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5625 | \n",
+ " 1 | \n",
+ " 1599854398602992102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 12 | \n",
+ " 20309692 | \n",
+ " 15 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5626 | \n",
+ " 1 | \n",
+ " 1599854398603004102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 15 | \n",
+ " 20309692 | \n",
+ " 18 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5627 | \n",
+ " 1 | \n",
+ " 1599854398603016102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 18 | \n",
+ " 20309692 | \n",
+ " 21 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5628 | \n",
+ " 1 | \n",
+ " 1599854398603028102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 21 | \n",
+ " 20309692 | \n",
+ " 24 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5629 | \n",
+ " 1 | \n",
+ " 1599854398603040102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 24 | \n",
+ " 20309692 | \n",
+ " 27 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5630 | \n",
+ " 1 | \n",
+ " 1599854398603052102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 27 | \n",
+ " 20309692 | \n",
+ " 30 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5631 | \n",
+ " 1 | \n",
+ " 1599854398603064102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 30 | \n",
+ " 20309692 | \n",
+ " 33 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5632 | \n",
+ " 1 | \n",
+ " 1599854398603076102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 33 | \n",
+ " 20309692 | \n",
+ " 36 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5633 | \n",
+ " 1 | \n",
+ " 1599854398603088102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 36 | \n",
+ " 20309692 | \n",
+ " 39 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5634 | \n",
+ " 1 | \n",
+ " 1599854398603100102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 39 | \n",
+ " 20309692 | \n",
+ " 42 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5635 | \n",
+ " 1 | \n",
+ " 1599854398603112102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 42 | \n",
+ " 20309692 | \n",
+ " 45 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5636 | \n",
+ " 1 | \n",
+ " 1599854398603124102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 45 | \n",
+ " 20309692 | \n",
+ " 48 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5637 | \n",
+ " 1 | \n",
+ " 1599854398603136102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 48 | \n",
+ " 20309692 | \n",
+ " 51 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5638 | \n",
+ " 1 | \n",
+ " 1599854398603148102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 51 | \n",
+ " 20309692 | \n",
+ " 54 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5639 | \n",
+ " 1 | \n",
+ " 1599854398603160102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 54 | \n",
+ " 20309692 | \n",
+ " 57 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5640 | \n",
+ " 1 | \n",
+ " 1599854398603172102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 57 | \n",
+ " 20309692 | \n",
+ " 60 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5641 | \n",
+ " 1 | \n",
+ " 1599854398603184102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 60 | \n",
+ " 20309692 | \n",
+ " 63 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5642 | \n",
+ " 1 | \n",
+ " 1599854398603196102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 63 | \n",
+ " 20309692 | \n",
+ " 66 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ " | 5643 | \n",
+ " 1 | \n",
+ " 1599854398603208102 | \n",
+ " +ESU0 +(ESU0-ESZ0) -ESZ0 | \n",
+ " 20309692 | \n",
+ " 66 | \n",
+ " 20309692 | \n",
+ " 69 | \n",
+ " 17.5 | \n",
+ " 17.5 | \n",
+ " 2918 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " run_id t_time polygon o_eid \\\n",
+ "5597 1 1599854398601848154 +ESU0 +(ESU0-ESZ0) -ESZ0 20309598 \n",
+ "5598 1 1599854398601867165 +ESU0 +(ESU0-ESZ0) -ESZ0 20309647 \n",
+ "5599 1 1599854398601879165 +ESU0 +(ESU0-ESZ0) -ESZ0 20309667 \n",
+ "5600 1 1599854398601970880 +ESU0 +(ESU0-ESZ0) -ESZ0 20309667 \n",
+ "5601 1 1599854398602128013 +ESU0 +(ESU0-ESZ0) -ESZ0 20309668 \n",
+ "5602 1 1599854398602143173 +ESU0 +(ESU0-ESZ0) -ESZ0 20309670 \n",
+ "5603 1 1599854398602246463 +ESU0 +(ESU0-ESZ0) -ESZ0 20309677 \n",
+ "5604 1 1599854398602351943 +ESU0 +(ESU0-ESZ0) -ESZ0 20309678 \n",
+ "5605 1 1599854398602428572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309689 \n",
+ "5606 1 1599854398602440572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5607 1 1599854398602452572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5608 1 1599854398602464572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5609 1 1599854398602476572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5610 1 1599854398602488572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5611 1 1599854398602500572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5612 1 1599854398602512572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5613 1 1599854398602524572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5614 1 1599854398602536572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5615 1 1599854398602548572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5616 1 1599854398602560572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5617 1 1599854398602572572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5618 1 1599854398602584572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5619 1 1599854398602596572 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5620 1 1599854398602676203 +ESU0 +(ESU0-ESZ0) -ESZ0 20309690 \n",
+ "5621 1 1599854398602944102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309691 \n",
+ "5622 1 1599854398602956102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5623 1 1599854398602968102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5624 1 1599854398602980102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5625 1 1599854398602992102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5626 1 1599854398603004102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5627 1 1599854398603016102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5628 1 1599854398603028102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5629 1 1599854398603040102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5630 1 1599854398603052102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5631 1 1599854398603064102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5632 1 1599854398603076102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5633 1 1599854398603088102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5634 1 1599854398603100102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5635 1 1599854398603112102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5636 1 1599854398603124102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5637 1 1599854398603136102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5638 1 1599854398603148102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5639 1 1599854398603160102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5640 1 1599854398603172102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5641 1 1599854398603184102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5642 1 1599854398603196102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "5643 1 1599854398603208102 +ESU0 +(ESU0-ESZ0) -ESZ0 20309692 \n",
+ "\n",
+ " o_eid_2 e_eid e_eid_2 target realized streak_id \n",
+ "5597 0 20309647 3 17.5 17.5 2918 \n",
+ "5598 3 20309667 3 17.5 17.5 2918 \n",
+ "5599 3 20309667 6 17.5 17.5 2918 \n",
+ "5600 6 20309668 3 17.5 17.5 2918 \n",
+ "5601 3 20309670 3 17.5 17.5 2918 \n",
+ "5602 3 20309677 3 17.5 17.5 2918 \n",
+ "5603 3 20309678 3 17.5 17.5 2918 \n",
+ "5604 3 20309689 3 17.5 17.5 2918 \n",
+ "5605 3 20309690 3 17.5 17.5 2918 \n",
+ "5606 3 20309690 6 17.5 17.5 2918 \n",
+ "5607 6 20309690 9 17.5 17.5 2918 \n",
+ "5608 9 20309690 12 17.5 17.5 2918 \n",
+ "5609 12 20309690 15 17.5 17.5 2918 \n",
+ "5610 15 20309690 18 17.5 17.5 2918 \n",
+ "5611 18 20309690 21 17.5 17.5 2918 \n",
+ "5612 21 20309690 24 17.5 17.5 2918 \n",
+ "5613 24 20309690 27 17.5 17.5 2918 \n",
+ "5614 27 20309690 30 17.5 17.5 2918 \n",
+ "5615 30 20309690 33 17.5 17.5 2918 \n",
+ "5616 33 20309690 36 17.5 17.5 2918 \n",
+ "5617 36 20309690 39 17.5 17.5 2918 \n",
+ "5618 39 20309690 42 17.5 17.5 2918 \n",
+ "5619 42 20309690 45 17.5 17.5 2918 \n",
+ "5620 45 20309691 3 17.5 17.5 2918 \n",
+ "5621 3 20309692 3 17.5 17.5 2918 \n",
+ "5622 3 20309692 6 17.5 17.5 2918 \n",
+ "5623 6 20309692 9 17.5 17.5 2918 \n",
+ "5624 9 20309692 12 17.5 17.5 2918 \n",
+ "5625 12 20309692 15 17.5 17.5 2918 \n",
+ "5626 15 20309692 18 17.5 17.5 2918 \n",
+ "5627 18 20309692 21 17.5 17.5 2918 \n",
+ "5628 21 20309692 24 17.5 17.5 2918 \n",
+ "5629 24 20309692 27 17.5 17.5 2918 \n",
+ "5630 27 20309692 30 17.5 17.5 2918 \n",
+ "5631 30 20309692 33 17.5 17.5 2918 \n",
+ "5632 33 20309692 36 17.5 17.5 2918 \n",
+ "5633 36 20309692 39 17.5 17.5 2918 \n",
+ "5634 39 20309692 42 17.5 17.5 2918 \n",
+ "5635 42 20309692 45 17.5 17.5 2918 \n",
+ "5636 45 20309692 48 17.5 17.5 2918 \n",
+ "5637 48 20309692 51 17.5 17.5 2918 \n",
+ "5638 51 20309692 54 17.5 17.5 2918 \n",
+ "5639 54 20309692 57 17.5 17.5 2918 \n",
+ "5640 57 20309692 60 17.5 17.5 2918 \n",
+ "5641 60 20309692 63 17.5 17.5 2918 \n",
+ "5642 63 20309692 66 17.5 17.5 2918 \n",
+ "5643 66 20309692 69 17.5 17.5 2918 "
+ ]
+ },
+ "execution_count": 63,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Get shots participating in the longest streak\n",
+ "longest_streak = streaks.streak_id.iloc[0]\n",
+ "first_event = shots[shots.streak_id==longest_streak].o_eid.min()\n",
+ "last_event = shots[shots.streak_id==longest_streak].o_eid.max()\n",
+ "shots = shots[shots.streak_id==longest_streak]\n",
+ "shots"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 92,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " | \n",
+ " t_time | \n",
+ " h_t_time | \n",
+ " entry_type | \n",
+ " eid1_x | \n",
+ " eid2_x | \n",
+ " val3 | \n",
+ " val4 | \n",
+ " val5 | \n",
+ " val6 | \n",
+ " val7 | \n",
+ " ... | \n",
+ " bid_z | \n",
+ " bid_qty_z | \n",
+ " ask_z | \n",
+ " ask_qty_z | \n",
+ " bid_u_z | \n",
+ " bid_qty_u_z | \n",
+ " ask_u_z | \n",
+ " ask_qty_u_z | \n",
+ " buy_p_value | \n",
+ " sell_p_value | \n",
+ "
\n",
+ " \n",
+ " | eid1 | \n",
+ " eid2 | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 20308598 | \n",
+ " 0 | \n",
+ " 1599854398473045923 | \n",
+ " 19:59:58.473045923 | \n",
+ " gw_order | \n",
+ " 20308598 | \n",
+ " 0 | \n",
+ " 1599854398472684897 | \n",
+ " updated | \n",
+ " ESU0-ESZ0 | \n",
+ " -1080.000000 | \n",
+ " 1 | \n",
+ " ... | \n",
+ " 333000.0 | \n",
+ " 155.0 | \n",
+ " 333025.0 | \n",
+ " 13.0 | \n",
+ " -1040.0 | \n",
+ " 9473.0 | \n",
+ " -1035.0 | \n",
+ " 5116.0 | \n",
+ " -20.0 | \n",
+ " -7.5 | \n",
+ "
\n",
+ " \n",
+ " | 20308599 | \n",
+ " 0 | \n",
+ " 1599854398473053752 | \n",
+ " 19:59:58.473053752 | \n",
+ " gw_order | \n",
+ " 20308599 | \n",
+ " 0 | \n",
+ " 1599854398472684897 | \n",
+ " updated | \n",
+ " ESU0 | \n",
+ " 334100.000000 | \n",
+ " 1 | \n",
+ " ... | \n",
+ " 333000.0 | \n",
+ " 155.0 | \n",
+ " 333025.0 | \n",
+ " 13.0 | \n",
+ " -1040.0 | \n",
+ " 9473.0 | \n",
+ " -1035.0 | \n",
+ " 5116.0 | \n",
+ " -20.0 | \n",
+ " -7.5 | \n",
+ "
\n",
+ " \n",
+ " | 20308600 | \n",
+ " 0 | \n",
+ " 1599854398473563030 | \n",
+ " 19:59:58.473563030 | \n",
+ " gw_order | \n",
+ " 20308600 | \n",
+ " 0 | \n",
+ " 1599854398472697817 | \n",
+ " removed | \n",
+ " ESZ0 | \n",
+ " 333025.000000 | \n",
+ " 0 | \n",
+ " ... | \n",
+ " 333025.0 | \n",
+ " 8.0 | \n",
+ " 333050.0 | \n",
+ " 158.0 | \n",
+ " -1040.0 | \n",
+ " 9473.0 | \n",
+ " -1035.0 | \n",
+ " 5116.0 | \n",
+ " -32.5 | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | 0 | \n",
+ " 1599854398473563030 | \n",
+ " 19:59:58.473563030 | \n",
+ " gw_order | \n",
+ " 20308600 | \n",
+ " 0 | \n",
+ " 1599854398472697817 | \n",
+ " removed | \n",
+ " ESZ0 | \n",
+ " 333025.000000 | \n",
+ " 0 | \n",
+ " ... | \n",
+ " 333025.0 | \n",
+ " 8.0 | \n",
+ " 333050.0 | \n",
+ " 158.0 | \n",
+ " -1040.0 | \n",
+ " 9473.0 | \n",
+ " -1035.0 | \n",
+ " 5116.0 | \n",
+ " -32.5 | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | 0 | \n",
+ " 1599854398473563030 | \n",
+ " 19:59:58.473563030 | \n",
+ " gw_order | \n",
+ " 20308600 | \n",
+ " 0 | \n",
+ " 1599854398472697817 | \n",
+ " removed | \n",
+ " ESZ0 | \n",
+ " 333025.000000 | \n",
+ " 0 | \n",
+ " ... | \n",
+ " 333025.0 | \n",
+ " 8.0 | \n",
+ " 333050.0 | \n",
+ " 158.0 | \n",
+ " -1040.0 | \n",
+ " 9473.0 | \n",
+ " -1035.0 | \n",
+ " 5116.0 | \n",
+ " -32.5 | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 20309692 | \n",
+ " 69 | \n",
+ " 1599854398603208102 | \n",
+ " 19:59:58.603208102 | \n",
+ " gw_fill | \n",
+ " 20309692 | \n",
+ " 69 | \n",
+ " 1599854398603208102 | \n",
+ " partial | \n",
+ " SIDE_SELL | \n",
+ " ESU0-ESZ0 | \n",
+ " -1035.000000 | \n",
+ " ... | \n",
+ " 332975.0 | \n",
+ " 151.0 | \n",
+ " 333025.0 | \n",
+ " 153.0 | \n",
+ " -1040.0 | \n",
+ " 9759.0 | \n",
+ " -1035.0 | \n",
+ " 5063.0 | \n",
+ " -45.0 | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | 69 | \n",
+ " 1599854398603208102 | \n",
+ " 19:59:58.603208102 | \n",
+ " gw_fill | \n",
+ " 20309692 | \n",
+ " 69 | \n",
+ " 1599854398603208102 | \n",
+ " filled | \n",
+ " SIDE_BUY | \n",
+ " ESU0-ESZ0 | \n",
+ " -1035.000000 | \n",
+ " ... | \n",
+ " 332975.0 | \n",
+ " 151.0 | \n",
+ " 333025.0 | \n",
+ " 153.0 | \n",
+ " -1040.0 | \n",
+ " 9759.0 | \n",
+ " -1035.0 | \n",
+ " 5063.0 | \n",
+ " -45.0 | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | 20309693 | \n",
+ " 0 | \n",
+ " 1599854398603824103 | \n",
+ " 19:59:58.603824103 | \n",
+ " gw_order | \n",
+ " 20309693 | \n",
+ " 0 | \n",
+ " 1599854398603739477 | \n",
+ " added | \n",
+ " ESU0 | \n",
+ " 334000.000000 | \n",
+ " 1 | \n",
+ " ... | \n",
+ " 332975.0 | \n",
+ " 151.0 | \n",
+ " 333025.0 | \n",
+ " 153.0 | \n",
+ " -1040.0 | \n",
+ " 9759.0 | \n",
+ " -1035.0 | \n",
+ " 5063.0 | \n",
+ " -45.0 | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | 0 | \n",
+ " 1599854398603824103 | \n",
+ " 19:59:58.603824103 | \n",
+ " gw_fill | \n",
+ " 20309693 | \n",
+ " 0 | \n",
+ " 1599854398603739477 | \n",
+ " partial | \n",
+ " SIDE_SELL | \n",
+ " ESU0 | \n",
+ " 334000.000000 | \n",
+ " ... | \n",
+ " 332975.0 | \n",
+ " 151.0 | \n",
+ " 333025.0 | \n",
+ " 153.0 | \n",
+ " -1040.0 | \n",
+ " 9759.0 | \n",
+ " -1035.0 | \n",
+ " 5063.0 | \n",
+ " -45.0 | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | 0 | \n",
+ " 1599854398603824103 | \n",
+ " 19:59:58.603824103 | \n",
+ " gw_fill | \n",
+ " 20309693 | \n",
+ " 0 | \n",
+ " 1599854398603739477 | \n",
+ " filled | \n",
+ " SIDE_BUY | \n",
+ " ESU0 | \n",
+ " 334000.000000 | \n",
+ " ... | \n",
+ " 332975.0 | \n",
+ " 151.0 | \n",
+ " 333025.0 | \n",
+ " 153.0 | \n",
+ " -1040.0 | \n",
+ " 9759.0 | \n",
+ " -1035.0 | \n",
+ " 5063.0 | \n",
+ " -45.0 | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
2094 rows × 28 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " t_time h_t_time entry_type eid1_x \\\n",
+ "eid1 eid2 \n",
+ "20308598 0 1599854398473045923 19:59:58.473045923 gw_order 20308598 \n",
+ "20308599 0 1599854398473053752 19:59:58.473053752 gw_order 20308599 \n",
+ "20308600 0 1599854398473563030 19:59:58.473563030 gw_order 20308600 \n",
+ " 0 1599854398473563030 19:59:58.473563030 gw_order 20308600 \n",
+ " 0 1599854398473563030 19:59:58.473563030 gw_order 20308600 \n",
+ "... ... ... ... ... \n",
+ "20309692 69 1599854398603208102 19:59:58.603208102 gw_fill 20309692 \n",
+ " 69 1599854398603208102 19:59:58.603208102 gw_fill 20309692 \n",
+ "20309693 0 1599854398603824103 19:59:58.603824103 gw_order 20309693 \n",
+ " 0 1599854398603824103 19:59:58.603824103 gw_fill 20309693 \n",
+ " 0 1599854398603824103 19:59:58.603824103 gw_fill 20309693 \n",
+ "\n",
+ " eid2_x val3 val4 val5 val6 \\\n",
+ "eid1 eid2 \n",
+ "20308598 0 0 1599854398472684897 updated ESU0-ESZ0 -1080.000000 \n",
+ "20308599 0 0 1599854398472684897 updated ESU0 334100.000000 \n",
+ "20308600 0 0 1599854398472697817 removed ESZ0 333025.000000 \n",
+ " 0 0 1599854398472697817 removed ESZ0 333025.000000 \n",
+ " 0 0 1599854398472697817 removed ESZ0 333025.000000 \n",
+ "... ... ... ... ... ... \n",
+ "20309692 69 69 1599854398603208102 partial SIDE_SELL ESU0-ESZ0 \n",
+ " 69 69 1599854398603208102 filled SIDE_BUY ESU0-ESZ0 \n",
+ "20309693 0 0 1599854398603739477 added ESU0 334000.000000 \n",
+ " 0 0 1599854398603739477 partial SIDE_SELL ESU0 \n",
+ " 0 0 1599854398603739477 filled SIDE_BUY ESU0 \n",
+ "\n",
+ " val7 ... bid_z bid_qty_z ask_z ask_qty_z \\\n",
+ "eid1 eid2 ... \n",
+ "20308598 0 1 ... 333000.0 155.0 333025.0 13.0 \n",
+ "20308599 0 1 ... 333000.0 155.0 333025.0 13.0 \n",
+ "20308600 0 0 ... 333025.0 8.0 333050.0 158.0 \n",
+ " 0 0 ... 333025.0 8.0 333050.0 158.0 \n",
+ " 0 0 ... 333025.0 8.0 333050.0 158.0 \n",
+ "... ... ... ... ... ... ... \n",
+ "20309692 69 -1035.000000 ... 332975.0 151.0 333025.0 153.0 \n",
+ " 69 -1035.000000 ... 332975.0 151.0 333025.0 153.0 \n",
+ "20309693 0 1 ... 332975.0 151.0 333025.0 153.0 \n",
+ " 0 334000.000000 ... 332975.0 151.0 333025.0 153.0 \n",
+ " 0 334000.000000 ... 332975.0 151.0 333025.0 153.0 \n",
+ "\n",
+ " bid_u_z bid_qty_u_z ask_u_z ask_qty_u_z buy_p_value \\\n",
+ "eid1 eid2 \n",
+ "20308598 0 -1040.0 9473.0 -1035.0 5116.0 -20.0 \n",
+ "20308599 0 -1040.0 9473.0 -1035.0 5116.0 -20.0 \n",
+ "20308600 0 -1040.0 9473.0 -1035.0 5116.0 -32.5 \n",
+ " 0 -1040.0 9473.0 -1035.0 5116.0 -32.5 \n",
+ " 0 -1040.0 9473.0 -1035.0 5116.0 -32.5 \n",
+ "... ... ... ... ... ... \n",
+ "20309692 69 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n",
+ " 69 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n",
+ "20309693 0 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n",
+ " 0 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n",
+ " 0 -1040.0 9759.0 -1035.0 5063.0 -45.0 \n",
+ "\n",
+ " sell_p_value \n",
+ "eid1 eid2 \n",
+ "20308598 0 -7.5 \n",
+ "20308599 0 -7.5 \n",
+ "20308600 0 5.0 \n",
+ " 0 5.0 \n",
+ " 0 5.0 \n",
+ "... ... \n",
+ "20309692 69 5.0 \n",
+ " 69 5.0 \n",
+ "20309693 0 5.0 \n",
+ " 0 5.0 \n",
+ " 0 5.0 \n",
+ "\n",
+ "[2094 rows x 28 columns]"
+ ]
+ },
+ "execution_count": 92,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "first_event = shots.o_eid.min() - 1000\n",
+ "last_event = shots.o_eid.max() + 1\n",
+ "\n",
+ "r_books = lined_up_books.loc[first_event:last_event]\n",
+ "\n",
+ "r_logs = gw_logs.loc[first_event:last_event]\n",
+ "long_streak_log = r_logs.merge(r_books,how='outer',left_index=True,right_index=True)\n",
+ "long_streak_log.to_csv(\"/home/ppinto/long_streak.csv\")\n",
+ "long_streak_log"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 93,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(20308598, 20309693)"
+ ]
+ },
+ "execution_count": 93,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "(first_event,last_event)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/src/server/sparkd/cmds/spartan/analysis/spartan.py b/src/server/sparkd/cmds/spartan/analysis/spartan.py
new file mode 100644
index 000000000..6d5f5694e
--- /dev/null
+++ b/src/server/sparkd/cmds/spartan/analysis/spartan.py
@@ -0,0 +1,212 @@
+#!/usr/bin/python3
+
+import pandas as pd
+import os
+import sqlite3
+import pathlib
+
+store_root = pathlib.Path.home() / pathlib.Path("sparkdata/data")
+
+def set_pcap_store(store):
+ store_root = store
+
+def get_db_path(channel, date):
+ return store_root / pathlib.Path(channel) / pathlib.Path(date+"_"+channel+"_24h.sqlite")
+
+def get_pcap_db(channel,date):
+ path = get_db_path(channel,date)
+ assert os.path.exists(path)
+ return sqlite3.connect(path)
+
+def get_sim_db():
+ from pathlib import Path
+ path = store_root / Path("databases") / Path("sim_db") /Path("sim_db.sqlite")
+ assert os.path.exists(path)
+ return sqlite3.connect(path)
+
+def qsim(query):
+ with get_sim_db() as conn:
+ return pd.read_sql_query(query,conn)
+
+def get_db(channel,store):
+ def querier(query):
+ with get_pcap_db(channel,store) as conn:
+ return pd.read_sql_query(query,conn)
+ return querier
+
+def clean_sim_books(books):
+ books = books.rename(columns={'val1':'eid1', 'val2':'eid2','val4':'symbol',
+ 'val5':'bid','val6':'bid_qty', 'val7':'ask','val8':'ask_qty'})
+ books = books.drop(['log_entry_order','t_time','h_t_time',
+ 'run_id','val3','entry_type','val9','val10'],axis=1)
+
+ books = books.astype({'eid1':'int32','eid2':'int32',
+ 'bid':'float64','bid_qty':'float64',
+ 'ask':'float64','ask_qty':'float64'})
+ return books
+
+def get_310_lined_prices(books):
+ # create an idx with all events in the range (this is so that we
+ # can have a row for each exchange state even if nothing visibly
+ # changed in those events)
+ idx = pd.DataFrame({'eid1': range(books.eid1.min(),books.eid1.max()+1)})
+ idx['eid2'] = 0
+
+ # Filter for the symbols we care
+ u=books[books.symbol=='ESU0'].drop(['symbol'],axis=1)
+ z=books[books.symbol=='ESZ0'].drop(['symbol'],axis=1)
+ u_z=books[books.symbol=='ESU0-ESZ0'].drop(['symbol'],axis=1)
+
+ # Line up symbols so we have an event time line that includes all symbols
+ prices = idx
+
+ prices = prices.merge(u,
+ how='outer',
+ left_on=['eid1','eid2'],
+ right_on=['eid1','eid2'])
+ prices = prices.rename({'bid':'bid_u','ask':'ask_u',
+ 'bid_qty':'bid_qty_u','ask_qty':'ask_qty_u'},axis=1)
+
+ prices = prices.merge(z,
+ how='outer',
+ left_on=['eid1','eid2'],
+ right_on=['eid1','eid2'])
+ prices = prices.rename({'bid':'bid_z','ask':'ask_z',
+ 'bid_qty':'bid_qty_z','ask_qty':'ask_qty_z'},axis=1)
+
+ prices = prices.merge(u_z,
+ how='outer',
+ left_on=['eid1','eid2'],
+ right_on=['eid1','eid2'])
+ prices = prices.rename({'bid':'bid_u_z','ask':'ask_u_z',
+ 'bid_qty':'bid_qty_u_z','ask_qty':'ask_qty_u_z'},axis=1)
+
+ # set eid1, eid2 as indexes and then sort by them so that ffill
+ # below will propagate the last price in event order
+ prices = prices.set_index(['eid1','eid2'],drop=False)
+ prices = prices.sort_index()
+
+ # Fill holes (every event only updates one symbol) with previous value of book
+ prices = prices.ffill()
+
+ # Add columns to calculate polygon values for buy and sell
+ prices['buy_p_value'] = (-prices.ask_z+prices.bid_u+prices.bid_u_z)/2
+ prices['sell_p_value'] = (prices.bid_z-prices.ask_u-prices.ask_u_z)/2
+ return prices
+
+def get_shot_outcomes(run_id=1):
+ with get_sim_db() as conn:
+ return pd.read_sql_query("select * from shot_outcome where run_id="+str(run_id),conn)
+
+
+def get_opp(*dates):
+ result = pd.DataFrame()
+
+ for date in dates:
+ with sqlite3.connect("/home/ppinto/310/"+date+"_310_24h.sqlite") as conn:
+ result = result.append(pd.read_sql_query(("select '{}' date,\"sum(merged_qty*merged_value)\" "
+ "opportunity from total_opportunity").format(date),conn))
+ return result
+
+
+book_cache = None;
+def books():
+ def books_from_db():
+ with get_md_conn() as conn:
+ df = pd.read_sql_query("select top_of_books.*,events.t_time "
+ "from top_of_books, events "
+ "where top_of_books.eid=events.eid",conn)
+ df = df.fillna(0)
+ df = df.astype({'eid':'int32','sid':'int32',
+ 't_time':'uint64',
+ 'bid':'int32','bid_qty':'int32',
+ 'ask':'int32','ask_qty':'int32'})
+ df = df.set_index('eid')
+ df = df.sort_index()
+ return df
+
+ global book_cache
+ if not book_cache is None:
+ return book_cache
+ elif os.path.exists("/tmp/books.parquet"):
+ book_cache = pd.read_parquet("/tmp/books.parquet")
+ return book_cache;
+ else:
+ book_cache = books_from_db()
+ book_cache.to_parquet("/tmp/books.parquet")
+ return book_cache
+
+def events_from_db():
+ with get_md_conn() as conn:
+ df = pd.read_sql_query("select * from events",conn)
+ df = df.fillna(0)
+ df = df.astype({'eid':'int32','t_time':'uint64',
+ 'first_sn':'uint64','last_sn':'uint64',
+ 'first_st':'uint64','last_st':'uint64'})
+ return df
+
+def securities():
+ with get_md_conn() as conn:
+ return pd.read_sql_query("select * from securities",conn)
+
+def get_event_range():
+ with get_md_conn() as conn:
+ with conn.cursor() as cursor:
+ cursor.execute("select min(eid),max(eid) from events")
+ min,max =cursor.fetchall()[0]
+ return int(min),int(max)
+
+def events_from_parquet():
+ return pd.read_parquet("/home/ppinto/events.parquet")
+
+def get_sec_book(symbol):
+ b = books()
+ s = b[b.symbol==symbol]
+ s = s.reindex(range(0,b.shape[0]+1),method='ffill')
+ s = s.fillna(0)
+ s = s.astype({'bid':'int32','bid_qty':'int32','ask_qty':'int32',
+ 'ask':'int32','t_time':'uint64'})
+ s = s.drop(columns=['symbol','sid'])
+ return s
+
+def line_up_books(*symbols):
+ result = None
+ for s in symbols:
+ df = get_sec_book(s)
+ df = df.add_suffix('_'+s)
+ if result is None:
+ result = df
+ else:
+ result = result.join(df)
+ return result
+
+
+def upload_lined_books(df):
+ book_f = "/tmp/lined_up_books.csv"
+ df.to_csv(book_f);
+ with get_md_conn() as conn:
+ with conn.cursor() as cursor:
+ with open(book_f) as f:
+ cursor.copy_from(f,"lined_up_books",sep=",")
+
+def price_series(symbol):
+ b = books()
+ s=b[b.symbol==symbol]
+ s = s[(s.shift(1).bid != s.bid) | (s.shift(1).ask != s.ask)]
+ s=s.copy()
+ s['delta'] = s.t_time - s.shift(1).t_time
+ return s
+
+def get_polygon_valuation():
+ lined_up = line_up_books('ESU0','ESZ0','ESU0-ESZ0')
+ lined_up['buy']=-lined_up.ask_ESZ0+lined_up.bid_ESU0+lined_up['bid_ESU0-ESZ0']
+ lined_up['sell']=lined_up.bid_ESZ0-lined_up.ask_ESU0-lined_up['ask_ESU0-ESZ0']
+ return lined_up
+
+def filter_for_opportunities(p):
+ p = p[ ((p.buy>0) | (p.shift(1).buy>0) | (p.shift(-1).buy>0)) |
+ ((p.sell>0) | (p.shift(1).sell>0) | (p.shift(-1).sell>0)) ]
+ return p
+
+
+
diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp
index 729fbd01c..14fae2fc2 100644
--- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp
+++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp
@@ -35,19 +35,17 @@ struct PolyObserver::Opportunity {
struct PolyObserver::PImpl {
ISimServices *sim = nullptr;
- std::unordered_map books;
+ std::unordered_map books;
std::vector> targets;
AlgoEventID event_id;
MDP3TimeStamp transact_time;
MDP3TimeStamp observable_time;
- size_t max_events;
};
PolyObserver::PolyObserver() : me(new PImpl()) {}
PolyObserver::~PolyObserver() {}
-
-void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) {
+void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &) {
me->sim = &s;
me->books.clear();
me->targets.clear();
@@ -55,8 +53,6 @@ void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) {
me->event_id = {0, 0};
me->transact_time = MDP3TimeStamp{0};
me->observable_time = MDP3TimeStamp{0};
- me->max_events =
- param_or_default(options, "MaxEvents", std::numeric_limits::max());
for (auto &p : s.polygons()) {
auto buy_t =
@@ -73,25 +69,21 @@ void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) {
}
void PolyObserver::on_notification(AlgoNotification const &n) {
- if (n.event_id.first >= me->max_events)
- me->sim->terminate_simulation();
- else {
- if (me->event_id.first) {
- for (auto i = me->event_id.first+1;i < n.event_id.first;i++) {
- me->event_id.first = i;
- me->event_id.second = 0;
- me->transact_time = n.transact_time;
- me->observable_time = n.observable_time;
- eval();
- rank();
- log();
- }
+ if (me->event_id.first) {
+ for (auto i = me->event_id.first + 1; i < n.event_id.first; i++) {
+ me->event_id.first = i;
+ me->event_id.second = 0;
+ me->transact_time = n.transact_time;
+ me->observable_time = n.observable_time;
+ eval();
+ rank();
+ log();
}
- me->event_id = n.event_id;
- me->transact_time = n.transact_time;
- me->observable_time = n.observable_time;
- std::visit(*this, n.event);
}
+ me->event_id = n.event_id;
+ me->transact_time = n.transact_time;
+ me->observable_time = n.observable_time;
+ std::visit(*this, n.event);
}
void PolyObserver::eval() {
@@ -134,16 +126,17 @@ void PolyObserver::rank() {
void PolyObserver::log() {
for (auto const &t : me->targets)
if (t->has_value())
- me->sim->log("opportunity", me->event_id.first, me->event_id.second,
- t->descriptor, t->value, t->quantity.value());
+ me->sim->log("poly_opportunity", me->event_id.first,
+ me->event_id.second, t->descriptor, t->value,
+ t->quantity.value());
}
-std::pair available(Book const&b) {
- std::pair result ;
+std::pair available(Book const &b) {
+ std::pair result;
result.first = md::IntPrice{};
- result.second= MDP3Quantity{0};
-
- for (size_t i = 0; i < b.size();i++) {
+ result.second = MDP3Quantity{0};
+
+ for (size_t i = 0; i < b.size(); i++) {
if (b[i].quantity.value() == 0)
break;
else {
@@ -154,22 +147,15 @@ std::pair available(Book const&b) {
return result;
}
-
void PolyObserver::operator()(const MBPUpdate &mbp) {
-
- auto avb = available(mbp.mbp_buy);
- auto avs = available(mbp.mbp_sell);
-
- me->sim->log("book", me->sim->get_sec(mbp.security_id).symbol(),
- me->event_id.first,
- mbp.mbp_buy[0].price.price(),
- mbp.mbp_buy[0].quantity.value(),
- mbp.mbp_sell[0].price.price(),
- mbp.mbp_sell[0].quantity.value(),
- avb.first.price(),
- avb.second.value(),
- avs.first.price(),
- avs.second.value());
+ me->sim->log(
+ "poly_book",
+ me->event_id.first,
+ me->event_id.second,
+ me->transact_time.value(),
+ me->sim->get_sec(mbp.security_id).symbol(),
+ mbp.mbp_buy[0].price.price(), mbp.mbp_buy[0].quantity.value(),
+ mbp.mbp_sell[0].price.price(), mbp.mbp_sell[0].quantity.value());
me->books[mbp.security_id] = mbp;
eval();
diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp
index 7adbb21bb..2523b81d7 100644
--- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp
+++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp
@@ -324,6 +324,15 @@ void Sniper::operator()(const MBPUpdate &mbp) {
if (it == me->books.end())
return;
+ // me->sim->log("book",
+ // me->current_event.first,
+ // me->current_event.second,
+ // me->sim->get_sec(mbp.security_id).symbol(),
+ // mbp.mbp_buy[0].price.price(),
+ // mbp.mbp_buy[0].quantity.value(),
+ // mbp.mbp_sell[0].price.price(),
+ // mbp.mbp_sell[0].quantity.value());
+
auto &book = it->second;
book.book = mbp;
diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp
index 1ac262ec1..ebd5d0b2b 100644
--- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp
+++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp
@@ -1,12 +1,12 @@
#include "SimOrchestrator.h"
#include "../simulator/SimInput.h"
#include "lib/utils/PBUtils.h"
+#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h"
-#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h"
#include
@@ -20,8 +20,9 @@ class SimOrchestrator::SimInputManager {
if (auto ptr = maybe_input.lock())
return ptr;
else {
- LOG(Spartan,Warn, "Loading input from {}", path);
- auto new_ptr = std::make_shared(make_sim_input_from_proto(path));
+ LOG(Spartan, Warn, "Loading input from {}", path);
+ auto new_ptr =
+ std::make_shared(make_sim_input_from_proto(path));
maybe_input = new_ptr;
return new_ptr;
}
@@ -123,7 +124,8 @@ SimOrchestrator::SimOrchestrator(protos::SimSessionOptions const &options)
auto &these_inputs = this_run.inputs;
these_options.set_algo(options.algo());
these_options.set_market_date(day);
-
+ these_options.set_log_all_transactions(options.log_all_transactions());
+ these_options.set_max_events(options.max_events());
// Set inputs
for (auto const &[channel, sim_input] : day_inputs) {
*these_options.add_channels() = channel;
@@ -170,7 +172,7 @@ std::unique_ptr make_algo(std::string const &algo) {
return std::make_unique();
if (algo == "PolyObserver")
return std::make_unique();
-
+
bts_assert(false);
return {};
}
@@ -189,8 +191,8 @@ void SimOrchestrator::run() {
auto algo = make_algo(r->options.algo());
auto rid = db.begin_run(sid, r->options);
int rr = remaining_runs;
- LOG(Spartan, Warn, "Starting run {} ({} remaining) {}", rid,
- rr, utils::pb::to_string(r->options));
+ LOG(Spartan, Warn, "Starting run {} ({} remaining) {}", rid, rr,
+ utils::pb::to_string(r->options));
auto logger = [rid, &db](auto ts, auto const &entry_type,
auto const &args) {
db.log(rid, ts, entry_type, args);
@@ -219,8 +221,7 @@ void SimOrchestrator::run() {
for (size_t i = 0; i < me->runs.size(); i++)
workers[i % workers.size()].assigned_runs.push_back(&me->runs[i]);
- auto session_id =
- me->db.create_session(me->options.algo());
+ auto session_id = me->db.create_session(me->options.algo());
std::atomic run_count = me->runs.size();
for (auto &w : workers)
diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto
index e9ef8555a..57419f262 100644
--- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto
+++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto
@@ -3,7 +3,6 @@ package bts.spartan.protos;
/*
algo: "Sniper"
-session_title: "A memorable title"
n_cores: 4
file: "/home/ppinto/360/2020-09-07_360_24h.sim"
@@ -31,6 +30,17 @@ message SimSessionOptions {
// The algo we are testing
string algo = 1;
+ // Wheter the simulator will record every order, order report and
+ // price update. This produces fantastic ammounts of information so
+ // it is off by default.
+ bool log_all_transactions = 9;
+
+ // The maximum number of events we will process before we terminate
+ // the simulation. This is usefull to speed initial prototyping of
+ // algos or analysis tools. Use 0 (the default) to signifiy all
+ // events.
+ uint64 max_events = 11;
+
// The sim input files we are using. File names should end with
// "__24h.sim". Files for the same day will
// grouped and used as a unit for each simulation run.
@@ -54,7 +64,7 @@ message SimSessionOptions {
// An estimate for the average transaction latency at the
// exchange. We fallback on this value when we cannot use the send
- // time of a packet to estimate for that purpose.
+ // time of a packet to estimate transaction duration.
repeated uint64 transaction_latency_ns = 8;
// Algo specific parameters used for this session. For each named
@@ -63,12 +73,14 @@ message SimSessionOptions {
message ParameterDomain {
repeated string item = 1;
}
- map parameter = 9;
+ map parameter = 10;
}
// The parameters for a single simulation run.
message SimRunOptions {
string algo = 1;
+ bool log_all_transactions = 10;
+ uint64 max_events = 11;
uint64 base_algo_latency_ns = 2;
uint64 order_update_delay_ns = 3;
uint64 mbp_update_delay_ns = 4;
diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp
index 25e2fdbbf..215b0d1f6 100644
--- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp
+++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp
@@ -48,6 +48,10 @@ SimDB::SimDB(std::string const &loc) {
db.exec(str);
}
me = std::make_unique(loc);
+ SQLite::Statement sync_off(me->db,"PRAGMA synchronous = OFF");
+ sync_off.executeStep();
+ SQLite::Statement journal_mem(me->db,"PRAGMA journal_mode = MEMORY");
+ journal_mem.executeStep();
me->logger_thread = std::thread{[this]() { do_work(); }};
}
diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp
index 5737368e6..74dfd8d85 100644
--- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp
+++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp
@@ -80,7 +80,8 @@ void ExchangeSim::prepare_run(std::vector const &algos,
// Init gateways, add them to the simulator
for (auto &i : me->inputs) {
- auto &gw = me->run_state->gateways.emplace_back(me->logger,i->instruments, options);
+ auto &gw = me->run_state->gateways.emplace_back(me->logger,
+ i->instruments, options);
me->run_state->simulator.add_agent(gw);
}
@@ -133,38 +134,45 @@ bool ExchangeSim::has_work() const {
void ExchangeSim::do_work() {
bts_assert(me->run_state);
me->run_state->simulator.run_once();
- auto now = Timer::now();
- const auto log_period_s = 10;
- if (ms_diff(now, me->run_state->last_time_log) > 1000 * log_period_s) {
+ if (me->run_state->options.max_events() &&
+ me->run_state->last_completed_count >=
+ me->run_state->options.max_events()) {
+ me->run_state->simulator.end_simulation();
+ } else {
+ auto now = Timer::now();
+ const auto log_period_s = 10;
+ if (ms_diff(now, me->run_state->last_time_log) > 1000 * log_period_s) {
- auto sim_time = me->run_state->simulator.simulation_time();
+ auto sim_time = me->run_state->simulator.simulation_time();
- size_t total_events = me->run_state->starting_event_count;
+ size_t total_events = me->run_state->starting_event_count;
- size_t remaining_events = me->run_state->simulator.pending_events_count();
+ size_t remaining_events =
+ me->run_state->simulator.pending_events_count();
- size_t completed_events = total_events > remaining_events
- ? total_events - remaining_events
- : total_events;
+ size_t completed_events = total_events > remaining_events
+ ? total_events - remaining_events
+ : total_events;
- size_t p_complete = (double)completed_events / total_events * 100;
+ size_t p_complete = (double)completed_events / total_events * 100;
- size_t completed_this_cycle =
- completed_events - me->run_state->last_completed_count;
+ size_t completed_this_cycle =
+ completed_events - me->run_state->last_completed_count;
- size_t elapsed_secs = sec_diff(now, me->run_state->run_start);
- auto eta = (remaining_events * elapsed_secs / completed_events);
- auto TPS = completed_this_cycle / log_period_s;
+ size_t elapsed_secs = sec_diff(now, me->run_state->run_start);
+ auto eta = (remaining_events * elapsed_secs / completed_events);
+ auto TPS = completed_this_cycle / log_period_s;
- me->run_state->last_time_log = now;
- me->run_state->last_sim_time = sim_time;
- me->run_state->last_completed_count = completed_events;
+ me->run_state->last_time_log = now;
+ me->run_state->last_sim_time = sim_time;
+ me->run_state->last_completed_count = completed_events;
- LOG(Spartan, Diag,
- "Run {} -> Time: {} TPS: {} Complete: {}% ETA: {} "
- "secs",
- me->run_id, to_time_string(MDP3TimeStamp{sim_time}), TPS, p_complete,
- eta);
+ LOG(Spartan, Diag,
+ "Run {} -> Time: {} TPS: {} Complete: {}% ETA: {} "
+ "secs",
+ me->run_id, to_time_string(MDP3TimeStamp{sim_time}), TPS,
+ p_complete, eta);
+ }
}
}
diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp
index 81be7b08a..71e9ad174 100644
--- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp
+++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp
@@ -117,6 +117,16 @@ void GatewayAgent::_handle(ExchangeTransactionEvent const &e) {
else
forward_clock_by(me->options.transaction_latency_ns());
+ if (me->options.log_all_transactions())
+ for (auto const &change : ev->order_changes) {
+ log(me->logger, time(), "gw_order", me->current_event.first,
+ me->current_event.second, ev->transact_time.value(),
+ change_name(change.change),
+ get_sec(change.order.security).symbol(),
+ change.order.price.price(), change.order.quantity.value(),
+ api::Side_Name(change.order.side), change.order.key.value());
+ }
+
// Now match the recorded orders and send out their effects as
// algo notification events
std::vector effects;
@@ -126,12 +136,34 @@ void GatewayAgent::_handle(ExchangeTransactionEvent const &e) {
std::visit(
overloaded{
[&](MBPUpdate &mbp) {
+ if (me->options.log_all_transactions()) {
+ log(me->logger, time(),
+ "gw_book",
+ me->current_event.first,
+ me->current_event.second,
+ ev->transact_time.value(),
+ get_sec(mbp.security_id).symbol(),
+ mbp.mbp_buy[0].price.price(),
+ mbp.mbp_buy[0].quantity.value(),
+ mbp.mbp_sell[0].price.price(),
+ mbp.mbp_sell[0].quantity.value());
+ }
emit_delayed_event(
me->options.mbp_update_delay_ns(),
AlgoNotification{me->current_event, ev->transact_time,
MDP3TimeStamp{time()}, mbp});
},
[&](OrderUpdate const &order_update) {
+ if (me->options.log_all_transactions()) {
+ log(me->logger, time(), "gw_fill",
+ me->current_event.first, me->current_event.second,
+ ev->transact_time.value(), order_update.status_str(),
+ api::Side_Name(order_update.side),
+ get_sec(order_update.security_id).symbol(),
+ order_update.last_price.price(),
+ order_update.last_fills.value(),
+ order_update.key.value());
+ }
emit_delayed_event(
me->options.order_update_delay_ns(),
AlgoNotification{me->current_event, ev->transact_time,
@@ -148,33 +180,55 @@ void GatewayAgent::_handle(AlgoActionEvent const &e) {
auto const &algo_action = e.ev;
auto const &change = algo_action.change;
-
- log(me->logger, time(), "gw_algo_order", algo_action.transact_time.value(),
- change_name(change.change), get_sec(change.order.security).symbol(),
- change.order.price.price(), change.order.quantity.value(),
- api::Side_Name(change.order.side), change.order.key.value(),
- change.order.security.value(), me->current_event.first,
- me->current_event.second);
+ if (me->options.log_all_transactions()) {
+ log(me->logger, time(), "gw_algo_order", me->current_event.first,
+ me->current_event.second, algo_action.transact_time.value(),
+ change_name(change.change), get_sec(change.order.security).symbol(),
+ change.order.price.price(), change.order.quantity.value(),
+ api::Side_Name(change.order.side), change.order.key.value());
+ }
forward_clock_by(me->options.transaction_latency_ns());
std::vector effects;
me->matcher.match(Matcher::ALGO, {algo_action.change}, &effects);
for (auto &ef : effects)
std::visit(
- overloaded{[&](MBPUpdate &mbp) {
- emit_delayed_event(
- me->options.mbp_update_delay_ns(),
- AlgoNotification{me->current_event,
- algo_action.transact_time,
- MDP3TimeStamp{time()}, mbp});
- },
- [&](OrderUpdate const &order_update) {
- emit_delayed_event(
- me->options.order_update_delay_ns(),
- AlgoNotification{
- me->current_event, algo_action.transact_time,
- MDP3TimeStamp{time()}, order_update});
- }},
+ overloaded{
+ [&](MBPUpdate &mbp) {
+ if (me->options.log_all_transactions()) {
+ log(me->logger, time(), "gw_book", me->current_event.first,
+ me->current_event.second,
+ algo_action.transact_time.value(),
+ get_sec(mbp.security_id).symbol(),
+ mbp.mbp_buy[0].price.price(),
+ mbp.mbp_buy[0].quantity.value(),
+ mbp.mbp_sell[0].price.price(),
+ mbp.mbp_sell[0].quantity.value());
+ }
+ emit_delayed_event(
+ me->options.mbp_update_delay_ns(),
+ AlgoNotification{me->current_event,
+ algo_action.transact_time,
+ MDP3TimeStamp{time()}, mbp});
+ },
+ [&](OrderUpdate const &order_update) {
+ if (me->options.log_all_transactions()) {
+ log(me->logger, time(), "gw_algo_fill", me->current_event.first,
+ me->current_event.second,
+ algo_action.transact_time.value(),
+ order_update.status_str(),
+ api::Side_Name(order_update.side),
+ get_sec(order_update.security_id).symbol(),
+ order_update.last_price.price(),
+ order_update.last_fills.value(),
+ order_update.key.value());
+ }
+ emit_delayed_event(
+ me->options.order_update_delay_ns(),
+ AlgoNotification{me->current_event,
+ algo_action.transact_time,
+ MDP3TimeStamp{time()}, order_update});
+ }},
ef);
}