diff --git a/src/server/sparkd/cmds/spartan/analysis/sniper_opportunity_analysis.ipynb b/src/server/sparkd/cmds/spartan/analysis/sniper_opportunity_analysis.ipynb index 265a75ed5..7e30b76b5 100644 --- a/src/server/sparkd/cmds/spartan/analysis/sniper_opportunity_analysis.ipynb +++ b/src/server/sparkd/cmds/spartan/analysis/sniper_opportunity_analysis.ipynb @@ -2,36 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 168, - "metadata": {}, - "outputs": [], - "source": [ - "# adds a new col called opp_p_id which is simply\n", - "# a numerical representation of [opp_id,pid]\n", - "def add_ids(poly_vals):\n", - " oid_map = {-1:{-1:0}}\n", - " oid_map.pop(-1)\n", - " curr_id=0\n", - " def get_new_id():\n", - " nonlocal curr_id\n", - " curr_id = curr_id+1\n", - " return curr_id\n", - " def get_opp_p_id(r):\n", - " oid = r['opp_id']\n", - " pid = r['pid']\n", - " if oid_map.get(oid)==None:\n", - " oid_map[oid]={-1:0}\n", - " pid_map = oid_map[oid]\n", - " if pid_map.get(pid)==None:\n", - " pid_map[pid]=get_new_id()\n", - " return oid_map[oid][pid]\n", - " poly_vals['opp_p_id'] = poly_vals.apply (lambda row: get_opp_p_id(row), axis=1)\n", - " return poly_vals" - ] - }, - { - "cell_type": "code", - "execution_count": 169, + "execution_count": 694, "metadata": {}, "outputs": [], "source": [ @@ -42,7 +13,136 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 695, + "metadata": {}, + "outputs": [], + "source": [ + "# converts series to dataframe\n", + "def tabularize_opp_summ(opp_summ_ser):\n", + " rv_df = opp_summ_ser.to_frame().reset_index()\n", + " rv_df.columns = ['opp_id','vals'] \n", + " rv_df['symbol'] = rv_df['vals'].apply(lambda x:x[0])\n", + " rv_df['opp_id'] = rv_df['vals'].apply(lambda x:x[1])\n", + " rv_df['opp_p_id'] = rv_df['vals'].apply(lambda x:x[2])\n", + " rv_df['start_eid'] = rv_df['vals'].apply(lambda x:x[3])\n", + " rv_df['start_t_time'] = rv_df['vals'].apply(lambda x:x[4])\n", + " rv_df['start_h_t_time'] = rv_df['vals'].apply(lambda x:x[5])\n", + " rv_df['first_send_opp_win'] = rv_df['vals'].apply(lambda x:x[6])\n", + " rv_df['last_send_opp_win'] = rv_df['vals'].apply(lambda x:x[7])\n", + " rv_df['merged_value'] = rv_df['vals'].apply(lambda x:x[8])\n", + " rv_df['merged_qty'] = rv_df['vals'].apply(lambda x:x[9])\n", + " rv_df['ticks'] = rv_df['vals'].apply(lambda x:x[10])\n", + " rv_df['is_direct'] = rv_df['vals'].apply(lambda x:x[11])\n", + " rv_df['poly_bid_bk'] = rv_df['vals'].apply(lambda x:x[12])\n", + " rv_df['poly_ask_bk'] = rv_df['vals'].apply(lambda x:x[13])\n", + " rv_df['leg_books'] = rv_df['vals'].apply(lambda x:x[14])\n", + " del rv_df['vals']\n", + " return rv_df" + ] + }, + { + "cell_type": "code", + "execution_count": 696, + "metadata": {}, + "outputs": [], + "source": [ + "# summarizes each opportunity\n", + "def summarize_opportunities(poly_vals):\n", + " def get_max_edge_row (opp_p_rows):\n", + " min_react_time = 50000\n", + " opp_p_rows_filt = opp_p_rows[opp_p_rows['t_opp_first_send'] >= min_react_time]\n", + " if (opp_p_rows_filt.shape[0] == 0):\n", + " return [0.0,0,0,False,'','','','']\n", + " else:\n", + " m_idx = opp_p_rows_filt['merged_value'].argmax()\n", + " m_max_row = opp_p_rows_filt.iloc[m_idx]\n", + " return [m_max_row['merged_value'], m_max_row['merged_qty'], m_max_row['ticks'],\n", + " m_max_row['direct_value'] == m_max_row['merged_value'],\n", + " m_max_row['poly_bid_bk'], m_max_row['poly_ask_bk'],\n", + " m_max_row['book_1_bid_bk'] + ';' + m_max_row['book_1_ask_bk'] + ' ' +\n", + " m_max_row['book_2_bid_bk'] + ';' + m_max_row['book_2_ask_bk'] + ' ' +\n", + " m_max_row['book_3_bid_bk'] + ';' + m_max_row['book_3_ask_bk'] + ' ' +\n", + " m_max_row['book_4_bid_bk'] + ';' + m_max_row['book_4_ask_bk']]\n", + " def summarize_single_opp_p_pid(opp_p_rows):\n", + " opp_idx = 0\n", + " # summary cols\n", + " poly = opp_p_rows['symbol'].iloc[opp_idx]\n", + " opp_id = opp_p_rows['opp_id'].iloc[opp_idx]\n", + " opp_p_id = opp_p_rows['opp_p_id'].iloc[opp_idx]\n", + " start_eid = opp_p_rows['eid'].iloc[opp_idx]\n", + " start_t_time = opp_p_rows['t_time'].iloc[opp_idx]\n", + " start_h_t_time = opp_p_rows['h_t_time'].iloc[opp_idx]\n", + " # win size\n", + " first_send_opp_win = opp_p_rows['t_opp_first_send'].max()\n", + " last_send_opp_win = opp_p_rows['t_opp_last_send'].max()\n", + " # edge values\n", + " edge_vals = get_max_edge_row(opp_p_rows)\n", + " return [poly, opp_id, opp_p_id, start_eid, start_t_time, start_h_t_time,\n", + " first_send_opp_win, last_send_opp_win,\n", + " edge_vals[0], edge_vals[1], edge_vals[2], edge_vals[3],\n", + " edge_vals[4], edge_vals[5], edge_vals[6]]\n", + " return poly_vals.groupby('opp_p_id').apply(lambda x: summarize_single_opp_p_pid(x))" + ] + }, + { + "cell_type": "code", + "execution_count": 697, + "metadata": {}, + "outputs": [], + "source": [ + "# returns the poly vals db table filtered for the products\n", + "def get_poly_vals_db(full_db_name, products):\n", + " where_txt = ''\n", + " i=0\n", + " for p in products:\n", + " p_filter = 'product=' + \"'\" + p + \"'\"\n", + " prefix_or = '' if (i==0) else ' or ' \n", + " where_txt = where_txt + prefix_or + p_filter\n", + " i=i+1\n", + " where_txt = '(' + where_txt + ')'\n", + " query = 'select * from polygon_value_events where ' + where_txt\n", + " print(query)\n", + " return pd.read_sql_query(query, sqlite3.connect(full_db_name))" + ] + }, + { + "cell_type": "code", + "execution_count": 698, + "metadata": {}, + "outputs": [], + "source": [ + "# adds a new col called opp_p_id which is simply\n", + "# a unique id for [opp_id,pid]\n", + "def add_opp_p_ids(poly_vals):\n", + " key_map = {'':0}\n", + " def get_opp_p_id(r):\n", + " key = str(r['opp_id']) + '_' + str(r['pid'])\n", + " if key_map.get(key)==None:\n", + " key_map[key]=len(key_map) + 1\n", + " return key_map[key]\n", + " poly_vals['opp_p_id'] = poly_vals.apply (lambda row: get_opp_p_id(row), axis=1)\n", + " return poly_vals" + ] + }, + { + "cell_type": "code", + "execution_count": 699, + "metadata": {}, + "outputs": [], + "source": [ + "# adds a new column called ticks (its hard coded to convert\n", + "# merged_value to ticks)\n", + "def add_ticks(poly_vals):\n", + " val_to_tick_map = {'GC':10, 'SI':5, 'HG':12.5}\n", + " def get_ticks(r):\n", + " return r['merged_value']/val_to_tick_map[r['product']]\n", + " poly_vals['ticks'] = poly_vals.apply (lambda row: get_ticks(row), axis=1)\n", + " return poly_vals" + ] + }, + { + "cell_type": "code", + "execution_count": 700, "metadata": {}, "outputs": [], "source": [ @@ -67,320 +167,107 @@ }, { "cell_type": "code", - "execution_count": 171, + "execution_count": 701, "metadata": {}, "outputs": [], "source": [ - "# returns the poly vals view filtered for the product\n", - "def get_poly_vals_db(full_db_name, products):\n", - " where_txt = ''\n", - " i=0\n", - " for p in products:\n", - " p_filter = 'product=' + \"'\" + p + \"'\"\n", - " prefix_or = '' if (i==0) else ' or ' \n", - " where_txt = where_txt + prefix_or + p_filter\n", - " i=i+1\n", - " where_txt = '(' + where_txt + ')'\n", - " query = 'select * from polygon_value_events where ' + where_txt\n", - " print(query)\n", - " return pd.read_sql_query(query, sqlite3.connect(full_db_name))" + "# filters out rows that dont meet the edge criteria specified\n", + "# in edge_map\n", + "def get_edge_filtered_rows(poly_vals, edge_map):\n", + " def apply_edge_filter(r):\n", + " return r['ticks'] >= edge_map[r['product']]\n", + " poly_vals['passes_filter'] = poly_vals.apply (lambda r: apply_edge_filter(r), axis=1)\n", + " poly_vals = poly_vals[poly_vals['passes_filter']==True]\n", + " del poly_vals['passes_filter']\n", + " return poly_vals" ] }, { "cell_type": "code", - "execution_count": 172, + "execution_count": 702, "metadata": {}, "outputs": [], "source": [ - "def enrich_poly_vals(poly_vals_db):\n", - " poly_vals = add_ids(poly_vals_db)\n", + "def enrich_poly_vals(poly_vals_db, edge_map): \n", + " # ticks for the edge\n", + " poly_vals = add_ticks(poly_vals_db)\n", + " # filter for ticks\n", + " poly_vals = get_edge_filtered_rows(poly_vals, edge_map)\n", " # add opp_start opp_first_s_time and last_first_s_time\n", " poly_vals = add_opp_start_times(poly_vals)\n", " poly_vals['proc_time'] = poly_vals['last_s_time']-poly_vals['t_time']\n", " poly_vals['n_packets'] = 1+poly_vals['last_sn']-poly_vals['first_sn']\n", " poly_vals['send_time_delta'] = poly_vals['last_s_time']-poly_vals['first_s_time']\n", - " poly_vals['t_time_from_opp_first_send'] = poly_vals['t_time']-poly_vals['opp_first_s_time']\n", - " poly_vals['t_time_from_opp_last_send'] = poly_vals['t_time']-poly_vals['opp_last_s_time']\n", - " # hide some cols\n", - " rv=poly_vals[['product' 'vid', 'pid', 'symbol', 'opp_id', 'opp_p_id', 'eid', 't_time', 'h_t_time',\n", + " poly_vals['t_opp_first_send'] = poly_vals['t_time']-poly_vals['opp_first_s_time']\n", + " poly_vals['t_opp_last_send'] = poly_vals['t_time']-poly_vals['opp_last_s_time']\n", + " # add opp_p_id\n", + " poly_vals = add_opp_p_ids(poly_vals)\n", + " # hide some cols \n", + " rv=poly_vals[['product', 'vid', 'pid', 'symbol', 'opp_id', 'opp_p_id', 'eid', 't_time', 'h_t_time',\n", " 'n_packets', 'send_time_delta', 'proc_time',\n", - " 't_time_from_opp_first_send', 't_time_from_opp_last_send',\n", - " 'side', 'direct_value', 'direct_qty', 'merged_value', 'merged_qty',\n", + " 't_opp_first_send', 't_opp_last_send',\n", + " 'side', 'direct_value', 'direct_qty', 'merged_value', 'merged_qty', 'ticks',\n", " 'poly_bid_bk', 'poly_ask_bk',\n", " 'contract', 'change', 'ok', 'order_side', 'price','qty',\n", " 'book_1_sid', 'book_1_eid', 'book_1_side','book_1_bid_bk', 'book_1_ask_bk', \n", " 'book_2_sid', 'book_2_eid','book_2_side', 'book_2_bid_bk', 'book_2_ask_bk',\n", " 'book_3_sid', 'book_3_eid', 'book_3_side', 'book_3_bid_bk', 'book_3_ask_bk',\n", " 'book_4_sid', 'book_4_eid', 'book_4_side', 'book_4_bid_bk', 'book_4_ask_bk']]\n", - " # sort by opp_id, pid, eid (to have consecutive rows for one polygon)\n", - " rv = rv.sort_values(['opp_id', 'pid', 'eid'], ascending=[True, True, True])\n", + " # sort by opp_id, pid, eid (to have consecutive rows for one polygon) \n", + " # rv = rv.sort_values(['opp_id', 'pid', 'eid'], ascending=[True, True, True])\n", + " rv = rv.sort_values(['opp_p_id', 'eid'], ascending=[True, True])\n", " return rv" ] }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 703, "metadata": {}, "outputs": [], "source": [ - "def get_enriched_poly_vals(full_db_name, products):\n", + "def get_enriched_poly_vals(full_db_name, edge_map):\n", + " products = edge_map.keys()\n", " poly_vals_db = get_poly_vals_db(full_db_name, products)\n", - " poly_vals = enrich_poly_vals(poly_vals_db)\n", + " poly_vals = enrich_poly_vals(poly_vals_db, edge_map) \n", + " poly_vals_summ = tabularize_opp_summ(summarize_opportunities(poly_vals))\n", " appended_products = ''\n", " for p in products:\n", " appended_products = appended_products + '_' + p\n", - " print(appended_products)\n", " file_name = full_db_name + '.sniper_opps' + appended_products + '.csv'\n", + " summ_file_name = full_db_name + '.sniper_opps_summ' + appended_products + '.csv'\n", " poly_vals.to_csv(file_name)\n", - " return poly_vals" + " poly_vals_summ.to_csv(summ_file_name)\n", + " return [poly_vals,poly_vals_summ]" ] }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 717, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "select * from polygon_value_events where (product='GC' or product='HG')\n", - "_GC_HG\n" + "select * from polygon_value_events where (product='GC' or product='HG' or product='SI')\n" ] - } - ], - "source": [ - "g_r=get_enriched_poly_vals('/home/taha/sparkdata/simulations/comex/2020-12-01_360_24h.sqlite',['GC','HG'])" - ] - }, - { - "cell_type": "code", - "execution_count": 166, - "metadata": {}, - "outputs": [ + }, { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
vidpidsymbolopp_idopp_p_ideidt_timeh_t_timen_packetssend_time_delta...book_3_sidbook_3_eidbook_3_sidebook_3_bid_bkbook_3_ask_bkbook_4_sidbook_4_eidbook_4_sidebook_4_bid_bkbook_4_ask_bk
0459651+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1)1012802734160690517752983560510:32:57.529835605684857...216612802723SIDE_BUY-22/2 -23/5 -24/6 -25/9 -26/10-20/5 -19/2 -18/8 -17/9 -16/14278452802734SIDE_BUY-25/6 -26/57 -27/60 -28/47 -29/81-24/28 -23/86 -22/68 -21/42 -20/56
4460051+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1)1012802735160690517753022468910:32:57.53022468928267...216612802723SIDE_BUY-22/2 -23/5 -24/6 -25/9 -26/10-20/5 -19/2 -18/8 -17/9 -16/14278452802734SIDE_BUY-25/6 -26/57 -27/60 -28/47 -29/81-24/28 -23/86 -22/68 -21/42 -20/56
7460351+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1)1012802736160690517753022526910:32:57.530225269314216...216612802723SIDE_BUY-22/2 -23/5 -24/6 -25/9 -26/10-20/5 -19/2 -18/8 -17/9 -16/14278452802734SIDE_BUY-25/6 -26/57 -27/60 -28/47 -29/81-24/28 -23/86 -22/68 -21/42 -20/56
10460651+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1)1012802737160690517753022711710:32:57.53022711710...216612802723SIDE_BUY-22/2 -23/5 -24/6 -25/9 -26/10-20/5 -19/2 -18/8 -17/9 -16/14278452802734SIDE_BUY-25/6 -26/57 -27/60 -28/47 -29/81-24/28 -23/86 -22/68 -21/42 -20/56
13460951+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1)1012802738160690517753022754910:32:57.53022754910...216612802723SIDE_BUY-22/2 -23/5 -24/6 -25/9 -26/10-20/5 -19/2 -18/8 -17/9 -16/14278452802734SIDE_BUY-25/6 -26/57 -27/60 -28/47 -29/81-24/28 -23/86 -22/68 -21/42 -20/56
\n", - "

5 rows × 46 columns

\n", - "
" - ], "text/plain": [ - " vid pid symbol opp_id opp_p_id \\\n", - "0 4596 51 +GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) 10 1 \n", - "4 4600 51 +GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) 10 1 \n", - "7 4603 51 +GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) 10 1 \n", - "10 4606 51 +GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) 10 1 \n", - "13 4609 51 +GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) 10 1 \n", - "\n", - " eid t_time h_t_time n_packets \\\n", - "0 2802734 1606905177529835605 10:32:57.529835605 6 \n", - "4 2802735 1606905177530224689 10:32:57.530224689 2 \n", - "7 2802736 1606905177530225269 10:32:57.530225269 3 \n", - "10 2802737 1606905177530227117 10:32:57.530227117 1 \n", - "13 2802738 1606905177530227549 10:32:57.530227549 1 \n", - "\n", - " send_time_delta ... book_3_sid book_3_eid book_3_side \\\n", - "0 84857 ... 21661 2802723 SIDE_BUY \n", - "4 8267 ... 21661 2802723 SIDE_BUY \n", - "7 14216 ... 21661 2802723 SIDE_BUY \n", - "10 0 ... 21661 2802723 SIDE_BUY \n", - "13 0 ... 21661 2802723 SIDE_BUY \n", - "\n", - " book_3_bid_bk book_3_ask_bk \\\n", - "0 -22/2 -23/5 -24/6 -25/9 -26/10 -20/5 -19/2 -18/8 -17/9 -16/14 \n", - "4 -22/2 -23/5 -24/6 -25/9 -26/10 -20/5 -19/2 -18/8 -17/9 -16/14 \n", - "7 -22/2 -23/5 -24/6 -25/9 -26/10 -20/5 -19/2 -18/8 -17/9 -16/14 \n", - "10 -22/2 -23/5 -24/6 -25/9 -26/10 -20/5 -19/2 -18/8 -17/9 -16/14 \n", - "13 -22/2 -23/5 -24/6 -25/9 -26/10 -20/5 -19/2 -18/8 -17/9 -16/14 \n", - "\n", - " book_4_sid book_4_eid book_4_side book_4_bid_bk \\\n", - "0 27845 2802734 SIDE_BUY -25/6 -26/57 -27/60 -28/47 -29/81 \n", - "4 27845 2802734 SIDE_BUY -25/6 -26/57 -27/60 -28/47 -29/81 \n", - "7 27845 2802734 SIDE_BUY -25/6 -26/57 -27/60 -28/47 -29/81 \n", - "10 27845 2802734 SIDE_BUY -25/6 -26/57 -27/60 -28/47 -29/81 \n", - "13 27845 2802734 SIDE_BUY -25/6 -26/57 -27/60 -28/47 -29/81 \n", - "\n", - " book_4_ask_bk \n", - "0 -24/28 -23/86 -22/68 -21/42 -20/56 \n", - "4 -24/28 -23/86 -22/68 -21/42 -20/56 \n", - "7 -24/28 -23/86 -22/68 -21/42 -20/56 \n", - "10 -24/28 -23/86 -22/68 -21/42 -20/56 \n", - "13 -24/28 -23/86 -22/68 -21/42 -20/56 \n", - "\n", - "[5 rows x 46 columns]" + "(394, 15)" ] }, - "execution_count": 166, + "execution_count": 717, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "g_r.head()" + "g_pvs=get_enriched_poly_vals('/home/taha/sparkdata/simulations/comex/2020-08-27_360_24h.sqlite',\n", + " {'GC':1, 'HG':1, 'SI':3})\n", + "g_pvs[0].shape\n", + "g_pvs[1].shape" ] }, {