diff --git a/src/server/sparkd/cmds/spartan/analysis/sniper_opportunity_analysis.ipynb b/src/server/sparkd/cmds/spartan/analysis/sniper_opportunity_analysis.ipynb index 265a75ed5..7e30b76b5 100644 --- a/src/server/sparkd/cmds/spartan/analysis/sniper_opportunity_analysis.ipynb +++ b/src/server/sparkd/cmds/spartan/analysis/sniper_opportunity_analysis.ipynb @@ -2,36 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 168, - "metadata": {}, - "outputs": [], - "source": [ - "# adds a new col called opp_p_id which is simply\n", - "# a numerical representation of [opp_id,pid]\n", - "def add_ids(poly_vals):\n", - " oid_map = {-1:{-1:0}}\n", - " oid_map.pop(-1)\n", - " curr_id=0\n", - " def get_new_id():\n", - " nonlocal curr_id\n", - " curr_id = curr_id+1\n", - " return curr_id\n", - " def get_opp_p_id(r):\n", - " oid = r['opp_id']\n", - " pid = r['pid']\n", - " if oid_map.get(oid)==None:\n", - " oid_map[oid]={-1:0}\n", - " pid_map = oid_map[oid]\n", - " if pid_map.get(pid)==None:\n", - " pid_map[pid]=get_new_id()\n", - " return oid_map[oid][pid]\n", - " poly_vals['opp_p_id'] = poly_vals.apply (lambda row: get_opp_p_id(row), axis=1)\n", - " return poly_vals" - ] - }, - { - "cell_type": "code", - "execution_count": 169, + "execution_count": 694, "metadata": {}, "outputs": [], "source": [ @@ -42,7 +13,136 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 695, + "metadata": {}, + "outputs": [], + "source": [ + "# converts series to dataframe\n", + "def tabularize_opp_summ(opp_summ_ser):\n", + " rv_df = opp_summ_ser.to_frame().reset_index()\n", + " rv_df.columns = ['opp_id','vals'] \n", + " rv_df['symbol'] = rv_df['vals'].apply(lambda x:x[0])\n", + " rv_df['opp_id'] = rv_df['vals'].apply(lambda x:x[1])\n", + " rv_df['opp_p_id'] = rv_df['vals'].apply(lambda x:x[2])\n", + " rv_df['start_eid'] = rv_df['vals'].apply(lambda x:x[3])\n", + " rv_df['start_t_time'] = rv_df['vals'].apply(lambda x:x[4])\n", + " rv_df['start_h_t_time'] = rv_df['vals'].apply(lambda x:x[5])\n", + " rv_df['first_send_opp_win'] = rv_df['vals'].apply(lambda x:x[6])\n", + " rv_df['last_send_opp_win'] = rv_df['vals'].apply(lambda x:x[7])\n", + " rv_df['merged_value'] = rv_df['vals'].apply(lambda x:x[8])\n", + " rv_df['merged_qty'] = rv_df['vals'].apply(lambda x:x[9])\n", + " rv_df['ticks'] = rv_df['vals'].apply(lambda x:x[10])\n", + " rv_df['is_direct'] = rv_df['vals'].apply(lambda x:x[11])\n", + " rv_df['poly_bid_bk'] = rv_df['vals'].apply(lambda x:x[12])\n", + " rv_df['poly_ask_bk'] = rv_df['vals'].apply(lambda x:x[13])\n", + " rv_df['leg_books'] = rv_df['vals'].apply(lambda x:x[14])\n", + " del rv_df['vals']\n", + " return rv_df" + ] + }, + { + "cell_type": "code", + "execution_count": 696, + "metadata": {}, + "outputs": [], + "source": [ + "# summarizes each opportunity\n", + "def summarize_opportunities(poly_vals):\n", + " def get_max_edge_row (opp_p_rows):\n", + " min_react_time = 50000\n", + " opp_p_rows_filt = opp_p_rows[opp_p_rows['t_opp_first_send'] >= min_react_time]\n", + " if (opp_p_rows_filt.shape[0] == 0):\n", + " return [0.0,0,0,False,'','','','']\n", + " else:\n", + " m_idx = opp_p_rows_filt['merged_value'].argmax()\n", + " m_max_row = opp_p_rows_filt.iloc[m_idx]\n", + " return [m_max_row['merged_value'], m_max_row['merged_qty'], m_max_row['ticks'],\n", + " m_max_row['direct_value'] == m_max_row['merged_value'],\n", + " m_max_row['poly_bid_bk'], m_max_row['poly_ask_bk'],\n", + " m_max_row['book_1_bid_bk'] + ';' + m_max_row['book_1_ask_bk'] + ' ' +\n", + " m_max_row['book_2_bid_bk'] + ';' + m_max_row['book_2_ask_bk'] + ' ' +\n", + " m_max_row['book_3_bid_bk'] + ';' + m_max_row['book_3_ask_bk'] + ' ' +\n", + " m_max_row['book_4_bid_bk'] + ';' + m_max_row['book_4_ask_bk']]\n", + " def summarize_single_opp_p_pid(opp_p_rows):\n", + " opp_idx = 0\n", + " # summary cols\n", + " poly = opp_p_rows['symbol'].iloc[opp_idx]\n", + " opp_id = opp_p_rows['opp_id'].iloc[opp_idx]\n", + " opp_p_id = opp_p_rows['opp_p_id'].iloc[opp_idx]\n", + " start_eid = opp_p_rows['eid'].iloc[opp_idx]\n", + " start_t_time = opp_p_rows['t_time'].iloc[opp_idx]\n", + " start_h_t_time = opp_p_rows['h_t_time'].iloc[opp_idx]\n", + " # win size\n", + " first_send_opp_win = opp_p_rows['t_opp_first_send'].max()\n", + " last_send_opp_win = opp_p_rows['t_opp_last_send'].max()\n", + " # edge values\n", + " edge_vals = get_max_edge_row(opp_p_rows)\n", + " return [poly, opp_id, opp_p_id, start_eid, start_t_time, start_h_t_time,\n", + " first_send_opp_win, last_send_opp_win,\n", + " edge_vals[0], edge_vals[1], edge_vals[2], edge_vals[3],\n", + " edge_vals[4], edge_vals[5], edge_vals[6]]\n", + " return poly_vals.groupby('opp_p_id').apply(lambda x: summarize_single_opp_p_pid(x))" + ] + }, + { + "cell_type": "code", + "execution_count": 697, + "metadata": {}, + "outputs": [], + "source": [ + "# returns the poly vals db table filtered for the products\n", + "def get_poly_vals_db(full_db_name, products):\n", + " where_txt = ''\n", + " i=0\n", + " for p in products:\n", + " p_filter = 'product=' + \"'\" + p + \"'\"\n", + " prefix_or = '' if (i==0) else ' or ' \n", + " where_txt = where_txt + prefix_or + p_filter\n", + " i=i+1\n", + " where_txt = '(' + where_txt + ')'\n", + " query = 'select * from polygon_value_events where ' + where_txt\n", + " print(query)\n", + " return pd.read_sql_query(query, sqlite3.connect(full_db_name))" + ] + }, + { + "cell_type": "code", + "execution_count": 698, + "metadata": {}, + "outputs": [], + "source": [ + "# adds a new col called opp_p_id which is simply\n", + "# a unique id for [opp_id,pid]\n", + "def add_opp_p_ids(poly_vals):\n", + " key_map = {'':0}\n", + " def get_opp_p_id(r):\n", + " key = str(r['opp_id']) + '_' + str(r['pid'])\n", + " if key_map.get(key)==None:\n", + " key_map[key]=len(key_map) + 1\n", + " return key_map[key]\n", + " poly_vals['opp_p_id'] = poly_vals.apply (lambda row: get_opp_p_id(row), axis=1)\n", + " return poly_vals" + ] + }, + { + "cell_type": "code", + "execution_count": 699, + "metadata": {}, + "outputs": [], + "source": [ + "# adds a new column called ticks (its hard coded to convert\n", + "# merged_value to ticks)\n", + "def add_ticks(poly_vals):\n", + " val_to_tick_map = {'GC':10, 'SI':5, 'HG':12.5}\n", + " def get_ticks(r):\n", + " return r['merged_value']/val_to_tick_map[r['product']]\n", + " poly_vals['ticks'] = poly_vals.apply (lambda row: get_ticks(row), axis=1)\n", + " return poly_vals" + ] + }, + { + "cell_type": "code", + "execution_count": 700, "metadata": {}, "outputs": [], "source": [ @@ -67,320 +167,107 @@ }, { "cell_type": "code", - "execution_count": 171, + "execution_count": 701, "metadata": {}, "outputs": [], "source": [ - "# returns the poly vals view filtered for the product\n", - "def get_poly_vals_db(full_db_name, products):\n", - " where_txt = ''\n", - " i=0\n", - " for p in products:\n", - " p_filter = 'product=' + \"'\" + p + \"'\"\n", - " prefix_or = '' if (i==0) else ' or ' \n", - " where_txt = where_txt + prefix_or + p_filter\n", - " i=i+1\n", - " where_txt = '(' + where_txt + ')'\n", - " query = 'select * from polygon_value_events where ' + where_txt\n", - " print(query)\n", - " return pd.read_sql_query(query, sqlite3.connect(full_db_name))" + "# filters out rows that dont meet the edge criteria specified\n", + "# in edge_map\n", + "def get_edge_filtered_rows(poly_vals, edge_map):\n", + " def apply_edge_filter(r):\n", + " return r['ticks'] >= edge_map[r['product']]\n", + " poly_vals['passes_filter'] = poly_vals.apply (lambda r: apply_edge_filter(r), axis=1)\n", + " poly_vals = poly_vals[poly_vals['passes_filter']==True]\n", + " del poly_vals['passes_filter']\n", + " return poly_vals" ] }, { "cell_type": "code", - "execution_count": 172, + "execution_count": 702, "metadata": {}, "outputs": [], "source": [ - "def enrich_poly_vals(poly_vals_db):\n", - " poly_vals = add_ids(poly_vals_db)\n", + "def enrich_poly_vals(poly_vals_db, edge_map): \n", + " # ticks for the edge\n", + " poly_vals = add_ticks(poly_vals_db)\n", + " # filter for ticks\n", + " poly_vals = get_edge_filtered_rows(poly_vals, edge_map)\n", " # add opp_start opp_first_s_time and last_first_s_time\n", " poly_vals = add_opp_start_times(poly_vals)\n", " poly_vals['proc_time'] = poly_vals['last_s_time']-poly_vals['t_time']\n", " poly_vals['n_packets'] = 1+poly_vals['last_sn']-poly_vals['first_sn']\n", " poly_vals['send_time_delta'] = poly_vals['last_s_time']-poly_vals['first_s_time']\n", - " poly_vals['t_time_from_opp_first_send'] = poly_vals['t_time']-poly_vals['opp_first_s_time']\n", - " poly_vals['t_time_from_opp_last_send'] = poly_vals['t_time']-poly_vals['opp_last_s_time']\n", - " # hide some cols\n", - " rv=poly_vals[['product' 'vid', 'pid', 'symbol', 'opp_id', 'opp_p_id', 'eid', 't_time', 'h_t_time',\n", + " poly_vals['t_opp_first_send'] = poly_vals['t_time']-poly_vals['opp_first_s_time']\n", + " poly_vals['t_opp_last_send'] = poly_vals['t_time']-poly_vals['opp_last_s_time']\n", + " # add opp_p_id\n", + " poly_vals = add_opp_p_ids(poly_vals)\n", + " # hide some cols \n", + " rv=poly_vals[['product', 'vid', 'pid', 'symbol', 'opp_id', 'opp_p_id', 'eid', 't_time', 'h_t_time',\n", " 'n_packets', 'send_time_delta', 'proc_time',\n", - " 't_time_from_opp_first_send', 't_time_from_opp_last_send',\n", - " 'side', 'direct_value', 'direct_qty', 'merged_value', 'merged_qty',\n", + " 't_opp_first_send', 't_opp_last_send',\n", + " 'side', 'direct_value', 'direct_qty', 'merged_value', 'merged_qty', 'ticks',\n", " 'poly_bid_bk', 'poly_ask_bk',\n", " 'contract', 'change', 'ok', 'order_side', 'price','qty',\n", " 'book_1_sid', 'book_1_eid', 'book_1_side','book_1_bid_bk', 'book_1_ask_bk', \n", " 'book_2_sid', 'book_2_eid','book_2_side', 'book_2_bid_bk', 'book_2_ask_bk',\n", " 'book_3_sid', 'book_3_eid', 'book_3_side', 'book_3_bid_bk', 'book_3_ask_bk',\n", " 'book_4_sid', 'book_4_eid', 'book_4_side', 'book_4_bid_bk', 'book_4_ask_bk']]\n", - " # sort by opp_id, pid, eid (to have consecutive rows for one polygon)\n", - " rv = rv.sort_values(['opp_id', 'pid', 'eid'], ascending=[True, True, True])\n", + " # sort by opp_id, pid, eid (to have consecutive rows for one polygon) \n", + " # rv = rv.sort_values(['opp_id', 'pid', 'eid'], ascending=[True, True, True])\n", + " rv = rv.sort_values(['opp_p_id', 'eid'], ascending=[True, True])\n", " return rv" ] }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 703, "metadata": {}, "outputs": [], "source": [ - "def get_enriched_poly_vals(full_db_name, products):\n", + "def get_enriched_poly_vals(full_db_name, edge_map):\n", + " products = edge_map.keys()\n", " poly_vals_db = get_poly_vals_db(full_db_name, products)\n", - " poly_vals = enrich_poly_vals(poly_vals_db)\n", + " poly_vals = enrich_poly_vals(poly_vals_db, edge_map) \n", + " poly_vals_summ = tabularize_opp_summ(summarize_opportunities(poly_vals))\n", " appended_products = ''\n", " for p in products:\n", " appended_products = appended_products + '_' + p\n", - " print(appended_products)\n", " file_name = full_db_name + '.sniper_opps' + appended_products + '.csv'\n", + " summ_file_name = full_db_name + '.sniper_opps_summ' + appended_products + '.csv'\n", " poly_vals.to_csv(file_name)\n", - " return poly_vals" + " poly_vals_summ.to_csv(summ_file_name)\n", + " return [poly_vals,poly_vals_summ]" ] }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 717, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "select * from polygon_value_events where (product='GC' or product='HG')\n", - "_GC_HG\n" + "select * from polygon_value_events where (product='GC' or product='HG' or product='SI')\n" ] - } - ], - "source": [ - "g_r=get_enriched_poly_vals('/home/taha/sparkdata/simulations/comex/2020-12-01_360_24h.sqlite',['GC','HG'])" - ] - }, - { - "cell_type": "code", - "execution_count": 166, - "metadata": {}, - "outputs": [ + }, { "data": { - "text/html": [ - "
| \n", - " | vid | \n", - "pid | \n", - "symbol | \n", - "opp_id | \n", - "opp_p_id | \n", - "eid | \n", - "t_time | \n", - "h_t_time | \n", - "n_packets | \n", - "send_time_delta | \n", - "... | \n", - "book_3_sid | \n", - "book_3_eid | \n", - "book_3_side | \n", - "book_3_bid_bk | \n", - "book_3_ask_bk | \n", - "book_4_sid | \n", - "book_4_eid | \n", - "book_4_side | \n", - "book_4_bid_bk | \n", - "book_4_ask_bk | \n", - "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", - "4596 | \n", - "51 | \n", - "+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) | \n", - "10 | \n", - "1 | \n", - "2802734 | \n", - "1606905177529835605 | \n", - "10:32:57.529835605 | \n", - "6 | \n", - "84857 | \n", - "... | \n", - "21661 | \n", - "2802723 | \n", - "SIDE_BUY | \n", - "-22/2 -23/5 -24/6 -25/9 -26/10 | \n", - "-20/5 -19/2 -18/8 -17/9 -16/14 | \n", - "27845 | \n", - "2802734 | \n", - "SIDE_BUY | \n", - "-25/6 -26/57 -27/60 -28/47 -29/81 | \n", - "-24/28 -23/86 -22/68 -21/42 -20/56 | \n", - "
| 4 | \n", - "4600 | \n", - "51 | \n", - "+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) | \n", - "10 | \n", - "1 | \n", - "2802735 | \n", - "1606905177530224689 | \n", - "10:32:57.530224689 | \n", - "2 | \n", - "8267 | \n", - "... | \n", - "21661 | \n", - "2802723 | \n", - "SIDE_BUY | \n", - "-22/2 -23/5 -24/6 -25/9 -26/10 | \n", - "-20/5 -19/2 -18/8 -17/9 -16/14 | \n", - "27845 | \n", - "2802734 | \n", - "SIDE_BUY | \n", - "-25/6 -26/57 -27/60 -28/47 -29/81 | \n", - "-24/28 -23/86 -22/68 -21/42 -20/56 | \n", - "
| 7 | \n", - "4603 | \n", - "51 | \n", - "+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) | \n", - "10 | \n", - "1 | \n", - "2802736 | \n", - "1606905177530225269 | \n", - "10:32:57.530225269 | \n", - "3 | \n", - "14216 | \n", - "... | \n", - "21661 | \n", - "2802723 | \n", - "SIDE_BUY | \n", - "-22/2 -23/5 -24/6 -25/9 -26/10 | \n", - "-20/5 -19/2 -18/8 -17/9 -16/14 | \n", - "27845 | \n", - "2802734 | \n", - "SIDE_BUY | \n", - "-25/6 -26/57 -27/60 -28/47 -29/81 | \n", - "-24/28 -23/86 -22/68 -21/42 -20/56 | \n", - "
| 10 | \n", - "4606 | \n", - "51 | \n", - "+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) | \n", - "10 | \n", - "1 | \n", - "2802737 | \n", - "1606905177530227117 | \n", - "10:32:57.530227117 | \n", - "1 | \n", - "0 | \n", - "... | \n", - "21661 | \n", - "2802723 | \n", - "SIDE_BUY | \n", - "-22/2 -23/5 -24/6 -25/9 -26/10 | \n", - "-20/5 -19/2 -18/8 -17/9 -16/14 | \n", - "27845 | \n", - "2802734 | \n", - "SIDE_BUY | \n", - "-25/6 -26/57 -27/60 -28/47 -29/81 | \n", - "-24/28 -23/86 -22/68 -21/42 -20/56 | \n", - "
| 13 | \n", - "4609 | \n", - "51 | \n", - "+GCZ0 -GCG1 -(GCF1-GCG1) -(GCZ0-GCF1) | \n", - "10 | \n", - "1 | \n", - "2802738 | \n", - "1606905177530227549 | \n", - "10:32:57.530227549 | \n", - "1 | \n", - "0 | \n", - "... | \n", - "21661 | \n", - "2802723 | \n", - "SIDE_BUY | \n", - "-22/2 -23/5 -24/6 -25/9 -26/10 | \n", - "-20/5 -19/2 -18/8 -17/9 -16/14 | \n", - "27845 | \n", - "2802734 | \n", - "SIDE_BUY | \n", - "-25/6 -26/57 -27/60 -28/47 -29/81 | \n", - "-24/28 -23/86 -22/68 -21/42 -20/56 | \n", - "
5 rows × 46 columns
\n", - "