Files
spark/src/data_files/files/spartan/md.sql
T

321 lines
10 KiB
SQL

/* An exchange transaction/event */
drop table if exists events;
CREATE TABLE events(
h_t_time TEXT, /* human readable t_time : 2020-08-24.14:49:29.146533601*/
eid numeric, /* event id*/
t_time numeric, /* exchange generated transaction time of the event ie when the exchange started to process it*/
first_sn numeric, /* seq no of the first physical packet that constitutes this event*/
last_sn numeric, /* seq no of the last physical packet that constitutes this event*/
first_st numeric, /* send time of the first packet*/
last_st numeric /* send time of the last packet*/
);
CREATE UNIQUE INDEX events_idx ON events(eid);
/* The parts that describe an exchange transaction/event */
drop table if exists event_items;
CREATE TABLE event_items(
eid numeric, /* id of the event (above) that this item belongs to (event_items:event is a n:1 relationship)*/
t_time numeric, /* transaction time for the event (==events.t_time), shared across event_items belonging to the same event*/
sid numeric, /* security id*/
h_t_time text, /* transaction time in human readeable form*/
symbol TEXT, /* Security symbol */
status TEXT, /* market status (PreOpen, TradingHalted, ReadyToTrade, Close)*/
impl numeric, /* if implied matching is allowed at this time*/
type TEXT, /* type of this event_item (order_change, fill, deduced_order)*/
change TEXT, /* order operation (init, added, removed, updated for type=order_change and added,updated for type=deduced_order)*/
ok numeric, /* order key*/
side TEXT, /* order side, does not apply to fills because each fill is represented by pairs of crossing pass/agg sides*/
pri numeric, /* exchange assigned priority (only for type=order_change)*/
price numeric, /* price, every row will have it*/
qty numeric, /* for type=order_change and type=deduced_order this is the qty of the order, for fills it is the fill size*/
tid numeric, /* all trades tied to the same agg order share the tid*/
agg TEXT /* for type=fill, indicates the cross/passive pairing (yes_buy,yes_sell for agg, no for passive)*/
);
/* The (changed) books for an instrument after a new event/transaction */
drop table if exists books;
CREATE TABLE books(
eid numeric,
t_time numeric,
sid numeric,
status TEXT,
impl numeric,
/* regular books */
/* ---------------------------------------------------- */
/* price qty n orders */
/* ---------------------------------------------------- */
bid_p_0 numeric, bid_q_0 numeric, bid_no_0 numeric,
bid_p_1 numeric, bid_q_1 numeric, bid_no_1 numeric,
bid_p_2 numeric, bid_q_2 numeric, bid_no_2 numeric,
bid_p_3 numeric, bid_q_3 numeric, bid_no_3 numeric,
bid_p_4 numeric, bid_q_4 numeric, bid_no_4 numeric,
bid_p_5 numeric, bid_q_5 numeric, bid_no_5 numeric,
bid_p_6 numeric, bid_q_6 numeric, bid_no_6 numeric,
bid_p_7 numeric, bid_q_7 numeric, bid_no_7 numeric,
bid_p_8 numeric, bid_q_8 numeric, bid_no_8 numeric,
bid_p_9 numeric, bid_q_9 numeric, bid_no_9 numeric,
ask_p_0 numeric, ask_q_0 numeric, ask_no_0 numeric,
ask_p_1 numeric, ask_q_1 numeric, ask_no_1 numeric,
ask_p_2 numeric, ask_q_2 numeric, ask_no_2 numeric,
ask_p_3 numeric, ask_q_3 numeric, ask_no_3 numeric,
ask_p_4 numeric, ask_q_4 numeric, ask_no_4 numeric,
ask_p_5 numeric, ask_q_5 numeric, ask_no_5 numeric,
ask_p_6 numeric, ask_q_6 numeric, ask_no_6 numeric,
ask_p_7 numeric, ask_q_7 numeric, ask_no_7 numeric,
ask_p_8 numeric, ask_q_8 numeric, ask_no_8 numeric,
ask_p_9 numeric, ask_q_9 numeric, ask_no_9 numeric,
/* implied books */
/* ---------------------------------------------------- */
/* price qty n orders */
/* ---------------------------------------------------- */
ibid_p_0 numeric, ibid_q_0 numeric, ibid_no_0 numeric,
ibid_p_1 numeric, ibid_q_1 numeric, ibid_no_1 numeric,
iask_p_0 numeric, iask_q_0 numeric, iask_no_0 numeric,
iask_p_1 numeric, iask_q_1 numeric, iask_no_1 numeric
);
CREATE UNIQUE INDEX book_idx ON books(eid,sid);
CREATE INDEX book_sid_idx ON books(sid);
/* All the orders we were able to identify as icebergs */
drop table if exists icebergs;
CREATE TABLE icebergs(
ok numeric,
eid numeric,
total_qty numeric,
show_qty numeric,
fill_qty numeric
);
/* All the securities*/
drop table if exists securities;
CREATE TABLE securities(
sid numeric,
symbol TEXT,
sec_type TEXT,
cme_sec_type TEXT,
sec_group TEXT,
asset TEXT,
implied_matching numeric,
match_algo TEXT,
display_factor REAL,
cash_per_point REAL,
min_price_increment REAL,
min_price_increment_ammount REAL,
n_legs NUMERIC,
update_time numeric
);
CREATE UNIQUE INDEX securities_idx ON securities(sid);
/* The leg constituents for spread securities */
drop table if exists spread_legs;
CREATE TABLE spread_legs(
ssid numeric, /* spread security id */
lsid numeric, /* leg security id*/
side TEXT, /* leg side */
ratio numeric /* leg ratio */
);
/* Polygons, that is combinations of contracts that net out to 0 */
drop table if exists polygons;
CREATE TABLE polygons(
pid numeric, /* polygon id */
symbol text /* descriptor */
);
drop table if exists polygon_legs;
CREATE TABLE polygon_legs(
pid numeric, /* polygon id */
lsid numeric, /* leg security id*/
side TEXT, /* leg side */
ratio numeric /* leg ratio */
);
drop table if exists polygon_values;
CREATE TABLE polygon_values(
vid numeric, /* valuation id */
pid numeric, /* polygon id */
symbol text, /* polygon symbol */
eid numeric, /* event in which this value occoured */
t_time numeric,
h_t_time text,
s_time numeric,
side TEXT,
direct_value real,
direct_qty numeric,
merged_value real,
merged_qty numeric,
min_n_orders numeric,
book_1_sid numeric,
book_1_eid numeric,
book_1_side text,
book_2_sid numeric,
book_2_eid numeric,
book_2_side text,
book_3_sid numeric,
book_3_eid numeric,
book_3_side text,
book_4_sid numeric,
book_4_eid numeric,
book_4_side text,
book_5_sid numeric,
book_5_eid numeric,
book_5_side text
);
CREATE UNIQUE INDEX polygon_value_idx ON polygon_values(vid);
drop table if exists polygon_opportunities;
create table polygon_opportunities(
opid numeric, /* opportunity id */
pid numeric, /* polygon id */
symbol text, /* polygon symbol */
side TEXT,
merged_value real,
merged_qty numeric,
direct_value real,
direct_qty numeric,
begin_vid numeric, /* first valuation where this opportunity appeared */
end_vid numeric, /* last valuation where this opportunity appeared */
begin_eid numeric, /* first event where this opportunity appeared */
end_eid numeric, /* last event where this opportunity appeared */
min_n_orders numeric /* smallest number of orders preset for each of the legs of the polygon */
);
CREATE UNIQUE INDEX polygon_op_value_idx ON polygon_opportunities(begin_vid,end_vid);
/* Usefull Views -------------------------------------------------------------------------------- */
/* top of book */
drop view if exists top_of_books;
create view top_of_books as
select
securities.symbol,
books.t_time,
books.eid,
books.sid,
status,
bid_p_0 bid,
bid_q_0 bid_qty,
ask_p_0 ask,
ask_q_0 ask_qty
from
securities,books
where
securities.sid=books.sid;
/* resolved polygon opportunities */
drop view if exists opportunities;
create view opportunities as
select
polygon_opportunities.*,
events_begin.h_t_time h_time,
events_begin.last_st begin_ts,
events_end.t_time end_ts,
(events_end.t_time - events_begin.last_st) / 1000 observable_duration,
(events_end.t_time - events_begin.first_st) / 1000 observable_duration_fst,
(events_end.t_time - events_begin.t_time) / 1000 duration,
events_begin.t_time t_time,
(events_begin.last_st - events_begin.t_time)/ 1000 transaction_duration
from
polygon_opportunities,
events events_begin,
events events_end
where
polygon_opportunities.begin_eid=events_begin.eid and
polygon_opportunities.end_eid+1=events_end.eid;
drop view if exists total_opportunity;
create view total_opportunity as
select
sum(merged_qty*merged_value),
100 *
(select count(*) from opportunities where observable_duration>0) / (select count(*) from opportunities) p_shootable
from
opportunities
where
observable_duration >20 and
merged_value > 5;
drop view if exists opportunities_detail;
create view opportunities_detail as
select
opportunities.*,
(polygon_values.s_time-polygon_values.t_time)/1000 detail_transaction_duration,
polygon_values.min_n_orders detail_n_orders,
polygon_values.merged_value detail_value,
polygon_values.merged_qty detail_qty
from
opportunities,
polygon_values
where
polygon_values.vid >= opportunities.begin_vid and
polygon_values.vid <= opportunities.end_vid;
drop view if exists opportunity_summary;
create view opportunity_summary as
select
opid,
min(merged_value) merged_value,
min(observable_duration) observable_duration,
min(transaction_duration) transaction_duration,
avg(detail_transaction_duration) avg_detail_transaction_duration,
max(detail_n_orders) max_n_orders,
max(detail_value) max_value,
max(detail_qty) max_detail_qt,
count(*) event_count
from
opportunities_detail
group by opid;
drop view if exists price_echoes;
create view price_echoes as
select
top_of_books.*,
(lag(eid) over (partition by sid,bid order by eid)) bid_echo_eid,
(lag(t_time) over (partition by sid,bid order by eid)) bid_echo_t_time,
(lag(eid) over (partition by sid,ask order by eid)) ask_echo_eid,
(lag(t_time) over (partition by sid,ask order by eid)) ask_echo_t_time
from
top_of_books;
drop view if exists p_of_1sec_price_echo;
create view p_of_1sec_price_echo as
with
total_events as (
select
symbol,
count(*) total_count
from
price_echoes
group by
symbol),
echo_events as (
select
count(*) echo_count,
symbol
from
price_echoes
where
(t_time-bid_echo_t_time) < 1000*1000*1000*1
group by symbol)
select
total_events.symbol,
1.0*echo_count/total_count p,
echo_count,
total_count
from
total_events,
echo_events
where
total_events.symbol=echo_events.symbol
order by
p;