chore: Get rid of old/unused code from Spartan

This commit is contained in:
John Colvin committed 2023-04-14 16:55:29 +00:00
1 parent a512460396
commit 72edd99cb0
614 files changed
+4 -500009

No files matched your search

@@ -1,13 +1,9 @@
#pragma once
#include "lib/domain_model/Security.h"
#include "lib/md/IntPrice.h"
// #include "lib/domain_model/Security.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/utils/ValueWrapper.h"
#include "lib/utils/overloaded.h"
#include "server/sparkd/cmds/spartan/domain/Price.h"
#include "server/sparkd/cmds/spartan/back_testing/common/SmallBookLevel.h"
// #include "server/sparkd/cmds/spartan/back_testing/common/mdp3_common.h"
namespace bts {
namespace orderman {
@@ -70,31 +66,6 @@ inline bool is_better_or_equal<md::IntPrice>(
return price == other || is_better(side, price, other);
}
template <>
inline bool is_better<bts::spartan::mdp3::Price>(
api::Side side, spartan::mdp3::Price price, spartan::mdp3::Price other) {
if (price.is_nan())
return false;
if (other.is_nan())
return true;
switch (side) {
case api::Side::SIDE_BUY:
return price > other;
case api::Side::SIDE_SELL:
return price < other;
default:
bts_assert_msg(
false,
"invalid price comparison, specified side other than Buy or Sell.");
return false;
}
}
template <>
inline bool is_better_or_equal<spartan::mdp3::Price>(
api::Side side, spartan::mdp3::Price price, spartan::mdp3::Price other) {
return price == other || is_better(side, price, other);
}
// This method is basically syntactic sugar;
// It is another name for the 'is_better_or_equal' method.
template <typename T>
@@ -102,16 +73,6 @@ inline bool is_cross(api::Side aggr_side, T aggr_price, T passive_price) {
return is_better_or_equal(aggr_side, aggr_price, passive_price);
}
inline bool is_equal(spartan::mdp3::Price a, spartan::mdp3::Price b) {
if (a.is_nan())
return b.is_nan();
if (b.is_nan())
return false;
double diff = a.to_double() - b.to_double();
return diff < securities::market_epsilon() &&
diff > -securities::market_epsilon();
}
inline bool is_equal(md::IntPrice a, md::IntPrice b) {
if (a.is_nan())
return b.is_nan();
@@ -154,15 +115,6 @@ struct SidePriceComp {
return price_comp::is_better(_side, lhs, rhs);
}
bool operator()(const spartan::mdp3::Price &lhs, const spartan::mdp3::Price &rhs) {
return price_comp::is_better(_side, lhs, rhs);
}
bool operator()(
const spartan::mdp3::SmallBookLevel &lhs,
const spartan::mdp3::SmallBookLevel &rhs) {
return price_comp::is_better(_side, lhs.price, rhs.price);
}
private:
api::Side _side;
};
-938
View File
@@ -1,938 +0,0 @@
#include "Benchmark.h"
#include "lib/api/services/OrderService.server.pb.h"
#include "lib/jetfix/jetfix.h"
#include "lib/md/PriceSize.h"
#include "lib/mercury/actors/Actor.h"
#include "lib/mercury/common/FatWrapper.h"
#include "lib/mercury/executor/Executor.h"
#include "lib/orderman/events/OrderStateUpdateEvent.h"
#include "lib/os/os.h"
#include "lib/utils/ConcurrentAppendCollection.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/Puid.h"
#include "lib/utils/SpinLock.h"
#include "lib/utils/TimeStamp.h"
#include "lib/utils/Uuid.h"
#include "lib/utils/mpmc_bounded_queue.h"
#include "system/Delegate.h"
#include "lib/tracing/PerfLog.h"
#include "system/ProducerConsumerQueue.h"
#include <benchmark/benchmark.h>
#include <boost/container/flat_map.hpp>
#include <boost/container/small_vector.hpp>
#include <boost/lockfree/queue.hpp>
#include <boost/lockfree/spsc_queue.hpp>
#include <boost/lockfree/stack.hpp>
#include <boost/pool/object_pool.hpp>
#include <boost/pool/pool.hpp>
#include <boost/pool/pool_alloc.hpp>
#include <boost/pool/singleton_pool.hpp>
#include <cstdlib>
#include <iostream>
#include <moodycamel/concurrentqueue.h>
#include <moodycamel/readerwriterqueue.h>
#include <mutex>
#include <sys/time.h>
#include <btree/set.h>
#include <set>
#include <unordered_set>
#include "logger/ILoggingSink.h"
#include "logger/Logger.h"
#include "logger/LoggerHub.h"
#include <date/date.h>
#include <tbb/concurrent_queue.h>
#include <tbb/concurrent_vector.h>
namespace bts::cmds {
// using namespace orderman;
// static void test_date_conversion(::benchmark::State &state) {
// auto lldt = datetime::ll_now();
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(datetime::utc_ll_to_timepoint(lldt));
// }
// }
// BENCHMARK(test_date_conversion);
// static void gen_id(::benchmark::State &state) {
// std::string scope = "benchmark";
// for (auto _ : state) {
// auto id = utils::reserve_ids(scope, 1);
// ::benchmark::DoNotOptimize(id + 1 > 0);
// }
// }
// BENCHMARK(gen_id);
// static void mutex_lock_unlock(::benchmark::State &state) {
// int i = 0;
// std::mutex lock;
// for (auto _ : state) {
// std::lock_guard<std::mutex> l(lock);
// ::benchmark::DoNotOptimize(i++);
// }
// }
// BENCHMARK(mutex_lock_unlock);
// static void spin_lock_unlock(::benchmark::State &state) {
// int i = 0;
// std::atomic_flag _mutex = ATOMIC_FLAG_INIT;
// for (auto _ : state) {
// while (_mutex.test_and_set(std::memory_order_acquire))
// ;
// ::benchmark::DoNotOptimize(i++);
// _mutex.clear(std::memory_order_release);
// }
// }
// BENCHMARK(spin_lock_unlock);
// static void bts_spinlock_lock_unlock(::benchmark::State &state) {
// int i = 0;
// utils::SpinLock l;
// for (auto _ : state) {
// std::lock_guard<utils::SpinLock> g(l);
// ::benchmark::DoNotOptimize(i++);
// }
// }
// BENCHMARK(bts_spinlock_lock_unlock);
// // static void serialize_update(::benchmark::State &state) {
// // api::orderman::Update update;
// // update.set_order_key(1);
// // update.set_price(1.2);
// // update.set_quantity(2);
// // std::string buffer;
// // buffer.resize(update.ByteSizeLong());
// // bts_assert(update.SerializeToArray(&buffer[0], buffer.size()));
// // for (auto _ : state) {
// // update.SerializeWithCachedSizesToArray((unsigned char *)&buffer[0]);
// // }
// // }
// // BENCHMARK(serialize_update);
// // struct IntId {
// // IntId() { value = std::rand(); }
// // bool operator<(IntId const &v) const { return value < v.value; }
// // uint32_t value;
// // };
// // static void binary_search_of_guid(::benchmark::State &state) {
// // using Id = utils::Uuid;
// // using Map = boost::container::flat_map<Id, int>;
// // auto sz = state.range(0);
// // Map m;
// // for (int i = 0; i < sz; i++)
// // m[Id()] = i;
// // auto target = Id();
// // int found_count = 0;
// // for (auto _ : state) {
// // auto it = m.find(target);
// // if (it != m.end())
// // found_count++;
// // ::benchmark::DoNotOptimize(found_count);
// // }
// // }
// // BENCHMARK(binary_search_of_guid)->Range(8, 8 << 10);
// // static void binary_search_of_int64(::benchmark::State &state) {
// // using Id = IntId;
// // using Map = boost::container::flat_map<Id, int>;
// // auto sz = state.range(0);
// // Map m;
// // for (int i = 0; i < sz; i++)
// // m[Id()] = i;
// // auto target = Id();
// // int found_count = 0;
// // for (auto _ : state) {
// // auto it = m.find(target);
// // if (it != m.end())
// // found_count++;
// // ::benchmark::DoNotOptimize(found_count);
// // }
// // }
// // BENCHMARK(binary_search_of_int64)->Range(8, 8 << 10);
// // static void binary_search_of_int32(::benchmark::State &state) {
// // using Id = IntId;
// // using Map = boost::container::flat_map<Id, int>;
// // auto sz = state.range(0);
// // Map m;
// // for (int i = 0; i < sz; i++)
// // m[Id()] = i;
// // auto target = Id();
// // int found_count = 0;
// // for (auto _ : state) {
// // auto it = m.find(target);
// // if (it != m.end())
// // found_count++;
// // ::benchmark::DoNotOptimize(found_count);
// // }
// // }
// // BENCHMARK(binary_search_of_int32)->Range(8, 8 << 10);
// // static void guid_gen(::benchmark::State &state) {
// // for (auto _ : state) {
// // ::benchmark::DoNotOptimize(utils::Uuid());
// // }
// // }
// // BENCHMARK(guid_gen);
// // struct A {
// // int x;
// // };
// // static void shared_ptr_cpy(::benchmark::State &state) {
// // auto a = std::make_shared<A>();
// // for (auto _ : state) {
// // std::shared_ptr<A> b;
// // ::benchmark::DoNotOptimize(b = a);
// // }
// // }
// // BENCHMARK(shared_ptr_cpy);
// // static void alloc_free(::benchmark::State &state) {
// // for (auto _ : state) {
// // auto a = std::make_shared<A>();
// // ::benchmark::DoNotOptimize(a.get());
// // }
// // }
// // BENCHMARK(alloc_free);
// // static void new_delete(::benchmark::State &state) {
// // for (auto _ : state) {
// // auto a = new A();
// // delete a;
// // ::benchmark::DoNotOptimize(a);
// // }
// // }
// // BENCHMARK(new_delete);
// // static void boost_pool(::benchmark::State &state) {
// // boost::pool<> p(sizeof(A));
// // for (auto _ : state) {
// // auto a = p.malloc();
// // p.free(a);
// // ::benchmark::DoNotOptimize(a);
// // }
// // }
// // BENCHMARK(boost_pool);
// // static void boost_object_pool(::benchmark::State &state) {
// // boost::object_pool<A> p;
// // for (auto _ : state) {
// // auto a = p.malloc();
// // p.free(a);
// // ::benchmark::DoNotOptimize(a);
// // }
// // }
// // BENCHMARK(boost_object_pool);
// // static void protected_boost_pool(::benchmark::State &state) {
// // boost::pool<> p(sizeof(A));
// // std::atomic_flag _mutex = ATOMIC_FLAG_INIT;
// // for (auto _ : state) {
// // while (_mutex.test_and_set(std::memory_order_acquire))
// // ;
// // auto a = p.malloc();
// // _mutex.clear(std::memory_order_release);
// // while (_mutex.test_and_set(std::memory_order_acquire))
// // ;
// // p.free(a);
// // _mutex.clear(std::memory_order_release);
// // ::benchmark::DoNotOptimize(a);
// // }
// // }
// // BENCHMARK(protected_boost_pool);
// // static void boost_singleton_pool(::benchmark::State &state) {
// // using Pool = boost::singleton_pool<struct Single, sizeof(A)>;
// // for (auto _ : state) {
// // auto a = Pool::malloc();
// // Pool::free(a);
// // ::benchmark::DoNotOptimize(a);
// // }
// // }
// // BENCHMARK(boost_singleton_pool);
// static void atomic_inc(::benchmark::State &state) {
// std::atomic<size_t> i(0);
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(i++);
// }
// }
// BENCHMARK(atomic_inc);
// static void mem_copy(::benchmark::State &state) {
// char *src = new char[state.range(0)];
// char *dst = new char[state.range(0)];
// memset(src, 'x', state.range(0));
// for (auto _ : state)
// memcpy(dst, src, state.range(0));
// state.SetBytesProcessed(
// int64_t(state.iterations()) * int64_t(state.range(0)));
// delete[] src;
// delete[] dst;
// }
// BENCHMARK(mem_copy)->Arg(8)->Arg(64)->Arg(128)->Arg(512)->Arg(1 << 10)->Arg(
// 8 << 10);
// // static void alloc_and_copy_from_pool(::benchmark::State &state) {
// // boost::pool<> p(sizeof(A));
// // for (auto _ : state) {
// // auto a = new (p.malloc()) A(); // NOLINT
// // a->~A(); // NOLINT
// // p.free(a);
// // ::benchmark::DoNotOptimize(a);
// // }
// // }
// // BENCHMARK(alloc_and_copy_from_pool);
// // static void lambda_copy(::benchmark::State &state) {
// // auto p = std::make_shared<A>();
// // std::function<int()> f = [p]() { return 1; };
// // std::array<std::function<int()>, 1> buffer;
// // for (auto _ : state) {
// // ::benchmark::DoNotOptimize(buffer[0] = f);
// // }
// // }
// // BENCHMARK(lambda_copy);
// // static void boost_pool_allocator(::benchmark::State &state) {
// // boost::fast_pool_allocator<A> p;
// // for (auto _ : state) {
// // auto ptr = std::allocate_shared<A>(p);
// // ::benchmark::DoNotOptimize(ptr.get());
// // }
// // }
// // BENCHMARK(boost_pool_allocator);
// static void boost_lockless_queue(::benchmark::State &state) {
// using FreeList = boost::lockfree::queue<void *>;
// FreeList queue(1000000);
// for (auto _ : state) {
// void *result;
// queue.push(nullptr);
// ::benchmark::DoNotOptimize(queue.pop(result));
// }
// }
// BENCHMARK(boost_lockless_queue);
// static void tbb_concurrent_queue(::benchmark::State &state) {
// using FreeList = tbb::concurrent_queue<void *>;
// FreeList queue;
// for (auto _ : state) {
// void *result;
// queue.push(nullptr);
// ::benchmark::DoNotOptimize(queue.try_pop(result));
// }
// }
// BENCHMARK(tbb_concurrent_queue);
static void boost_spsc_str(::benchmark::State &state) {
boost::lockfree::spsc_queue<std::string> q(10000);
std::string x;
std::string y;
for (auto _ : state) {
q.push(x);
::benchmark::DoNotOptimize(q.pop(y));
}
}
BENCHMARK(boost_spsc_str);
static void folly_spsc(::benchmark::State &state) {
folly::ProducerConsumerQueue<int> q(10000);
int x;
for (auto _ : state) {
q.write(1);
::benchmark::DoNotOptimize(q.read(x));
}
}
BENCHMARK(folly_spsc);
static void boost_spsc(::benchmark::State &state) {
boost::lockfree::spsc_queue<int> q(10000);
int i = 0;
for (auto _ : state) {
q.push(12);
::benchmark::DoNotOptimize(q.pop(i));
}
}
BENCHMARK(boost_spsc);
// // static void boost_lockless_stack(::benchmark::State &state) {
// // using FreeList = boost::lockfree::stack<void *>;
// // FreeList stack;
// // for (auto _ : state) {
// // void *result;
// // stack.push(nullptr);
// // ::benchmark::DoNotOptimize(stack.pop(result));
// // }
// // }
// // BENCHMARK(boost_lockless_stack);
// // using Msg = mercury::FatWrapper<80>;
// // struct PPQueue {
// // std::atomic_flag _mutex = ATOMIC_FLAG_INIT;
// // std::vector<Msg *> v;
// // void push(Msg *e) {
// // while (_mutex.test_and_set(std::memory_order_acquire))
// // ;
// // v.push_back(e);
// // _mutex.clear(std::memory_order_release);
// // }
// // bool pop(Msg **e) {
// // auto result = false;
// // while (_mutex.test_and_set(std::memory_order_acquire))
// // ;
// // if (v.size()) {
// // *e = v.back();
// // v.pop_back();
// // result = true;
// // }
// // _mutex.clear(std::memory_order_release);
// // return result;
// // }
// // };
// static void get_thread_id(::benchmark::State &state) {
// for (auto _ : state) {
// auto tid = std::this_thread::get_id();
// ::benchmark::DoNotOptimize(tid == std::this_thread::get_id());
// }
// }
// BENCHMARK(get_thread_id);
// // static void native_order_fix_data_create(::benchmark::State &state) {
// // auto sz = state.range(0);
// // for (auto _ : state) {
// // auto p = new uint8_t[sz];
// // delete[] p;
// // ::benchmark::DoNotOptimize(p);
// // }
// // }
// // BENCHMARK(native_order_fix_data_create)->Range(8, 8 << 14);
// // static void native_order_fix_data_fragment(::benchmark::State &state) {
// // for (auto _ : state) {
// // auto p = new jetfix::FIXFragment();
// // delete p;
// // ::benchmark::DoNotOptimize(p);
// // }
// // }
// // BENCHMARK(native_order_fix_data_fragment);
// // static void small_vec(::benchmark::State &state) {
// // auto i = 0;
// // for (auto _ : state) {
// // boost::container::small_vector<size_t, 1> vec;
// // vec.emplace_back(i++);
// // ::benchmark::DoNotOptimize(vec[0]);
// // }
// // }
// // BENCHMARK(small_vec);
// static void inter_thread_spsc(::benchmark::State &state) {
// boost::lockfree::spsc_queue<int> q1(10);
// boost::lockfree::spsc_queue<int> q2(10);
// auto t2 = std::thread([&q1, &q2]() {
// int i = 1;
// cpu_set_t cpuset;
// CPU_ZERO(&cpuset);
// CPU_SET(1, &cpuset);
// pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
// do {
// while (!q1.pop(i))
// ;
// q2.push(i);
// } while (i);
// });
// cpu_set_t cpuset;
// CPU_ZERO(&cpuset);
// CPU_SET(0, &cpuset);
// pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
// sleep_for(2s);
// int i = 0;
// for (auto _ : state) {
// q1.push(++i);
// while (!q2.pop(i))
// ;
// }
// q1.push(0);
// t2.join();
// }
// BENCHMARK(inter_thread_spsc);
// // static void inter_thread_mpmc(::benchmark::State &state) {
// // using Queue = boost::lockfree::queue<int>;
// // Queue q1(100), q2(100);
// // auto t2 = std::thread([&q1, &q2]() {
// // int i = 1;
// // cpu_set_t cpuset;
// // CPU_ZERO(&cpuset);
// // CPU_SET(1, &cpuset);
// // pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
// // do {
// // while (!q1.pop(i))
// // ;
// // q2.push(i);
// // } while (i);
// // });
// // cpu_set_t cpuset;
// // CPU_ZERO(&cpuset);
// // CPU_SET(0, &cpuset);
// // pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
// // sleep_for(2s);
// // int i = 0;
// // for (auto _ : state) {
// // q1.push(++i);
// // while (!q2.pop(i))
// // ;
// // }
// // q1.push(0);
// // t2.join();
// // }
// // BENCHMARK(inter_thread_mpmc);
// static void inter_thread_mpmc_spdlog(::benchmark::State &state) {
// using Queue = spdlog::details::mpmc_bounded_queue<int>;
// Queue q1(8), q2(8);
// auto t2 = std::thread([&q1, &q2]() {
// int i = 1;
// cpu_set_t cpuset;
// CPU_ZERO(&cpuset);
// CPU_SET(1, &cpuset);
// pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
// do {
// while (!q1.dequeue(i))
// ;
// q2.enqueue(i);
// } while (i);
// });
// cpu_set_t cpuset;
// CPU_ZERO(&cpuset);
// CPU_SET(0, &cpuset);
// pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
// sleep_for(2s);
// int i = 0;
// for (auto _ : state) {
// q1.enqueue(++i);
// while (!q2.dequeue(i))
// ;
// }
// q1.enqueue(0);
// t2.join();
// }
// BENCHMARK(inter_thread_mpmc_spdlog);
// static void same_thread_mpmc_spdlog(::benchmark::State &state) {
// using Queue = spdlog::details::mpmc_bounded_queue<int>;
// Queue q1(8);
// int i = 0;
// for (auto _ : state) {
// q1.enqueue(++i);
// q1.dequeue(i);
// }
// }
// BENCHMARK(same_thread_mpmc_spdlog);
// static void moodycamel_concurrent_queue(::benchmark::State &state) {
// using Q = moodycamel::ConcurrentQueue<int>;
// Q queue(8);
// int i = 0;
// for (auto _ : state) {
// queue.enqueue(++i); // NOLINT
// ::benchmark::DoNotOptimize(queue.try_dequeue(i));
// }
// }
// BENCHMARK(moodycamel_concurrent_queue);
// static void moodycamel_spsc_queue(::benchmark::State &state) {
// using Queue = moodycamel::ReaderWriterQueue<int>;
// Queue q1(8);
// int i = 0;
// for (auto _ : state) {
// q1.enqueue(++i);
// ::benchmark::DoNotOptimize(q1.try_dequeue(i));
// }
// }
// BENCHMARK(moodycamel_spsc_queue);
// // struct MyBase {
// // virtual int inc(int &x) = 0;
// // };
// // struct MyDerived : MyBase {
// // int inc(int &x) override { return x += 1; };
// // };
// // struct MyDirect {
// // int inc(int &x) { return x += 1; };
// // };
// // static void derived_call(::benchmark::State &state) {
// // int x = 0;
// // MyDerived d;
// // MyBase &b = d;
// // for (auto _ : state) {
// // ::benchmark::DoNotOptimize(b.inc(x));
// // }
// // }
// // BENCHMARK(derived_call);
// // static void direct_call(::benchmark::State &state) {
// // int x = 0;
// // MyDerived b;
// // for (auto _ : state) {
// // ::benchmark::DoNotOptimize(b.inc(x));
// // }
// // }
// // BENCHMARK(direct_call);
// // static void get_time_of_day(::benchmark::State &state) {
// // std::string result(strlen("20121204-14:44:02.999"), ' ');
// // struct timeval tv;
// // for (auto _ : state) {
// // gettimeofday(&tv, nullptr);
// // auto tm = localtime(&tv.tv_sec);
// // sprintf(
// // &result[0], "%d%02d%02d-%02d:%02d:%02d.%03d", tm->tm_year,
// // tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec,
// 0);
// // }
// // }
// // BENCHMARK(get_time_of_day);
// // static void fix_time_stamp(::benchmark::State &state) {
// // std::string r(128, 32);
// // auto now = datetime::now();
// // for (auto _ : state) {
// // datetime::FIX_timestamp(now, &r);
// // }
// // }
// // BENCHMARK(fix_time_stamp);
// // static void perf_log(::benchmark::State &state) {
// // trace::PerfLog l;
// // for (auto _ : state) {
// // l.log(trace::CorID{"sim", 0}, "this", 1, "txt", 3);
// // }
// // }
// // BENCHMARK(perf_log);
// // void int_price_less(::benchmark::State &state) {
// // for (auto _ : state) {
// // state.PauseTiming();
// // int64_t count = state.range(0);
// // std::vector<md::IntPrice> prices;
// // prices.reserve((int)count);
// // for (int64_t x = 0; x < count; ++x) {
// // prices.push_back({x % 100, (int8_t) - (x % 5)});
// // }
// // state.ResumeTiming();
// // std::sort(prices.begin(), prices.end());
// // }
// // }
// // BENCHMARK(int_price_less)->Range(8, 8 << 11);
// // void set_hashset_insert(::benchmark::State &state) {
// // std::unordered_set<uintptr_t> s;
// // size_t x = 0;
// // for (auto _ : state) {
// // x = std::rand();
// // s.insert(x);
// // }
// // }
// // void set_stdset_insert(::benchmark::State &state) {
// // std::set<uintptr_t> s;
// // size_t x = 0;
// // for (auto _ : state) {
// // x = std::rand();
// // s.insert(x);
// // }
// // }
// // void set_btreeset_insert(::benchmark::State &state) {
// // btree::set<uintptr_t> s;
// // size_t x = 0;
// // for (auto _ : state) {
// // x = std::rand();
// // s.insert(x);
// // }
// // }
// // void set_hashset_iterate(::benchmark::State &state) {
// // std::unordered_set<uintptr_t> s;
// // size_t x = 0;
// // for (auto _ : state) {
// // x = std::rand();
// // s.insert(x);
// // for (const auto &p : s) {
// // ::benchmark::DoNotOptimize(p);
// // }
// // }
// // }
// // void set_stdset_iterate(::benchmark::State &state) {
// // std::set<uintptr_t> s;
// // size_t x = 0;
// // for (auto _ : state) {
// // x = std::rand();
// // s.insert(x);
// // for (const auto &p : s) {
// // ::benchmark::DoNotOptimize(p);
// // }
// // }
// // }
// // void set_btreeset_iterate(::benchmark::State &state) {
// // btree::set<uintptr_t> s;
// // size_t x = 0;
// // for (auto _ : state) {
// // x = std::rand();
// // s.insert(x);
// // for (const auto &p : s) {
// // ::benchmark::DoNotOptimize(p);
// // }
// // }
// // }
// // BENCHMARK(set_hashset_insert);
// // BENCHMARK(set_stdset_insert);
// // BENCHMARK(set_btreeset_insert);
// // BENCHMARK(set_hashset_iterate);
// // BENCHMARK(set_stdset_iterate);
// // BENCHMARK(set_btreeset_iterate);
// // struct Chunk {
// // size_t a, b, c, d;
// // };
// // void tbb_concurrent_vector_insert(::benchmark::State &state) {
// // tbb::concurrent_vector<Chunk> vec;
// // size_t x = 0;
// // for (auto _ : state) {
// // vec.push_back({++x, ++x, ++x, ++x});
// // }
// // }
// // BENCHMARK(tbb_concurrent_vector_insert);
// // void tbb_concurrent_vector_iterate(::benchmark::State &state) {
// // tbb::concurrent_vector<Chunk> vec;
// // for (size_t x = 0; x < 9999; ++x) {
// // vec.push_back({++x, ++x, ++x, ++x});
// // }
// // for (auto _ : state) {
// // for (const auto &v : vec) {
// // ::benchmark::DoNotOptimize(v);
// // }
// // }
// // }
// // BENCHMARK(tbb_concurrent_vector_iterate);
// // void tbb_concurrent_vector_index(::benchmark::State &state) {
// // tbb::concurrent_vector<Chunk> vec;
// // for (size_t x = 0; x < 9999; ++x) {
// // vec.push_back({++x, ++x, ++x, ++x});
// // }
// // for (auto _ : state) {
// // for (size_t x = 0; x < vec.size(); ++x) {
// // ::benchmark::DoNotOptimize(vec[x]);
// // }
// // }
// // }
// // BENCHMARK(tbb_concurrent_vector_index);
// // void concurrent_append_collection_insert(::benchmark::State &state) {
// // ConcurrentAppendCollection<Chunk> vec;
// // size_t x = 0;
// // for (auto _ : state) {
// // vec.push_back({++x, ++x, ++x, ++x});
// // }
// // }
// // BENCHMARK(concurrent_append_collection_insert);
// // void concurrent_append_collection_iterate(::benchmark::State &state) {
// // ConcurrentAppendCollection<Chunk> vec;
// // for (size_t x = 0; x < 9999; ++x) {
// // vec.push_back({++x, ++x, ++x, ++x});
// // }
// // for (auto _ : state) {
// // for (const auto &v : vec) {
// // ::benchmark::DoNotOptimize(v);
// // }
// // }
// // }
// // BENCHMARK(concurrent_append_collection_iterate);
// // void concurrent_append_collection_index(::benchmark::State &state) {
// // ConcurrentAppendCollection<Chunk> vec;
// // for (size_t x = 0; x < 9999; ++x) {
// // vec.push_back({++x, ++x, ++x, ++x});
// // }
// // for (auto _ : state) {
// // for (size_t x = 0; x < vec.size(); ++x) {
// // ::benchmark::DoNotOptimize(vec[x]);
// // }
// // }
// // }
// // BENCHMARK(concurrent_append_collection_index);
// // void utils_timestamp(::benchmark::State &state) {
// // for (auto _ : state) {
// // ::benchmark::DoNotOptimize(utils::TimeStamp::now());
// // }
// // }
// // BENCHMARK(utils_timestamp);
// // void bts_chrono_now(::benchmark::State &state) {
// // for (auto _ : state) {
// // ::benchmark::DoNotOptimize(Timer::now());
// // }
// // }
// // BENCHMARK(bts_chrono_now);
// // void unix_nano_now(::benchmark::State &state) {
// // for (auto _ : state) {
// // ::benchmark::DoNotOptimize(datetime::unix_nano_now());
// // }
// // }
// // BENCHMARK(unix_nano_now);
// struct TestHandler {
// int g(int x) { return 2 * x; }
// static int z(int x);
// };
// int TestHandler::z(int x) { return 2 * x; }
// void function_handler(::benchmark::State &state) {
// TestHandler h;
// std::function<int(int)> f = [&h](int x) { return h.g(x); };
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(f(1));
// }
// }
// BENCHMARK(function_handler);
// void raw_handler(::benchmark::State &state) {
// TestHandler h;
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(h.g(1));
// }
// }
// BENCHMARK(raw_handler);
// void method_pointer(::benchmark::State &state) {
// TestHandler h;
// spartan::Delegate<int(int)> d;
// d.bind<&TestHandler::g>(h);
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(d(1));
// }
// }
// BENCHMARK(method_pointer);
// void static_method(::benchmark::State &state) {
// spartan::Delegate<int(int)> d;
// d.bind<&TestHandler::z>();
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(d(1));
// }
// }
// BENCHMARK(static_method);
// void lambda_method(::benchmark::State &state) {
// spartan::Delegate<int(int)> d;
// auto op = [](int x) { return 2 * x; };
// d.bind(op);
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(d(1));
// }
// }
// BENCHMARK(lambda_method);
// int fd(int x) { return 2 * x; }
// void function_delegate(::benchmark::State &state) {
// spartan::Delegate<int(int)> d;
// d.bind<&fd>();
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(d(1));
// }
// }
// BENCHMARK(function_delegate);
// void direct_static(::benchmark::State &state) {
// for (auto _ : state) {
// ::benchmark::DoNotOptimize(fd(1));
// }
// }
// BENCHMARK(direct_static);
struct Benchmark::Args {
std::vector<std::string> args;
};
Benchmark::Benchmark()
: CLI::App(
"Run micro-benchmarks. Use --benchmark_filter=regex to run only "
"specific benchmarks.",
"benchmark"),
_args(new Benchmark::Args()) {
add_option("args", _args->args);
callback([&] {
bts_assert(_args->args.size() < 99);
int argc = _args->args.size() + 1;
char *argv[100];
argv[0] = (char *)"spark";
for (size_t i = 0; i < _args->args.size(); i++)
::benchmark::Initialize(&argc, argv);
::benchmark::RunSpecifiedBenchmarks();
});
}
Benchmark::~Benchmark() {}
} // namespace bts::cmds
-16
View File
@@ -1,16 +0,0 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::cmds {
struct Benchmark : public CLI::App {
Benchmark();
~Benchmark();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::cmds
+3 -5
View File
@@ -1,15 +1,13 @@
#include "QuickTest.h"
#include "lib/logging/logging.h"
#include <thread>
#include "logger/SLOG.h"
namespace bts::cmds {
static void execute() {
using namespace spartan;
for (int i =0 ; i < 100;i++) {
SLOG(MD,Warn,"The rain in [spain] falls mainly in the {}", "plain");
ThreadLocalLogger::process_log_entries();
LOG(md, Warn, "The rain in [spain] falls mainly in the plain");
}
}
-39
View File
@@ -1,39 +0,0 @@
#include "Spartan.h"
#include "cmds/BookStats.h"
#include "cmds/CSV.h"
#include "cmds/Compress.h"
#include "cmds/FindPolys.h"
#include "cmds/Icebergs.h"
#include "cmds/Json.h"
#include "cmds/ListSec.h"
#include "cmds/PolygonBooks.h"
#include "cmds/MatcherCmd.h"
#include "cmds/SQL.h"
#include "cmds/Sim.h"
#include "cmds/Stats.h"
#include "cmds/ValuePolys.h"
#include "cmds/Test.h"
namespace bts::cmds {
using namespace spartan::cmds;
Spartan::Spartan() : CLI::App("Futures trading system for devs", "spartan") {
// subcommands
add_subcommand(CLI::App_p(new BookStats()));
add_subcommand(CLI::App_p(new Compress()));
add_subcommand(CLI::App_p(new CSV()));
add_subcommand(CLI::App_p(new FindPolys()));
add_subcommand(CLI::App_p(new Icebergs()));
add_subcommand(CLI::App_p(new Json()));
add_subcommand(CLI::App_p(new ListSec()));
add_subcommand(CLI::App_p(new PolygonBooks()));
add_subcommand(CLI::App_p(new MatcherCmd()));
add_subcommand(CLI::App_p(new Sim()));
add_subcommand(CLI::App_p(new SQL()));
add_subcommand(CLI::App_p(new Stats()));
add_subcommand(CLI::App_p(new ValuePolys()));
add_subcommand(CLI::App_p(new Test()));
}
} // namespace bts::cmds
-10
View File
@@ -1,10 +0,0 @@
#pragma once
#include <CLI11/CLI11.hpp>
namespace bts::cmds {
struct Spartan : public CLI::App {
Spartan();
};
} // namespace bts::cmds
+1 -1
View File
@@ -1,5 +1,5 @@
#include "Validators.h"
#include "server/sparkd/cmds/spartan/utils/FileUtils.h"
#include "lib/utils/FileUtils.h"
#include <fmt/format.h>
namespace bts::cmds {
@@ -2,7 +2,6 @@
#include "DeribitDemoConfiguration.h"
#include "lib/logging/logging.h"
#include "server/sparkd/cmds/crypto/DeribitDemoListener.h"
#include "server/sparkd/cmds/spartan/utils/FileUtils.h"
#include <csignal>
#include <emumba-deribit-connector/deribit_exception.h>
#include <emumba-deribit-connector/deribit_handler.h>
@@ -6,7 +6,6 @@
#include "lib/logging/logging.h"
#include "server/sparkd/cmds/common/Validators.h"
#include "server/sparkd/cmds/crypto/LiveLadderDemo.h"
#include "server/sparkd/cmds/spartan/utils/FileUtils.h"
#include <CLI11/CLI11.hpp>
#include <optional>
@@ -1 +0,0 @@
#include "AlgoManager.h"
@@ -1,57 +0,0 @@
#pragma once
#include "domain/OrderHandle.h"
#include "feed_handler/InitialSnapshot.h"
#include "system/Delegate.h"
#include <map>
#include <memory>
#include <vector>
namespace bts::spartan {
class NewOrder;
class CancelReplace;
class BookUpdate;
class TradeSummaryUpdate;
class IAlgo;
class AlgoParam;
class TimeStamp;
class OrderUpdate;
class SpartanError;
// This class manages a set of algos, routing relevant market data and
// order updates as well as providing facilities for algos to create
// and manage their own orders. This class handles only the "happy
// path", that is it assumes that live, valid connections to the
// exchange have already been established and no errors will occurr in
// these connections. It is up to external controller classes to
// handle both setup and errors.
class AlgoManager {
public:
AlgoManager();
~AlgoManager();
// Call these first, before run
void init(InitialSnapshot const &, std::map<IAlgo *, AlgoParam> const &);
Delegate<OrderHandle(const NewOrder &)> on_queue_new_order;
Delegate<void(OrderHandle, CancelReplace)> on_queue_cancel_replace;
Delegate<void(OrderHandle)> on_queue_cancel;
Delegate<void()> on_send_queued;
Delegate<void(SpartanError)> on_error;
// Call these after initialization is complete
void run();
// These should only be called after run is called
void handle(const BookUpdate &);
void handle(const TradeSummaryUpdate &);
void handle(TimeStamp);
void handle(const OrderUpdate &);
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan
@@ -1 +0,0 @@
#include "IAlgo.h"
@@ -1,37 +0,0 @@
#pragma once
#include "common/TimeStamp.h"
namespace bts::spartan {
class AlgoParams;
class IAlgoInitService;
class IAlgoRunService;
class BookUpdate;
class TradeSummaryUpdate;
class TimerUpdate;
class BookUpdate;
class TimeStamp;
class OrderUpdate;
class IAlgo {
public:
// Called once only, before any other method. The passed
// IAlgoInitService can only be used during this call and should
// not be cached by the algo.
virtual void init(AlgoParams const &, IAlgoInitService &) = 0;
// Called once after init(). The passed IAlgoRunService
// should be cached by the algo and used to send/update/cancel
// orders
virtual void run(IAlgoRunService &) = 0;
// These will only be called after run()
virtual void handle(const BookUpdate &){};
virtual void handle(const TradeSummaryUpdate &) {}
virtual void handle(TimeStamp) {}
virtual void handle(const OrderUpdate &) {}
};
} // namespace bts::spartan
@@ -1 +0,0 @@
#include "IAlgoInitService.h"
@@ -1,17 +0,0 @@
#include<string>
namespace bts::spartan {
class SecurityID;
class SecuritySnapshot;
class Polygon;
class IAlgoInitService {
public:
virtual void subscribe_security(SecurityID) = 0;
virtual SecuritySnapshot const &get_instrument(SecurityID const&) = 0;
virtual SecuritySnapshot const &get_instrument(std::string const&) = 0;
virtual Polygon const &get_polygon(std::string const&) = 0;
};
}
@@ -1 +0,0 @@
#include "IAlgoRunService.h"
@@ -1,37 +0,0 @@
#pragma once
#include <string>
namespace bts::spartan {
class OrderHandle;
class CancelReplace;
class Cancel;
class NewOrder;
class TimeStamp;
class IAlgoRunService{
public:
OrderHandle queue_new_order(NewOrder const &);
void queue_cancel_replace(OrderHandle, CancelReplace);
void queue_cancel(OrderHandle);
void send_queued();
// Add a tracepoint in the current trace. A trace is a sequence of
// events starting in an external event such as market data or
// order updates and ending with an order change. Traces are logged
// automatically after the correponsing messages are sent to the
// exchange,
template <class... T> void add_tracepoint(const char *tp, T &&... args) ;
// Log algo state
template <class... T> void log(const char *entry_type, T &&... args) ;
// This is only used when the algo is running under simulation. In
// production this is a no-op but during back testing this will
// cause the algo to be stopped and rebuilt at later point in the
// pcap file.
virtual void reset_if_simulation() = 0;
};
}
@@ -1,312 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"import spartan \n",
"import pandas as pd\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Log entire log from DB (this can easily be 10GB, so it might take a while)\n",
"log = spartan.qsim(\"select * from log\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Extract and clean up simulator book entries\n",
"sim_books = log[log.entry_type=='gw_book']\n",
"clean_sim_books = spartan.clean_sim_books(books)\n",
"\n",
"# Create an event timeline for the books that form our main poligon\n",
"sim_lined_up_books = spartan.get_310_lined_prices(clean_sim_books)\n",
"sim_lined_up_books"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# Grab and clean raw pcap books\n",
"pcap_books = spartan.get_pcap_books(\"310\",\"2020-09-10\")\n",
"clean_pcap_books = spartan.clean_pcap_books(pcap_books)\n",
"\n",
"# Create an event timeline for the books that form our main poligon\n",
"pcap_lined_up_books = spartan.get_310_lined_prices(clean_pcap_books)\n",
"pcap_lined_up_books"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"# print out an interesting event range\n",
"pcap_lined_up_books.loc[17295234-1:(17295234+105)].to_csv(\"/home/ppinto/big_streak.csv\")"
]
},
{
"cell_type": "code",
"execution_count": 136,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<AxesSubplot:>"
]
},
"execution_count": 136,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA300lEQVR4nO3deXhb93ng++8LcJNIkRR3UKR2UpQsibSt2ErsxLbk3VacZncytdqm8TM3bm866b1TZ9rezCQ302Tu06ZxMk0ncdzYnTaW4ixeYluiJSuJHW/yImolQG0WJRGkSJECSXEDfvePcyBBEheQBHCwvJ/nwQPgdxa8R4T48vxWMcaglFIqs7mcDkAppZTzNBkopZTSZKCUUkqTgVJKKTQZKKWUArKcDmCmysrKzOLFi50OQymlUsbbb799xhhTPt62lE0GixcvZvfu3U6HoZRSKUNEjk+0TauJlFJKaTJQSimlyUAppRSaDJRSSqHJQCmlFJoMlFJKoclAKaUUmgxUGgqGDE+++T5Do0GnQ1EqZWgyUGnn9SPdPPyLvfzinZNOh6JUytBkoNLOoY4AADsPdTociVKpQ5OBSjs+v5UMXm07o1VFSkVJk4FKO15/gDnZbs6PBnntSLfT4SiVEqJKBiJSLCJPicghETkoIh8UkRIRaRYRn/08395XROQREWkTkRYRuSbiPJvt/X0isjmi/FoR2Wsf84iISOwvVWUCYww+fz8fbaxmTrabnQe1qkipaER7Z/Bd4EVjTAPQCBwEHgZ2GGPqgB32e4C7gDr78SDwAwARKQG+BlwPXAd8LZxA7H2+GHHcnbO7LJWpTvcNERgeY3VNETfWlbHzUCfGGKfDSm9v/giOv+Z0FGqWpkwGIlIEfAT4MYAxZsQY0wvcBzxu7/Y48DH79X3AE8byOlAsIh7gDqDZGNNjjDkLNAN32tsKjTGvG+t/7RMR51JqWrx2e8GKynlsbKjgZO95Wu0yFQfnz8IL/xl+/z2nI1GzFM2dwRKgC/gXEXlXRB4VkXyg0hhz2t6nA6i0Xy8ATkQc326XTVbePk75FUTkQRHZLSK7u7q6oghdZZpwMqivLOCWhgoAdmhVUfwc3gkmBKf3OB2JmqVokkEWcA3wA2PM1cAAF6uEALD/oo/7vbgx5ofGmHXGmHXl5eMu1qMynNffT8W8XIrn5lBZmMeaBUXaxTSevNut53PtMHDG2VjUrESTDNqBdmPMG/b7p7CSg9+u4sF+Dv+POwnURhxfY5dNVl4zTrlS0+bzB6ivnHfh/YaGCt55/yw9AyMORpWmQiFoa4aihdb70+85Go6anSmTgTGmAzghIivsoo3AAeAZINwjaDPwtP36GeABu1fReqDPrk7aBtwuIvPthuPbgW32tnMist7uRfRAxLmUilooZPD6+6mrLLhQtnFlBcbArla9O4i5U+/CYDfc+GXrvVYVpbRo10D+c+DfRCQHOAL8MVYi2SoiXwCOA5+2930euBtoAwbtfTHG9IjIN4C37P2+bozpsV9/CfgJMAd4wX4oNS0ne89zfjTIiog7g9XVRZTPy2XHoU4+fk3NJEerafNtB3HBVR+H339fk0GKiyoZGGPeA9aNs2njOPsa4KEJzvMY8Ng45buB1dHEotREWu1pKOoikoHLJWxYUcHze08zGgyR7dZxljHj2w41H4C5JeBp1GqiFKf/M1Ta8HaGk0HBJeUbVlYQGB7jrWM94x2mZqK/E069A3W3We+rm+DsMaurqUpJmgxU2vD5+6kuyqMwL/uS8huXl5Hjdulo5Fhqe8l6rrvdevY0Ws+nW5yJR82aJgOVNlo7ApdUEYXl52axflkpO7UROXZ826GgCqrWWu+rwslA2w1SlSYDlRaCIcPhrn7qL6siCtvYUMGRrgGOnhlIcGRpKDgGbTutKqLwNGL5pVBUq8kghWkyUGnh/Z5BhsdCl4wxiLTBHo2sA9BioP1NGO67WEUU5mnUZJDCNBmotBDuSTRRMqgtmUt9ZQE7D/kTGVZ68m4DVxYsvfnSck8TdLfBsM4FlYo0Gai0EF7QZnnF+NVEABsaKnnjSA+BodFEhZWefM2w8IOQV3hpuacRMNCx15Gw1OxoMlBpwdvZT23JHPJzJx46s3FlBWMhw+98OofOjPW1Q+d+qL/jym0ebUROZZoMVFrwdgSorxi/iijs6tpiiudm6yyms+Frtp4vby8AmFdp9TDSZJCSNBmolDcaDHHkTP+43UojZbld3Fxfzq7WToIhXfBmRnzNULwQyurH366NyClLk4FKece7BxgNGlZUTdxeEHZLQwXdAyPsae+Nf2DpZmwYjuyy7gomWpm2ugm6DsHIYCIjUzGgyUClvNaOfgDqpqgmAripvhy3S3Q08kwcfxVGB8avIgrzNFqL3fj3Jy4uFROaDFTK8/oDuGTynkRhxXNzuHbRfHboeIPp8zVDVh4s/vDE+1xoRH4vISGp2NFkoFKerzPAotJ88rLdUe2/saGCg6fPcar3fJwjSzO+7VYiyJk78T6FC2BuqbYbpCBNBirltXYEqIviriBs40odjTxt3YetAWWTVRGB1ZagjcgpSZOBSmnDY0GOdQ9OOPJ4PMvKC1hYMleTwXRc6FJ629T7epqg86DV4KxShiYDldKOnhkgGDLUV0WfDESEDQ0VvNp2hvMjwThGl0Z826G0DkqWTL2vpxFCo9B5IP5xqZjRZKBS2sU5iaKvJgKrqmh4LMTvD+to5CmNDMCxV8YfdTweHYmckjQZqJTm8/fjdglLyvKnddx1S0rIz3Frr6JoHP0dBIejqyICmL8Ycos0GaQYTQYqpXn9AZaU5ZObFV1PorDcLDcfritn58FOrGW71YR82yCnwJqcLhoi4FmrySDFaDJQKc3rD0y7iihsw8oKOs4NceD0uRhHlUaMsRqPl94MWbnRH1fdBB37IKgzxKYKTQYqZQ2NBjneMxjVyOPx3LLC7mKqo5En1nUI+k5EX0UU5mmyqpa6WuMSloo9TQYqZbV19mMMrJhGT6JI5fNyaawt1naDyfi2W89TjS+4nDYip5yokoGIHBORvSLynojststKRKRZRHz283y7XETkERFpE5EWEbkm4jyb7f19IrI5ovxa+/xt9rETzIKl1EVe/8x6EkXa2FDBnvZeugLaJ35cvmaoXAOF1dM7rmSZ1c6gySBlTOfO4BZjTJMxZp39/mFghzGmDthhvwe4C6izHw8CPwAreQBfA64HrgO+Fk4g9j5fjDjuzhlfkcoYXn8/2W5hUen0ehJF2tBQgTGwq1XvDq4w1Afvvzb9KiIAlwuq1mgySCGzqSa6D3jcfv048LGI8ieM5XWgWEQ8wB1AszGmxxhzFmgG7rS3FRpjXjdWt44nIs6l1IR8/gDLygvIds/8a3xVdSGVhbk6Gnk8h1+G0Nj0q4jCPE3Q0QIhHdiXCqL9X2SA7SLytog8aJdVGmNO2687gEr79QLgRMSx7XbZZOXt45RfQUQeFJHdIrK7q6srytBVumr1B6Zc0GYq1mjkSn7nO8PIWChGkaUJXzPkFUPNB2Z2vKcRRgetOY1U0os2GdxojLkGqwroIRH5SORG+y/6uHfWNsb80Bizzhizrry8PN4fp5LYwPAY7WfPUz+NCeomsrGhgv7hMd461hODyNJEKARtzbB8I7gnXld6UtqInFKiSgbGmJP2cyfwS6w6f79dxYP9HL7PPgnURhxeY5dNVl4zTrlSE2rrtBa0mc6cRBO5YXkZuVkuXRs5UkcL9PtnXkUE1tKYWXmaDFLElMlARPJFZF74NXA7sA94Bgj3CNoMPG2/fgZ4wO5VtB7os6uTtgG3i8h8u+H4dmCbve2ciKy3exE9EHEupcbVeqEn0eyTwZwcNx9aVsqOQ34djRzm2w4ILNs483O4s6BytSaDFBHNnUEl8IqI7AHeBH5tjHkR+BZwm4j4gFvt9wDPA0eANuBHwJcAjDE9wDeAt+zH1+0y7H0etY85DLww+0tT6cznD5Cb5WJhySQLrUzDhpWVHO8e5MiZgZicL+X5tsOCa6BgltWx1U1WMghpe0yym7Iy0BhzBGgcp7wbuOLPBrv94KEJzvUY8Ng45buB1VHEqxRgdStdXlGA2xWbISkbGir4W6zRyMvKZ98OkdIGuqF9N9z81dmfy9MIbz0KZ49C6bLZn0/FjY5AVinJmpNo9lVEYQuK59BQNY8dh/wxO2fKOrwDMDMbX3A5bUROGZoMVMo5NzTK6b4h6mYx8ng8GxoqeOvYWfrOZ/jkat5tkF9ujROYrfKV4MrWZJACNBmolOPzWz2JVsTwzgCsBW+CIcNvvRk8hiUUhLaXYPlt1iji2crKgcpVcPq92Z9LxZUmA5VyvDHsSRSpqXY+8+dmZ/Zo5PbdMNQbmyqiME+TdWegPbWSmiYDlXK8/gBzst0sKJ4T0/O6XcItKyp4ubWTYChDf3H5toO4YdmG2J3T0wjnz1pTYaukpclApRyfv5/6ygJcMepJFGnDygp6B0d59/2zMT93SvBth4XrYU5x7M4ZbnvQdoOkpslApZxYzEk0kQ/XlZPlksxc4+DcaWvkcSyriMBqMxC3JoMkp8lApZTewRG6AsOzWsNgMkVzsvnA4pLMXP2srdl6ns0UFOPJngPlDXDqvdieV8WUJgOVUrx2T6JYNx5H2riyglZ/gBM9g3H7jKTk2w6FC6BiVezPXd1k9SjSRuSkpclApZRYzkk0kQ0N1trIL2fSgjdjI3B4l3VXEI+FBj2NMNAFgY7Yn1vFhCYDlVJ8/gDzcrPwFOXF7TOWlhewpCw/s2YxPfE6jARiX0UUpiORk54mA5VSWjsC1FUWEO9lsjc0VPDa4W4Ghsfi+jlJw7sN3Dmw5CNT7zsTlasB0WSQxDQZqJTi6+yPaxVR2MaGCkaCIV5tOxP3z0oKvmZYdAPkxmmSvtwCKKvTkchJTJOBShln+ofpGRhJSDJYt7iEeblZmTELine truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"active_books = pcap_lined_up_books[100000:200000].copy()\n",
"active_books['ask_syn']=active_books.ask_z-active_books.bid_u\n",
"active_books['bid_syn']=active_books.bid_z-active_books.ask_u\n",
"active_books.bid_syn.value_counts().sort_index().plot()\n",
"active_books.ask_syn.value_counts().sort_index().plot()\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 85,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<AxesSubplot:>"
]
},
"execution_count": 85,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD5CAYAAADSiMnIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzYUlEQVR4nO3dd3wUdf7H8dcnnV4DJARISCgiGITQkabSlGJDrDQpiufd6f0Uz/PU0/P0vDvLASJFATtWAqKIEkApQqJ0BBJqSCABQofU7++PGbyI2TSymU3283w89pHZ70557zLsZ2fmOzNijEEppZQqiI/TAZRSSnkuLRJKKaVc0iKhlFLKJS0SSimlXNIioZRSyiUtEkoppVzyK85IIlIbmA20BQwwFtgJfAiEA/uAEcaYDBER4FVgMHAOGG2M+dGezyjgL/ZsnzPGzLPbOwJzgSrAEuD3poi+ufXr1zfh4eHFe5dKKaVISEg4aowJLsk0xSoSWF/6XxljbhWRAKAq8GfgW2PMCyIyBZgCPAYMAlrYjy7A60AXEakLPAXEYBWaBBGJNcZk2OOMB37AKhIDgS8LCxQeHk58fHxJ3qtSSnk1Edlf0mmK3N0kIrWAXsAcAGNMljHmBDAMmGePNg8Ybg8PA+YbyzqgtoiEAAOAZcaY43ZhWAYMtF+raYxZZ289zM83L6WUUg4qzjGJCCAdeEtEfhKR2SJSDWhojEm1xzkMNLSHGwMH802fbLcV1p5cQPtviMgEEYkXkfj09PRiRFdKKXU5ilMk/IAOwOvGmKuBs1i7ln5hbwG4/foexpiZxpgYY0xMcHCJdqsppZQqheIUiWQg2Rjzg/38Y6yiccTeVYT9N81+/RDQJN/0YXZbYe1hBbQrpZRyWJFFwhhzGDgoIq3spmuB7UAsMMpuGwUstIdjgXvF0hU4ae+WWgr0F5E6IlIH6A8stV87JSJd7Z5R9+abl1JKKQcVt3fT74B37Z5Ne4AxWAVmgYiMA/YDI+xxl2B1f03E6gI7BsAYc1xEngU22OP9zRhz3B5+gP91gf2SIno2KaWUKh9SUS8VHhMTY7QLrFJKFZ+IJBhjYkoyjVedcX0hO5eZq5L4Yc8xp6MopVSF4FVFQgTe/H4fL3+zy+koSilVIXhVkQj082V8r+as23OchP3Hi55AKaW8nFcVCYA7OjehbrUApi5PdDqKUkp5PK8rElUD/BjbI5y4nelsSznpdByllPJoXlckAO7pFk6NQD+mxyU5HUUppTyaVxaJWlX8uadbM5ZsTSUx7YzTcZRSymN5ZZEAGNczgkA/H2as1K0JpZRyxWuLRL3qgdzRuSmf/3SI5IxzTsdRSimP5LVFAmBCr+aIwBsr9zgdRSmlPJJXF4mQWlW4pUMYH8YfJO30BafjKKWUx/HqIgEwqXckObl5zPlur9NRlFLK43h9kQivX40h0aG8s24/J85lOR1HKaU8itcXCYAH+kRxNiuXuWv2OR1FKaU8ihYJoFWjGlzfpiFvrd7Hmcwcp+MopZTH0CJhm9w3ipPns3l33X6noyillMfQImFr36Q2PaPqM+u7vVzIznU6jlJKeQQtEvlM7hvF0TOZfBR/0OkoSinlEbRI5NO1eV06NqvDjJV7yM7NczqOUko5TotEPiLC5L6RHDpxns9/OuR0HKWUcpwWiUv0bdWANiE1eX1lErl5xuk4SinlKC0Sl7C2JqLYk36Wr7YedjqOUko5SotEAQa2bUTz4GpMjUvEGN2aUEp5Ly0SBfD1Ee7vHcmO1FOs2JnudByllHKMFgkXhl/dmMa1q3j31kROJix8ENbPcjqJUsohWiRc8Pf1YWLv5iTsz2DdnuNOx3GGbwCcToWvn4SjiU6nUUo5QItEIUbENKF+9UCmr/DSL0gRGPpf8AuAzydBrl7XSilvo0WiEEH+voy/JoLvdh9l08ETTsdxRs1QGPxvSN4Aa151Oo1SqpwVq0iIyD4R2SIiG0Uk3m6rKyLLRGS3/beO3S4i8pqIJIrIZhHpkG8+o+zxd4vIqHztHe35J9rTSlm/0dK6q2szalXxZ2qcl25NALS7FdoMg7h/wOEtTqdRSpWjkmxJ9DXGtDfGxNjPpwDfGmNaAN/azwEGAS3sxwTgdbCKCvAU0AXoDDx1sbDY44zPN93AUr+jMlY90I/R3cNZtv0IPx8+5XQcZ4jADS9DlTrw2STrgLZSyitczu6mYcA8e3geMDxf+3xjWQfUFpEQYACwzBhz3BiTASwDBtqv1TTGrDNWN6L5+eblEcb0CKdagC+vr0hyOopzqtWDoa/Bka2w4gWn0yilyklxi4QBvhaRBBGZYLc1NMak2sOHgYb2cGMg/2VUk+22wtqTC2j/DRGZICLxIhKfnl5+5y/UrhrA3V2bsWhTCvuOni235XqcVoOg/d2w+hU4uN7pNEqpclDcItHTGNMBa1fSZBHplf9FewvA7ScTGGNmGmNijDExwcHB7l7cr4zrGYGfrw8zVnrx1gTAwH9AzTBrt1OWFxdMpbxEsYqEMeaQ/TcN+AzrmMIRe1cR9t80e/RDQJN8k4fZbYW1hxXQ7lEa1Azi9pgmfPJjMqknzzsdxzlBNWH4NDieBN887XQapZSbFVkkRKSaiNS4OAz0B7YCscDFHkqjgIX2cCxwr93LqStw0t4ttRToLyJ17APW/YGl9munRKSr3avp3nzz8igTezfHGJi5ao/TUZwV0Qu63A/rZ8KeFU6nUUq5UXG2JBoC34vIJmA98IUx5ivgBeB6EdkNXGc/B1gC7AESgVnAAwDGmOPAs8AG+/E3uw17nNn2NEnAl5f/1speWJ2qDGvfmPfXH+DoGS/v4XPdU1CvBXw+GS6cdDqNUspNpKJelygmJsbEx8eX+3KT0s9w3X9W8kCfSP5vQOtyX75HSU6AOdfDVbfDTa87nUYpVQQRSch3GkOx6BnXJRQZXJ3BbUOYv2Y/J89nOx3HWWEd4ZqHYdN78PMXTqdRSrmBFolSuL9PJKczc3h77T6noziv16PQ6CpY9Hs4e9TpNEqpMqZFohTaNq5F31bBvLl6H+eyvPyid34BcNMb1nGJxX+ACrr7UilVMC0SpfRgvyiOn83i/fUHix65smvYBvo+ATsWweYFTqdRSpUhLRKl1LFZXbo2r8vMVUlk5uQ6Hcd53X8HTbrCkv+Dkx53motSqpS0SFyGyX2jOHIqk08S9EsRH18YPh3ysiH2Qd3tpFQloUXiMvSMqk90WC1mrEwiJzfP6TjOqxcJ/Z+FpOUQP8fpNEqpMqBF4jKICJP7RnHg+DkWb04tegJvEDMOIvtZtzw95uXXuVKqEtAicZmuu6IhLRtWZ1pcInl5uovFuuXpVPDxh88fgDw9XqNURaZF4jL5+FhbE7vTzrBsxxGn43iGWo1h8EtwcB2s+a/TaZRSl0GLRBm4oV0IzepVZVpcIhX1Midl7qoRcMUQiPs7HNnmdBqlVClpkSgDfr4+TOodyebkk3y3W886BqzdTje+AkG14NOJkJPldCKlVClokSgjN3doTKOaQUyLS3Q6iueoVh+GvApHtsDKF51Oo5QqBS0SZSTQz5cJvZrzw97jxO87XvQE3qL1DRB9J3z/H0gu/6v2KqUujxaJMjSycxPqVgtgqm5N/NqgF6BGKHw2EbLOOZ1GKVUCWiTKUNUAP8b1jGDFznS2HtIb8fwiqJZ1y9NjifDtM06nUUqVgBaJMnZPt2bUCPJj+grdmviV5n2g8wT4YQbsWel0GqVUMWmRKGM1g/wZ1S2cL7ceJjHttNNxPMt1z0C9KFiotzxVqqLQIuEGY3qEE+Tny/QVelmKXwmoCsNnwKlD8NWfnU6jlCoGLRJuUK96IHd0bsrCjSkcPK4Han+lSSfo+UfY+A7s/NLpNEqpImiRcJMJvZrjI/DGKt2a+I3eU6BhO4h9CM4eczqNUqoQWiTcpFGtIG7tGMaC+GTSTl1wOo5n8QuAm2bA+Qz44o967wmlPJgWCTea1DuSnNw8Zn+/1+konqdRW+j7Z9i+ELZ87HQapZQLWiTcqFm9agyNDuWddfvJOKvXLvqNHr+HsM6w5BE4leJ0GqVUAbRIuNn9faI4l5XLW2v2OR3F8/j4WrudcrMh9ne620kpD6RFws1aNapB/zYNmbt6L2cyc5yO43nqRcL1f4PEbyDhLafTKKUuoUWiHEzuG8WpCzm8s26/01E8U8w464zspX+B43ucTqOUykeLRDmIblKba1rUZ/Z3e7mQrbfz/A0fHxg2DXz89JanSnmYYhcJEfEVkZ9EZLH9PEJEfhCRRBH5UEQC7PZA+3mi/Xp4vnk8brfvFJEB+doH2m2JIjKlDN+fx5jcN4qjZzJZEH/Q6SieqVYYDHoRDqyFtdOcTqOUspVkS+L3wI58z18EXjbGRAEZwDi7fRyQYbe/bI+HiLQBRgJXAgOB6Xbh8QWmAYOANsAd9riVSpeIusQ0q8MbK/eQnZvndBzPFD0SWt8Iy5+FtB1Fj6+UcrtiFQkRCQNuAGbbzwXoB1zs4D4PGG4PD7OfY79+rT3+MOADY0ymMWYvkAh0th+Jxpg9xpgs4AN73EpFRJjcL4pDJ87z2U+HnI7jmS7e8jSwpnXvidxspxMp5fWKuyXxCvAocPEncD3ghDHmYnedZKCxPdwYOAhgv37SHv+X9kumcdX+GyIyQUTiRSQ+PT29mNE9R5+WwVwZWpPXVySRm6fdPQtUPRiGvAKpm2DVS06nUcrrFVkkRORGIM0Yk1AOeQpljJlpjIkxxsQEBwc7HafERITJfaPYe/QsX25NdTqO57piCFw1Elb9Cw45vtop5dWKsyXRAxgqIvuwdgX1A14FaouInz1OGHBxH8ohoAmA/Xot4Fj+9kumcdVeKQ28shGRwdWYFpeE0ZPHXBv0ItRoBJ9NguzzTqdRymsVWSSMMY8bY8KMMeFYB56XG2PuAuKAW+3RRgEL7eFY+zn268uN9W0Line truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"active_books.bid_u_z.value_counts().sort_index().plot()\n",
"active_books.ask_u_z.value_counts().sort_index().plot()"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<AxesSubplot:xlabel='eid1,eid2'>"
]
},
"execution_count": 86,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA44ElEQVR4nO2deZwUxdn4v8/MXpxyI4KyqHig4sF6oMZoREUxP09ejSZi1Bij5vzpGzwSSX7BMzFv1BjPeCZ54xEiCR4BRPFCXBQF5FoEFJT7EjmW3anfH90z2zM793TPdM88389nd7qrq6uerqqup+qp6ioxxqAoiqIoAKFSC6AoiqL4B1UKiqIoSgxVCoqiKEoMVQqKoihKDFUKiqIoSoyqUgtQKL169TL19fWlFkNRFCVQzJo1a50xpneie+CVQn19PY2NjaUWQ1EUJVCIyPJk7mo+UhRFUWKoUlAURVFiqFJQFEVRYqhSUBRFUWKoUlAURVFieKYURORQEXlHROaIyL9EpKvtXi8i20Vktv33gOOeYbb/JhG5R0TEK/kURVGU9njZU3gEGGuMOQSYAFzvuLbEGHOY/XeVw/1PwPeAwfbfSA/lUxRFURLwUinsB0y3jycD56XzLCL9gK7GmBnGWs/7SeBsD+UrS16eu4q1X+4stRiKogQUL5XCPOAs+3g0sKfj2iAR+UBEXheRr9lu/YEVDj8rbLd2iMiVItIoIo1r1651W+7Asq25hauensV3Hn231KIoihJQCvqiWUSmALsnuXQTcBlwj4j8ApgINNvXvgD2MsasF5FhwD9F5KBc4jXGPAQ8BNDQ0KC7BNm0Rqyk+GzDthJLoihKUClIKRhjRmTwciqAiOwHjLLv2QnstI9nicgSLFPTSmCA494BtpuSJdFxedWSiqLki5ezj/rYvyHgZuAB+7y3iITt472xBpQ/McZ8AWwRkWPsWUeXAC94JV85ErLnaukOq4qi5IuXYwrfEpFFwALgc+Ax2/0E4CMRmQ08B1xljNlgX7saa9ZSE7AEeMlD+coOwdIKEdUKiqLkiWerpBpj/gD8IYn788DzKe5pBA72SqZyJ/pVh6oERVHyRb9oLkdUKyiKkieqFMoQNR8pipIvqhTKEFUJiqLkiyqFMsRoT0FRlDxRpVCGRFQnKIqSJ6oUFEVRlBiqFBRFUZQYqhQURVGUGKoUFEVRlBiqFBRFUZQYqhQURVGUGKoUFEVRlBiqFBRFUZQYFa8U/jbzU656ahb/eH8FT89YTv3YSYybOI8T75rGb/79cczfW03ruOuVBfz3cx+yrbml6HIaY6gfO4kxf56Zlf/6sZOIePwV2/RFa6kfO4n1W0uzJ/Smbc3Uj53E64t0S1Ylnoenf8J0LRd54dnS2UHhhn/MAeDleatibo+/vQyAR95cys1nDgHg4kfa9j0e3KcL3zth7+IJCazctB0gpwpw8Zqt7L97F69E4hJbQd3+0gLuGn2oZ/Gk4g9TFwMw5s8zWXb7qKLHr/iX8S/OB9BykQcV31PIB1OCJefyWc4our+C15RqVQ1d4klR3EeVQkDwcwVYKtl04T9FcR9VCnkQlLqoSB0FrZwVpYxQpRAQSmGyypaSmY9KFK+ilDOqFPKgFJWRr8cUStRT0A6KoriPKoWA4Of6T/dvUJTyQZVCHpSihepnu33pzEf+TRNFCSqqFAKCn6s/NR8pSvmgSiEPgvKdQrHws2yKouSGKoXAkHvNW6zKWs04ilI+qFLIg6C0jIs1+ygSKU48iQQkGxQlUKhSCAj5KCLtKSiKkiuqFAKCn6d9lm6Zi9LEqyjljCqFgBDxcQ1YOoXl3zRRlKCiSiEPSjEF089KQStnRSkfVCkEBH8vc1GcePwSr6KUMwUrBREZLSLzRCQiIg0J124QkSYRWSgipzncR9puTSIy1uE+SETetd3/LiI1hcrnBaX5orn4cWZLqXoxfk4TRQkqbvQU5gLnAtOdjiIyBLgQOAgYCdwvImERCQN/BE4HhgDfsv0C3AH83hizL7ARuNwF+cqCVh/XgP6VTFGUXBG37OMi8hpwnTGm0T6/AcAYc5t9/gowzvY+zhhzmtMfcDuwFtjdGNMiIsOd/lLR0NBgGhsbc5b3+mc/5NlZK3K+LxkPfPsIfj95MQtXf9nu2p3nDeW4wb047vZXcwqzS20VJ+zXm/lfbOGTdV9l9H/zqAPZo1sHrv7L+znF4yaPXNLAFU8mz4vbzj0ktvVplJ+dsh93T16UMryrvr4PD7y+BIDnrhrO+Q+8A1hbLNaPnZRRnl6da7jo6IH87JT92l27+Z9zaFy2kVOG9OXeV5t4+vKjOX5wL8Da+jSaX9OvP4lR975BbVWYbc0tfPzrkXHh7H/zS+xsiTDpR8dz0B67AfDE28u4ZeI8rjh+EJ9v3s6XO1p46vKjM8qbC//93IfMWbmF+V9sAdJvOzl35WbOvPdNpl13IoN6dWp3fdQ9b7B9VyufrP2KmTedzNjn5xASeGTMkTE/5/3pbWYt38ism0fQs3Nt3nIPHfcKW3a00DT+dKrC2bVJR9z9Ok1rtvKdYwYy+ePVrNqyI63/6defxAl3TYtz++c1x3HYnt0AOOf+tzjr0D0Y96+PY/736tkxzv+mbc0c9uvJAHw07lS61lXz3cdmMm3hWh78zjB++LcPuPH0A7j0uEEZ5Y+W1Vk3j2DYb6bE3Ht1rqHx5lMAOPa2qXy+uf1zHTWoB898f3jGOPJBRGYZYxoS3b0cU+gPfOY4X2G7pXLvCWwyxrQkuLdDRK4UkUYRaVy7Nr/Nued9viWv+5LxP1OSKwSA216az1tN63IO88udLUya80VWCgHgN5Pmc+OEOZk9eshv/7Mw5bVb7T1znaRTCEBMIQA8+ubSnOVZt7WZe+x9nBN5esanLFj1Jfe+2gTAn99qC3/Ois2x49cWreHLHS2s27qTbc2t7cLZ2WJ9uffq/DUxt1smzgOsPb5fnLOKNxbnnv+ZeKZxRUwhZOIf768EYOr81Umvz/t8C5+stcrZ+8s38eqCNUxxPA/ArOUbAfho5eZ29+fClh3W6/1VkrRMRdOarQA8NWN5RoUA8PqiNe3c/v5eW5XzwaebYgoBrDxO5GNH2i6x45+20KprfvefhTS3ROLCyIbEOmfd1ubYcTKFADBz6Yac4nCDrJSCiEwRkblJ/s7yWsBkGGMeMsY0GGMaevfunVcYX98/v/uSy5PmGsXbAa1Y8aSiJd3cVJ/bmPw9u6swohMOXHlEt5LJw+TONehk7404XCVhxkZLa/mWFYCqbDwZY0bkEfZKYE/H+QDbjRTu64FuIlJl9xac/l2naF/7mvaFyitCxZpulAd+r3RbHQrNx8mYEmNMynIWdXXjy3O38tHLr+AjLnw440zKxFT1e1kuFC/NRxOBC0WkVkQGAYOBmcB7wGB7plEN1mD0RGMNbkwDzrfvHwO84KF8RaGYBahYyicfCk0Fr5PRmU/OVAzK+59OTjeLhVvp4eUHj24Ena6BFZ304ePXrSDcmJJ6joisAIYDk+wBZYwx84BngI+Bl4FrjDGtdi/gWuAVYD7wjO0X4OfAz0SkCWuM4dFC5UtF0dbrMaU36/gBv1eupVrUzy3SJW+0sZBNHmSq6FyzHnlYINwIOq6nkJAmQS8rmcjKfJQOY8wEYEKKa+OB8UncXwReTOL+CXBUoTJlRfF0QtFaFCEfa59Ce0xep2FcTyGATUCrkk1vPnKjde6e+cg7cpYxQ35LQrpGFVrwSkl2VOwXzcVquBZzSQw/12VBNR8FhbTJEx1odqHUu2c+8nfXMV0Z8PPilG5QuUqhSIWymD0FX+PzFynVi+7nvbGdpB1TyEHNZfLpWnp4OaaQa0chmVsa85GfPyR1gwpWCsWLJ5eXshB09lH+xJuPSiiIh7iRBQGYkZq2rGWr1NKZEIPSUMiXylUKroaVphAWsYlc6ros3cvi99fI2VPIRymU+vnSlTM3lZxrYwolmn2ULN5k6ZMuycpcJ1SwUijqdwrFicvPA6StPjfEpprb7uc0dZKuPEcnILjRwnXPelSa2Uf5KLV2s49iU1KDUTZypXKVgouFMp15qJitilKX0SC/JHEfrznyM7EidZ47j/385NHn8ZX5yMP3Iq35KIlbsvfXWZYTr/u8fVMwFasUikmxKssA18kZ8doMF1eRpElHpzc/mRGykcWVVS5cemi/L7ceSjPQHO1VuvG6+XF8omKVQjE3tS/e2kdlrBV8iJ8Gz7MZU3Clp+CW+cjLnkKapny2eRa/9lF+YWSDj4pQjIpVCsWiksxHXuK1wos3H6XGpDguNemnpNp+fLT2kZe4IWG6dylaVNx43/yYmhWrFIrVbYsYU7yB5uJEUxK8Nh855547zX3pYg1CBQnkVHtlMnUG4eO1XAeaMyVP+zEFkzGepOEkicePZahylUIR49HvFPxPtu9m/ECzR8LkQTpRYj0FHWh2Za9zN2X3UxmKUrlKoYhTUouG6oS8ydZ85MRPL3S6nm/bmIJ/zEdeJl2uvbtk+Z2ugdU2JTVHwZJQzO+YsqXgBfGCipuZkWrXtSgvzvnClXj2lRV8Zvqwk5qk16O7Z5WK6A5ZXuDcveyVeatyuvfxt5bSu0sdnWrDrN/aTN+ude38fLphG5M/Xs26rTv5amdLzH3SR/F598/Zn1MdFjrVVNHc2rZc5tQFa+jWsZoD+nVNKsPkj1ezeM2X7Nm9I9071rBu6052tUZY8+VOdu9ax46WVqpDIRCo79mJVVt20KE6jABf7txFc0uE2qowddVhendpvx3mPz9YiYiwZ4+OVIeEiIEN25ppjURiu9bd82oTvbvWsWX7Lj7bsI3D9+rG+q+a48Jxpu20BWs4oF8XPvxsU8xt+qK1bG9uZWDPjlSHQ2zZvotQSOhaV8225ha27myhtirMrtYIe3SrY8GqL+laV82qzTviKtH3lm7g0w3bCAls2rYLgM61VezLine truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"active_books.bid_syn.plot()\n",
"active_books.ask_syn.plot()"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<AxesSubplot:xlabel='eid1,eid2'>"
]
},
"execution_count": 87,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEGCAYAAACdJRn3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlnUlEQVR4nO3de7xcZX3v8c+XAAEjmCAhQEJMqEElKhS34aDIAUFCowcMLziNLTVIOeklcDy9qOTFsXrgpFKtbamXmkjxUimX0kZyCCYkKEYrEHYkCEkI2RBqdrgkXKQikpDkd/6YZ4eVyczee9bM7D0z6/t+vea1Z5611nPJgvnNep5nrUcRgZmZWS32G+4KmJlZ+3HwMDOzmjl4mJlZzRw8zMysZg4eZmZWs/2HuwJD5fDDD49JkyYNdzXMzNrG6tWrn42IsZW2FSZ4TJo0ie7u7uGuhplZ25D0H9W2udvKzMxq5uBhZmY1c/AwM7OaOXiYmVnNHDzMzKxmdQcPSRdKWitpt6Susm3zJPVI2iBpetm2EZIekHR7Ju2bkjZJWpNeJ1Ypc7akjek1u942mJlZbRoxVfdh4HxgQTZR0vHALGAqcDSwQtJxEbEr7fJxYD1waFl+n4iIW6sVJukw4DNAFxDAakmLI+KFBrTFzMwGoe7gERHrASSVbzoPuCkitgObJPUA04B7JE0APgjMB/60xiKnA8sj4vlU7nLgHODG3I2o14u98MB3YPeugfctt/9IePelcPDohlfLzKxZmnmT4Hjg3szn3pQG8HfAJ4FDKhw3X9JfAHcBV6TgU57v5ir57kXSHGAOwMSJE2usfg0evBHu/lxfqTUcmNZSGTMJ3nFBgytlZtY8gwoeklYAR1bYdGVE3FZLgZI+BGyNiNWSTi/bPA94GjgQWAh8CriqlvyzImJhyoeurq7mrXrVd8XxFy/AfjUMIz33GHzppHxXLGZmw2hQwSMizsqR9xbgmMznCSntXOBcSTOAg4BDJX0nIi6KiKfSvtslfQP48yr5nl6W79056mdmZjk1c6ruYmCWpJGSJgNTgFURMS8iJkTEJEoD6t+PiIsAJB2V/gr4MKXB+HLLgLMljZE0Bjg7pQ2fvqV89x33GWwGDauKmdlQaMRU3ZmSeoFTgCWSlgFExFrgFmAdsBSYm5lpVc0Nkh4CHgIOB/5vKqNL0nUp3+eBq4H70+uqvsFzMzMbGo2YbbUIWFRl23xKM6qqHXs3mS6niHh/lf26gUszn68Hrs9V4abIeeXRt3/4ysPM2ovvMDczs5o5eDRC7iuHvisVX3mYWXtx8GiIoLb7OxJ3W5lZm3LwMDOzmjl4NEJEzmm67rYys/bk4GFmZjVz8GgIj3mYWbE4eDRK7rvLzczaj4NHI3iqrpkVjINHQ7jbysyKxcHDzMxq5uDRCJ6qa2YF4+DRMB4wN7PicPBoiJxXDh7zMLM25eDRKJ6qa2YF4uDRCJ6qa2YFU1fwkHShpLWSdkvqKts2T1KPpA2SppdtGyHpAUm3Z9J+JGlNej0p6btVytyV2W9xPfVvHE/VNbNiqXclwYeB84EF2URJx1Nan3wqcDSwQtJxmWVoPw6sBw7tOyYi3pc5/l+B26qU+euIOLHOejeeu63MrEDquvKIiPURsaHCpvOAmyJie0RsAnqAaQCSJgAfBK6rlKekQ4H3A9+tp25Dyt1WZlYwzRrzGA9sznzuTWkAfwd8Ethd5dgPA3dFxH9W2X6QpG5J90r6cH+VkDQn7du9bdu2wdY9J195mFlxDBg8JK2Q9HCF13m1FibpQ8DWiFjdz24fAW7sZ/ubIqIL+B3g7yT9RrUdI2JhRHRFRNfYsWNrrW7zeczDzNrUgGMeEXFWjny3AMdkPk9IaecC50qaARwEHCrpOxFxEYCkwyl1b83spz5b0t/HJd0N/CbwWI46NpbHPMysQJrVbbUYmCVppKTJwBRgVUTMi4gJETGJ0oD69/sCR3IBcHtEvFIpU0ljJI1M7w8H3gusa1IbBi9yzrZyV5eZtal6p+rOlNQLnAIskbQMICLWArdQ+mJfCszNzLTqzyzKuqwkdUnqG1x/G9At6UHgB8A1ETH8wcN3mJtZwdQ1VTciFgGLqmybD8zv59i7gbvL0k6vsF83cGl6/xPgHXnr21TutjKzAvEd5o3gqbpmVjAOHg3jKw8zKw4Hj4aInOPlHvMws/bk4GFmZjVz8GiE3FN192TQqJqYmQ0JB4+G8FRdMysWB49G8VRdMysQB49GqPsOc195mFl7cfAwM7OaOXg0ROTrtvKYh5m1KQePRvAd5mZWMA4eDeMBczMrDgePhnC3lZkVi4OHmZnVzMGjETxV18wKxsHDzMxqVnfwkHShpLWSdkvqKts2T1KPpA2SpmfSn5D0kKQ1kroz6YdJWi5pY/o7pkqZs9M+GyXNrrcN9fOYh5kVS10rCSYPA+cDC7KJko6ntKzsVOBoYIWk4zLL0Z4REc+W5XUFcFdEXCPpivT5U2X5HgZ8Buii1N+zWtLiiHihAW3Z10tbB97n1V+zO+C5X26vLe9XdzAWYMdLgyunXWkE7DcCDh493DUxswapO3hExHoA7fvL+zzgpojYDmyS1ANMA+7pJ7vzgNPT+29RWqb2U2X7TAeWR8TzqdzlwDmUrX3eMNeeAK++POBuT8bhnDp/RU1ZH8irPHLQfuy38guw8gt5a9g+PnobHHv6cNfCzBqgEVce1YwH7s187k1pULpiuFNSAAsiYmFKHxcRT6X3TwPjquS7uUq+e5E0B5gDMHHixDxtgHM+B7t39rvL9x/ZxlfWjeSq86ZWCqJVXbtiI18b95f88YkH5KtbO9i1E5am+P/LZ4a3LmbWMIMKHpJWAEdW2HRlRNyWo9xTI2KLpCOA5ZIeiYiV2R0iIlJwyS0FpYUAXV1d+fJ618UD7vKzFx9l9dqN/MvJb2K//QYfPL7575tYO+pkePdJuarWFnZufy14eFaZWccYVPCIiLNy5L0FOCbzeUJKIyL6/m6VtIhSd9ZK4BlJR0XEU5KOAioNBGzhta6tvnzvzlE/MzPLqZlTdRcDsySNlDQZmAKskjRK0iEAkkYBZ1MadO87pm/21Gyg0lXNMuBsSWPSbKyzU9qw6ZssVeuEK0kF+DGe+UfxrDKzjtGIqbozJfUCpwBLJC0DiIi1wC3AOmApMDfNtBoH/FjSg8AqYElELE3ZXQN8QNJG4Kz0GUldkq5L+T4PXA3cn15X9Q2eD5e+r8Raxjug9LUanR499vo36fC2mhVII2ZbLQIWVdk2H5hflvY4cEKV/Z8DzqyQ3g1cmvl8PXB9/lqbmVk9fId5I+TsjpGK0JPjbiuzTuTgYWZmNXPwaIAg59NJUOf/GPeYh1lHcvBokLxLQXX8gHlWx0dKs+Jw8GiAvN+Jea5W2k8hGmlWOA4eDRBEzdN09xzb6T/G3W1l1pEcPMzMrGYOHg0QkXMdQanzf4vLU3XNOpGDR4PkHb8o1vdpoRpr1tEcPBog71eih5LNrF05eDSI6pisWxjFuswy62gOHg3gqbpmVjQOHg0Q5BsxL8azrbIK1Vizjubg0SDutBqEYkVKs47m4NEIebutPGRuZm2qruAh6UJJayXtltRVtm2epB5JGyRNz6Q/IekhSWskdWfSvyDpEUk/k7RI0ugqZVY8frjln6pbpF/jRWqrWWer98rjYeB8SuuP7yHpeGAWMBU4B/iqpBGZXc6IiBMjIhtwlgNvj4h3Ao8C8/opt9Lxwyb3VF1feJhZm6oreETE+ojYUGHTecBNEbE9IjYBPcC0AfK6MyJ2po/3AhPqqdtQiojcXVCF+i1eqKsss87WrDGP8cDmzOfelAal78s7Ja2WNKfK8ZcA36uybTDHAyBpjqRuSd3btm2rofq1y7eeh79Pzaw9DbiGuaQVwJEVNl0ZEbflKPPUiNgi6QhguaRHImJPt5ekK4GdwA15js+KiIXAQoCurq6mfU3nDgDutzKzNjVg8IiIs3LkuwU4JvN5QkojIvr+bpW0iFJ31koASRcDHwLOjCojyf0dP5w8VXcQfJll1jGa1W21GJglaaSkycAUYJWkUZIOAZA0Cjib0qA7ks4BPgmcGxEvV8q0v+OHk59tZWZFU+9U3ZmSeoFTgCWSlgFExFrgFmAdsBSYGxG7gHHAjyU9CKwClkTE0pTdl4FDKHVFrZH0tVTG0ZLuSPv0d/ywyr8YVJF+jReprWadbcBuq/5ExCJgUZVt84H5ZWmPAydU2f/NVdKfBGYMdPxwyr+eR8Or0toKFSjNOpvvMG+AyPmLumixw8w6h4NHo3gxqEEoVGPNOpqDRwPkfyS7rz3MrD05eDRI/qm6Bfo1XqzLLLOO5uDRIHmuIop3h3mhGmvW0Rw8zMysZg4eDRAR+Z5tVbSVBAvVWLPO5uDRAPnvMPeAuZm1JwePBvGA+WAUqa1mnc3BowHyP1W3YD05hWqsWWdz8GgQ37NhZkXi4NEAQeR7thVF68gpVmvNOpmDh5mZ1czBowEici5DW7RLD495mHUMB48GqGeqrmdbmVk7cvBoGA+Ym1lx1LuS4IWS1kraLamrbNs8ST2SNkiankl/QtJDabXA7kz6ZyVtSelrJM2oUuY5Kc8eSVfUU/9GqafbqlA9OYVqrFlnq2slQUrrh58PLMgmSjoemAVMBY4GVkg6Li1FC3BGRDxbIb+/jYi/rlaYpBHAV4APAL3A/ZIWR8S6OtthZmY1qHcZ2vVQ8R6H84CbImI7sElSDzANuKee8lIePWk5WiTdlMpqWvD4+E0PsGPn7n73eWjLi7mXoe3Z9hJ/9J3V+SrXJv4h/d314M2MePKBYa3LHi9sgqcfgred25z8j/2v8O5LG5vn9l/CX02G3a/uW+/9RsBpn4BxUxtbplkV9V55VDMeuDfzuTelQWnU9E5JASyIiIWZ/S6T9FGgG/iziHihQr6by/I9uVolJM0B5gBMnDgxTzvLine truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"\n",
"active_books.bid_u_z.plot()\n",
"active_books.ask_u_z.plot()\n"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<AxesSubplot:xlabel='eid1,eid2'>"
]
},
"execution_count": 88,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzPElEQVR4nO2dd7gV1dW433XvhUsTkCodVFBRUeGi2GJDwJKgRiLGFhsmGpOYxARijCkf0fRPk1j4aYzts8RINKKo2LABghVEBEEQREVBUfrlrt8fZ85h7rmnTrln5pz1Ps95zsyua8+e2Wv3LaqKYRiGYQBUlVoAwzAMIzqYUjAMwzBSmFIwDMMwUphSMAzDMFKYUjAMwzBS1JRaAL906dJF+/fvX2oxDMMwYsW8efM+UdWu6eaxVwr9+/dn7ty5pRbDMAwjVojI8kzm1n1kGIZhpDClYBiGYaQwpWAYhmGkMKVgGIZhpDClYBiGYaQwpWAYhmGkMKVgGIZhpDClUEaoKv+et5JNW7eXWhTDMGKKKYUy4qWln/Kjf73Ob6a9VWpRDMOIKaYUyogvN9cD8PH6zSWWxDCMuGJKoYwQEQDsMD3DMLxiSqGMEOffdIJhGF4xpVBGiOR3YxiGkQtTCmWIWv+RYRgeMaVQRiRbCqYSDMPwiimFMkKwgWbDMPxhSqGcsJaCYRg+MaVQRtg4s2EYfglNKYjIfiLykoi8KSL/FZH2Lrshjt0Cx76VYz7MuV8iIteJ2HwaL9hAs2EYXgmzpXAzMFFV9wWmApcDiEgNcCfwbVXdGzgS2Ob4uQG4EBjo/MaEKF/ZYTrUMAy/hKkUBgEznesngK8716OAN1T1dQBV/VRVt4tID6C9qs7SRFX3duCkEOUrO1KL16yhYBiGR8JUCguAsc71OKCPcz0IUBF5TEReEZGfOOa9gJUu/ysdsyaIyAQRmSsic9esWROC6PFkx5RU0wqGYXijxo9nEZkB7JLB6grgPOA6EbkSeAjY6orzMGA4sBF4UkTmAZ8XGq+qTgGmANTV1VkJ6GBTUg3D8IsvpaCqI/M4GQUgIoOAExyzlcBMVf3EsXsEGEpinKG3y29vYJUf+SoNG1IwDMMvYc4+6ub8VwE/B250rB4D9hWRNs6g8xHAW6q6GlgvIiOcWUdnAw+GJV85Yy0FwzC8EuaYwuki8g7wNvABcCuAqq4D/gy8DLwGvKKq0xw/F5OYtbQEeBd4NET5yo4du6SaVjAMwxu+uo9yoarXAtdmsbuTRHdRuvlcYJ+wZCp7kgPNphMMw/CIrWguI1IDzSWWwzCM+GJKoYwQO2XHMAyfmFIoI2zykWEYfjGlUIbYQLNhGF4xpVBGJPc+soFmwzC8YkqhjLCT1wzD8IsphTJix4Z4phYMw/CGKYUywra5MAzDL6YUyhBrJxiG4RVTCmWFDTQbhuEPUwplhA00G4bhF1MKZURqSMGaCoZheMSUQhmRWqdQYjkMw4gvFa8Ups9fzc3PLeWtD9bzwCsrGT/lJf77+gf84J5XeX7xJyl385avY/r81fzf7BUlkbOhQek/cRrPvpP9+NFkS+GNlZ9z7J+fDV2ml979lAtvn0tDQ2nU0NoNWznl+hdY88WWksRvRJepr65k1WebSi1GLAlt6+y48O07X2liNmvpWgD+89oHvHdN4sC4r9/wYsp+aL+O7LlL++YR0OH2l94D4Jx/zEnJlE6rFtWp68Uffxm6TKf/v1kAPLbgQ47bt0fo8aVz6o0vsnTNBo750zO88cvRzR6/EU221jdw2b2v06tja16YeHSpxYkdFd9S8MK2+uavGX+2aVteN9VVpVmosHHr9pLE++HnmwFYv7m+JPEb0SS595e1IL1hSiEmlKiHpiBKJZqt1TNyYRtDesOUggdK8rJFeEaRbathRAmx6oIvQlMKIrKfiLwkIm+KyH9FpL1j3kJEbnPMF4rIJJefMSKySESWiMjEsGSLI1bsNkVsXw8jA9ZC8EeYLYWbgYmqui8wFbjcMR8H1Drmw4CLRKS/iFQDfweOAwYDp4vI4BDlixUNEa6NR1cywzCKJUylMAiY6Vw/AXzduVagrYjUAK2BrcB64EBgiaouVdWtwD3A2BDl80wpyucI6wTTCkaksO4jf4SpFBawo1AfB/Rxru8HNgCrgRXAH1V1LdALeN/lf6Vj1gQRmSAic0Vk7po12eftG81DqZrr9ukbmbDuI3/4UgoiMkNE5mf4jQXOAy4WkXnATiRaBJBoEWwHegIDgB+JyK7FxKuqU1S1TlXrunbt6icJsSHSs48iLJthGMXha/Gaqo7M42QUgIgMApIrrr4JTFfVbcDHIvICUEeildDH5bc3sMqPfOVElGs/JZPMmgpGBqz7yB9hzj7q5vxXAT8HbnSsVgBHO3ZtgRHA28DLwEARGSAiLYHxwENhyeeHkhSC0dUJhmGUEWGOKZwuIu+QKPA/AG51zP8OtBORBSQUwa2q+oaq1gPfBR4DFgL3qeqCEOWLFVHWCdZ9ZESJKLeq40Boex+p6rXAtRnMvyQx8JzJzyPAI2HJFGdKtelcIdhAs2GUD7ai2QOlWMEbXZVgLQUjmth76Q1TCjEhyi94yfY+shXNhhE4phRigvWTGkZhRLkCFQdMKXigFO9cYS96ib6GEn2F1lAwjOAxpWD4xrbONozywZRCTIjy9tQRFs0wjCIxpeCBkmyI1/xRFkypFJYNNBtG8JhSiAlRro1HWDTDMIrElEJMiPJ5CqXC2gmGETymFGJClFVCqfRVlJ+JUXrs/fCGKQVPlGBFc4Tf8AiLZlQgUf5W4oAphZgQ7dlHpZEtys/EMOKKKYWYYOWfYRjNgSkFD1gBbRhGuWJKISZEee8jG2g2okSUv5U4YEohJkT4OAX7CA2jjDCl4IHobohXGkrWUojwMzGMuOJLKYjIOBFZICINIlKXZjdJRJaIyCIRGe0yH+OYLRGRiS7zASIy2zG/1zmn2XCIcm08upIZlYzNTvOG35bCfOAUYKbbUEQGA+OBvYExwPUiUi0i1STOaD4OGEziHOfBjrffAX9R1d2BdcD5PmUrK6L8fpeupRDhh2KUDHst/OHrjGZVXQgZNyYbC9yjqluAZSKyBDjQsVuiqksdf/cAY0VkIXA08E3HzW3AL4Eb/MiXixueeZffTX87r7v+E6fRsqax7hx340ucMKQH095YzX59OnLrt4Yzff6H1Dc08IsHF3DQgE7MXraWrwzqyt++eQBVIpx6w4tcPnoPvnf3q2zYuh2Acw/tz39f/4BPvtzaKPw9d9mJEbt25ug9u7Fi7Ubumr2ChavXp+zPuHkWLyz5NHX/1f16ctVXB7Ni7cYmsueidYtqNm3bnvcZ5ON309/ms41b+c6RuzFl5lJuf2k5X26pT9nX1lQx54qRnP/Pl5m7fB0A7WprePjSwzjyj89kDPO28w5ky7btTLhjHkP7duS+iw5m5J+f5U/f2J9h/XZm8rS3WL95Rxxj//4C6zZsRVHWfrmVySfvy7qNWxm99y707Ni6SfjPLPqYFtVV1FQJv3hwAQ9/7zBaVCfyee2GrQz9zRMcMagr/zx3ODc/t4x+ndtQ36Acv2+PVBiqyk/uf4OFH67nPxcfSo3j/9UV67j07lf5+zeH8sFnm+jeoRVD++7s+zm7eXLhR7RuUc0/XniPYwd347ThfbO6/fiLzTz46gdccPiAjJsI3vvyCrq3b8WKtRs5++DEO9m3Uxv269Mx5eZfc9/nkTdXc+u5BzbxXwxTZr7L26u/4M+n7V+wn/+bvYI5yz7lgsN35dl31nDtk4vZWt+Q0W2Xdi155vKjgMQ4XP+J0+jZoRXP/fRoqqsEVeVvTy3ha/v35GdT3+Tdjzfw0qSjmzyXLfXb+en9bzC0386cfXB/AP7z6irunrOCG88cxj0vv89pw/vQqW3+Do2f3v8GfTu34eyD+3HBbXOZvWwtPTu04ldj9+HYwd2BpuVRdZVwyG6d+cHIQQzrF+y7kw8JorYlIs8AP1bVuc7934BZqnqnc38L8KjjfIyqXuCYnwUcREIBzHJaCYhIH+BRVd0nS3wTgAkAffv2HbZ8+fKiZc5XYBbD947eneueWpLR7tRhvWndopo7ZhUvY7edavn4iy0FuT18YBeeW/xJ0XEESa7ncPIBvZj66irPYY8b1pt/zVsJwLOXH8kRf3imIH+Durfj8cuOaGKenv+/Hrt36uP/8xPvcN2TiwG4/oyhXHzXKyl3711zQup6/eZtDPnl4wD8+zuHpD7eZNh7dN+JRR990cRfEKTLnyv80256idnL1jL9B4ez5y7tc4b1+lWj2O9XjzcJM+lm0f+Mobam2rfcxTyPYr/Vhy89jBP/+nwjsxvOGMpx+/Zg1WebOPSapxp9LzN+eAS7d2vX2L2rkE7KmpTjkN068+K7nzJyr27cfM7wguWfdNyeXP1o44poetiZCPrdSSIi81S1Lt08b/eRiMwQkfkZfmNDkbQAVHWKqtapal3Xrl09hTG0b8fA5Pl807asdhu21LNha31We6/hpvPFZm9xBMnajVuz2rlbDl5w+68vYipWoc9ws6vFtN7lZ0t99paUuiqrmTYs/ODzTQXFHTbJd6N+ewHPLY+TOHTNZHo/tm5PZFaDY7fskw0pu0x5l6sF/dH6zUDx73Qx33Mpydt9pKojPYS7Cujjuu/tmJHF/FOgo4jUqGp9mvtQaK53WxXE436exRwXEPVvNcjCpJinWWi8XuSL8uC/m+R7FEQexEEpFNv7kel9yvWOxeAR+CKsKakPAeNFpFZEBgADgTnAy8BAZ6ZRSxKD0Q9pIhefBk51/J8DPBiSbLHBqzKJIiXbH8mDH7cyzpUHcSgggyYOijCINT05K2TRfwS+8Dsl9WQRWQkcDEwTkccAVHUBcB/wFjAduERVtzutgO8CjwELgfsctwA/BX7oDEp3Bm7xI1s+muuDVtTzAfNF+Yt4CeVXOnfLine truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"active_books.bid_syn.plot()\n",
"active_books.bid_u_z.plot()\n"
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<AxesSubplot:xlabel='eid1,eid2'>"
]
},
"execution_count": 89,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEGCAYAAABhMDI9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAx8klEQVR4nO2deZwdVZn3v7/uTncWAlkJCUkIS1gVIrTIooASCKssiuIKbuAAr+OC85JhUF5nUARHR8EFXMYFdUAlyhgBCYKIKNgJURJCIIEGErYQCJBAlu5+3j9udXfV7bvUvVV37ef7+dzuqrM+p07Veeqc55xTMjMcx3Ecp5+WWgvgOI7j1BeuGBzHcZwIrhgcx3GcCK4YHMdxnAiuGBzHcZwIbbUWICmTJk2yWbNm1VoMx3GchmLx4sXPm9nkXH4NrxhmzZpFV1dXrcVwHMdpKCQ9ns/Ph5Icx3GcCK4YHMdxnAiuGBzHcZwIrhgcx3GcCK4YHMdxnAiJFIOkMyQtl9QnqTPLb76kVZJWSpoXuM2QdIekB4N4/xwKf6mktZKWBr8TksjmOI7jlEfS6arLgNOBa8KOkvYFzgT2A6YBiyTtCfQAnzGzJZLGAosl3WZmDwZRv2ZmX0kok+M4jpOARD0GM1thZitzeJ0C/I+ZbTGzx4BVwMFm9rSZLQnivgKsAHZOIsNw4gv/+yC3LHum1mI4jtPkVMrGsDPwZOh8DVkKQNIs4A3AvSHnCyT9Q9IPJI3Pl7ikcyR1Sepat25dimLXNz/482N8/LrFtRbDcZwmp6hikLRI0rIcv1PKzVTSdsCvgE+a2cuB87eB3YE5wNPAf+aLb2bXmlmnmXVOnpxzRbfjOI5TJkVtDGY2t4x01wIzQufTAzckjSCjFH5qZjeG8nm2/1jSd4HflpGv4ziOk5BKDSXdBJwpqUPSrsBs4D5JAr4PrDCzr4YjSJoaOj2NjGHbcRzHqTKJZiVJOg24CpgMLJS01MzmmdlySTcAD5KZiXS+mfVKejPwAeABSUuDZP7VzH4HXCFpDmBAN3BuEtkcx3Gc8kikGMxsAbAgj99lwGVZbncDyhP+A0lkcRzHcdLBVz47juM4EVwxOI7jOBFcMTiO4zgRXDE4juM4EVwxOI7jOBFcMTiO4zgRXDE4juM4EVwxOI7jOBFcMTiO4zgRXDE4juM4EVwxOI7jOBFcMTiO4zgRXDE4juM4EVwxNAhmVmsRHMcZJrhiaBBcLziOUy1cMTiO4zgREn2oZ1jww5PguRXlxd1zHpz6rVTE8A6D4zjVwhVDIfr6oPtPMO1AmPaG0uI+9kfovrsycjmO41QQVwyF6NuW+b/3iXDEhaXF/fX5sPoPqYnixmfHcapFYhuDpDMkLZfUJ6kzy2++pFWSVkqaF3LvlvSApKWSukLuEyTdJumR4P/4pPIlojdQDK0jSo/b2jaoWBzHcRqINIzPy4DTgbvCjpL2Bc4E9gOOA74lqTUU5K1mNsfMwsrkIuB2M5sN3B6c147+hr2lDMXQMmJQsaSA9xccx6kWiYeSzGwFgKRsr1OA/zGzLcBjklYBBwN/KZDcKcBRwfGPgDuB/5tUxlw81b2SR1f+g1Htrax9cTNTx43E+oz2thZGd7TxwqYtdGx7hTcAW62FJ557hSVPbGCvKWPZa6exjBzRmjftVc+9wpRtsF3PFrT6DgC6129ih1EjGD+6PRLWgGdf3kxri9iuo432tlaef2ULW3v72K6jdTB8r3F4ywOZ49WjeXVbL8+8tJmJ27Wzw8hBxbW1t49Hnt1I9/pNHLPvFNpbo7p/S08fm7b00AdsP7KN9tYWVj77CjMnjKa9rZWHnn6ZWZPHsPG1HtrbxIbXtjF6RBsbXtvKuFEj6DNjx7EjeX7jVlpa4O4HVjH7wKOYMG13Joxpxww2vLaVqTuMYvO2Xm76+1OcOmdn2tvyv4Pcs+p5Dpo1no62/Nc0F8++vJmlT25gh1EjOGS3iZnraUb3+lfZddKYktJyqsPm4L6dVYX6efql1xg3qp1R7aXdV5VmW28fz7y0mRkTRkfcu5/fxMwJo3n8hdrfv0pr7FrSncCFZtYVnF8N/NXMrgvOvw/cbGa/lPQY8CKZdvEaM7s2CLPBzMYFxwJe7D/Pyusc4ByAmTNnHvT444+XLO/N18zn+KfjzRj6j5Gf4XsbDho4n7vPFL53VmfOsGs3vMbhl/+BC1oXcOGIX5QsVyOywcYwZ8t3mbvPjrz8Wg/3db9A9+Un8vpLb+WVzT0AdF9+Ys64tyx7ho9ft7hgmHzMumjhwPE/Lj2W7UeO4Cd/fZxLfr2MX/3ToRy0y4QyS+RUin/55d+5oWtNyXVdDrMuWsjBsyZww8cPrXhepTD/xgf4+X1PsPRzxzAuePF7+NlXOPZrdzFnxjiWPrmB77z/QI573dSKyiFpcdaIzQCxegySFgE75fC62Mx+U4ZMbzaztZJ2BG6T9JCZRYaizMwk5dRagSK5FqCzs7Mszfbw5Hl8v3tS0XDbaOOBzbtF3BateDZv+PUbtwBwbe9J3NO3Hzd+/BCeeuk1PvHz+wH45ccPi4S//JaH6Op+YeB89x23Y/VzGwfO+8PfeP8afnbvEwB88bTX868LHhgSBuBd1/6Fvj7L6Qfwzu/cEzn/6rvn8OnrlwIwdmTbQEMeh38bcR1zWh5lnDYBsGjFcxH/OGktW/tS7PwK8eqWXrYfOYL7n3gRgEfXbXLFUIfc0LWmqvndF3q26oW7Hl4HwMYtPQOKYe2LrwGw9MkNADz41MsVVwyFiKUYzGxuGWmvBWaEzqcHbphZ///nJC0gM8R0F/CspKlm9rSkqcBzVIgxk2fSZZsqlTxbGcES2xN2OZStz2+iyzIVzy7Rt5dVI0fQZYOKZtuIHfi7hRrLIPxTqx+hyzJdz007vZEu2zYkDEBX3wv0hVVlVn5dFn1QNk89mC7bDMCElnZesK2xy/iUTWIOj8YOX0kssMIIBeeOU98UGqyp9f1byZXPNwFnSuqQtCswG7hP0hhJYwEkjQGOJWPA7o9zVnB8FlBObyQWOWwiFaNQVlUUIyfhm7PGoqTCwPWs9ZPlOA1MYuOzpNOAq4DJwEJJS81snpktl3QD8CDQA5xvZr2SpgALgoa5DfiZmd0SJHc5cIOkjwCPA+9KKl9euSuVcM68Ssgtj6aINOAFtMlwbw8H9cJwvxL1jZlV9eWs0aj1sqU0ZiUtABbk8bsMuCzL7VHggDzh1wNHJ5UpDtW8J6uZVzVvqL6Qwruy7TtRz18v5Mq2JyPnuThu7Uvs0vZywTD5CKe/w62/gfY23rX2Ba4ceTPP/3l/WPv6ktKLRVsHHHkRjJ2Sbrp/+z4s/DTscjiMnxX1m7w3HP6JdPNz6ppav9gM25XPSdrqSjb0+ZK2GGES511iwj/smcdJrffSYy0c2vpg1POxxzi09bXIeS5mvLaNca09BcPkI5x++5OroaWF2a9mjP+TNvwDHnuxpPSK0rsNNj4DMw6BA96dbtoLP535//ifYcMTg+6bX4KlP2s6xWBW+2HUWtEI5R6+iqFObAz1RWmCdtnezNr8s8HYGuyxdH/qRN4cmk7a/anc0xOvvfUhvnnH6oJh8hFO/56PvI1p40Zx+S/u5/LlR2UcP7Usd8RyefFx+Pr+lV/RHpb7zi/DnV+Evl5oqa/5+E4yChqfazyUNGy33a7uUFL+zLJ98gWt9Y1SKdIuV0XrtX9rlBRXtBfPs636eVaBJr2dY9EIL4rDVzE0WF7hMcd6vbHKESvtBqKiPcH+rVH64q/1SC/PJlMMzfqmkxK1vjrDVjEkaV1LjVlKVtVu82utcNJuH6ySt3Qt3t5r0UtxqkLEwJz17NVabw5fG0NV80qeW3S9QWWkT5qqwkaGmKQ9+0ItmVJsGrkTqe8209qR+f/7i2HR59NOPTdtQZ5f2bN+u4olsLKjD4DWL1b+nbQ/L/69vt5/b+/twzqg/ZstAw/dkWas7Bh8FlrvE3TFqO/jr4DOD6Uu4/BVDHU6XdXndpdH/yMl4Nytn+TYI0/gHWln0j4aTv46vNiddsrw1FJYvxpOuDLqvvfJ8Moz0Bt/RXo98/07MxMNzj18N1pbKnuv9+d13lt2r2g+pXLDX5/g5c3beO9BMxk3OtMjfHL9qyx84OmBMHOmjeOw3ScWT2ynCkzJZjgrhjpZ5xtXD1SjZ1kTnVQB4/OtfQdz+MiU1xn0c9DZlUk3H9tNhrf+a3XzrCBXLMrMJPvo246ntcCOu2nmdd7cym/YVwrfvf8OHt/4KiccdhTjgl1UH135HFfc/7eBMOfO2I3D5u5TKxGHr42hqj2GlMNWSvakyrIujM91ovAdp5EZtoohCSUP96QyLakyfYZaG7kqNTul1uVyClPrlb31QKErUOur44qhCpTyFusmhmT0Xz+fDunUK3Ee8Vrfv64YqkAajX2lbpOwbEnlLCd+Wvd//4Pkm6s2Bq63a9/4F8IVQxUo1F5m9yaqPUae5rbb5cheqQVudfzMOcOcOEPRtb5/XTFUgZJsEjG2xPDhJsdxKokrhjIoeeVzRaRIn1qsoUhvKCnrPJ1kHadiFNoxudb3ryuGOiP/ttu1vlViUI6NIe2Vz258bgiGc/XEMz5XXIyCuGJoQHyufn782jiNQuFvPvuspGFN9uhNtbfdrrXtolLbbtf6jcspTK0bvprSAO8urhgahMh4ZJ3eWPUgln/zuTFwxV3fuGIog8p+2rO6zWuaZSlvHUOlbAypJus4FSD/TVrr+zeRYpB0hqTlkvokdWb5zZe0StJKSfMCt70kLQ39Xpb0ycDvUklrQ34nJJGt2ajGUFItqNg6hpTTddJlONdPPfSsi5F0d9VlwOnANWFHSfsCZwL7AdOARZL2NLOVwJwgTCuwFlgQivo1M/tKQpkqTm9f5rbuC/63tIie3sze79kNbW+f0RdyNDN6+4zWFtGX4+nozXI0M8yiQyNLine truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"active_books.ask_syn.plot()\n",
"active_books.ask_u_z.plot()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
@@ -1,204 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import spartan \n",
"import pandas as pd\n",
"import sqlite3"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def get_uz_lined_books(events,books): \n",
" # cleanup books\n",
" books = books.drop(['t_time','sid','status'],axis=1)\n",
" books = books.astype({'bid':'float64','bid_qty':'float64',\n",
" 'ask':'float64','ask_qty':'float64'})\n",
" # Filter for the symbols we care\n",
" u=books[books.symbol=='ESU0']\n",
" z=books[books.symbol=='ESZ0']\n",
" u_z=books[books.symbol=='ESU0-ESZ0']\n",
"\n",
" # Line up symbols so we have an event time line that includes all symbols\n",
" prices = events\n",
"\n",
" prices = prices.merge(u,\n",
" how='outer',\n",
" left_on=['eid'],\n",
" right_on=['eid'])\n",
" prices = prices.rename({'symbol':'u','bid':'bid_u','ask':'ask_u',\n",
" 'bid_qty':'bid_qty_u','ask_qty':'ask_qty_u'},axis=1)\n",
"\n",
" prices = prices.merge(z,\n",
" how='outer',\n",
" left_on=['eid'],\n",
" right_on=['eid'])\n",
" prices = prices.rename({'symbol':'z','bid':'bid_z','ask':'ask_z',\n",
" 'bid_qty':'bid_qty_z','ask_qty':'ask_qty_z'},axis=1)\n",
" \n",
" prices = prices.merge(u_z,\n",
" how='outer',\n",
" left_on=['eid'],\n",
" right_on=['eid'])\n",
" prices = prices.rename({'symbol':'u_z','bid':'bid_u_z','ask':'ask_u_z',\n",
" 'bid_qty':'bid_qty_u_z','ask_qty':'ask_qty_u_z'},axis=1)\n",
" \n",
" # set eid1, eid2 as indexes and then sort by them so that ffill\n",
" # below will propagate the last price in event order\n",
" prices = prices.set_index(['eid'],drop=False)\n",
" prices = prices.sort_index()\n",
" \n",
" # Fill holes (every event only updates one symbol) with previous value of book\n",
" prices = prices.ffill()\n",
" prices = prices.rename_axis(None)\n",
" return prices"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# return schema : ['opp_idx', 'opp_id', 'opp_obs_dur', 'opp_begin_ts', 'opp_end_ts',\n",
"# 'opp_beg_first_st', 'opp_beg_last_st', 'eid']\n",
"def get_opp_events(pid, min_edge, opps_db_df, events_db_df): \n",
" opps = opps_db_df[(opps_db_df.pid==pid) & (opps_db_df.direct_value >= min_edge)]\n",
" # putting send times that created the opp next to it\n",
" opps = opps.merge(events_db_df,\n",
" how='inner',\n",
" left_on=['begin_eid'],\n",
" right_on=['eid'])\n",
" opps = opps[['opid','observable_duration','begin_eid','end_eid','begin_ts','end_ts','first_st','last_st']]\n",
" # expand to include all events in the opp\n",
" recs = []\n",
" oidx = 0\n",
" padding = 1\n",
" for index, r in opps.iterrows():\n",
" for eid in range(r['begin_eid']-padding,r['end_eid']+padding+1):\n",
" recs.append([oidx,r['opid'],r['observable_duration'],r['begin_ts'],r['end_ts'],r['first_st'],r['last_st'],eid])\n",
" oidx = oidx + 1\n",
" return pd.DataFrame.from_records(recs,columns=['opp_idx','opp_id','opp_obs_dur',\n",
" 'opp_begin_ts','opp_end_ts',\n",
" 'opp_beg_first_st','opp_beg_last_st','eid'])\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"def add_comp_cols (books_with_opp_events_df):\n",
" comp_set = books_with_opp_events_df\n",
" # t_time of each event to the start of event send times\n",
" comp_set['t_time_from_opp_first_send'] = comp_set['t_time'] - comp_set['opp_beg_first_st']\n",
" comp_set['t_time_from_opp_last_send'] = comp_set['t_time'] - comp_set['opp_beg_last_st']\n",
" # add synthetic book prices\n",
" comp_set['poly_bid'] = (-comp_set.ask_u+comp_set.bid_z-comp_set.ask_u_z)/2\n",
" comp_set['poly_bid_qty'] = comp_set[['ask_qty_u','bid_qty_z','ask_qty_u_z']].min(axis=1)\n",
" comp_set['poly_ask'] = (-comp_set.bid_u+comp_set.ask_z-comp_set.bid_u_z)/2\n",
" comp_set['poly_ask_qty'] = comp_set[['bid_qty_u','ask_qty_z','bid_qty_u_z']].min(axis=1)\n",
" # rearrange columns\n",
" comp_set = comp_set[['opp_idx', 'opp_id', 'opp_obs_dur', 'opp_begin_ts', 'opp_end_ts',\n",
" 'opp_beg_first_st', 'opp_beg_last_st', 'eid', 'h_t_time', 't_time',\n",
" 'first_st', 'last_st', 'proc_time', 'send_time_delta', 'n_packets', \n",
" 't_time_from_opp_first_send', 't_time_from_opp_last_send',\n",
" 'u','bid_u', 'bid_qty_u', 'ask_u', 'ask_qty_u', 'z', 'bid_z', 'bid_qty_z',\n",
" 'ask_z', 'ask_qty_z', 'u_z', 'bid_u_z', 'bid_qty_u_z', 'ask_u_z',\n",
" 'ask_qty_u_z', 'poly_bid', 'poly_bid_qty', 'poly_ask',\n",
" 'poly_ask_qty']]\n",
" return comp_set"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def get_uz_opps(db_name,min_edge,pid):\n",
" # 2020-09-10_310_24h.sqlite\n",
" con = sqlite3.connect(\"/home/taha/sparkdata/simulations/\" + db_name)\n",
" # books\n",
" books_db = pd.read_sql_query(\"select * from top_of_books\", con)\n",
" # deduced orders\n",
" deduced_orders_db = pd.read_sql_query(\"select * from event_items where type='deduced_order'\", con) \n",
" deduced_orders_db = deduced_orders_db[['eid', 'ok', 'symbol', 'change', 'side', 'price', 'qty']]\n",
" #events\n",
" all_events_db = pd.read_sql_query(\"select * from events\", con)\n",
" all_events_db['proc_time'] = all_events_db['last_st'] - all_events_db['t_time']\n",
" all_events_db['send_time_delta'] = all_events_db['last_st'] - all_events_db['first_st']\n",
" all_events_db['n_packets'] = 1 + all_events_db['last_sn'] - all_events_db['first_sn']\n",
" all_events_db = all_events_db.drop(['first_sn','last_sn'],axis=1)\n",
" # polygon opps\n",
" opps_db = pd.read_sql_query(\"select * from opportunities\", con)\n",
" \n",
" # lined up books\n",
" uz_books_df = get_uz_lined_books(all_events_db,books_db)\n",
" # opp events\n",
" opp_events_df = get_opp_events(pid,min_edge,opps_db,all_events_db)\n",
" # combining books with events\n",
" uz_books_with_opp_events_df = opp_events_df.merge(uz_books_df,\n",
" how='inner',\n",
" left_on=['eid'],\n",
" right_on=['eid'])\n",
" # \n",
" uz_books_with_opp_events_calc_df = add_comp_cols(uz_books_with_opp_events_df)\n",
" # add events deduction to each lined up book\n",
" lined_books_with_orders = uz_books_with_opp_events_calc_df.merge(deduced_orders_db,\n",
" how='left',\n",
" left_on=['eid'],\n",
" right_on=['eid'])\n",
" return lined_books_with_orders"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"rv=get_uz_opps('2020-09-17_310_24h.sqlite',10,23)\n",
"rv.to_csv('2020-09-17_detailed_opps_10.csv')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
@@ -1,312 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"def add_eff_edge_cols(single_opp_df):\n",
" def get_eff_edge(r):\n",
" poly_bid = r['poly_bid']\n",
" if(r['poly_bid']>0):\n",
" return [r['poly_bid'],r['poly_bid_qty']]\n",
" else:\n",
" if(r['poly_ask']<0):\n",
" return [abs(r['poly_ask']),r['poly_ask_qty']]\n",
" else:\n",
" return [0.0,0]\n",
" def get_eff_edge_pr(r): \n",
" return get_eff_edge(r)[0]\n",
" def get_eff_edge_qty(r): \n",
" return get_eff_edge(r)[1]\n",
" single_opp_df['eff_edge'] = single_opp_df.apply (lambda row: get_eff_edge_pr(row), axis=1)\n",
" single_opp_df['eff_edge_qty'] = single_opp_df.apply (lambda row: get_eff_edge_qty(row), axis=1)\n",
" return single_opp_df"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# returns the entry_edge, entry_edge_qty, distinct edge prices\n",
"# cumm qty and weighted egde prices for a single opp\n",
"def comp_edge_info(single_opp_df,min_edge):\n",
" edge_map = {0:0}\n",
" edge_map.pop(0)\n",
" entry_edge=0.0\n",
" entry_qty=0\n",
" # add effective edge to dict\n",
" for index,r in single_opp_df.iterrows():\n",
" eff_edge = r['eff_edge']\n",
" eff_edge_qty = r['eff_edge_qty']\n",
" if eff_edge >= min_edge:\n",
" if entry_qty == 0:\n",
" entry_edge=eff_edge\n",
" entry_qty=eff_edge_qty\n",
" if edge_map.get(eff_edge)==None:\n",
" edge_map[eff_edge] = eff_edge_qty \n",
" cumm_qty=0\n",
" avg_price=0\n",
" for p,q in edge_map.items():\n",
" cumm_qty = cumm_qty + q\n",
" avg_price = avg_price + p*q \n",
" avg_price = avg_price / cumm_qty\n",
" return [entry_edge,entry_qty,len(edge_map.values()),cumm_qty,avg_price]"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# finds the achievable edge when reacting to first send and when reacting\n",
"# to last send\n",
"def find_achievable_edge(single_opp_df,time_ns):\n",
" first_send_eff_edge = 0\n",
" first_send_eff_edge_qty = 0\n",
" last_send_eff_edge = 0\n",
" last_send_eff_edge_qty = 0\n",
" # first end\n",
" for index,r in single_opp_df.iterrows():\n",
" t = r['t_time_from_opp_first_send'] \n",
" if(t >= time_ns):\n",
" # since this is the row that takes away the\n",
" # the opp, we compute what the opp was just before this\n",
" first_send_eff_edge = prev_row['eff_edge']\n",
" first_send_eff_edge_qty = prev_row['eff_edge_qty']\n",
" break\n",
" else:\n",
" prev_row = r\n",
" # last send\n",
" for index,r in single_opp_df.iterrows():\n",
" t = r['t_time_from_opp_last_send']\n",
" if(t >= time_ns):\n",
" last_send_eff_edge = r['eff_edge']\n",
" last_send_eff_edge_qty = r['eff_edge_qty']\n",
" break\n",
" \n",
" return [first_send_eff_edge,first_send_eff_edge_qty,\n",
" last_send_eff_edge,last_send_eff_edge_qty]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"def summarize_opps(opps_df):\n",
" # this is the groupby method that acts on a single opp\n",
" def analyze_single_opp(single_opp_df):\n",
" sz = len(single_opp_df['proc_time'])\n",
" # last event before opp\n",
" pre_opp_ev_proc_time = single_opp_df['proc_time'].iloc[0]\n",
" # first event creating the opp\n",
" first_ev_proc_time = single_opp_df['proc_time'].iloc[1]\n",
" first_ev_n_packets = single_opp_df['n_packets'].iloc[1]\n",
" first_ev_send_delta = single_opp_df['send_time_delta'].iloc[1]\n",
" # opp times\n",
" last_t_time = single_opp_df['t_time'].iloc[sz-1]\n",
" first_send_time = single_opp_df['first_st'].iloc[1]\n",
" last_send_time = single_opp_df['last_st'].iloc[1]\n",
" opp_duration_first_send = last_t_time-first_send_time\n",
" opp_duration_last_send = last_t_time-last_send_time\n",
" # cumm qty 10 and over \n",
" edge_info = comp_edge_info(single_opp_df,10.0)\n",
" entry_edge = edge_info[0]\n",
" entry_qty = edge_info[1]\n",
" cumm_n_levels = edge_info[2]\n",
" cumm_qty = edge_info[3]\n",
" cumm_avg_price = edge_info[4]\n",
" # achievable price\n",
" pr_qty = find_achievable_edge(single_opp_df,25000)\n",
" return [pre_opp_ev_proc_time,first_ev_proc_time,first_ev_n_packets,first_ev_send_delta,\n",
" opp_duration_first_send,opp_duration_last_send,\n",
" entry_edge,entry_qty,cumm_n_levels,cumm_qty,cumm_avg_price,\n",
" pr_qty[0],pr_qty[1],pr_qty[2],pr_qty[3]]\n",
" opps_df = add_eff_edge_cols(opps_df)\n",
" rv = opps_df.groupby('opp_id').apply(lambda x: analyze_single_opp(x))\n",
" rv_df = rv.to_frame().reset_index()\n",
" rv_df.columns = ['opp_id','val_anal']\n",
" rv_df['pre_opp_ev_proc_time'] = rv_df['val_anal'].apply(lambda x:x[0])\n",
" rv_df['first_ev_proc_time'] = rv_df['val_anal'].apply(lambda x:x[1])\n",
" rv_df['first_ev_n_packets'] = rv_df['val_anal'].apply(lambda x:x[2])\n",
" rv_df['first_ev_send_delta'] = rv_df['val_anal'].apply(lambda x:x[3])\n",
" rv_df['opp_duration_first_send'] = rv_df['val_anal'].apply(lambda x:x[4])\n",
" rv_df['opp_duration_last_send'] = rv_df['val_anal'].apply(lambda x:x[5])\n",
" rv_df['entry_edge'] = rv_df['val_anal'].apply(lambda x:x[6])\n",
" rv_df['entry_qty'] = rv_df['val_anal'].apply(lambda x:x[7])\n",
" rv_df['cumm_n_levels'] = rv_df['val_anal'].apply(lambda x:x[8])\n",
" rv_df['cumm_qty'] = rv_df['val_anal'].apply(lambda x:x[9])\n",
" rv_df['cumm_avg_price'] = rv_df['val_anal'].apply(lambda x:x[10])\n",
"\n",
"\n",
" rv_df['ach_edge_first_send'] = rv_df['val_anal'].apply(lambda x:x[11])\n",
" rv_df['ach_edge_qty_first_send'] = rv_df['val_anal'].apply(lambda x:x[12])\n",
" rv_df['ach_edge_last_send'] = rv_df['val_anal'].apply(lambda x:x[13])\n",
" rv_df['ach_edge_qty_last_send'] = rv_df['val_anal'].apply(lambda x:x[14])\n",
"\n",
" rv_df = rv_df.drop(['val_anal'],axis=1)\n",
" return rv_df"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def add_edge_summary_cols(opps_df):\n",
" rv = opps_df\n",
" def get_qty (r, des_qty, is_first_fs):\n",
" if is_first_fs:\n",
" return min(r['ach_edge_qty_first_send'],des_qty)\n",
" else:\n",
" return min(r['ach_edge_qty_last_send'],des_qty)\n",
" def is_stable(r):\n",
" entry_edge = r['entry_edge']\n",
" entry_qty = r['entry_qty']\n",
" return entry_edge >= 30 or (entry_edge >= 17.5 and entry_qty >= 10) or (entry_edge >= 12.5 and entry_qty >= 20 )\n",
" rv['is_stable'] = rv.apply(lambda r : is_stable(r), axis=1)\n",
" # 1 lot qty fs\n",
" # 1 lot edge fs\n",
" # same for 2, 5, full \n",
" rv['one_lot_qty_fs'] = rv.apply(lambda r : get_qty(r,1,True), axis=1)\n",
" rv['one_lot_edge_fs'] = rv['one_lot_qty_fs']*rv['ach_edge_first_send']\n",
" rv['two_lot_qty_fs'] = rv.apply(lambda r : get_qty(r,2,True), axis=1)\n",
" rv['two_lot_edge_fs'] = rv['two_lot_qty_fs']*rv['ach_edge_first_send']\n",
" rv['five_lot_qty_fs'] = rv.apply(lambda r : get_qty(r,5,True), axis=1)\n",
" rv['five_lot_edge_fs'] = rv['five_lot_qty_fs']*rv['ach_edge_first_send']\n",
" rv['full_lot_qty_fs'] = rv.apply(lambda r : get_qty(r,1000000,True), axis=1)\n",
" rv['full_lot_edge_fs'] = rv['full_lot_qty_fs']*rv['ach_edge_first_send']\n",
" # 1 lot qty ls\n",
" # 1 lot edge ls\n",
" # same for 2, 5, full\n",
" rv['one_lot_qty_ls'] = rv.apply(lambda r : get_qty(r,1,False), axis=1)\n",
" rv['one_lot_edge_ls'] = rv['one_lot_qty_fs']*rv['ach_edge_last_send']\n",
" rv['two_lot_qty_ls'] = rv.apply(lambda r : get_qty(r,2,False), axis=1)\n",
" rv['two_lot_edge_ls'] = rv['two_lot_qty_fs']*rv['ach_edge_last_send']\n",
" rv['five_lot_qty_ls'] = rv.apply(lambda r : get_qty(r,5,False), axis=1)\n",
" rv['five_lot_edge_ls'] = rv['five_lot_qty_fs']*rv['ach_edge_last_send']\n",
" rv['full_lot_qty_ls'] = rv.apply(lambda r : get_qty(r,1000000,False), axis=1)\n",
" rv['full_lot_edge_ls'] = rv['full_lot_qty_fs']*rv['ach_edge_last_send']\n",
" return rv"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"def get_final_per_opp_df(file_name):\n",
" opp_summary_df = summarize_opps(pd.read_csv(file_name))\n",
" return add_edge_summary_cols(opp_summary_df)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"def add_up_edges(filter_for_stability, is_stable, opps_df):\n",
" # rv_filt = opps_df[opps_df.is_stable==is_stable]\n",
" rv_filt = opps_df\n",
" if(filter_for_stability):\n",
" rv_filt = rv_filt[rv_filt.is_stable == is_stable]\n",
" fee = 4.16\n",
" hit_ratio_fs = 100.0*rv_filt['one_lot_qty_fs'].sum()/len(rv_filt['one_lot_qty_fs'])\n",
" # fs\n",
" one_lot_qty_fs = rv_filt['one_lot_qty_fs'].sum()\n",
" one_lot_edge_fs = rv_filt['one_lot_edge_fs'].sum()\n",
" one_lot_edge_fee_fs = one_lot_edge_fs - fee*one_lot_qty_fs\n",
"\n",
" two_lot_qty_fs = rv_filt['two_lot_qty_fs'].sum()\n",
" two_lot_edge_fs = rv_filt['two_lot_edge_fs'].sum()\n",
" two_lot_edge_fee_fs = two_lot_edge_fs - fee*two_lot_qty_fs\n",
"\n",
" five_lot_qty_fs = rv_filt['five_lot_qty_fs'].sum()\n",
" five_lot_edge_fs = rv_filt['five_lot_edge_fs'].sum()\n",
" five_lot_edge_fee_fs = five_lot_edge_fs - fee*five_lot_qty_fs\n",
" # ls\n",
" hit_ratio_ls = 100.0*rv_filt['one_lot_qty_ls'].sum()/len(rv_filt['one_lot_qty_ls'])\n",
"\n",
" one_lot_qty_ls = rv_filt['one_lot_qty_ls'].sum()\n",
" one_lot_edge_ls = rv_filt['one_lot_edge_ls'].sum()\n",
" one_lot_edge_fee_ls = one_lot_edge_ls - fee*one_lot_qty_ls\n",
"\n",
" two_lot_qty_ls = rv_filt['two_lot_qty_ls'].sum()\n",
" two_lot_edge_ls = rv_filt['two_lot_edge_ls'].sum()\n",
" two_lot_edge_fee_ls = two_lot_edge_fs - fee*two_lot_qty_ls\n",
"\n",
" five_lot_qty_ls = rv_filt['five_lot_qty_ls'].sum()\n",
" five_lot_edge_ls = rv_filt['five_lot_edge_ls'].sum()\n",
" five_lot_edge_fee_ls = five_lot_edge_ls - fee*five_lot_qty_ls\n",
" \n",
" return [hit_ratio_fs,\n",
" one_lot_qty_fs, one_lot_edge_fs,one_lot_edge_fee_fs,\n",
" two_lot_qty_fs, two_lot_edge_fs,two_lot_edge_fee_fs,\n",
" five_lot_qty_fs, five_lot_edge_fs,five_lot_edge_fee_fs,\n",
" hit_ratio_ls,\n",
" one_lot_qty_ls, one_lot_edge_ls,one_lot_edge_fee_ls,\n",
" two_lot_qty_ls, two_lot_edge_ls,two_lot_edge_fee_ls,\n",
" five_lot_qty_ls, five_lot_edge_ls,five_lot_edge_fee_ls]"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[41.37931034482759, 36.0, 285.0, 135.24, 62.0, 470.0, 212.07999999999998, 126.0, 900.0, 375.84000000000003, 16.091954022988507, 14.0, 85.0, 26.759999999999998, 27.0, 157.5, 357.68, 61.0, 345.0, 91.23999999999998]\n"
]
}
],
"source": [
"import pandas as pd\n",
"\n",
"opps_df = get_final_per_opp_df('2020-09-14_detailed_opps_10.csv')\n",
"opps_df.to_csv('2020-09-14_summary_opps_10.csv')\n",
"edge_summ = add_up_edges(False,True,opps_df)\n",
"print(edge_summ)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
File diff suppressed because it is too large. Load diff
@@ -1,815 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" x y z\n",
"0 1 10 20.0\n",
"1 2 10 20.0\n",
"2 3 10 20.0\n",
" x y\n",
"0 1 10\n",
"1 2 10\n",
"2 3 10\n",
"3 4 10\n",
"4 5 10\n",
" x y\n",
"0 1 10\n",
"1 2 10\n",
"2 3 10\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>x</th>\n",
" <th>y</th>\n",
" <th>z</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1200</td>\n",
" <td>1200</td>\n",
" <td>1200.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1200</td>\n",
" <td>1200</td>\n",
" <td>1200.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>10</td>\n",
" <td>20.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>10</td>\n",
" <td>20.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>10</td>\n",
" <td>20.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" x y z\n",
"0 1200 1200 1200.0\n",
"1 1200 1200 1200.0\n",
"2 3 10 20.0\n",
"3 4 10 20.0\n",
"4 5 10 20.0"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import pandas as pd\n",
"import sqlite3\n",
"%config IPCompleter.greedy=True\n",
"\n",
"\n",
"d = {'x': [1,2,3,4,5], 'y': 10, 'z': 20.0}\n",
"df = pd.DataFrame(d)\n",
"\n",
"#rows 0 to 2\n",
"print(df[0:3])\n",
"#cols x and y\n",
"print(df[['x','y']])\n",
"#rows 0 to 2, cols x and y\n",
"print(df.loc[0:2,['x','y']])\n",
"df.loc[0:1,['x','y','z']]=1200\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>x</th>\n",
" <th>y</th>\n",
" <th>z</th>\n",
" <th>diff</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1200</td>\n",
" <td>1200</td>\n",
" <td>1200.0</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1200</td>\n",
" <td>1200</td>\n",
" <td>1200.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>10</td>\n",
" <td>20.0</td>\n",
" <td>-1197.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>10</td>\n",
" <td>20.0</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>10</td>\n",
" <td>20.0</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" x y z diff\n",
"0 1200 1200 1200.0 NaN\n",
"1 1200 1200 1200.0 0.0\n",
"2 3 10 20.0 -1197.0\n",
"3 4 10 20.0 1.0\n",
"4 5 10 20.0 1.0"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#df=df[df.x < 1200]\n",
"df.head()\n",
"df['diff'] = df['x']-df.x.shift(1)\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"recs = []\n",
"recs.append(['x',1])\n",
"recs.append(['x',2])\n",
"recs.append(['y',10])\n",
"recs.append(['y',20])\n",
"\n",
"df = pd.DataFrame.from_records(recs,columns=['category','val'])\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def test_boo(opp_grp) :\n",
" rv = [opp_grp['val'].min(),opp_grp['val'].min(),opp_grp['val'].median()]\n",
" return rv\n",
"rv = df.groupby('category').apply(lambda x: test_boo(x))\n",
"rv_df = rv.to_frame().reset_index()\n",
"rv_df.columns = ['category','val_anal']\n",
"rv_df['min'] = rv_df['val_anal'].apply(lambda x:x[0])\n",
"rv_df['max'] = rv_df['val_anal'].apply(lambda x:x[1])\n",
"rv_df['mean'] = rv_df['val_anal'].apply(lambda x:x[2])\n",
"\n",
"rv_df = rv_df.drop(['val_anal'],axis=1)\n",
"rv_df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dict = {0:0}\n",
"print(type(dict))\n",
"for k in dict.keys():\n",
" print(k)\n",
"dict[1]=1\n",
"dict[1]=2\n",
"print(dict.get(1))\n",
"print(dict.get(2))"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>x</th>\n",
" <th>y</th>\n",
" <th>z</th>\n",
" <th>a</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>y1</td>\n",
" <td>20.0</td>\n",
" <td>a1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>y1</td>\n",
" <td>20.0</td>\n",
" <td>a2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2</td>\n",
" <td>y2</td>\n",
" <td>20.0</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3</td>\n",
" <td>y3</td>\n",
" <td>20.0</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4</td>\n",
" <td>y4</td>\n",
" <td>20.0</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" x y z a\n",
"0 1 y1 20.0 a1\n",
"1 1 y1 20.0 a2\n",
"2 2 y2 20.0 NaN\n",
"3 3 y3 20.0 NaN\n",
"4 4 y4 20.0 NaN"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"l = {'x': [1,2,3,4,5], 'y': ['y1','y2','y3','y4','y5'], 'z': 20.0}\n",
"l_df = pd.DataFrame(l)\n",
"\n",
"#r = {'x': [1,1,3,4,5], 'a': ['a1','a2','a3','a4','a5']}\n",
"r = {'x': [1,1,100], 'a': ['a1','a2','a100']}\n",
"r_df = pd.DataFrame(r)\n",
"\n",
"l_df.head()\n",
"r_df.head()\n",
"\n",
"join_res = l_df.merge(r_df,\n",
" how='left',\n",
" left_on=['x'],\n",
" right_on=['x'])\n",
"join_res.head()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>x</th>\n",
" <th>y</th>\n",
" <th>z</th>\n",
" <th>x_sq</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>y1</td>\n",
" <td>20.0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>y2</td>\n",
" <td>20.0</td>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>y3</td>\n",
" <td>20.0</td>\n",
" <td>9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>y4</td>\n",
" <td>20.0</td>\n",
" <td>16</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>y5</td>\n",
" <td>20.0</td>\n",
" <td>25</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" x y z x_sq\n",
"0 1 y1 20.0 1\n",
"1 2 y2 20.0 4\n",
"2 3 y3 20.0 9\n",
"3 4 y4 20.0 16\n",
"4 5 y5 20.0 25"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"l = {'x': [1,2,3,4,5], 'y': ['y1','y2','y3','y4','y5'], 'z': 20.0}\n",
"l_df = pd.DataFrame(l)\n",
"l_df.head()\n",
"\n",
"\n",
"\n",
"def func(df):\n",
" def foo(r):\n",
" return r['x']*r['x']\n",
" df['x_sq'] = df.apply (lambda row: foo(row), axis=1)\n",
" return df\n",
"\n",
"rv = func(l_df)\n",
"rv.head()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import spartan \n",
"import pandas as pd\n",
"import sqlite3\n",
"\n",
"con = sqlite3.connect(\"/home/taha/sparkdata/simulations/2020-09-10_310_24h.sqlite\")\n",
"events_db = pd.read_sql_query(\"select * from events\", con)\n"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"opps_db = pd.read_sql_query(\"select * from opportunities\", con)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"events_db['first_st_time_diff'] = events_db['first_st'] - events_db.first_st.shift(1)\n",
"events_db['last_st_time_diff'] = events_db['last_st'] - events_db.last_st.shift(1)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>h_t_time</th>\n",
" <th>eid</th>\n",
" <th>t_time</th>\n",
" <th>first_sn</th>\n",
" <th>last_sn</th>\n",
" <th>first_st</th>\n",
" <th>last_st</th>\n",
" <th>first_st_time_diff</th>\n",
" <th>last_st_time_diff</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>20647952</th>\n",
" <td>2020-09-11.20:59:59.701115855</td>\n",
" <td>20647953</td>\n",
" <td>1599857999701115855</td>\n",
" <td>76848775</td>\n",
" <td>76848775</td>\n",
" <td>1599857999702940018</td>\n",
" <td>1599857999702940018</td>\n",
" <td>24832.0</td>\n",
" <td>24832.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20647953</th>\n",
" <td>2020-09-11.20:59:59.701117761</td>\n",
" <td>20647954</td>\n",
" <td>1599857999701117761</td>\n",
" <td>76848775</td>\n",
" <td>76848775</td>\n",
" <td>1599857999702940018</td>\n",
" <td>1599857999702940018</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" h_t_time eid t_time \\\n",
"20647952 2020-09-11.20:59:59.701115855 20647953 1599857999701115855 \n",
"20647953 2020-09-11.20:59:59.701117761 20647954 1599857999701117761 \n",
"\n",
" first_sn last_sn first_st last_st \\\n",
"20647952 76848775 76848775 1599857999702940018 1599857999702940018 \n",
"20647953 76848775 76848775 1599857999702940018 1599857999702940018 \n",
"\n",
" first_st_time_diff last_st_time_diff \n",
"20647952 24832.0 24832.0 \n",
"20647953 0.0 0.0 "
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#fil = events_db[events_db.first_st_time_diff==0]\n",
"#fil.tail(10)\n",
"#20647954\n",
"\n",
"fil = events_db[(events_db.eid==20647954) | (events_db.eid==20647953)]\n",
"fil.head()\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"3116600 3584.0\n",
"681514 3840.0\n",
"1078545 3840.0\n",
"1112266 3840.0\n",
"1320756 3840.0\n",
" ... \n",
"10872539 4352.0\n",
"10885258 4352.0\n",
"10888645 4352.0\n",
"10898749 4352.0\n",
"10909548 4352.0\n",
"Name: first_st_time_diff, Length: 2000, dtype: float64"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"events_db.head(10)\n",
"time_diff = events_db.first_st_time_diff\n",
"time_diff = time_diff[time_diff > 0]\n",
"time_diff.nsmallest(2000)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"con = sqlite3.connect(\"/home/taha/sparkdata/simulations/2020-09-10_310_24h.sqlite\")\n",
"event_items_db = pd.read_sql_query(\"select * from event_items where (type='deduced_order' and (change='added' or change='updated')) or type='fill'\", con)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"def find_sweeps(filter_sid,sid,event_items_db):\n",
" def summarize_items(single_ev_df):\n",
" n_fills = single_ev_df[single_ev_df.type=='fill']['type'].count()\n",
" if n_fills == 0:\n",
" return [single_ev_df['eid'].iloc[0],None,None,None]\n",
" # price vs qty of fills, if this has > 1 entry\n",
" # then it is a sweep\n",
" fill_map = {0:0}\n",
" fill_map.pop(0)\n",
" # all orders that were added or modified\n",
" deduced_orders = [[0.0,0,'buy']]\n",
" deduced_orders.clear()\n",
" eid = 0\n",
" sid = 0\n",
" for index, r in single_ev_df.iterrows():\n",
" eid = r['eid']\n",
" sid = r['sid']\n",
" r_type = r['type']\n",
" r_qty = r['qty']\n",
" r_price = r['price']\n",
" if(r_type == 'fill'): \n",
" if fill_map.get(r_price)==None:\n",
" fill_map[r_price] = r_qty\n",
" else:\n",
" fill_map[r_price] = fill_map.get(r_price) + r_qty\n",
" else:\n",
" if(r_type == 'deduced_order'):\n",
" if (r['change'] == 'added' or r['change'] == 'updated'):\n",
" deduced_orders.append([r_price,r_qty,r['side']])\n",
" return [eid,sid,fill_map,deduced_orders]\n",
" rv = event_items_db\n",
" if filter_sid:\n",
" rv = events_db[events_db.sid==sid]\n",
" rv = rv[(rv.type=='fill') | (rv.type=='deduced_order')]\n",
" # group by\n",
" sweeps = rv.groupby('eid').apply(lambda x: summarize_items(x))\n",
" return sweeps"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>eid</th>\n",
" <th>0</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1455</td>\n",
" <td>[1455, 12181, {333475: 28, -1050: 916, 334525:...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1456</td>\n",
" <td>[1456, None, None, None]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1457</td>\n",
" <td>[1457, None, None, None]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1459</td>\n",
" <td>[1459, None, None, None]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1460</td>\n",
" <td>[1460, None, None, None]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" eid 0\n",
"0 1455 [1455, 12181, {333475: 28, -1050: 916, 334525:...\n",
"1 1456 [1456, None, None, None]\n",
"2 1457 [1457, None, None, None]\n",
"3 1459 [1459, None, None, None]\n",
"4 1460 [1460, None, None, None]"
]
},
"execution_count": 39,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"rv = event_items_db.tail(100000)\n",
"#11:18\n",
"rv = event_items_db\n",
"#rv = event_items_db[event_items_db.eid==853423]\n",
"rv = find_sweeps(False,0,rv)\n",
"rv_df = rv.to_frame().reset_index()\n",
"rv_df.head()\n",
"#print(rv)"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"rv_df.to_csv('sweeps.csv')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
@@ -1,368 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 157,
"metadata": {},
"outputs": [],
"source": [
"import spartan \n",
"import pandas as pd\n",
"import sqlite3"
]
},
{
"cell_type": "code",
"execution_count": 158,
"metadata": {},
"outputs": [],
"source": [
"#def get_enriched_poly_vals(inp_df, edge_map, full_file_name, summ_file_name):\n",
"# poly_vals = enrich_poly_vals(inp_df, edge_map)\n",
"# poly_vals_summ = tabularize_opp_summ(summarize_opportunities(poly_vals)) \n",
"# poly_vals.to_csv(full_file_name)\n",
"# poly_vals_summ.to_csv(summ_file_name)\n",
"# return [poly_vals,poly_vals_summ]"
]
},
{
"cell_type": "code",
"execution_count": 159,
"metadata": {},
"outputs": [],
"source": [
"# converts series to dataframe\n",
"def tabularize_opp_summ(opp_summ_ser):\n",
" rv_df = opp_summ_ser.to_frame().reset_index()\n",
" rv_df.columns = ['opp_id','vals'] \n",
" rv_df['symbol'] = rv_df['vals'].apply(lambda x:x[0])\n",
" rv_df['opp_id'] = rv_df['vals'].apply(lambda x:x[1])\n",
" rv_df['opp_p_id'] = rv_df['vals'].apply(lambda x:x[2])\n",
" rv_df['start_eid'] = rv_df['vals'].apply(lambda x:x[3])\n",
" rv_df['start_t_time'] = rv_df['vals'].apply(lambda x:x[4])\n",
" rv_df['start_h_t_time'] = rv_df['vals'].apply(lambda x:x[5])\n",
" rv_df['first_send_opp_win'] = rv_df['vals'].apply(lambda x:x[6])\n",
" rv_df['last_send_opp_win'] = rv_df['vals'].apply(lambda x:x[7])\n",
" rv_df['merged_value'] = rv_df['vals'].apply(lambda x:x[8])\n",
" rv_df['merged_qty'] = rv_df['vals'].apply(lambda x:x[9])\n",
" rv_df['ticks'] = rv_df['vals'].apply(lambda x:x[10])\n",
" rv_df['is_direct'] = rv_df['vals'].apply(lambda x:x[11])\n",
" rv_df['poly_bid_bk'] = rv_df['vals'].apply(lambda x:x[12])\n",
" rv_df['poly_ask_bk'] = rv_df['vals'].apply(lambda x:x[13])\n",
" rv_df['leg_books'] = rv_df['vals'].apply(lambda x:x[14])\n",
" del rv_df['vals']\n",
" return rv_df"
]
},
{
"cell_type": "code",
"execution_count": 160,
"metadata": {},
"outputs": [],
"source": [
"# summarizes each opportunity\n",
"def summarize_opportunities(poly_vals):\n",
" def get_max_edge_row (opp_p_rows):\n",
" min_react_time = 50000\n",
" opp_p_rows_filt = opp_p_rows[opp_p_rows['t_opp_first_send'] >= min_react_time]\n",
" if (opp_p_rows_filt.shape[0] == 0):\n",
" return [0.0,0,0,False,'','','','']\n",
" else:\n",
" m_idx = opp_p_rows_filt['merged_value'].argmax()\n",
" m_max_row = opp_p_rows_filt.iloc[m_idx]\n",
" return [m_max_row['merged_value'], m_max_row['merged_qty'], m_max_row['ticks'],\n",
" m_max_row['direct_value'] == m_max_row['merged_value'],\n",
" m_max_row['poly_bid_bk'], m_max_row['poly_ask_bk'],\n",
" m_max_row['book_1_bid_bk'] + ';' + m_max_row['book_1_ask_bk'] + ' : ' +\n",
" m_max_row['book_2_bid_bk'] + ';' + m_max_row['book_2_ask_bk'] + ' : ' +\n",
" m_max_row['book_3_bid_bk'] + ';' + m_max_row['book_3_ask_bk'] + ' : ' +\n",
" m_max_row['book_4_bid_bk'] + ';' + m_max_row['book_4_ask_bk']]\n",
" def summarize_single_opp_p_pid(opp_p_rows):\n",
" opp_idx = 1\n",
" # summary cols\n",
" poly = opp_p_rows['symbol'].iloc[opp_idx]\n",
" opp_id = opp_p_rows['opp_id'].iloc[opp_idx]\n",
" opp_p_id = opp_p_rows['opp_p_id'].iloc[opp_idx]\n",
" start_eid = opp_p_rows['eid'].iloc[opp_idx]\n",
" start_t_time = opp_p_rows['t_time'].iloc[opp_idx]\n",
" start_h_t_time = opp_p_rows['h_t_time'].iloc[opp_idx]\n",
" # win size\n",
" first_send_opp_win = opp_p_rows['t_opp_first_send'].max()\n",
" last_send_opp_win = opp_p_rows['t_opp_last_send'].max()\n",
" # edge values\n",
" edge_vals = get_max_edge_row(opp_p_rows)\n",
" return [poly, opp_id, opp_p_id, start_eid, start_t_time, start_h_t_time,\n",
" first_send_opp_win, last_send_opp_win,\n",
" edge_vals[0], edge_vals[1], edge_vals[2], edge_vals[3],\n",
" edge_vals[4], edge_vals[5], edge_vals[6]]\n",
" return poly_vals.groupby('opp_p_id').apply(lambda x: summarize_single_opp_p_pid(x))"
]
},
{
"cell_type": "code",
"execution_count": 161,
"metadata": {},
"outputs": [],
"source": [
"# returns the poly vals db table filtered for the products\n",
"def get_poly_vals_db(full_db_name, products):\n",
" where_txt = ''\n",
" i=0\n",
" for p in products:\n",
" p_filter = 'product=' + \"'\" + p + \"'\"\n",
" prefix_or = '' if (i==0) else ' or ' \n",
" where_txt = where_txt + prefix_or + p_filter\n",
" i=i+1\n",
" where_txt = '(' + where_txt + ')'\n",
" query = 'select * from polygon_value_events where ' + where_txt\n",
" print(query)\n",
" return pd.read_sql_query(query, sqlite3.connect(full_db_name))"
]
},
{
"cell_type": "code",
"execution_count": 162,
"metadata": {},
"outputs": [],
"source": [
"# adds a new col called opp_p_id which is simply\n",
"# a unique id for every [opp_id,pid]\n",
"def add_opp_p_ids(poly_vals):\n",
" key_map = {'':0}\n",
" def get_opp_p_id(r):\n",
" key = str(r['opp_id']) + '_' + str(r['pid'])\n",
" if key_map.get(key)==None:\n",
" key_map[key]=len(key_map) + 1\n",
" return key_map[key]\n",
" poly_vals['opp_p_id'] = poly_vals.apply (lambda row: get_opp_p_id(row), axis=1)\n",
" return poly_vals"
]
},
{
"cell_type": "code",
"execution_count": 163,
"metadata": {},
"outputs": [],
"source": [
"# adds a new column called ticks (its hard coded to convert\n",
"# merged_value to ticks)\n",
"def add_ticks(poly_vals):\n",
" val_to_tick_map = {'GC':10, 'SI':5, 'HG':5}\n",
" tick_to_dollar_map = {'GC':10, 'SI':5, 'HG':12.5}\n",
" def get_ticks(r):\n",
" return r['merged_value']/tick_to_dollar_map[r['product']]\n",
" def get_cash_value(r, is_merged):\n",
" col_name = \"merged_value\" if is_merged else \"direct_value\"\n",
" norm_value = r[col_name]\n",
" if r['side'] == 'Buy':\n",
" norm_value = -1*norm_value\n",
" return (norm_value/val_to_tick_map[r['product']]) * tick_to_dollar_map[r['product']] \n",
" poly_vals['merged_value'] = poly_vals.apply (lambda row: get_cash_value(row, True), axis=1)\n",
" poly_vals['direct_value'] = poly_vals.apply (lambda row: get_cash_value(row, False), axis=1)\n",
" poly_vals['ticks'] = poly_vals.apply (lambda row: get_ticks(row), axis=1)\n",
" return poly_vals"
]
},
{
"cell_type": "code",
"execution_count": 164,
"metadata": {},
"outputs": [],
"source": [
"# adds opp first send time, last send time across\n",
"# all rows\n",
"def add_opp_start_times(poly_vals):\n",
" # this is the groupby method that acts on a single opp\n",
" def analyze_single_opp(single_opp_pvs):\n",
" opp_first_p_st = single_opp_pvs['first_s_time'].iloc[1]\n",
" opp_last_p_st = single_opp_pvs['last_s_time'].iloc[1]\n",
" return [opp_first_p_st,opp_last_p_st]\n",
" rv = poly_vals.groupby('opp_id').apply(lambda x: analyze_single_opp(x))\n",
" st_df = rv.to_frame().reset_index()\n",
" st_df.columns = ['opp_id','st']\n",
" st_df['opp_first_s_time'] = st_df['st'].apply(lambda x:x[0])\n",
" st_df['opp_last_s_time'] = st_df['st'].apply(lambda x:x[1])\n",
" st_df=st_df[['opp_id','opp_first_s_time','opp_last_s_time']]\n",
" # now join it with opps to repeat the values across \n",
" return poly_vals.merge(st_df,how='inner',left_on=['opp_id'],right_on=['opp_id'])"
]
},
{
"cell_type": "code",
"execution_count": 165,
"metadata": {},
"outputs": [],
"source": [
"# filters out rows that dont meet the edge criteria specified\n",
"# in edge_map\n",
"def get_edge_filtered_rows(poly_vals, edge_map):\n",
" def apply_edge_filter(r):\n",
" return r['ticks'] >= edge_map[r['product']]\n",
" poly_vals['passes_filter'] = poly_vals.apply (lambda r: apply_edge_filter(r), axis=1)\n",
" poly_vals = poly_vals[poly_vals['passes_filter']==True]\n",
" del poly_vals['passes_filter']\n",
" return poly_vals"
]
},
{
"cell_type": "code",
"execution_count": 166,
"metadata": {},
"outputs": [],
"source": [
"def enrich_poly_vals(poly_vals_db, edge_map): \n",
" # ticks for the edge\n",
" poly_vals = add_ticks(poly_vals_db)\n",
" # filter for ticks\n",
" poly_vals = get_edge_filtered_rows(poly_vals, edge_map)\n",
" # add opp_start opp_first_s_time and last_first_s_time\n",
" poly_vals = add_opp_start_times(poly_vals)\n",
" poly_vals['proc_time'] = poly_vals['last_s_time']-poly_vals['t_time']\n",
" poly_vals['n_packets'] = 1+poly_vals['last_sn']-poly_vals['first_sn']\n",
" poly_vals['send_time_delta'] = poly_vals['last_s_time']-poly_vals['first_s_time']\n",
" poly_vals['t_opp_first_send'] = poly_vals['t_time']-poly_vals['opp_first_s_time']\n",
" poly_vals['t_opp_last_send'] = poly_vals['t_time']-poly_vals['opp_last_s_time']\n",
" # add opp_p_id\n",
" poly_vals = add_opp_p_ids(poly_vals)\n",
" # hide some cols \n",
" rv=poly_vals[['status', 'product', 'vid', 'pid', 'symbol', 'opp_id', 'opp_p_id', 'eid', 't_time', 'h_t_time',\n",
" 'n_packets', 'send_time_delta', 'proc_time',\n",
" 't_opp_first_send', 't_opp_last_send',\n",
" 'side', 'direct_value', 'direct_qty', 'merged_value', 'merged_qty', 'ticks',\n",
" 'poly_bid_bk', 'poly_ask_bk',\n",
" 'contract', 'change', 'ok', 'order_side', 'price','qty',\n",
" 'book_1_sid', 'book_1_eid', 'book_1_side','book_1_bid_bk', 'book_1_ask_bk', \n",
" 'book_2_sid', 'book_2_eid','book_2_side', 'book_2_bid_bk', 'book_2_ask_bk',\n",
" 'book_3_sid', 'book_3_eid', 'book_3_side', 'book_3_bid_bk', 'book_3_ask_bk',\n",
" 'book_4_sid', 'book_4_eid', 'book_4_side', 'book_4_bid_bk', 'book_4_ask_bk']]\n",
" # sort by opp_id, pid, eid (to have consecutive rows for one polygon) \n",
" # rv = rv.sort_values(['opp_id', 'pid', 'eid'], ascending=[True, True, True])\n",
" rv = rv.sort_values(['opp_p_id', 'eid'], ascending=[True, True])\n",
" return rv"
]
},
{
"cell_type": "code",
"execution_count": 167,
"metadata": {},
"outputs": [],
"source": [
"def get_enriched_poly_vals(full_db_name, edge_map):\n",
" products = edge_map.keys()\n",
" poly_vals_db = get_poly_vals_db(full_db_name, products)\n",
" poly_vals = enrich_poly_vals(poly_vals_db, edge_map)\n",
" poly_vals_summ = tabularize_opp_summ(summarize_opportunities(poly_vals))\n",
" appended_products = ''\n",
" for p in products:\n",
" appended_products = appended_products + '_' + p\n",
" file_name = full_db_name + '.sniper_opps' + appended_products + '.csv'\n",
" summ_file_name = full_db_name + '.sniper_opps_summ' + appended_products + '.csv'\n",
" poly_vals.to_csv(file_name)\n",
" poly_vals_summ.to_csv(summ_file_name)\n",
" return [poly_vals,poly_vals_summ]"
]
},
{
"cell_type": "code",
"execution_count": 168,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"select * from polygon_value_events where (product='xxxGC' or product='HG' or product='xxxSI')\n"
]
}
],
"source": [
"g_pvs=get_enriched_poly_vals('/home/taha/sparkdata/simulations/comex/2020-12-01_360_24h.sqlite',\n",
" {'xxxGC':1, 'HG':-1000000000000, 'xxxSI':3})\n",
"#g_r = pd.read_csv('/home/taha/sparkdata/simulations/comex/Output/2020-12-01-HG_new.csv')\n",
"#g_r = enrich_poly_vals(g_r,{'HG':-1000000000000})\n",
"#g_r.to_csv('/home/taha/sparkdata/simulations/comex/Output/2020-12-01-HG_new_out.csv')\n",
"#g_r.columns\n",
"#g_r_summ = summarize_opportunities(g_r)\n",
"#g_r_summ.to_csv('/home/taha/sparkdata/simulations/comex/Output/2020-12-01-HG_new_summ_out.csv')"
]
},
{
"cell_type": "code",
"execution_count": 110,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"select * from polygon_value_events where (product='HG')\n"
]
}
],
"source": [
"g_pvs=get_enriched_poly_vals('/home/taha/sparkdata/simulations/comex/2020-12-01_360_24h.sqlite',\n",
" {'HG':-100000000000000})"
]
},
{
"cell_type": "code",
"execution_count": 156,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0 0.000000e+00\n",
"1 1.250000e+01\n",
"2 1.250000e+01\n",
"3 1.250000e+01\n",
"4 1.250000e+01\n",
" ... \n",
"62 -2.305842e+10\n",
"55 -2.305842e+10\n",
"56 1.250000e+01\n",
"59 1.250000e+01\n",
"61 -2.305842e+10\n",
"Name: merged_value, Length: 63, dtype: float64"
]
},
"execution_count": 156,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"g_pvs[0].shape\n",
"g_pvs[0].merged_value"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
@@ -1,331 +0,0 @@
#!/usr/bin/python3
import pandas as pd
import os
import sqlite3
import pathlib
import re
store_root = pathlib.Path.home() / pathlib.Path("sparkdata/data")
def set_pcap_store(store):
store_root = store
def get_channel_path(channel):
return store_root / "pcaps" / pathlib.Path(channel)
def get_db_path(channel, date):
return get_channel_path(channel) / pathlib.Path(date+"_"+channel+"_24h.sqlite")
def get_pcap_db(channel,date):
path = get_db_path(channel,date)
assert os.path.exists(path)
return sqlite3.connect(path)
def get_available_days(channel):
result = []
rx = "^(\\d\\d\\d\\d-\\d\\d-\\d\\d)_{}_24h\\.sqlite$".format(channel)
for f in os.listdir(get_channel_path(channel)):
m = re.match(rx,f)
if m:
result.append(m.groups()[0])
result.sort()
return result
def get_sim_db():
from pathlib import Path
path = store_root / Path("databases") / Path("sim_db") /Path("sim_db.sqlite")
assert os.path.exists(path)
return sqlite3.connect(path)
def qsim(query):
with get_sim_db() as conn:
return pd.read_sql_query(query,conn)
def get_db(channel,date):
def querier(query):
with get_pcap_db(channel,date) as conn:
return pd.read_sql_query(query,conn)
return querier
def query_pcap(channel,date,query):
return get_db(channel,date)(query)
def get_pcap_books(channel,date):
return get_db(channel,date)("select * from top_of_books")
def clean_pcap_books(books):
books = books.rename(columns={'eid':'eid1'})
books = books.drop(['t_time','sid','status'],axis=1)
books = books.astype({'bid':'float64','bid_qty':'float64',
'ask':'float64','ask_qty':'float64'})
books['eid2'] = 0
return books
def clean_sim_books(books):
books = books.rename(columns={'val1':'eid1', 'val2':'eid2','val4':'symbol',
'val5':'bid','val6':'bid_qty', 'val7':'ask','val8':'ask_qty'})
books = books.drop(['log_entry_order','t_time','h_t_time',
'run_id','val3','entry_type','val9','val10'],axis=1)
books = books.astype({'eid1':'int32','eid2':'int32',
'bid':'float64','bid_qty':'float64',
'ask':'float64','ask_qty':'float64'})
return books
def get_pcap_ops(channel,date):
def clean_pcap_ops(ops):
clean_ops = ops[['symbol',
'merged_value',
'merged_qty',
'begin_eid',
'min_n_orders',
'end_eid',
'observable_duration',
'duration',
'transaction_duration']].copy()
clean_ops['event_len']=clean_ops.end_eid-clean_ops.begin_eid+1
clean_ops=clean_ops.rename({'merged_value':'val',
'merged_qty':'qty',
'min_n_orders':'n_ord',
'observable_duration':'o_duration',
'transaction_duration':'t_duration'},axis=1)
clean_ops=clean_ops.set_index('begin_eid')
clean_ops=clean_ops[['val','qty','n_ord',
'duration','o_duration',
't_duration','event_len']]
clean_ops = clean_ops.assign(total_val=clean_ops.val*clean_ops.qty)
return clean_ops
return clean_pcap_ops(get_db(channel,date)("select * from opportunities"))
def get_estimated_total_opportunity(channel,date):
ops = get_pcap_ops(channel,date)
return ops[(ops.val>=7) & (ops.o_duration>20)].total_val.sum()
def get_sim_ops():
def clean_sim_ops(ops):
clean_ops=ops.rename({'val2':'val','val3':'qty','val4':'o_duration',
'val5':'duration','val6':'t_duration','val7':'event_len','val8':'begin_eid'},axis=1)
clean_ops=clean_ops.astype({'begin_eid':'int64'})
clean_ops = clean_ops.set_index('begin_eid')
clean_ops=clean_ops[['val','qty','duration','o_duration','t_duration','event_len']]
clean_ops=clean_ops.astype('float')
clean_ops = clean_ops.assign(total_val=clean_ops.val*clean_ops.qty)
return clean_ops
raw_sim_ops = qsim("select * from log where entry_type='opportunity_end'")
return clean_sim_ops(raw_sim_ops)
def get_310_lined_prices(books):
# create an idx with all events in the range (this is so that we
# can have a row for each exchange state even if nothing visibly
# changed in those events)
idx = pd.DataFrame({'eid1': range(books.eid1.min(),books.eid1.max()+1)})
idx['eid2'] = 0
# Filter for the symbols we care
u=books[books.symbol=='ESU0'].drop(['symbol'],axis=1)
z=books[books.symbol=='ESZ0'].drop(['symbol'],axis=1)
u_z=books[books.symbol=='ESU0-ESZ0'].drop(['symbol'],axis=1)
# Line up symbols so we have an event time line that includes all symbols
prices = idx
prices = prices.merge(u,
how='outer',
left_on=['eid1','eid2'],
right_on=['eid1','eid2'])
prices = prices.rename({'bid':'bid_u','ask':'ask_u',
'bid_qty':'bid_qty_u','ask_qty':'ask_qty_u'},axis=1)
prices = prices.merge(z,
how='outer',
left_on=['eid1','eid2'],
right_on=['eid1','eid2'])
prices = prices.rename({'bid':'bid_z','ask':'ask_z',
'bid_qty':'bid_qty_z','ask_qty':'ask_qty_z'},axis=1)
prices = prices.merge(u_z,
how='outer',
left_on=['eid1','eid2'],
right_on=['eid1','eid2'])
prices = prices.rename({'bid':'bid_u_z','ask':'ask_u_z',
'bid_qty':'bid_qty_u_z','ask_qty':'ask_qty_u_z'},axis=1)
# set eid1, eid2 as indexes and then sort by them so that ffill
# below will propagate the last price in event orderG
prices = prices.set_index(['eid1','eid2'],drop=False)
prices = prices.sort_index()
# Fill holes (every event only updates one symbol) with previous value of book
prices = prices.ffill()
# Add columns to calculate polygon values for buy and sell
prices['buy_p_value'] = (-prices.ask_z+prices.bid_u+prices.bid_u_z)/2
prices['sell_p_value'] = (prices.bid_z-prices.ask_u-prices.ask_u_z)/2
return prices
def get_shot_outcomes(run_id=1):
with get_sim_db() as conn:
return pd.read_sql_query("select * from shot_outcome where run_id="+str(run_id),conn)
def get_opp(*dates):
result = pd.DataFrame()
for date in dates:
with sqlite3.connect("/home/ppinto/310/"+date+"_310_24h.sqlite") as conn:
result = result.append(pd.read_sql_query(("select '{}' date,\"sum(merged_qty*merged_value)\" "
"opportunity from total_opportunity").format(date),conn))
return result
book_cache = None;
def books():
def books_from_db():
with get_md_conn() as conn:
df = pd.read_sql_query("select top_of_books.*,events.t_time "
"from top_of_books, events "
"where top_of_books.eid=events.eid",conn)
df = df.fillna(0)
df = df.astype({'eid':'int32','sid':'int32',
't_time':'uint64',
'bid':'int32','bid_qty':'int32',
'ask':'int32','ask_qty':'int32'})
df = df.set_index('eid')
df = df.sort_index()
return df
global book_cache
if not book_cache is None:
return book_cache
elif os.path.exists("/tmp/books.parquet"):
book_cache = pd.read_parquet("/tmp/books.parquet")
return book_cache;
else:
book_cache = books_from_db()
book_cache.to_parquet("/tmp/books.parquet")
return book_cache
def events_from_db():
with get_md_conn() as conn:
df = pd.read_sql_query("select * from events",conn)
df = df.fillna(0)
df = df.astype({'eid':'int32','t_time':'uint64',
'first_sn':'uint64','last_sn':'uint64',
'first_st':'uint64','last_st':'uint64'})
return df
def securities():
with get_md_conn() as conn:
return pd.read_sql_query("select * from securities",conn)
def get_event_range():
with get_md_conn() as conn:
with conn.cursor() as cursor:
cursor.execute("select min(eid),max(eid) from events")
min,max =cursor.fetchall()[0]
return int(min),int(max)
def events_from_parquet():
return pd.read_parquet("/home/ppinto/events.parquet")
def get_sec_book(symbol):
b = books()
s = b[b.symbol==symbol]
s = s.reindex(range(0,b.shape[0]+1),method='ffill')
s = s.fillna(0)
s = s.astype({'bid':'int32','bid_qty':'int32','ask_qty':'int32',
'ask':'int32','t_time':'uint64'})
s = s.drop(columns=['symbol','sid'])
return s
def line_up_books(*symbols):
result = None
for s in symbols:
df = get_sec_book(s)
df = df.add_suffix('_'+s)
if result is None:
result = df
else:
result = result.join(df)
return result
def upload_lined_books(df):
book_f = "/tmp/lined_up_books.csv"
df.to_csv(book_f);
with get_md_conn() as conn:
with conn.cursor() as cursor:
with open(book_f) as f:
cursor.copy_from(f,"lined_up_books",sep=",")
def price_series(symbol):
b = books()
s=b[b.symbol==symbol]
s = s[(s.shift(1).bid != s.bid) | (s.shift(1).ask != s.ask)]
s=s.copy()
s['delta'] = s.t_time - s.shift(1).t_time
return s
def get_polygon_valuation():
lined_up = line_up_books('ESU0','ESZ0','ESU0-ESZ0')
lined_up['buy']=-lined_up.ask_ESZ0+lined_up.bid_ESU0+lined_up['bid_ESU0-ESZ0']
lined_up['sell']=lined_up.bid_ESZ0-lined_up.ask_ESU0-lined_up['ask_ESU0-ESZ0']
return lined_up
def filter_for_opportunities(p):
p = p[ ((p.buy>0) | (p.shift(1).buy>0) | (p.shift(-1).buy>0)) |
((p.sell>0) | (p.shift(1).sell>0) | (p.shift(-1).sell>0)) ]
return p
def count_ibs(fills,begin_eid,event_len):
result = { 'SIDE_SELL' : {} , 'SIDE_BUY':{}}
icebergs = spartan.query_pcap("310","2020-09-10","select * from icebergs")
ib_set = set(icebergs.ok)
fills_in_range = fills.loc[begin_eid:begin_eid+event_len]
for index,row in fills_in_range.iterrows():
ok=row.ok
fill_count=row.fill_count
side_result = result[row.side]
if ok in ib_set:
side_result[ok] = side_result.get(ok,0) + fill_count
else:
side_result[0] = side_result.get(0,0) + fill_count
return result
def ib_percentage_side(side_ibs):
ib_fills=0
total_fills=0
for (ok,fill_count) in side_ibs.items():
total_fills += fill_count
if ok!=0:
ib_fills+=fill_count
if total_fills >0:
return ib_fills*100/total_fills
else:
return 0
def ib_percentage(fills,begin_eid,event_len):
ibs = count_ibs(fills,begin_eid,event_len)
return {
'SIDE_SELL':ib_percentage_side(ibs['SIDE_SELL']),
'SIDE_BUY':ib_percentage_side(ibs['SIDE_BUY'])
}
def parse_latency_logs(file,out_path):
rx = ".*\|Latency.*\|PROD\|(.*)"
with open(out_path,mode='w') as out_file:
with open(file) as f:
for l in f.readlines():
m = re.match(rx,l)
if m:
log = m.groups()[0]
log = log.strip()
log = log.replace("::","_")
row = { nv[0]:nv[1] for nv in [ kv.split(':') for kv in log.strip().split(" ") ] }
if row['sn'] != '0':
print (row['sn'],",",row['entry_type'],",",row['app_arrival'],",",row['now'],file=out_file)
@@ -1,878 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import spartan\n",
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"pd.set_option('display.float_format', lambda x: '%.3f' % x)"
]
},
{
"cell_type": "code",
"execution_count": 155,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>val</th>\n",
" <th>qty</th>\n",
" <th>duration</th>\n",
" <th>o_duration</th>\n",
" <th>t_duration</th>\n",
" <th>event_len</th>\n",
" <th>total_val</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>count</th>\n",
" <td>13033.000</td>\n",
" <td>13033.000</td>\n",
" <td>13033.000</td>\n",
" <td>13033.000</td>\n",
" <td>13033.000</td>\n",
" <td>13033.000</td>\n",
" <td>13033.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean</th>\n",
" <td>4.893</td>\n",
" <td>2.298</td>\n",
" <td>23741.119</td>\n",
" <td>23214.200</td>\n",
" <td>527.129</td>\n",
" <td>15.662</td>\n",
" <td>16.126</td>\n",
" </tr>\n",
" <tr>\n",
" <th>std</th>\n",
" <td>4.719</td>\n",
" <td>3.935</td>\n",
" <td>637805.002</td>\n",
" <td>637815.784</td>\n",
" <td>784.926</td>\n",
" <td>111.326</td>\n",
" <td>62.237</td>\n",
" </tr>\n",
" <tr>\n",
" <th>min</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>0.000</td>\n",
" <td>-16068.000</td>\n",
" <td>69.000</td>\n",
" <td>1.000</td>\n",
" <td>2.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25%</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>45.000</td>\n",
" <td>-288.000</td>\n",
" <td>204.000</td>\n",
" <td>2.000</td>\n",
" <td>2.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50%</th>\n",
" <td>5.000</td>\n",
" <td>1.000</td>\n",
" <td>204.000</td>\n",
" <td>-89.000</td>\n",
" <td>316.000</td>\n",
" <td>5.000</td>\n",
" <td>5.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75%</th>\n",
" <td>5.000</td>\n",
" <td>2.000</td>\n",
" <td>372.000</td>\n",
" <td>10.000</td>\n",
" <td>516.000</td>\n",
" <td>13.000</td>\n",
" <td>10.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>max</th>\n",
" <td>105.000</td>\n",
" <td>134.000</td>\n",
" <td>60505274.000</td>\n",
" <td>60505187.000</td>\n",
" <td>16467.000</td>\n",
" <td>9375.000</td>\n",
" <td>1620.000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" val qty duration o_duration t_duration event_len \\\n",
"count 13033.000 13033.000 13033.000 13033.000 13033.000 13033.000 \n",
"mean 4.893 2.298 23741.119 23214.200 527.129 15.662 \n",
"std 4.719 3.935 637805.002 637815.784 784.926 111.326 \n",
"min 2.500 1.000 0.000 -16068.000 69.000 1.000 \n",
"25% 2.500 1.000 45.000 -288.000 204.000 2.000 \n",
"50% 5.000 1.000 204.000 -89.000 316.000 5.000 \n",
"75% 5.000 2.000 372.000 10.000 516.000 13.000 \n",
"max 105.000 134.000 60505274.000 60505187.000 16467.000 9375.000 \n",
"\n",
" total_val \n",
"count 13033.000 \n",
"mean 16.126 \n",
"std 62.237 \n",
"min 2.500 \n",
"25% 2.500 \n",
"50% 5.000 \n",
"75% 10.000 \n",
"max 1620.000 "
]
},
"execution_count": 155,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sim_ops=spartan.get_sim_ops()\n",
"sim_ops.describe()"
]
},
{
"cell_type": "code",
"execution_count": 170,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>val</th>\n",
" <th>qty</th>\n",
" <th>n_ord</th>\n",
" <th>duration</th>\n",
" <th>o_duration</th>\n",
" <th>t_duration</th>\n",
" <th>event_len</th>\n",
" <th>total_val</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>count</th>\n",
" <td>12567.000</td>\n",
" <td>12567.000</td>\n",
" <td>12567.000</td>\n",
" <td>12567.000</td>\n",
" <td>12567.000</td>\n",
" <td>12567.000</td>\n",
" <td>12567.000</td>\n",
" <td>12567.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean</th>\n",
" <td>5.029</td>\n",
" <td>2.246</td>\n",
" <td>1.001</td>\n",
" <td>1792.471</td>\n",
" <td>1243.548</td>\n",
" <td>549.167</td>\n",
" <td>11.710</td>\n",
" <td>13.247</td>\n",
" </tr>\n",
" <tr>\n",
" <th>std</th>\n",
" <td>6.251</td>\n",
" <td>4.264</td>\n",
" <td>0.039</td>\n",
" <td>69127.349</td>\n",
" <td>69136.590</td>\n",
" <td>806.276</td>\n",
" <td>22.684</td>\n",
" <td>44.818</td>\n",
" </tr>\n",
" <tr>\n",
" <th>min</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>1.000</td>\n",
" <td>0.000</td>\n",
" <td>-16068.000</td>\n",
" <td>69.000</td>\n",
" <td>1.000</td>\n",
" <td>2.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25%</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>1.000</td>\n",
" <td>42.000</td>\n",
" <td>-309.000</td>\n",
" <td>213.000</td>\n",
" <td>2.000</td>\n",
" <td>2.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50%</th>\n",
" <td>5.000</td>\n",
" <td>1.000</td>\n",
" <td>1.000</td>\n",
" <td>193.000</td>\n",
" <td>-101.000</td>\n",
" <td>325.000</td>\n",
" <td>5.000</td>\n",
" <td>5.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75%</th>\n",
" <td>5.000</td>\n",
" <td>2.000</td>\n",
" <td>1.000</td>\n",
" <td>349.000</td>\n",
" <td>0.000</td>\n",
" <td>538.000</td>\n",
" <td>13.000</td>\n",
" <td>10.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>max</th>\n",
" <td>197.500</td>\n",
" <td>134.000</td>\n",
" <td>3.000</td>\n",
" <td>5270850.000</td>\n",
" <td>5270731.000</td>\n",
" <td>16467.000</td>\n",
" <td>646.000</td>\n",
" <td>1520.000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" val qty n_ord duration o_duration t_duration \\\n",
"count 12567.000 12567.000 12567.000 12567.000 12567.000 12567.000 \n",
"mean 5.029 2.246 1.001 1792.471 1243.548 549.167 \n",
"std 6.251 4.264 0.039 69127.349 69136.590 806.276 \n",
"min 2.500 1.000 1.000 0.000 -16068.000 69.000 \n",
"25% 2.500 1.000 1.000 42.000 -309.000 213.000 \n",
"50% 5.000 1.000 1.000 193.000 -101.000 325.000 \n",
"75% 5.000 2.000 1.000 349.000 0.000 538.000 \n",
"max 197.500 134.000 3.000 5270850.000 5270731.000 16467.000 \n",
"\n",
" event_len total_val \n",
"count 12567.000 12567.000 \n",
"mean 11.710 13.247 \n",
"std 22.684 44.818 \n",
"min 1.000 2.500 \n",
"25% 2.000 2.500 \n",
"50% 5.000 5.000 \n",
"75% 13.000 10.000 \n",
"max 646.000 1520.000 "
]
},
"execution_count": 170,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pcap_ops = spartan.get_pcap_ops(\"310\",\"2020-09-10\")\n",
"pcap_ops.describe()"
]
},
{
"cell_type": "code",
"execution_count": 158,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"count 13033.000\n",
"mean 16.126\n",
"std 62.237\n",
"min 2.500\n",
"25% 2.500\n",
"50% 5.000\n",
"75% 10.000\n",
"max 1620.000\n",
"Name: total_val, dtype: float64"
]
},
"execution_count": 158,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sim_ops.total_val.describe()\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 160,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"count 12567.000\n",
"mean 13.247\n",
"std 44.818\n",
"min 2.500\n",
"25% 2.500\n",
"50% 5.000\n",
"75% 10.000\n",
"max 1520.000\n",
"Name: total_val, dtype: float64"
]
},
"execution_count": 160,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pcap_ops.total_val.describe()"
]
},
{
"cell_type": "code",
"execution_count": 164,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>val</th>\n",
" <th>qty</th>\n",
" <th>duration</th>\n",
" <th>o_duration</th>\n",
" <th>t_duration</th>\n",
" <th>event_len</th>\n",
" <th>total_val</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>count</th>\n",
" <td>2936.000</td>\n",
" <td>2936.000</td>\n",
" <td>2936.000</td>\n",
" <td>2936.000</td>\n",
" <td>2936.000</td>\n",
" <td>2936.000</td>\n",
" <td>2936.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean</th>\n",
" <td>4.929</td>\n",
" <td>2.616</td>\n",
" <td>104627.286</td>\n",
" <td>104322.678</td>\n",
" <td>304.103</td>\n",
" <td>41.622</td>\n",
" <td>21.446</td>\n",
" </tr>\n",
" <tr>\n",
" <th>std</th>\n",
" <td>5.233</td>\n",
" <td>5.822</td>\n",
" <td>1340822.742</td>\n",
" <td>1340827.561</td>\n",
" <td>341.782</td>\n",
" <td>231.198</td>\n",
" <td>87.040</td>\n",
" </tr>\n",
" <tr>\n",
" <th>min</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>98.000</td>\n",
" <td>21.000</td>\n",
" <td>70.000</td>\n",
" <td>1.000</td>\n",
" <td>2.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25%</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>266.000</td>\n",
" <td>49.000</td>\n",
" <td>105.000</td>\n",
" <td>8.000</td>\n",
" <td>2.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50%</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>469.500</td>\n",
" <td>134.000</td>\n",
" <td>210.000</td>\n",
" <td>16.000</td>\n",
" <td>5.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75%</th>\n",
" <td>5.000</td>\n",
" <td>2.000</td>\n",
" <td>1061.250</td>\n",
" <td>598.250</td>\n",
" <td>329.000</td>\n",
" <td>32.250</td>\n",
" <td>10.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>max</th>\n",
" <td>60.000</td>\n",
" <td>134.000</td>\n",
" <td>60505274.000</td>\n",
" <td>60505187.000</td>\n",
" <td>3849.000</td>\n",
" <td>9375.000</td>\n",
" <td>1520.000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" val qty duration o_duration t_duration event_len \\\n",
"count 2936.000 2936.000 2936.000 2936.000 2936.000 2936.000 \n",
"mean 4.929 2.616 104627.286 104322.678 304.103 41.622 \n",
"std 5.233 5.822 1340822.742 1340827.561 341.782 231.198 \n",
"min 2.500 1.000 98.000 21.000 70.000 1.000 \n",
"25% 2.500 1.000 266.000 49.000 105.000 8.000 \n",
"50% 2.500 1.000 469.500 134.000 210.000 16.000 \n",
"75% 5.000 2.000 1061.250 598.250 329.000 32.250 \n",
"max 60.000 134.000 60505274.000 60505187.000 3849.000 9375.000 \n",
"\n",
" total_val \n",
"count 2936.000 \n",
"mean 21.446 \n",
"std 87.040 \n",
"min 2.500 \n",
"25% 2.500 \n",
"50% 5.000 \n",
"75% 10.000 \n",
"max 1520.000 "
]
},
"execution_count": 164,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sim_ops[sim_ops.o_duration>20].describe()"
]
},
{
"cell_type": "code",
"execution_count": 165,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>val</th>\n",
" <th>qty</th>\n",
" <th>duration</th>\n",
" <th>o_duration</th>\n",
" <th>t_duration</th>\n",
" <th>event_len</th>\n",
" <th>total_val</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>count</th>\n",
" <td>2386.000</td>\n",
" <td>2386.000</td>\n",
" <td>2386.000</td>\n",
" <td>2386.000</td>\n",
" <td>2386.000</td>\n",
" <td>2386.000</td>\n",
" <td>2386.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean</th>\n",
" <td>4.580</td>\n",
" <td>2.508</td>\n",
" <td>8494.438</td>\n",
" <td>8179.782</td>\n",
" <td>314.147</td>\n",
" <td>26.096</td>\n",
" <td>15.946</td>\n",
" </tr>\n",
" <tr>\n",
" <th>std</th>\n",
" <td>5.591</td>\n",
" <td>6.128</td>\n",
" <td>158496.958</td>\n",
" <td>158499.021</td>\n",
" <td>352.171</td>\n",
" <td>39.265</td>\n",
" <td>71.275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>min</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>98.000</td>\n",
" <td>21.000</td>\n",
" <td>73.000</td>\n",
" <td>1.000</td>\n",
" <td>2.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25%</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>255.000</td>\n",
" <td>43.000</td>\n",
" <td>109.000</td>\n",
" <td>8.000</td>\n",
" <td>2.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50%</th>\n",
" <td>2.500</td>\n",
" <td>1.000</td>\n",
" <td>420.000</td>\n",
" <td>106.000</td>\n",
" <td>217.000</td>\n",
" <td>15.000</td>\n",
" <td>5.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75%</th>\n",
" <td>5.000</td>\n",
" <td>2.000</td>\n",
" <td>786.750</td>\n",
" <td>392.750</td>\n",
" <td>339.000</td>\n",
" <td>28.000</td>\n",
" <td>7.500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>max</th>\n",
" <td>105.000</td>\n",
" <td>134.000</td>\n",
" <td>5270850.000</td>\n",
" <td>5270731.000</td>\n",
" <td>3849.000</td>\n",
" <td>646.000</td>\n",
" <td>1520.000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" val qty duration o_duration t_duration event_len \\\n",
"count 2386.000 2386.000 2386.000 2386.000 2386.000 2386.000 \n",
"mean 4.580 2.508 8494.438 8179.782 314.147 26.096 \n",
"std 5.591 6.128 158496.958 158499.021 352.171 39.265 \n",
"min 2.500 1.000 98.000 21.000 73.000 1.000 \n",
"25% 2.500 1.000 255.000 43.000 109.000 8.000 \n",
"50% 2.500 1.000 420.000 106.000 217.000 15.000 \n",
"75% 5.000 2.000 786.750 392.750 339.000 28.000 \n",
"max 105.000 134.000 5270850.000 5270731.000 3849.000 646.000 \n",
"\n",
" total_val \n",
"count 2386.000 \n",
"mean 15.946 \n",
"std 71.275 \n",
"min 2.500 \n",
"25% 2.500 \n",
"50% 5.000 \n",
"75% 7.500 \n",
"max 1520.000 "
]
},
"execution_count": 165,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pcap_ops[pcap_ops.o_duration>20].describe()"
]
},
{
"cell_type": "code",
"execution_count": 171,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>val</th>\n",
" <th>qty</th>\n",
" <th>n_ord</th>\n",
" <th>duration</th>\n",
" <th>o_duration</th>\n",
" <th>t_duration</th>\n",
" <th>event_len</th>\n",
" <th>total_val</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>val</th>\n",
" <td>1.000</td>\n",
" <td>0.073</td>\n",
" <td>0.037</td>\n",
" <td>0.013</td>\n",
" <td>0.010</td>\n",
" <td>0.254</td>\n",
" <td>0.225</td>\n",
" <td>0.417</td>\n",
" </tr>\n",
" <tr>\n",
" <th>qty</th>\n",
" <td>0.073</td>\n",
" <td>1.000</td>\n",
" <td>0.124</td>\n",
" <td>-0.003</td>\n",
" <td>-0.003</td>\n",
" <td>0.068</td>\n",
" <td>0.235</td>\n",
" <td>0.648</td>\n",
" </tr>\n",
" <tr>\n",
" <th>n_ord</th>\n",
" <td>0.037</td>\n",
" <td>0.124</td>\n",
" <td>1.000</td>\n",
" <td>-0.000</td>\n",
" <td>-0.001</td>\n",
" <td>0.058</td>\n",
" <td>0.086</td>\n",
" <td>0.142</td>\n",
" </tr>\n",
" <tr>\n",
" <th>duration</th>\n",
" <td>0.013</td>\n",
" <td>-0.003</td>\n",
" <td>-0.000</td>\n",
" <td>1.000</td>\n",
" <td>1.000</td>\n",
" <td>-0.006</td>\n",
" <td>0.112</td>\n",
" <td>0.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>o_duration</th>\n",
" <td>0.010</td>\n",
" <td>-0.003</td>\n",
" <td>-0.001</td>\n",
" <td>1.000</td>\n",
" <td>1.000</td>\n",
" <td>-0.017</td>\n",
" <td>0.109</td>\n",
" <td>-0.001</td>\n",
" </tr>\n",
" <tr>\n",
" <th>t_duration</th>\n",
" <td>0.254</td>\n",
" <td>0.068</td>\n",
" <td>0.058</td>\n",
" <td>-0.006</td>\n",
" <td>-0.017</td>\n",
" <td>1.000</td>\n",
" <td>0.286</td>\n",
" <td>0.122</td>\n",
" </tr>\n",
" <tr>\n",
" <th>event_len</th>\n",
" <td>0.225</td>\n",
" <td>0.235</td>\n",
" <td>0.086</td>\n",
" <td>0.112</td>\n",
" <td>0.109</td>\n",
" <td>0.286</td>\n",
" <td>1.000</td>\n",
" <td>0.322</td>\n",
" </tr>\n",
" <tr>\n",
" <th>total_val</th>\n",
" <td>0.417</td>\n",
" <td>0.648</td>\n",
" <td>0.142</td>\n",
" <td>0.000</td>\n",
" <td>-0.001</td>\n",
" <td>0.122</td>\n",
" <td>0.322</td>\n",
" <td>1.000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" val qty n_ord duration o_duration t_duration event_len \\\n",
"val 1.000 0.073 0.037 0.013 0.010 0.254 0.225 \n",
"qty 0.073 1.000 0.124 -0.003 -0.003 0.068 0.235 \n",
"n_ord 0.037 0.124 1.000 -0.000 -0.001 0.058 0.086 \n",
"duration 0.013 -0.003 -0.000 1.000 1.000 -0.006 0.112 \n",
"o_duration 0.010 -0.003 -0.001 1.000 1.000 -0.017 0.109 \n",
"t_duration 0.254 0.068 0.058 -0.006 -0.017 1.000 0.286 \n",
"event_len 0.225 0.235 0.086 0.112 0.109 0.286 1.000 \n",
"total_val 0.417 0.648 0.142 0.000 -0.001 0.122 0.322 \n",
"\n",
" total_val \n",
"val 0.417 \n",
"qty 0.648 \n",
"n_ord 0.142 \n",
"duration 0.000 \n",
"o_duration -0.001 \n",
"t_duration 0.122 \n",
"event_len 0.322 \n",
"total_val 1.000 "
]
},
"execution_count": 171,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pcap_ops.corr(method='pearson')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
@@ -1,183 +0,0 @@
#include "PolyObserver.h"
#include "back_testing/sim/ISim.h"
#include "back_testing/sim/sim_common.h"
#include "domain/SecurityDefinition.h"
#include "domain/polygons/polygon_utils.h"
namespace bts::spartan::mdp3 {
static auto constexpr Worthless = -1.0;
struct PolyObserver::Target {
size_t polygon_id;
Polygon const polygon;
std::string const descriptor;
api::Side side;
std::string const product;
double value = Worthless;
Quantity quantity = Quantity{0};
void mark_as_worthless() { value = Worthless; }
bool has_value() const { return value > securities::market_epsilon(); }
};
struct PolyObserver::Opportunity {
Target *target;
double value;
Quantity quantity;
EventHeader start;
};
struct PolyObserver::PImpl {
std::vector<std::unique_ptr<Target>> targets;
std::optional<Opportunity> current_opportunity;
std::unordered_map<SecurityID, SimBookUpdate> books;
EventHeader last_event;
ISim *sim = nullptr;
};
PolyObserver::PolyObserver() : me(new PImpl()) {}
PolyObserver::~PolyObserver() {}
void PolyObserver::init(ISim &s, protos::SimRunOptions const &) {
me->sim = &s;
for (auto &p : s.polygons()) {
auto descriptor = polygon_descriptor(p.second, s.securities());
auto buy_t = Target{
p.first, p.second, descriptor, api::SIDE_BUY,
polygon_product(p.second, s.securities())};
auto sell_t = Target{
p.first, flip(p.second), descriptor, api::SIDE_SELL,
polygon_product(p.second, s.securities())};
me->targets.emplace_back(std::make_unique<Target>(buy_t));
me->targets.emplace_back(std::make_unique<Target>(sell_t));
}
}
void PolyObserver::finit() {}
void PolyObserver::on_replay_start(
EventHeader const &current_event,
std::vector<SimBookUpdate> const &current_books) {
me->sim->log("replay_start", current_event.event_id.first);
me->books.clear();
for (auto const &b : current_books)
me->books[b.security_id] = b;
}
void PolyObserver::eval() {
for (auto &t : me->targets) {
t->value = 0;
int qty = std::numeric_limits<int>::max();
for (auto &l : t->polygon.parts) {
auto mkt_side = side_from_ratio(l.ratio);
auto &mkt = me->books[l.sec_id].mbp_side(mkt_side);
auto this_qty = mkt[0].quantity.value();
auto this_ratio = this_qty / std::abs(l.ratio);
if (this_ratio < qty)
qty = this_ratio;
if (qty == 0) {
t->mark_as_worthless();
break;
} else {
auto cash = me->sim->get_security(l.sec_id).to_cash(mkt[0].price);
t->quantity.value() = qty;
t->value += -l.ratio * cash;
}
}
}
}
void PolyObserver::rank() {
std::sort(
me->targets.begin(), me->targets.end(),
[](std::unique_ptr<Target> const &ptr1,
std::unique_ptr<Target> const &ptr2) {
auto val1 = ptr1->value; // * ptr1->quantity.value();
auto val2 = ptr2->value; // * ptr2->quantity.value();
if (std::abs(val1 - val2) > securities::market_epsilon())
return val1 > val2;
else if (ptr1->polygon.parts.size() != ptr2->polygon.parts.size())
return ptr1->polygon.parts.size() < ptr2->polygon.parts.size();
else
return ptr1->polygon_id < ptr2->polygon_id;
});
}
void PolyObserver::try_start_new_opportunity() {
bts_assert(!me->current_opportunity);
auto best = me->targets.begin()->get();
if (best->has_value()) {
// We have an opportunity lets record it
me->current_opportunity =
Opportunity{best, best->value, best->quantity, me->last_event};
me->sim->log(
"opportunity_begin", best->polygon_id, best->descriptor,
api::Side_Name(best->side), best->value, best->quantity.value(), 0, 0,
(me->last_event.observable_time - me->last_event.transact_time)
.micros(),
me->last_event.event_id.first, me->last_event.event_id.first);
}
}
void PolyObserver::analyze() {
if (me->current_opportunity) {
auto &opportunity = *me->current_opportunity;
auto &target = *opportunity.target;
if (target.has_value()) {
// Add value and still has value, opportunity is still ongoing
me->sim->log(
"opportunity_continue", target.polygon_id, target.descriptor,
api::Side_Name(target.side), target.value, target.quantity.value(),
(me->last_event.observable_time -
opportunity.start.observable_time)
.micros(),
(me->last_event.transact_time - opportunity.start.transact_time)
.micros(),
(opportunity.start.observable_time -
opportunity.start.transact_time)
.micros(),
opportunity.start.event_id.first);
} else {
// Had value but not anymore, opportunity is concluded
me->sim->log(
"opportunity_end", target.polygon_id, target.descriptor,
api::Side_Name(target.side), opportunity.value,
opportunity.quantity.value(),
(me->last_event.observable_time -
opportunity.start.observable_time)
.micros(),
(me->last_event.transact_time - opportunity.start.transact_time)
.micros(),
(opportunity.start.observable_time -
opportunity.start.transact_time)
.micros(),
opportunity.start.event_id.first,
me->last_event.event_id.first - 1);
me->current_opportunity.reset();
try_start_new_opportunity();
}
} else {
// We did not have an opportunity before. Maybe we do now?
try_start_new_opportunity();
}
}
void PolyObserver::on_order_update(
EventHeader const &, SimOrderUpdate const &) {
bts_assert(false);
}
void PolyObserver::on_mbp(EventHeader const &h, SimBookUpdate const &mbp) {
me->last_event = h;
me->books[mbp.security_id] = mbp;
eval();
rank();
analyze();
}
void PolyObserver::on_time() {}
} // namespace bts::spartan::mdp3
@@ -1,34 +0,0 @@
#pragma once
#include "../sim/IAlgoModel.h"
#include <memory>
#include <optional>
namespace bts::spartan::mdp3 {
struct Polygon;
class PolyObserver : public IAlgoModel {
public:
PolyObserver();
~PolyObserver();
void init(ISim &, protos::SimRunOptions const &) override;
void finit() override;
void on_replay_start(
EventHeader const &, std::vector<SimBookUpdate> const &) override;
void on_order_update(EventHeader const &, SimOrderUpdate const &) override;
void on_mbp(EventHeader const &, SimBookUpdate const &) override;
void on_time() override;
private:
struct Target;
struct Opportunity;
struct PImpl;
void eval();
void rank();
void analyze();
void try_start_new_opportunity();
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
@@ -1,562 +0,0 @@
#include "SniperModel.h"
#include "domain/book/SyntheticBook.h"
#include "back_testing/sim/ISim.h"
#include "back_testing/sim/PandL.h"
#include "back_testing/sim/sim_common.h"
#include "domain/SecurityDefinition.h"
#include "domain/polygons/polygon_utils.h"
#include "back_testing/common/mdp3_utils.h"
#include "lib/utils/helper_macros.h"
#include <boost/algorithm/string.hpp>
#include <boost/container/small_vector.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
namespace bts::spartan::mdp3 {
enum class SniperModel::State { W_OPPORTUNITY, SHOOTING, PAUSED };
struct SniperModel::Target {
constexpr static double Worthless = -1.0;
size_t polygon_id;
Polygon const polygon;
boost::container::small_vector<DependentTargets const *, 3> leg_sec_info;
std::string const descriptor;
api::Side side;
double value = Worthless;
Quantity quantity = Quantity{0};
void mark_as_worthless() { value = Worthless; }
bool has_value(double v) const { return value > v; }
};
struct SniperModel::WorkingOrder {
SmallOrderChange order;
std::vector<SimOrderUpdate> fills;
};
struct SniperModel::Shot {
Target &target;
EventHeader observed;
EventHeader concluded;
double value_at_observation_ts;
Quantity quantity;
bool forced_completion;
std::unordered_map<OrderKey, WorkingOrder> working;
size_t transaction_duration;
};
struct SniperModel::DependentTargets {
SecurityDefinition definition;
SimBookUpdate book;
std::vector<Target *> dependencies;
};
struct SniperModel::PImpl {
ISim *sim = nullptr;
protos::SimRunOptions options;
int min_orders_at_top;
int max_qty;
int min_depth;
double min_value;
size_t max_transaction_latency_us;
bool double_dip;
size_t ms_to_work_shot;
size_t min_cumm_qty;
double min_cumm_edge;
size_t pay_up_ticks;
size_t throttle_ns;
double max_shot_loss;
std::set<std::string> polys;
EventHeader current_event;
size_t streak_id = 0;
State state = State::W_OPPORTUNITY;
std::unordered_map<SecurityID, DependentTargets> books = {};
std::vector<std::unique_ptr<Target>> targets = {};
std::optional<Shot> current_shot{};
};
SniperModel::SniperModel() : me(new PImpl()) {}
SniperModel::~SniperModel() {}
void SniperModel::init(ISim &s, protos::SimRunOptions const &options) {
me->sim = &s;
me->options = options;
}
void SniperModel::on_replay_start(
EventHeader const &h, std::vector<SimBookUpdate> const &books) {
auto parse_poly_filter = [](std::string const &pf) {
std::vector<std::string> polys;
boost::split(polys, pf, boost::is_any_of(","));
std::set<std::string> result;
for (std::string &s : polys) {
boost::trim(s);
if (s.size())
result.insert(s);
}
return result;
};
auto sim = me->sim;
auto options = me->options;
auto streak_id = me->streak_id;
me.reset(new PImpl{
sim, options, get_param<int>(me->options, "min_orders_at_top"),
get_param<int>(me->options, "max_qty"),
get_param<int>(me->options, "min_depth"),
get_param<double>(me->options, "min_value"),
get_param<size_t>(me->options, "max_transaction_latency_us"),
get_param<bool>(me->options, "double_dip"),
get_param<size_t>(me->options, "ms_to_work_shot"),
get_param<size_t>(me->options, "min_cumm_qty"),
get_param<double>(me->options, "min_cumm_edge"),
get_param<size_t>(me->options, "pay_up_ticks"),
get_param<size_t>(me->options, "throttle_ns"),
get_param<double>(me->options, "max_shot_loss"),
parse_poly_filter(get_param(me->options, "polys")), h, streak_id});
for (auto &p : me->sim->polygons()) {
auto desc = polygon_descriptor(p.second, me->sim->securities());
if (me->polys.size() && !me->polys.count(desc))
continue;
auto buy_t = std::make_unique<Target>(
Target{p.first, p.second, {}, desc, api::SIDE_BUY});
auto sell_t = std::make_unique<Target>(
Target{p.first, flip(p.second), {}, desc, api::SIDE_SELL});
me->targets.emplace_back(std::move(buy_t));
me->targets.emplace_back(std::move(sell_t));
}
for (auto &t : me->targets) {
for (auto &pp : t->polygon.parts) {
auto &dt = me->books[pp.sec_id];
dt.book.security_id = pp.sec_id;
dt.definition = me->sim->get_security(pp.sec_id);
t->leg_sec_info.push_back(&dt);
auto &dependencies = dt.dependencies;
dependencies.push_back(t.get());
}
}
me->sim->log("replay_start");
for (auto const &mbp : books) {
auto it = me->books.find(mbp.security_id);
if (it != me->books.end()) {
me->sim->log(
"book_at_replay_start",
me->sim->get_security(mbp.security_id).symbol(),
mbp.mbp_buy[0].price.to_double(), mbp.mbp_buy[0].quantity.value(),
mbp.mbp_sell[0].price.to_double(),
mbp.mbp_sell[0].quantity.value());
it->second.book = mbp;
}
}
}
void SniperModel::on_mbp(EventHeader const &h, const SimBookUpdate &mbp) {
me->current_event = h;
auto it = me->books.find(mbp.security_id);
if (it == me->books.end())
return;
auto &book = it->second;
book.book = mbp;
eval(book.dependencies);
switch (me->state) {
case State::W_OPPORTUNITY: {
auto best_shot = pick();
if (best_shot) {
me->streak_id++;
me->sim->start_simulation();
transition(State::SHOOTING);
fire(best_shot.value());
}
break;
}
case State::SHOOTING:
break;
case State::PAUSED:
check_if_pause_over();
break;
}
}
void SniperModel::on_order_update(
EventHeader const &h, const SimOrderUpdate &update) {
me->current_event = h;
bts_assert(me->state == State::SHOOTING);
bts_assert(me->current_shot);
auto &in_flight = me->current_shot.value();
bts_assert(in_flight.working.count(update.key));
in_flight.working[update.key].fills.push_back(update);
auto pending_orders = in_flight.working.size();
bool clean_fills = true;
for (auto const &p : in_flight.working) {
auto &these_fills = p.second.fills;
for (auto &f : these_fills) {
clean_fills =
clean_fills && (f.fill_type == SimOrderUpdate::AGGRESSIVE);
if (f.status == SimOrderUpdate::CANCELED ||
f.status == SimOrderUpdate::FILLED) {
pending_orders--;
break;
}
}
}
if (pending_orders == 0) {
std::vector<SimOrderUpdate> all_fills;
for (auto const &p : in_flight.working) {
auto &these_fills = p.second.fills;
all_fills.insert(
all_fills.end(), these_fills.begin(), these_fills.end());
}
PandL pandl(me->sim->securities());
auto cash = pandl.net_cash(all_fills);
auto commission = pandl.commission_cost(
map_commission_model(me->options.commission_model()), all_fills);
auto net = cash - commission;
in_flight.concluded = h;
me->sim->log(
"shot_outcome", in_flight.target.descriptor,
api::Side_Name(in_flight.target.side),
in_flight.observed.event_id.first, in_flight.observed.event_id.second,
in_flight.value_at_observation_ts * in_flight.quantity.value(), net,
commission, in_flight.quantity.value(),
clean_fills && !me->current_shot->forced_completion,
me->current_shot->forced_completion, in_flight.transaction_duration);
me->sim->log_pandl();
if (!me->double_dip)
transition(State::PAUSED);
else {
auto best_shot = pick();
if (best_shot) {
me->sim->log("double_dip", me->streak_id);
fire(best_shot.value());
} else
transition(State::PAUSED);
}
}
}
void SniperModel::on_time() {
switch (me->state) {
case State::W_OPPORTUNITY: {
break;
}
case State::SHOOTING: {
maybe_force_fills();
break;
}
case State::PAUSED:
check_if_pause_over();
break;
}
}
void SniperModel::maybe_force_fills() {
bts_assert(me->state == State::SHOOTING);
bts_assert(me->current_shot);
auto ts = me->sim->time();
auto shot_observed_ts = me->current_shot->observed.observable_time;
auto shot_duration = ts.value() - shot_observed_ts.value();
if (me->current_shot->forced_completion)
return;
if (me->ms_to_work_shot) {
if (shot_duration > 1000 * 1000 * me->ms_to_work_shot) {
me->sim->log("forcing_fills_timeout", shot_observed_ts.value());
me->current_shot->forced_completion = true;
} else if (me->current_shot->target.value < -me->max_shot_loss) {
me->sim->log("forcing_fills_maxloss", shot_observed_ts.value());
me->current_shot->forced_completion = true;
}
}
if (!me->current_shot->forced_completion)
return;
auto &in_flight = me->current_shot.value();
for (auto &p : in_flight.working) {
auto &order = p.second.order;
auto &these_fills = p.second.fills;
auto total_fills = 0;
for (auto &f : these_fills) {
bts_assert(f.status != SimOrderUpdate::CANCELED);
total_fills += f.last_fills.value();
}
if (order.order.quantity.value() > total_fills) {
Price worst_price = order.order.price;
if (order.order.side == Side::Buy)
worst_price += me->sim->get_security(order.order.security)
.min_price_increment_int_price() *
100;
else
worst_price -= me->sim->get_security(order.order.security)
.min_price_increment_int_price() *
100;
order.change = OrderChange::updated_price;
order.order.price = worst_price;
me->sim->send(order);
}
}
}
void SniperModel::eval(std::vector<Target *> const &targets) {
for (auto &t : targets) {
t->value = 0;
int qty = me->max_qty;
size_t p_idx = 0;
for (auto &l : t->polygon.parts) {
auto mkt_side = other_side(side_from_ratio(l.ratio));
auto &leg_sec_info = t->leg_sec_info[p_idx++];
auto &mkt = leg_sec_info->book.mbp_side(mkt_side);
auto this_qty = mkt[0].quantity.value();
auto this_ratio = this_qty / std::abs(l.ratio);
if (this_ratio < qty)
qty = this_ratio;
if (qty == 0 || mkt[0].n_orders < me->min_orders_at_top ||
mkt.total_depth().value() < me->min_depth) {
t->mark_as_worthless();
break;
} else {
auto cash =
leg_sec_info->definition.to_cash(mkt[0].price.to_double());
t->quantity.value() = qty;
t->value += -l.ratio * cash;
}
}
}
}
bool SniperModel::within_safety_margin(Target const &t) const {
SyntheticBook synth_book;
for (auto &leg : t.polygon.parts) {
auto leg_book_side = other_side(side_from_ratio(leg.ratio));
synth_book.add_leg(LegDefinition {
leg.sec_id,
other_side(side_from_ratio(leg.ratio)),
Quantity(leg.ratio)
}, &me->books[leg.sec_id].book.mbp_side(leg_book_side));
}
auto cumm_qty = 0ul;
auto min_price = me->min_cumm_edge;
// This is not right, exactly, we are assuming all the legs are
// expressed in the same terms as the first.
auto &first_leg_def = t.leg_sec_info[0]->definition;
auto p_value = t.value;
// The synth book calculation uses a different convention. if a
// polygon has value the synth book will have a < 0 price.
auto s_value = -(first_leg_def.to_cash(synth_book[0].price));
// The synth book should match the first level of the polygon value.
bts_assert(std::abs(p_value - s_value) < securities::market_epsilon());
for (auto &level : synth_book) {
// Again, we need to flip the sign because the convention of the
// synth book calculation is to return <0 for polygons with
// value.
auto level_cash_price = first_leg_def.to_cash(-level.price);
// We crossed the threshold that allows us to ignore the quantity.
if (level_cash_price >= min_price)
return true;
else if (level_cash_price >= me->min_value)
cumm_qty += level.quantity.value();
else
break;
}
return cumm_qty >= me->min_cumm_qty;
}
SniperModel::MaybeShot SniperModel::pick() {
std::sort(
me->targets.begin(), me->targets.end(),
[](std::unique_ptr<Target> const &ptr1,
std::unique_ptr<Target> const &ptr2) {
auto val1 = ptr1->value * ptr1->quantity.value();
auto val2 = ptr2->value * ptr2->quantity.value();
if (std::abs(val1 - val2) > securities::market_epsilon())
return val1 > val2;
else if (ptr1->polygon.parts.size() != ptr2->polygon.parts.size())
return ptr1->polygon.parts.size() < ptr2->polygon.parts.size();
else
return ptr1->polygon_id < ptr2->polygon_id;
});
auto &best_target = *me->targets[0];
if (best_target.has_value(me->min_value)) {
bool has_any_qty = me->max_qty;
auto transaction_duration = me->current_event.observable_time.value() -
me->current_event.transact_time.value();
bool transaction_is_fresh =
transaction_duration < 1000 * me->max_transaction_latency_us;
if (has_any_qty && transaction_is_fresh &&
within_safety_margin(best_target))
return Shot{
best_target,
me->current_event,
{},
best_target.value,
best_target.quantity,
false,
{},
transaction_duration};
else
return {};
} else
return {};
}
void SniperModel::fire(Shot const &s) {
auto &shot = me->current_shot.emplace(s);
me->sim->log(
"shot_begin", s.target.descriptor, api::Side_Name(s.target.side),
s.value_at_observation_ts, s.quantity.value(), me->streak_id);
for (auto const &b : me->books) {
auto const &mbp = b.second.book;
me->sim->log(
"book_at_shot_start", me->sim->get_security(mbp.security_id).symbol(),
mbp.mbp_buy[0].price.to_double(), mbp.mbp_buy[0].quantity.value(),
mbp.mbp_sell[0].price.to_double(), mbp.mbp_sell[0].quantity.value());
}
if (me->ms_to_work_shot == 0) {
shot.forced_completion = true;
for (auto &p : shot.target.polygon.parts) {
auto part_side = from_ratio(p.ratio);
auto mkt_side = other_side(part_side);
auto &book = me->books[p.sec_id].book.mbp_side(mkt_side);
Price worst_price;
size_t available_qty = 0;
for (int i = book.size() - 1; i >= 0; i--) {
if (book[i].quantity.value()) {
worst_price = book[i].price;
for (int n = 0; n <= i; n++)
available_qty += book[n].quantity.value();
break;
}
}
UNUSED(available_qty);
if (part_side == Side::Buy)
worst_price += me->sim->get_security(p.sec_id)
.min_price_increment_int_price() *
100;
else
worst_price -= me->sim->get_security(p.sec_id)
.min_price_increment_int_price() *
100;
auto order_change = SmallOrderChange{};
order_change.order.security = p.sec_id;
order_change.order.price = worst_price;
order_change.order.quantity =
Quantity{s.quantity.value() * std::abs(p.ratio)};
order_change.order.side = part_side;
order_change.change = OrderChange::added;
order_change.order.key = me->sim->send(order_change);
shot.working[order_change.order.key].order = order_change;
order_change.change = OrderChange::removed;
me->sim->send(order_change);
}
} else {
for (auto &p : shot.target.polygon.parts) {
auto part_side = from_ratio(p.ratio);
auto mkt_side = other_side(part_side);
auto &book = me->books[p.sec_id].book.mbp_side(mkt_side);
auto limit_price = book[0].price;
auto pay_up_adj =
me->sim->get_security(p.sec_id).min_price_increment_int_price() *
me->pay_up_ticks;
if (part_side == Side::Buy)
limit_price += pay_up_adj;
else
limit_price -= pay_up_adj;
auto order_change = SmallOrderChange{};
order_change.order.security = p.sec_id;
order_change.order.price = limit_price;
order_change.order.quantity =
Quantity{s.quantity.value() * std::abs(p.ratio)};
order_change.order.side = part_side;
order_change.change = OrderChange::added;
order_change.order.key = me->sim->send(order_change);
me->sim->log(
"sent_order", Side_Name(part_side),
me->sim->get_security(p.sec_id).symbol(),
book[0].price.to_double(), book[0].price.to_double(),
book[0].quantity.value());
shot.working[order_change.order.key].order = order_change;
}
}
}
void SniperModel::check_if_pause_over() {
bts_assert(me->state == State::PAUSED);
bts_assert(me->current_shot);
bts_assert(me->throttle_ns);
auto ns_since_shot_concluded =
me->sim->time().value() -
me->current_shot->concluded.observable_time.value();
if (ns_since_shot_concluded > me->throttle_ns)
me->sim->stop_simulation();
else
me->sim->log("paused");
}
const char *SniperModel::state_name(State s) {
switch (s) {
case State::SHOOTING:
return "WAIT_FILLS";
case State::W_OPPORTUNITY:
return "WAIT_OPPORTUNITY";
case State::PAUSED:
return "WAIT_OPPORTUNITY_CLEAR";
}
return nullptr;
}
void SniperModel::transition(State new_state) {
me->sim->log("transition", state_name(me->state), state_name(new_state));
me->state = new_state;
}
protos::SimRunOptions SniperModel::fill_defaults(
protos::SimRunOptions const &in) {
protos::SimRunOptions result = in;
set_param_if_empty(&result, "min_orders_at_top", 1);
set_param_if_empty(&result, "max_qty", 1);
set_param_if_empty(&result, "min_depth", 0);
set_param_if_empty(&result, "min_value", 2.0);
set_param_if_empty(&result, "max_transaction_latency_us", 10000);
set_param_if_empty(&result, "double_dip", false);
set_param_if_empty(&result, "ms_to_work_shot", 0);
set_param_if_empty(&result, "min_cumm_edge", 17.5);
set_param_if_empty(&result, "min_cumm_qty", 10);
set_param_if_empty(&result, "pay_up_ticks", 10);
set_param_if_empty(&result, "throttle_ns", 1 * 1000 * 1000 * 1000);
set_param_if_empty(&result, "max_shot_loss", 200);
set_param_if_empty(&result, "polys", "");
return result;
}
} // namespace bts::spartan::mdp3
@@ -1,47 +0,0 @@
#pragma once
#include <memory>
#include <optional>
#include "back_testing/sim/IAlgoModel.h"
namespace bts::spartan::mdp3 {
struct Polygon;
}
namespace bts::spartan::mdp3 {
class SniperModel : public IAlgoModel {
public:
SniperModel();
~SniperModel();
void init(ISim &, protos::SimRunOptions const &) override;
protos::SimRunOptions fill_defaults(protos::SimRunOptions const &) override;
void on_replay_start(
EventHeader const &, std::vector<SimBookUpdate> const &) override;
void on_mbp(EventHeader const &, SimBookUpdate const &) override;
void on_order_update(EventHeader const &, SimOrderUpdate const &) override;
void on_time() override;
private:
enum class State;
struct PImpl;
struct Target;
struct Shot;
struct WorkingOrder;
struct DependentTargets;
using MaybeShot = std::optional<Shot>;
void eval(std::vector<Target *> const &);
MaybeShot pick();
void fire(Shot const &);
bool within_safety_margin(Target const &) const;
void check_if_pause_over();
void maybe_force_fills();
void transition(State);
const char *state_name(State);
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
@@ -1,25 +0,0 @@
#include "TestAlgo.h"
namespace bts::spartan::mdp3 {
struct TestAlgo::PImpl {};
TestAlgo::TestAlgo() : _me(new TestAlgo::PImpl()) {}
TestAlgo::~TestAlgo() {}
void TestAlgo::init(ISim &, protos::SimRunOptions const &) {}
void TestAlgo::finit() {}
void TestAlgo::on_replay_start(
EventHeader const &, std::vector<SimBookUpdate> const &) {}
void TestAlgo::on_order_update(
EventHeader const &, SimOrderUpdate const &) {}
void TestAlgo::on_mbp(EventHeader const &, SimBookUpdate const &) {}
void TestAlgo::on_time() {}
} // namespace bts::spartan::mdp3
@@ -1,30 +0,0 @@
#pragma once
#include "../sim/IAlgoModel.h"
#include <memory>
namespace bts::spartan::mdp3 {
/**
* An empty implementation of IAlgoModel that serves as a placeholder for
* experimental/test code.
*/
class TestAlgo : public IAlgoModel {
public:
TestAlgo();
~TestAlgo();
void init(ISim &, protos::SimRunOptions const &) override;
void finit() override;
void on_replay_start(
EventHeader const &, std::vector<SimBookUpdate> const &) override;
void on_order_update(EventHeader const &, SimOrderUpdate const &) override;
void on_mbp(EventHeader const &, SimBookUpdate const &) override;
void on_time() override;
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
};
} // namespace bts::spartan::mdp3
@@ -1,165 +0,0 @@
// #include "SimOrderGateway.h"
// #include "SimSpreader.h"
// #include "back_testing/simulator/ISimServices.h"
// #include "lib/api/services/OrderService.pb.h"
// #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
// using namespace bts::orderman;
// using namespace bts::api::orderman;
// namespace bts::spartan::mdp3 {
// struct OrderTuple{
// const OrderKey key;
// OrderChange ochange;
// uint32_t total_fills = 0;
// double avg_filled_price = 0;
// bool order_send_acked = false;
// OrderTuple (OrderKey key,
// OrderChange ochange)
// :
// key(key),
// ochange(ochange)
// {}
// };
// OrderStatus translate_status(const OrderUpdate& ord){
// switch(ord.status){
// case(OrderUpdate::Status::CANCELED):
// return OrderStatus::ORDER_STATUS_CANCELED;
// case(OrderUpdate::Status::FILLED):
// return OrderStatus::ORDER_STATUS_COMPLETED;
// case(OrderUpdate::Status::PARTIALLY_FILLED):
// return OrderStatus::ORDER_STATUS_WORKING;
// }
// bts_assert(false);
// return OrderStatus::ORDER_STATUS_WORKING;
// }
// OrderTuple* find_ord(OrderKey key,
// std::vector<OrderTuple>& orders){
// for(auto& o : orders)
// if(o.key == key)
// return &o;
// return nullptr;
// }
// OrderTuple* find_ord(OrderKey key,
// std::vector<OrderTuple>& orders){
// for(auto& o : orders)
// if(o.ochange.order.key == key)
// return &o;
// return nullptr;
// }
// struct SimOrderGateway::PImpl {
// ISimServices* sim_services = nullptr;
// SpreaderOrder* spreader = nullptr;
// std::vector<OrderTuple> orders;
// };
// SimOrderGateway::SimOrderGateway()
// :
// me(new PImpl()){
// }
// void SimOrderGateway::init(ISimServices& sim_services,
// SpreaderOrder& spreader){
// me->sim_services = &sim_services;
// me->spreader = &spreader;
// }
// void SimOrderGateway:: place_order(SecurityKey skey,
// OrderKey okey,
// api::Side side,
// Price p,
// Quantity q){
// bts_assert(me->sim_services!=nullptr);
// const auto& t =
// me->orders.emplace_back(
// okey,
// OrderChange{
// MDP3Order(),
// MDP3Order{
// me->sim_services->generate_order_key(),
// SecurityID{static_cast<int>(skey.value())},
// side,
// MDP3OrderPriority{},
// Price_from_double<7>(p.value()),
// Quantity{static_cast<int>(q.value())},
// {}
// },
// OrderChange::added
// });
// me->sim_services->send(t.ochange);
// }
// void SimOrderGateway::modify_order(OrderKey okey,
// Price p,
// Quantity q){
// bts_assert(me->sim_services!=nullptr);
// OrderTuple* t = find_ord(okey,me->orders);
// bts_assert(t != nullptr);
// bts_assert(t->order_send_acked);
// auto& ochange = t->ochange;
// ochange.previous_order = ochange.order;
// ochange.order.price = Price_from_double<7>(p.value());
// ochange.order.quantity = Quantity{static_cast<int>(q.value())};
// ochange.change = OrderChange::updated;
// me->sim_services->send(ochange);
// }
// void SimOrderGateway::cancel_order(OrderKey okey){
// bts_assert(me->sim_services!=nullptr);
// OrderTuple* t = find_ord(okey,me->orders);
// bts_assert(t != nullptr);
// bts_assert(t->order_send_acked);
// auto& ochange = t->ochange;
// ochange.change = OrderChange::removed;
// me->sim_services->send(ochange);
// }
// OrderKey SimOrderGateway::generate_key(){
// auto rv =
// static_cast<uint32_t>(bts::Timer::now().time_since_epoch().count());
// return OrderKey(rv);
// }
// void SimOrderGateway::poll(){
// for(auto& t : me->orders){
// if(!t.order_send_acked){
// t.order_send_acked=true;
// route_update_to_spreader(
// OrderUpdate{
// t.ochange.order.key,
// OrderUpdate::Status::PARTIALLY_FILLED,
// t.ochange.order.security,
// t.ochange.order.side,
// Quantity{},
// Price()
// });
// }
// }
// }
// void SimOrderGateway::route_update_to_spreader(const OrderUpdate& u){
// OrderTuple* t = find_ord(u.key,me->orders);
// bts_assert(t!=nullptr);
// uint32_t total_fills = t->total_fills +
// static_cast<uint32_t>(u.last_fills.value());
// double avg_filled_price = 0;
// if(total_fills > 0){
// avg_filled_price = (t->total_fills*t->avg_filled_price) +
// (u.last_fills.value() * u.last_price.price());
// avg_filled_price = avg_filled_price / total_fills;
// }
// t->total_fills = total_fills;
// t->avg_filled_price = avg_filled_price;
// me->spreader->test_only_on_child_order_update(
// t->key,
// translate_status(u),
// Quantity(t->total_fills),
// Price(t->avg_filled_price)
// );
// }
// }
@@ -1,41 +0,0 @@
// #pragma once
// #include "lib/orderman/orderman_utils/orderman_common.h"
// #include "lib/domain_model/Security.pb.h"
// #include "back_testing/simulator/AlgoEvent.h"
// #include "back_testing/common/mdp3_common.h"
// #include
// "lib/orderman/orders/spreader_orders/spreader/external_interfacing/IAlternateOrderGateway.h"
// namespace bts { namespace orderman {class SpreaderOrder;}}
// namespace bts::spartan::mdp3 {
// class ISimServices;
// class SimOrderGateway : public bts::orderman::IAlternateOrderGateway{
// private:
// struct PImpl;
// PImpl* me;
// public:
// SimOrderGateway();
// ~SimOrderGateway(){}
// void init(ISimServices&,
// bts::orderman::SpreaderOrder&);
// // IAlternateOrderGateway functions
// // (SpreaderChildOrderState's calls need to end up here)
// void place_order(SecurityKey,
// bts::orderman::OrderKey,
// api::Side,
// Price,
// Quantity) override;
// void modify_order(bts::orderman::OrderKey, Price, Quantity) override;
// void cancel_order(bts::orderman::OrderKey) override;
// bts::orderman::OrderKey generate_key() override;
// // Sends ISimServices's updates to spreader
// void poll();
// void route_update_to_spreader(const OrderUpdate&);
// };
// }
@@ -1,117 +0,0 @@
// #include "SimSpreader.h"
// #include "lib/domain_model/Security.pb.h"
// #include "lib/api/services/OrderService.pb.h"
// #include "lib/sparkproto/SparkProtos.pb.h"
// #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
// #include
// "lib/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.h"
// #include "lib/orderman/orderman_utils/Book.h"
// #include "SimOrderGateway.h"
// using namespace bts::orderman;
// using namespace bts::api::orderman;
// namespace bts::spartan::mdp3 {
// void translate_book(const MBPUpdate& u_in,
// bts::orderman::Book& b_out){
// size_t i=-1;
// for (auto const& bl : u_in.mbp_buy)
// if(++i<5)
// b_out.side(api::Side::SIDE_BUY)[i] =
// md::PriceSize (bl.price,bl.quantity.value());
// i=-1;
// for (auto const& bl : u_in.mbp_sell)
// if(++i<5)
// b_out.side(api::Side::SIDE_SELL)[i] =
// md::PriceSize (bl.price,bl.quantity.value());
// }
// bool compare_keys (SecurityKey spark_k,
// SecurityID spartan_k){
// return static_cast<int>(spark_k.value()) == spartan_k.value();
// }
// bts::orderman::Book* find_book(SecurityID id,
// std::vector<bts::orderman::Book>& books){
// for (auto& b : books)
// if(compare_keys(b.security_key(),id))
// return &b;
// return nullptr;
// }
// struct SimSpreader::PImpl {
// const OrderDefinition order_def;
// const SpreaderPreActivationOrderServices pao_services;
// SpreaderOrderServices so_services;
// ISimServices* sim_services = nullptr;
// SimOrderGateway sim_order_gateway;
// SpreaderOrder spreader;
// Price price;
// Quantity quantity;
// std::vector<bts::orderman::Book> books;
// PImpl(const OrderDefinition& order_def,
// const bts::api::Security& sec,
// Price price,
// Quantity quantity)
// :
// order_def(order_def),
// pao_services(sec),
// so_services(nullptr),
// sim_order_gateway(),
// spreader(&sim_order_gateway),
// price(price),
// quantity(quantity),
// books(){
// for (const auto& l : sec.synthetic().legs())
// books.emplace_back(SecurityKey(l.security().id()));
// }
// };
// SimSpreader::SimSpreader(const OrderDefinition& order_def,
// const bts::api::Security& sec,
// Price price,
// Quantity qty)
// :
// me(new PImpl(order_def,sec,price,qty)){
// }
// const SpreaderOrder& SimSpreader::spreader() const{
// return me->spreader;
// }
// // IAlgoModel
// /////////////////////////////////////////////////////////////////
// ///////////////////////////////////////////////////////////////////////////////
// void SimSpreader::init(ISimServices& sim_services, protos::SimRunOptions
// const&){
// me->sim_services = &sim_services;
// me->sim_order_gateway.init(sim_services,me->spreader);
// // TODO (no test_ blah)
// me->spreader.test_only_on_init_new (me->order_def,
// me->pao_services);
// me->spreader.test_only_on_activate(me->price,
// me->quantity,
// me->so_services);
// }
// void SimSpreader::on_time(TimeStamp){
// me->sim_order_gateway.poll();
// }
// void SimSpreader::on_notification(AlgoNotification const& ev) {
// if(std::holds_alternative<MBPUpdate>(ev.event)){
// // book update
// const MBPUpdate& bk = std::get<MBPUpdate>(ev.event);
// // copy book
// bts::orderman::Book* leg_bk = find_book(bk.security_id, me->books);
// if(leg_bk != nullptr){
// auto& leg_bk_r = *leg_bk;
// translate_book(bk, leg_bk_r);
// me->spreader.test_only_on_book_update(leg_bk_r);
// }
// }
// else{
// const OrderUpdate& u = std::get<OrderUpdate>(ev.event);
// me->sim_order_gateway.route_update_to_spreader(u);
// }
// }
// }
@@ -1,33 +0,0 @@
// #pragma once
// #include "back_testing/common/mdp3_common.h"
// #include "back_testing/simulator/AlgoEvent.h"
// #include "back_testing/simulator/IAlgoModel.h"
// #include "SimOrderGateway.h"
// namespace bts { namespace api { namespace orderman {class OrderDefinition;}}}
// namespace bts { namespace api { class OrderDefinition;}}
// namespace bts { namespace orderman { class SpreaderOrder;}}
// namespace bts::spartan::mdp3 {
// class ISimServices;
// class SimSpreader : public IAlgoModel {
// private:
// struct PImpl;
// //std::unique_ptr<PImpl> me;
// PImpl* me;
// public:
// SimSpreader(const bts::api::orderman::OrderDefinition&,
// const bts::api::Security&,
// Price, Quantity);
// const bts::orderman::SpreaderOrder& spreader() const;
// // IAlgoModel
// void init(ISimServices&, protos::SimRunOptions const&) override;
// void on_time(TimeStamp) override;
// void on_notification(AlgoNotification const&) override;
// ~SimSpreader(){}
// };
// }
@@ -1,366 +0,0 @@
// #include <catch/catch.hpp>
// #include "lib/domain_model/Security.pb.h"
// #include
// "back_testing/algos/spreader/SimSpreader.h"
// #include "back_testing/common/mdp3_common.h"
// #include "back_testing/simulator/AlgoEvent.h"
// #include "back_testing/common/SecurityDefinition.h"
// #include "back_testing/simulator/ISimServices.h"
// #include "lib/orderman/orders/spreader_orders/tests/tests_common.h"
// #include
// "back_testing/orchestrator/SimSessionOptions.pb.h"
// using namespace bts::spartan::mdp3;
// namespace bts {
// namespace orderman {
// using namespace api;
// using namespace spark;
// class TestSimServices : public bts::spartan::mdp3::ISimServices{
// private:
// SecurityMap _instruments;
// std::unordered_map<size_t,Polygon> _polygons;
// protected:
// void _log(std::string const &,
// std::initializer_list<std::string> const &) override {}
// public:
// SmallOrderChange last_change;
// size_t n_updates = 0;
// size_t key;
// public:
// TestSimServices ()
// :
// key(0){
// }
// SecurityMap const &instruments() const override{
// bts_assert(false);
// return _instruments;
// }
// void terminate_simulation() override {}
// void log_pandl() override {}
// std::unordered_map<size_t,Polygon> const&polygons() const override {
// return _polygons;
// }
// spartan::mdp3::Book book(SecurityID,api::Side) const override {
// return {};
// }
// OrderKey generate_order_key() override{
// return OrderKey(key++);
// }
// TimeStamp now() const override{
// return TimeStamp(0);
// }
// void wait(uint64_t) override{
// }
// void send(const SmallOrderChange& c, uint64_t) override{
// last_change = c;
// n_updates++;
// }
// ~TestSimServices(){}
// };
// bts::spartan::mdp3::MBPUpdate get_book_update(
// const std::vector<std::tuple<double,uint32_t>>& bids,
// const std::vector<std::tuple<double,uint32_t>>& asks,
// uint32_t key){
// bts::spartan::mdp3::MBPUpdate rv;
// size_t i=0;
// for(auto[p,q] : bids)
// rv.mbp_buy[i++] = BookLevel{
// Price_from_double<7>(p),
// bts::spartan::mdp3::Quantity(q),
// 0
// };
// i=0;
// for(auto[p,q] : asks)
// rv.mbp_sell[i++] = BookLevel{
// Price_from_double<7>(p),
// bts::spartan::mdp3::Quantity(q),
// 0
// };
// rv.security_id = SecurityID(key);
// return rv;
// }
// TEST_CASE ("sim_spreader", "[spreader]"){
// bts::orderman::OrderKey order_key(1);
// // security
// ContractLeg leg_1, leg_2;
// fill_future_contract_leg (leg_1, 1, "CL",
// bts::api::Venue::VENUE_GLBX,
// 2020, 11, 15,
// 1.0, 1.0,
// api::Side::SIDE_BUY,
// 1.0, 0.01);
// fill_future_contract_leg (leg_2, 2, "CL",
// bts::api::Venue::VENUE_GLBX,
// 2020, 12, 15,
// -1.0, -1.0,
// api::Side::SIDE_SELL,
// 1.0, 0.01);
// std::vector<ContractLeg> legs;
// legs.push_back(leg_1);
// legs.push_back(leg_2);
// Security sec;
// fill_synthetic_security (sec, 2000, "simspreader_2legs", legs);
// // spreader params
// LegSpreaderParams lsp_1, lsp_2, lsp_3, lsp_4;
// fill_leg_spreader_params(lsp_1, 1.0, false, 1.0);
// fill_leg_spreader_params(lsp_2, -1.0, true, 1.0);
// // post hedge time out here
// std::vector<LegSpreaderParams> params;
// params.push_back(lsp_1);
// params.push_back(lsp_2);
// // putting it together in the order def
// bts::api::orderman::OrderDefinition order_def;
// fill_spreader_order_definition(order_def, Side::SIDE_BUY, sec, params);
// bts::spartan::mdp3::SimSpreader
// sim_spreader(order_def,sec,Price(-10),Quantity(1));
// //ISimServices* sim_services = new TestSimServices();
// TestSimServices sim_services;
// spartan::protos::SimRunOptions options;
// sim_spreader.init(*(ISimServices*)&sim_services,options);
// SECTION("requoting"){
// std::vector<std::tuple<double,uint32_t>> bids =
// {
// {100,5},
// {101,5},
// {102,5}
// },
// asks =
// {
// {103,5},
// {104,5},
// {105,5}
// };
// auto md_udpate_leg_1 =
// get_book_update(bids,asks,leg_1.security().id());
// // send leg_1 book update => leg_2 should quote
// sim_spreader.on_notification(
// {
// {},
// md_udpate_leg_1
// }
// );
// sim_spreader.on_time(bts::spartan::mdp3::TimeStamp());
// REQUIRE(sim_services.n_updates==1);
// REQUIRE(sim_services.last_change.order.side == api::Side::SIDE_SELL);
// REQUIRE(sim_services.last_change.order.quantity ==
// bts::spartan::mdp3::Quantity(1)); REQUIRE(
// bts::securities::is_same(Price(sim_services.last_change.order.price.price()),
// Price(113)));
// // send multiple book updates
// double p = 103;
// for(int i=0; i<100; i++){
// asks =
// {
// {p,5},
// {p+1,5},
// {p+2,5}
// };
// auto md_udpate_leg_1 =
// get_book_update(bids,asks,leg_1.security().id());
// sim_spreader.on_notification(
// {
// {},
// md_udpate_leg_1
// });
// REQUIRE(sim_services.n_updates==i+1); // +1 because order_place is
// also counted REQUIRE(sim_services.last_change.order.security ==
// bts::spartan::SecurityID(2));
// REQUIRE(sim_services.last_change.order.side ==
// api::Side::SIDE_SELL);
// REQUIRE(sim_services.last_change.order.quantity ==
// bts::spartan::mdp3::Quantity(1)); REQUIRE(
// bts::securities::is_same(Price(sim_services.last_change.order.price.price()),
// Price(p+10)));
// p++;
// }
// //force a quoting leg fill
// auto q_ord = sim_services.last_change.order;
// sim_spreader.on_notification(
// {
// {},
// bts::spartan::mdp3::OrderUpdate{
// q_ord.key,
// bts::spartan::mdp3::OrderUpdate::FILLED,
// q_ord.security,
// q_ord.side,
// bts::spartan::mdp3::Quantity(1),
// Price_from_double<7>(202)
// }
// });
// // the above should have sent a hedge order, verify that
// auto hedge_ord = sim_services.last_change.order;
// REQUIRE(hedge_ord.security == bts::spartan::SecurityID(1));
// REQUIRE(hedge_ord.side == api::Side::SIDE_BUY);
// REQUIRE(hedge_ord.quantity == bts::spartan::mdp3::Quantity(1));
// REQUIRE(
// bts::securities::is_same(Price(hedge_ord.price.price()),
// Price(192)));
// REQUIRE(sim_spreader.spreader().test_only_func_spreader_state() !=
// SpreaderState::END);
// sim_spreader.on_time(bts::spartan::mdp3::TimeStamp());
// // send the hedge fill and it should go to end state
// sim_spreader.on_notification(
// {
// {},
// bts::spartan::mdp3::OrderUpdate{
// hedge_ord.key,
// bts::spartan::mdp3::OrderUpdate::FILLED,
// hedge_ord.security,
// hedge_ord.side,
// bts::spartan::mdp3::Quantity(1),
// Price_from_double<7>(192)
// }
// });
// REQUIRE(sim_spreader.spreader().test_only_func_spreader_state() ==
// SpreaderState::END);
// }
// }
// class RV
// {
// public:
// int i;
// RV(int i_arg)
// :
// i(i_arg){
// }
// RV()=delete;
// RV(const RV&&)=delete;
// RV (RV&& other){
// i=other.i;
// other.i=0;
// }
// RV& operator=(const RV&)=delete;
// RV& operator=(RV &&src){
// i=src.i;
// src.i=0;
// return *this;
// }
// int foo (double d1, double d2){
// return static_cast<int>(d1+d2);
// }
// };
// class TTemp{
// };
// class TInterface{
// virtual void foo() = 0;
// };
// template <class T, size_t N>
// class TestContainer : public TInterface{
// private:
// T _t;
// std::array<T,N> _at;
// public:
// void add (T &t){
// _at[0] = t;
// }
// T get() {
// return _t;
// }
// void foo () override{
// }
// };
// class RVUser{
// public:
// RV rv;
// RVUser(RV& rv_arg)
// : rv(std::move(rv_arg))
// {
// }
// };
// TEST_CASE ("scratch_pad", "[spreader]"){
// int (RV::*f_ptr)(double, double) = &RV::foo;
// char* f_addr = (char*&)f_ptr;
// TInterface* r = nullptr;
// int n = gen_rand(0.0, 10.0);
// if (n <= 5)
// r = new TestContainer<TTemp,5>();
// else
// r = new TestContainer<TTemp,10>();
// REQUIRE(r != nullptr);
// RV rv_1(100);
// RVUser rvuser_1(rv_1);
// REQUIRE(rv_1.i==0);
// REQUIRE(rvuser_1.rv.i==100);
// }
// TEST_CASE ("scratch_pad_2", "[spreader]"){
// time_t rawtime;
// time (&rawtime);
// struct tm* ptm = gmtime (&rawtime);
// LOG(OrderMan, Info, "hrs:{} mins:{} sec:{}", ptm->tm_hour, ptm->tm_min,
// ptm->tm_sec);
// }
// // vector insert //////////////////////////////////////
// ///////////////////////////////////////////////////////
// class Elem{
// public:
// int i;
// Elem()
// :
// i(-1)
// {}
// Elem (int i_arg)
// :
// i(i_arg)
// {}
// Elem(const Elem& other){
// i = other.i;
// }
// Elem (Elem&& other){
// i=other.i;
// }
// Elem& operator=(const Elem& other){
// i=other.i;
// return *this;
// }
// Elem& operator=(Elem &&src){
// i = src.i;
// return *this;
// }
// bool operator==(const Elem& e) const {return i == e.i;}
// bool operator!=(const Elem& e) const {return i != e.i;}
// bool operator<(const Elem& e) const {return i < e.i;}
// bool operator>(const Elem& e) const {return i > e.i;}
// };
// TEST_CASE ("scratch_pad_3", "[spreader]"){
// std::vector<Elem> v;
// for(int i=1;i<10;i++)
// v.emplace_back(i);
// Elem e(0);
// // move op= is called when inserting or erasing
// v.insert(v.begin()+2,e);
// v.erase(v.begin()+2);
// std::random_shuffle(v.begin(),v.end());
// // flat set
// boost::container::flat_set<Elem> fs;
// for(int i=0;i<20;i+=2){
// Elem e(i);
// fs.insert(e);
// }
// Elem e9(9);
// // move op is called when inserting
// fs.insert(e9);
// for (const auto& e : fs)
// std::cout << e.i << std::endl;
// // vector assignment vs move assignment
// std::vector<Elem> vs;
// int n_elems = 10;
// for(int i=1;i<n_elems;i++)
// vs.emplace_back(i);
// std::vector<Elem> vt;
// // new elems are cresated in vt from the elem's in vs,
// // with their copy cons
// vt = vs;
// std::vector<Elem> vt_2;
// vt_2 = std::move(vs);
// }
// }}
@@ -1,9 +0,0 @@
#pragma once
#include "domain/EventID.h"
#include <boost/container/flat_map.hpp>
namespace bts::spartan::mdp3 {
using EventRangeMap = boost::container::flat_map<EventID, std::size_t>;
} // namespace bts::spartan::mdp3
@@ -1,9 +0,0 @@
#include "EventRangeMonitor.h"
namespace bts::spartan::mdp3 {
constexpr EventID &EventRangeMonitor::event() { return _event; }
constexpr const EventID &EventRangeMonitor::event() const { return _event; }
} // namespace bts::spartan::mdp3
@@ -1,35 +0,0 @@
#include "EventRangeMonitor.h"
namespace bts::spartan::mdp3 {
EventRangeMonitor::EventRangeMonitor() : _event(0), _observed_events() {}
EventRangeMonitor::EventRangeMonitor(
boost::container::flat_map<EventID, std::size_t> observed_events)
: _event(0), _observed_events(observed_events) {}
EventRangeMonitor::State EventRangeMonitor::state() const {
return peek(event());
}
EventRangeMonitor::State EventRangeMonitor::peek(EventID event_id) const {
//
if (_observed_events.empty())
return EventRangeMonitor::State::None;
auto it = _observed_events.upper_bound(event_id);
if (it == _observed_events.end()) {
it--;
return event_id < (it->first + it->second)
? EventRangeMonitor::State::Listening
: EventRangeMonitor::State::Done;
} else if (it == _observed_events.begin()) {
return EventRangeMonitor::State::Idle;
} else {
it--;
return event_id >= it->first && event_id < (it->first + it->second)
? EventRangeMonitor::State::Listening
: EventRangeMonitor::State::Idle;
}
}
} // namespace bts::spartan::mdp3
@@ -1,53 +0,0 @@
#pragma once
#include "domain/EventID.h"
#include <boost/container/flat_map.hpp>
namespace bts::spartan::mdp3 {
/**
* Signals callers when events falls within one or more particular event
* ranges
*/
class EventRangeMonitor {
public:
EventRangeMonitor();
EventRangeMonitor(
boost::container::flat_map<EventID, std::size_t> observed_events);
enum class State {
None,
Idle,
Listening,
Done
};
/**
* Gets the state associated with the current event
*
* @see event
*/
State state() const;
/**
* Gets the state associated with a particular event
*/
State peek(EventID) const;
/**
* Gets the current event
*/
constexpr EventID &event();
/**
* Gets the current event
*/
constexpr const EventID &event() const;
private:
EventID _event;
boost::container::flat_map<EventID, std::size_t> _observed_events;
};
} // namespace bts::spartan::mdp3
#include "EventRangeMonitor.cc"
@@ -1,21 +0,0 @@
#include "IcebergQuantityModification.h"
namespace bts::spartan::mdp3 {
inline bool IcebergQuantityModification::has_changes() const {
return show.has_value() || total.has_value();
}
inline std::ostream &operator<<(
std::ostream &o, const IcebergQuantityModification &m) {
if (m.show.has_value()) {
o << "show: " << m.show.value();
if (m.total.has_value())
o << ", total: " << m.total.value();
} else if (m.total.has_value()) {
o << "total: " << m.total.value();
}
return o;
}
} // namespace bts::spartan::mdp3
@@ -1,24 +0,0 @@
#include "IcebergQuantityModification.h"
#include "server/sparkd/cmds/spartan/domain/IcebergQuantity.h"
namespace bts::spartan::mdp3 {
IcebergQuantityModification IcebergQuantityModification::deduce(
const IcebergQuantity &old_iceberg,
const IcebergQuantity &new_iceberg) {
IcebergQuantityModification mod;
if (new_iceberg.show != old_iceberg.show)
mod.show = new_iceberg.show;
if (new_iceberg.total != old_iceberg.total)
mod.total = new_iceberg.total;
return mod;
}
void IcebergQuantityModification::apply(IcebergQuantity &target) const {
if (show.has_value())
target.show = show.value();
if (total.has_value())
target.total = total.value();
}
} // namespace bts::spartan::mdp3
@@ -1,40 +0,0 @@
#pragma once
#include "domain/Quantity.h"
#include <iostream>
#include <optional>
namespace bts::spartan {
struct IcebergQuantity;
}
namespace bts::spartan::mdp3 {
struct IcebergQuantityModification {
/**
* Initialize an IcebergQuantityModification with no changes
*/
static IcebergQuantityModification empty();
/**
* Initialize an IcebergQuantityModification that represents the transition
* between two states
*/
static IcebergQuantityModification deduce(
const IcebergQuantity &old_iceberg, const IcebergQuantity &new_iceberg);
/**
* Determine whether or not the modification has any changes
*/
bool has_changes() const;
/**
* Apply modifications to a particular iceberg quantity
*/
void apply(IcebergQuantity &) const;
std::optional<Quantity> show;
std::optional<Quantity> total;
};
std::ostream &operator<<(std::ostream &, const IcebergQuantityModification &);
} // namespace bts::spartan::mdp3
#include "IcebergQuantityModification.cc"
@@ -1,10 +0,0 @@
#pragma once
#include "SimBookUpdate.h"
#include "SimOrderUpdate.h"
#include <variant>
namespace bts::spartan::mdp3 {
using MatchEffect = std::variant<SimBookUpdate, SimOrderUpdate>;
} // namespace bts::spartan::mdp3
@@ -1,24 +0,0 @@
#include "OrderModification.h"
namespace bts::spartan::mdp3 {
inline OrderModification OrderModification::empty() {
return OrderModification();
}
inline bool OrderModification::has_changes() const {
return price.has_value() || quantity.has_value();
}
inline std::ostream &operator<<(std::ostream &o, const OrderModification &m) {
if (m.price.has_value()) {
o << "price: " << m.price.value();
if (m.quantity.has_value())
o << ", quantity: " << m.quantity.value();
} else if (m.quantity.has_value()) {
o << "quantity: " << m.quantity.value();
}
return o;
}
} // namespace bts::spartan::mdp3
@@ -1,38 +0,0 @@
#include "OrderModification.h"
#include "mdp3_common.h"
namespace bts::spartan::mdp3 {
OrderModification OrderModification::deduce(
const MDP3Order &old_order, const MDP3Order &new_order) {
OrderModification mod;
if (new_order.quantity != old_order.quantity)
mod.quantity = new_order.quantity;
if (new_order.price != old_order.price)
mod.price = new_order.price;
if (new_order.iceberg_quantity.has_value()) {
if (old_order.iceberg_quantity.has_value()) {
if (new_order.iceberg_quantity.value().show !=
old_order.iceberg_quantity.value().show)
mod.iceberg_quantity.show = new_order.iceberg_quantity.value().show;
if (new_order.iceberg_quantity.value().total !=
old_order.iceberg_quantity.value().total)
mod.iceberg_quantity.total = new_order.iceberg_quantity.value().total;
} else {
mod.iceberg_quantity.show = new_order.iceberg_quantity.value().show;
mod.iceberg_quantity.total = new_order.iceberg_quantity.value().total;
}
}
return mod;
}
void OrderModification::apply(MDP3Order &target) const {
if (price.has_value())
target.price = price.value();
if (quantity.has_value())
target.quantity = quantity.value();
if (target.iceberg_quantity.has_value())
iceberg_quantity.apply(target.iceberg_quantity.value());
}
} // namespace bts::spartan::mdp3
@@ -1,41 +0,0 @@
#pragma once
#include "IcebergQuantityModification.h"
#include "domain/Price.h"
#include "domain/Quantity.h"
#include <iostream>
#include <optional>
namespace bts::spartan::mdp3 {
struct Order;
struct OrderModification {
/**
* Initialize an OrderModification with no changes
*/
static OrderModification empty();
/**
* Initialize an OrderModification that represents the transition between two
* states
*/
static OrderModification deduce(
const Order &old_order, const Order &new_order);
/**
* Determine whether or not the modification has any changes
*/
bool has_changes() const;
/**
* Apply modifications to a particular order
*/
void apply(Order &) const;
std::optional<Price> price;
std::optional<Quantity> quantity;
IcebergQuantityModification iceberg_quantity;
};
std::ostream &operator<<(std::ostream &, const OrderModification &);
} // namespace bts::spartan::mdp3
#include "OrderModification.cc"
@@ -1,12 +0,0 @@
#include "SimBookUpdate.h"
namespace bts::spartan::mdp3 {
inline BookSide const &SimBookUpdate::mbp_side(Side side) const {
return side == Side::Buy ? mbp_buy : mbp_sell;
}
inline const ImpliedBookSide &SimBookUpdate::implied_mbp_side(Side side) const {
return side == Side::Buy ? implied_mbp_buy : implied_mbp_sell;
}
} // namespace bts::spartan::mdp3
@@ -1,22 +0,0 @@
#pragma once
#include "domain/SecurityID.h"
#include "domain/book/Book.h"
namespace bts::spartan::mdp3 {
/**
* Represents a market-by-price update for a single security
*/
struct SimBookUpdate {
SecurityID security_id;
BookSide mbp_buy;
BookSide mbp_sell;
ImpliedBookSide implied_mbp_buy;
ImpliedBookSide implied_mbp_sell;
BookSide const &mbp_side(Side) const;
const ImpliedBookSide &implied_mbp_side(Side) const;
};
} // namespace bts::spartan::mdp3
#include "SimBookUpdate.cc"
@@ -1,55 +0,0 @@
#include "SimOrderUpdate.h"
namespace bts::spartan::mdp3 {
constexpr const char *SimOrderUpdate::status_str() const {
return Status_Name(status);
}
constexpr const char *SimOrderUpdate::Status_Name(const Status &status) {
switch (status) {
case FILLED:
return "filled";
case PARTIALLY_FILLED:
return "partial";
case CANCELED:
return "canceled";
default:
bts_assert(false);
return "";
}
}
inline std::ostream &operator<<(
std::ostream &out, const SimOrderUpdate::Status &status) {
out << SimOrderUpdate::Status_Name(status);
return out;
}
inline const char* FillType_Name(SimOrderUpdate::FillType ft) {
switch (ft) {
case SimOrderUpdate::FillType::AGGRESSIVE: return "AGGRESSIVE";
case SimOrderUpdate::FillType::PASSIVE: return "PASSIVE";
default: return "UNKNOWN";
}
}
inline std::ostream& operator<<(std::ostream& out, const SimOrderUpdate::FillType& ft) {
out << FillType_Name(ft);
return out;
}
inline std::ostream& operator<<(std::ostream& out, const std::optional<SimOrderUpdate::FillType>& ft) {
if (ft.has_value())
out << ft.value();
return out;
}
inline std::ostream &operator<<(std::ostream &ostr, const SimOrderUpdate &upd) {
ostr << "key: " << upd.key.value() << ", status: " << upd.status
<< ", last fills: " << upd.last_fills.value()
<< ", last price: " << upd.last_price.to_double();
return ostr;
}
} // namespace bts::spartan::mdp3
@@ -1,35 +0,0 @@
#pragma once
#include "domain/order/OrderKey.h"
#include "domain/Price.h"
#include "domain/Quantity.h"
#include "domain/SecurityID.h"
#include "domain/Side.h"
#include <optional>
#include <ostream>
namespace bts::spartan::mdp3 {
struct SimOrderUpdate {
OrderKey key;
enum Status { FILLED, PARTIALLY_FILLED, CANCELED };
Status status;
SecurityID security_id;
Side side;
Quantity last_fills;
Price last_price;
enum FillType { PASSIVE, AGGRESSIVE };
// non null when Status == FILLED or PARTIALLY_FILLED
std::optional<FillType> fill_type;
constexpr const char *status_str() const;
static constexpr const char *Status_Name(const Status &);
static constexpr const char *FillType_Name(const Status &);
};
std::ostream &operator<<(std::ostream &, const SimOrderUpdate::Status &);
std::ostream& operator<<(std::ostream &, const SimOrderUpdate::FillType&);
std::ostream& operator<<(std::ostream &, const std::optional<SimOrderUpdate::FillType> &);
std::ostream &operator<<(std::ostream &, const SimOrderUpdate &);
} // namespace bts::spartan::mdp3
#include "SimOrderUpdate.cc"
@@ -1,32 +0,0 @@
#pragma once
#include "server/sparkd/cmds/spartan/domain/Price.h"
#include "server/sparkd/cmds/spartan/domain/Quantity.h"
#include <vector>
namespace bts::spartan::mdp3 {
struct SmallBookLevel {
SmallBookLevel() : price(), quantity(0) {}
SmallBookLevel(Price price, Quantity quantity)
: price(price), quantity(quantity) {}
Price price;
Quantity quantity;
};
inline bool operator==(const SmallBookLevel &lhs, const SmallBookLevel &rhs) {
return lhs.price == rhs.price &&
lhs.quantity.value() == rhs.quantity.value();
}
inline bool operator!=(const SmallBookLevel &lhs, const SmallBookLevel &rhs) {
return lhs.price != rhs.price ||
lhs.quantity.value() != rhs.quantity.value();
}
inline std::ostream &operator<<(std::ostream &out, const SmallBookLevel &sbl) {
out << sbl.quantity.value() << " @ " << sbl.price;
return out;
}
using SmallBook = std::vector<SmallBookLevel>;
} // namespace bts::spartan::mdp3
@@ -1,22 +0,0 @@
#include "Trace.h"
namespace bts::spartan::mdp3 {
template <class TracedOrders, class TracedEvents, class IgnoredEvents>
inline Trace Trace::create(
const TracedOrders &traced_orders,
const TracedEvents &traced_events,
const IgnoredEvents &ignored_events) {
boost::container::flat_set<OrderKey> traced_orders_set;
boost::container::flat_set<EventID> traced_events_set;
boost::container::flat_set<EventID> ignored_events_set;
for (auto o : traced_orders)
traced_orders_set.emplace(o);
for (auto e : traced_events)
traced_events_set.emplace(e);
for (auto e : ignored_events)
ignored_events_set.emplace(e);
return {traced_orders_set, traced_events_set, ignored_events_set};
}
} // namespace bts::spartan::mdp3
@@ -1,55 +0,0 @@
#include "Trace.h"
namespace bts::spartan::mdp3 {
bool Trace::log(OrderKey key, EventID eid) const {
return traced_orders.find(key) != traced_orders.end() ||
traced_events.find(eid) != traced_events.end();
}
bool Trace::log(OrderKey key, AlgoEventID aeid) const {
return log(key, aeid.first);
}
bool Trace::log(EventID eid) const {
return traced_events.find(eid) != traced_events.end();
}
bool Trace::log(AlgoEventID aeid) const { return log(aeid.first); }
bool Trace::is_ignored_event(EventID eid) const {
return ignored_events.find(eid) != ignored_events.end();
}
bool Trace::is_ignored_event(AlgoEventID aeid) const {
return is_ignored_event(aeid.first);
}
void Trace::log_incoming_change(const SmallOrderChange &c, AlgoEventID aeid) {
if (traced_orders.find(c.order.key) != traced_orders.end() ||
traced_events.find(aeid.first) != traced_events.end())
LOG(OrderMan, Diag,
"[event {}]Matcher incoming change:{} ok:{} side:{} price:{} qty:{} "
"ib:{}",
aeid, c.change, c.order.key, c.order.side, c.order.price,
c.order.quantity, c.order.iceberg_quantity);
}
void Trace::log_incoming_change(const SmallOrderChange &c, EventID eid) {
log_incoming_change(c, {eid, 0});
}
void Trace::log_outgoing_update(const SimOrderUpdate &u, AlgoEventID aeid) {
if (traced_orders.find(u.key) != traced_orders.end() ||
traced_events.find(aeid.first) != traced_events.end())
LOG(OrderMan, Diag,
"[event {}]Matcher outgoing update:{} ok:{} side:{} last_fp:{} "
"last_fq:{}",
aeid, u.status, u.key, u.side, u.last_price, u.last_fills);
}
void Trace::log_outgoing_update(const SimOrderUpdate &u, EventID eid) {
log_outgoing_update(u, {eid, 0});
}
} // namespace bts::spartan::mdp3
@@ -1,45 +0,0 @@
#pragma once
#include "back_testing/sim/sim_common.h"
#include "domain/order/OrderKey.h"
#include <boost/container/flat_set.hpp>
#include <cstdint>
namespace bts::spartan::mdp3 {
class Trace {
public:
template <
class TracedOrders = std::vector<std::uint64_t>,
class TracedEvents = std::vector<EventID>,
class IgnoredEvents = std::vector<EventID>>
static Trace create(
const TracedOrders & = {},
const TracedEvents & = {},
const IgnoredEvents & = {});
bool log(OrderKey key, EventID eid = no_event()) const;
bool log(OrderKey key, AlgoEventID eid) const;
bool log(EventID eid) const;
bool log(AlgoEventID eid) const;
bool is_ignored_event(EventID eid) const;
bool is_ignored_event(AlgoEventID eid) const;
void log_incoming_change(const SmallOrderChange &, EventID = no_event());
void log_incoming_change(const SmallOrderChange &, AlgoEventID);
void log_outgoing_update(const SimOrderUpdate &, EventID = no_event());
void log_outgoing_update(const SimOrderUpdate &, AlgoEventID);
const boost::container::flat_set<OrderKey> traced_orders =
boost::container::flat_set<OrderKey>();
const boost::container::flat_set<EventID> traced_events =
boost::container::flat_set<EventID>();
const boost::container::flat_set<EventID> ignored_events =
boost::container::flat_set<EventID>();
};
} // namespace bts::spartan::mdp3
#include "Trace.cc"
@@ -1,45 +0,0 @@
#pragma once
#include <boost/container/flat_map.hpp>
#include <map>
namespace bts::spartan {
enum class EventOrdinal {
// signals to the observer that it needs to perform state maintenance,
// but does not need to perform expensive validation.
Waiting,
// signals to the observer that it needs to perform state maintenance and
// validation.
Interesting,
// signals to the observer that it no longer needs to maintain any state
// because all interesting events have been processed.
Done
};
// the interesting events map's keys indicate the beginning of an
// "interesting range of events," the corresponding vaule indicates
// that event range's length (e.g., the key-value-pair {200, 17} means
// events 200-216 are "interesting")
static EventOrdinal get_event_ordinal(
const boost::container::flat_map<size_t, size_t> &interesting_events,
size_t event_id,
size_t min_observed_event = 0) {
//
if (interesting_events.empty()) {
return event_id < min_observed_event ? EventOrdinal::Waiting
: EventOrdinal::Interesting;
}
auto it = interesting_events.upper_bound(event_id);
if (it == interesting_events.end()) {
it--;
return event_id < (it->first + it->second) ? EventOrdinal::Interesting
: EventOrdinal::Done;
} else if (it == interesting_events.begin()) {
return EventOrdinal::Waiting;
} else {
it--;
return event_id >= it->first && event_id < (it->first + it->second)
? EventOrdinal::Interesting
: EventOrdinal::Waiting;
}
}
} // namespace bts::spartan
@@ -1,7 +0,0 @@
#include "mdp3_common.h"
#include <boost/container/small_vector.hpp>
namespace bts::spartan::mdp3 {
} // namespace bts::spartan::mdp3
@@ -1,90 +0,0 @@
#pragma once
#include "lib/domain_model/Security.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/utils/ValueWrapper.h"
#include "lib/utils/overloaded.h"
#include "server/sparkd/cmds/spartan/back_testing/common/SmallBookLevel.h"
#include "server/sparkd/cmds/spartan/common/TimeStamp.h"
#include "server/sparkd/cmds/spartan/domain/AlgoEventID.h"
#include "server/sparkd/cmds/spartan/domain/EventID.h"
#include "server/sparkd/cmds/spartan/domain/Price.h"
#include "server/sparkd/cmds/spartan/domain/Quantity.h"
#include "server/sparkd/cmds/spartan/domain/SecurityID.h"
#include "server/sparkd/cmds/spartan/domain/book/Book.h"
#include "server/sparkd/cmds/spartan/domain/fill/Fill.h"
#include "server/sparkd/cmds/spartan/domain/order/Order.h"
#include "server/sparkd/cmds/spartan/domain/order/OrderChange.h"
#include "server/sparkd/cmds/spartan/domain/order/OrderKey.h"
#include "server/sparkd/cmds/spartan/feed_handler/FeedEvent.h"
#include <boost/container/flat_set.hpp>
#include <boost/container/small_vector.hpp>
#include <variant>
namespace bts::spartan::mdp3 {
inline std::string str_from_fixed_array(const char *ptr, size_t max_len) {
std::string result;
result.reserve(max_len);
for (size_t i = 0; i < max_len; i++)
if (ptr[i] != 0)
result.append(1, ptr[i]);
else
break;
return result;
}
using Pricer = std::function<double(SecurityID, Quantity, Side)>;
inline std::ostream &operator<<(std::ostream &s, BookLevel const &l) {
s << " " << l.quantity.value() << " @ " << l.price.to_double() << " ("
<< l.n_orders << " orders)";
return s;
}
inline std::ostream &operator<<(std::ostream &s, BookSide const &b) {
for (auto const &l : b)
s << l << "\n";
return s;
}
inline std::ostream &operator<<(std::ostream &s, ImpliedBookSide const &b) {
for (auto const &l : b)
s << l << "\n";
return s;
}
inline std::string to_str(std::string const &s) { return s; }
inline std::string to_str(const char *s) { return std::string(s); }
template <class T>
std::string to_str(T t) {
return std::to_string(t);
}
inline std::ostream &operator<<(std::ostream &ostr, const api::Side &side) {
switch (side) {
case api::Side::SIDE_BUY:
ostr << "Buy";
break;
case api::Side::SIDE_SELL:
ostr << "Sell";
break;
case api::Side::SIDE_UNDEFINED:
ostr << "Undef";
break;
case api::Side::Side_INT_MIN_SENTINEL_DO_NOT_USE_:
ostr << "MinSen";
break;
case api::Side::Side_INT_MAX_SENTINEL_DO_NOT_USE_:
ostr << "MaxSen";
break;
default:
ostr << "SIDE_" << static_cast<int>(side);
break;
}
return ostr;
}
} // namespace bts::spartan::mdp3
@@ -1,806 +0,0 @@
#include "mdp3_utils.h"
#include "domain/SecuritySnapshot.h"
#include "domain/polygons/PolygonFinder.h"
#include "domain/polygons/PolygonOpportunityFinder.h"
#include "feed_handler/FeedHandler.h"
#include "feed_handler/detail/AB_Arbitrageur.h"
#include "feed_handler/detail/MDP3Packet.h"
#include "feed_handler/detail/MDP3Parser.h"
#include "feed_handler/detail/gen/src/MDIncrementalRefreshOrderBook47.h"
#include "feed_handler/detail/gen/src/MDInstrumentDefinitionFuture54.h"
#include "feed_handler/detail/gen/src/MDInstrumentDefinitionOption55.h"
#include "feed_handler/detail/gen/src/MDInstrumentDefinitionSpread56.h"
#include "feed_handler/detail/gen/src/SnapshotFullRefreshOrderBook53.h"
#include "back_testing/feed_observers/DefinitionCollector.h"
#include "back_testing/feed_observers/IcebergFinder.h"
#include "lib/config/config.h"
#include "lib/config/protos/spartan.pb.h"
#include "lib/jetfix/codec/f_toa.h"
#include "lib/utils/TempFile.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/overloaded.h"
#include "system/PCapReader.h"
#include <Pcap++/PcapFileDevice.h>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/date_time/posix_time/conversion.hpp>
#include <boost/date_time/posix_time/ptime.hpp>
#include <boost/filesystem.hpp>
#include <boost/process.hpp>
#include <boost/process/start_dir.hpp>
#include <fmt/format.h>
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
#include <sys/stat.h>
#include <sys/types.h>
namespace bts::spartan::mdp3 {
std::string drop_extensions(std::string const &str) {
size_t lastindex = str.find_last_of(".");
auto result = str.substr(0, lastindex);
if (result != str)
return drop_extensions(result);
else
return result;
}
struct Guesser {
FeedMap result;
PCapReader::PacketInfo current_packet;
void packet_begin(MDP3PacketView const &) {}
void packet_end(MDP3PacketView const &) {}
void set_current_source(const PCapReader::PacketInfo &info) {
current_packet = info;
}
template <class M>
void operator()(M const &) {
if (result.incremental[0].port == 0) {
result.incremental[0].port = current_packet.dst_port;
result.incremental[0].ip = current_packet.dst_ip;
return;
}
if (result.incremental[0].port == current_packet.dst_port &&
result.incremental[0].ip == current_packet.dst_ip)
return;
if (result.incremental[1].port == 0) {
result.incremental[1].port = current_packet.dst_port;
result.incremental[1].ip = current_packet.dst_ip;
}
}
void operator()(::mdp3::SnapshotFullRefresh52 const &) {
if (result.snapshot.port == 0) {
result.snapshot.port = current_packet.dst_port;
result.snapshot.ip = current_packet.dst_ip;
}
}
void operator()(::mdp3::SnapshotFullRefreshOrderBook53 const &) {
if (result.snapshot.port == 0) {
result.snapshot.port = current_packet.dst_port;
result.snapshot.ip = current_packet.dst_ip;
}
}
void operator()(::mdp3::MDInstrumentDefinitionFuture54 const &) {
on_definition();
}
void operator()(::mdp3::MDInstrumentDefinitionOption55 const &) {
on_definition();
}
void operator()(::mdp3::MDInstrumentDefinitionSpread56 const &) {
on_definition();
}
void on_definition() {
if (result.instrument.port)
return;
if (result.incremental[0].port == 0)
return;
if (result.incremental[1].port == 0)
return;
if (result.incremental[0].ip == current_packet.dst_ip &&
result.incremental[0].port == current_packet.dst_port)
return;
if (result.incremental[1].ip == current_packet.dst_ip &&
result.incremental[1].port == current_packet.dst_port)
return;
result.instrument.port = current_packet.dst_port;
result.instrument.ip = current_packet.dst_ip;
}
};
FeedMap guess_feed_map(std::string const &pcap_path) {
Guesser guesser;
auto reader = PCapReader{pcap_path};
reader.on_packet() = [&](auto const &info, auto const &payload) {
guesser.set_current_source(info);
MDP3Parser::parse(MDP3PacketView{payload}, guesser);
if (guesser.result)
return false;
else
return true;
};
while (reader.read_next())
;
return guesser.result;
}
std::ostream &operator<<(std::ostream &o, FeedMap const &fm) {
o << "IA:" << fm.incremental[0].ip << ":" << fm.incremental[0].port
<< " IB:" << fm.incremental[1].ip << ":" << fm.incremental[1].port
<< " N:" << fm.instrument.ip << ":" << fm.instrument.port
<< " SMBO:" << fm.snapshot.ip << ":" << fm.snapshot.port;
return o;
}
int compress(std::string const &file, std::string const &out_file) {
using namespace bts::spartan;
using namespace bts::spartan::mdp3;
struct Ctrl {
bool instrument_feed = false;
bool snapshot_feed = false;
bool incremental_feed = false;
void start_instrument_stream() { instrument_feed = true; }
void stop_instrument_stream() { instrument_feed = false; }
void start_snapshot_stream() { snapshot_feed = true; }
void stop_snapshot_stream() { snapshot_feed = false; }
void start_incremental_stream() { incremental_feed = true; }
void stop_incremental_stream() { incremental_feed = false; }
} ctrl;
struct Obs : FeedObserver {
bool observed_loss = false;
bool recovery_complete = false;
size_t previous_packet = 0;
size_t previous_send_ts = 0;
bts::TimeStamp last_print;
void on_recovery_complete(
std::vector<SecuritySnapshot const *> const &) override {
recovery_complete = true;
observed_loss = false;
}
void on_event_end(FeedEvent const &ev) override {
if (ms_diff(Timer::now(), last_print) >= 1000) {
last_print = Timer::now();
LOG(Spartan, Info, "Time: {} -> {} pkt/sec ({}x real time)",
ev.transact_time.to_time_string(),
(ev.first_packet_sn - previous_packet),
(ev.first_packet_send_time.value() - previous_send_ts) /
1000000000);
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
}
void on_feed_error(SpartanError const &) override {
observed_loss = true;
recovery_complete = false;
}
} obs;
struct IncrementalFeedBuffer {
bool written_once = false;
pcpp::RawPacket rp;
mdp3::MDP3Packet p;
};
std::array<IncrementalFeedBuffer, 2> inc_buffer;
Guesser guesser;
auto reader = PCapReader{file};
pcpp::PcapNgFileWriterDevice writer(out_file.c_str());
writer.open();
reader.on_packet() = [&](auto const &info, auto const &payload) {
guesser.set_current_source(info);
MDP3Parser::parse(MDP3PacketView{payload}, guesser);
writer.writePacket(*info.rp);
if (guesser.result)
return false;
else
return true;
};
// Run through the file long enough to be able to map feeds to IP's
while (reader.read_next())
;
auto feed_map = guesser.result;
if (!feed_map) {
LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting",
file);
return -1;
}
// Now continue reading, feeding each packet to our channel handler
mdp3::AB_Arbitrageur ab;
FeedHandler handler;
obs.bind(handler);
handler.start_incremental_stream.bind<&Ctrl::start_incremental_stream>(ctrl);
handler.stop_incremental_stream.bind<&Ctrl::stop_incremental_stream>(ctrl);
handler.start_snapshot_stream.bind<&Ctrl::start_snapshot_stream>(ctrl);
handler.stop_snapshot_stream.bind<&Ctrl::stop_snapshot_stream>(ctrl);
handler.start_instrument_stream.bind<&Ctrl::start_instrument_stream>(ctrl);
handler.stop_instrument_stream.bind<&Ctrl::stop_instrument_stream>(ctrl);
handler.run();
// For every packet extractred from the pcap decide if we need to
// write it out to our output or not.
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
inc_buffer[0].rp = *info.rp;
inc_buffer[0].p.set_bytes(payload);
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
info.dst_port == feed_map.incremental[1].port)) {
inc_buffer[1].rp = *info.rp;
inc_buffer[1].p.set_bytes(payload);
} else if (
info.dst_ip == feed_map.instrument.ip &&
info.dst_port == feed_map.instrument.port) {
if (ctrl.instrument_feed) {
handler.handle_instrument_packet(MDP3PacketView{payload});
writer.writePacket(*info.rp);
}
return true;
} else if (
info.dst_ip == feed_map.snapshot.ip &&
info.dst_port == feed_map.snapshot.port) {
if (ctrl.snapshot_feed) {
handler.handle_snapshot_packet(MDP3PacketView{payload});
writer.writePacket(*info.rp);
}
return true;
} else
return true;
// If we reached this point then we are dealing with an
// incremental feed packet.
auto picked = handler.handle_incremental_packet(
inc_buffer[0].p.view(), inc_buffer[1].p.view());
if (picked == PickedFeed::A ||
((!inc_buffer[0].written_once || !obs.recovery_complete) &&
!inc_buffer[0].p.view().empty())) {
inc_buffer[0].written_once = true;
writer.writePacket(inc_buffer[0].rp);
}
if (picked == PickedFeed::B ||
((!inc_buffer[1].written_once || !obs.recovery_complete) &&
!inc_buffer[1].p.view().empty())) {
inc_buffer[1].written_once = true;
writer.writePacket(inc_buffer[1].rp);
}
return true;
};
while (reader.read_next())
if (obs.observed_loss) {
LOG(Spartan, Error, "Observed loss in {}. Recovery.", file);
};
return 0;
}
struct JSON_Writer {
size_t first_packet = 0;
size_t packet_count = 0;
std::stringstream json_buffer;
nlohmann::json json_obj;
MDP3PacketView packet;
JSON_Writer(size_t first_packet = 0, size_t packet_count = 0)
: first_packet(first_packet), packet_count(packet_count) {}
void packet_begin(MDP3PacketView const &p) { packet = p; }
void packet_end(MDP3PacketView const &) {}
template <class M>
void operator()(M const &m) {
if (first_packet == 0 ||
(packet.seq_num() >= first_packet &&
packet.seq_num() <= first_packet + packet_count)) {
json_obj["SeqNum"] = packet.seq_num();
json_obj["SendTime"] =
TimeStamp{packet.sending_time()}.to_datetime_string();
json_obj["Size"] = packet.size();
json_buffer.str("");
json_buffer.clear();
json_buffer << m;
json_obj["Msg"] = nlohmann::json::parse(json_buffer.str());
std::cout << json_obj.dump(4) << std::endl;
}
}
};
struct PacketBuffers {
MDP3Packet i;
MDP3Packet n;
};
int to_json(std::string const &pcap, size_t first_packet, size_t count) {
auto reader = PCapReader{pcap};
JSON_Writer writer(first_packet, count);
reader.on_packet() = [&](auto const &, auto const &payload) {
MDP3Parser::parse(MDP3PacketView{payload}, writer);
return true;
};
while (reader.read_next())
;
return 0;
}
struct ObsWrapper : mdp3::FeedObserver {
ObsWrapper(mdp3::FeedObserver &obs) : real_obs(obs) {}
FeedObserver &real_obs;
size_t previous_packet = 0;
size_t previous_send_ts = 0;
bts::TimeStamp last_print = Timer::now();
void init() override { real_obs.init(); }
void finit() override { real_obs.finit(); }
void on_recovery_complete(
std::vector<SecuritySnapshot const *> const &s) override {
real_obs.on_recovery_complete(s);
}
void on_packet_begin(MDP3PacketView const &pv) override {
real_obs.on_packet_begin(pv);
}
void on_event_end(FeedEvent const &ev) override {
auto period_ms = 10000;
if (ms_diff(Timer::now(), last_print) >= period_ms) {
last_print = Timer::now();
std::stringstream str2;
str2 << "Packet time: " << ev.transact_time.to_time_string()
<< " Event ID: " << ev.event_id << " ("
<< (ev.first_packet_sn - previous_packet) / (period_ms / 1000)
<< " pkt/sec, "
<< (ev.first_packet_send_time.value() - previous_send_ts) /
1000000000 / (period_ms / 1000)
<< "x speedup)";
LOG(Spartan, Info, "{}", str2.str());
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
real_obs.on_event_end(ev);
}
};
struct PCapStreamer::PImpl {
PImpl(std::string const &file, mdp3::FeedObserver &obs)
: obs_wrapper(obs), reader(file), handler() {
obs.bind(handler);
}
ObsWrapper obs_wrapper;
PCapReader reader;
FeedHandler handler;
std::array<PacketBuffers, 2> buffers;
};
PCapStreamer::PCapStreamer(std::string const &file, mdp3::FeedObserver &obs)
: me(new PImpl(file, obs)) {
Guesser guesser;
me->reader.on_packet() = [&](auto const &info, auto const &payload) {
guesser.set_current_source(info);
MDP3Parser::parse(MDP3PacketView{payload}, guesser);
if (guesser.result)
return false;
else
return true;
};
while (me->reader.read_next())
;
auto feed_map = guesser.result;
if (!feed_map) {
LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting",
file);
bts_assert(false);
}
me->obs_wrapper.init();
me->handler.run();
me->reader.on_packet() = [this,
feed_map](auto const &info, auto const &payload) {
if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
me->buffers[0].i.set_bytes(payload);
me->handler.handle_incremental_packet(
me->buffers[0].i.view(), me->buffers[1].i.view());
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
info.dst_port == feed_map.incremental[1].port)) {
me->buffers[1].i.set_bytes(payload);
me->handler.handle_incremental_packet(
me->buffers[0].i.view(), me->buffers[1].i.view());
} else if ((info.dst_ip == feed_map.instrument.ip &&
info.dst_port == feed_map.instrument.port)) {
me->handler.handle_instrument_packet(MDP3PacketView{payload});
} else if ((info.dst_ip == feed_map.snapshot.ip &&
info.dst_port == feed_map.snapshot.port)) {
me->handler.handle_snapshot_packet(MDP3PacketView{payload});
}
return true;
};
}
PCapStreamer::~PCapStreamer() { me->obs_wrapper.finit(); }
bool PCapStreamer::next() { return me->reader.read_next(); }
void observe_channel(std::string const &file, mdp3::FeedObserver &obs) {
PCapStreamer streamer(file, obs);
while (streamer.next())
;
}
void observe_channel(
std::string const &pcap, std::initializer_list<FeedObserver *> observers) {
struct Composite : FeedObserver {
std::initializer_list<FeedObserver *> const &obs;
Composite(std::initializer_list<FeedObserver *> const &obs) : obs(obs) {}
void init() override {
for (auto o : obs)
o->init();
}
void finit() override {
for (auto o : obs)
o->finit();
}
void on_recovery_complete(InitialSnapshot const &s) override {
for (auto o : obs)
o->on_recovery_complete(s);
}
void on_packet_begin(MDP3PacketView const &pv) override {
for (auto o : obs)
o->on_packet_begin(pv);
}
void on_event_end(FeedEvent const &ev) override {
for (auto o : obs)
o->on_event_end(ev);
}
} comp(observers);
observe_channel(pcap, comp);
}
struct CSVObs : FeedObserver {
CSVObs(std::string const &pcap)
: write_books(config::get<config::MarketDataDownload>().write_books()),
events(drop_extensions(pcap) + ".events.csv"),
event_items(drop_extensions(pcap) + ".event_items.csv"),
securities(drop_extensions(pcap) + ".securities.csv"),
books(drop_extensions(pcap) + ".books.csv"),
packets(drop_extensions(pcap) + ".packets.csv"),
spread_legs(drop_extensions(pcap) + ".spread_legs.csv") {}
const bool write_books;
std::ofstream events;
std::ofstream event_items;
std::ofstream orders;
std::ofstream securities;
std::ofstream books;
std::ofstream packets;
std::ofstream spread_legs;
std::unordered_set<SecurityID> known_securities;
std::stringstream ts_buffer;
template <class B>
void print_book(B const &b) {
char buffer[128];
for (auto level : b) {
d_toa(level.price.to_double(), buffer, sizeof(buffer), 0);
books << "," << buffer << "," << level.quantity.value() << ","
<< level.n_orders;
}
}
void on_recovery_complete(
std::vector<SecuritySnapshot const *> const &secs) override {
for (auto s : secs) {
securities << s->security_id().value() << "," << s->symbol() << ","
<< api::SecurityType_Name(s->definition().security_type())
<< "," << s->definition().cme_security_type() << ","
<< s->definition().security_group() << ","
<< s->definition().asset() << ","
<< s->definition().implied_matching() << ","
<< s->definition().match_algorithm() << ","
<< s->definition().display_factor() << ","
<< s->definition().cash_per_point() << ","
<< s->definition().min_price_increment() << ","
<< s->definition().min_price_increment_ammount() << ","
<< s->definition().legs().size() << ","
<< s->last_update_time() << std::endl;
for (auto const &l : s->definition().legs())
spread_legs << s->security_id().value() << ","
<< l.security_id.value() << "," << Side_Name(l.side)
<< "," << l.ratio.value() << std::endl;
known_securities.insert(s->security_id());
if (write_books) {
books << 0 << "," // event_id
<< 0 << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< s->implied_matching_status(); // implied matching status
print_book(s->book_side(Side::Buy));
print_book(s->book_side(Side::Sell));
print_book(s->implied_book_side(Side::Buy));
print_book(s->implied_book_side(Side::Sell));
books << std::endl;
}
for (auto const &[id, order] : s->orders())
event_items << 0 << "," // event_id
<< 0 << "," // transact time
<< s->security_id().value() << "," // security id
<< "," // human transact time
<< s->symbol() << "," // symbol
<< s->trading_status() << "," // trading status
<< s->implied_matching_status()
<< "," // implied matching status
<< "order_change," // type
<< "init," // change
<< id << "," // order key
<< Side_Name(order.side) << "," // side
<< order.priority.value() << "," // priority
<< order.price.to_double() << "," // price
<< order.quantity.value() << "," // quantity
<< "0," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
void on_packet_begin(MDP3PacketView const &pv) override {
packets << pv.sending_time() << ","
<< TimeStamp(pv.sending_time()).to_datetime_string() << ","
<< pv.seq_num() << "," << pv.size() << std::endl;
}
void on_event_end(FeedEvent const &ev) override {
events << ev.transact_time.to_datetime_string()
<< "," // human transact time
<< ev.event_id << "," // event id
<< ev.transact_time << "," // transact time
<< ev.first_packet_sn << "," // first packet_sn
<< ev.last_packet_sn << "," // last_packet_sn
<< ev.first_packet_send_time << "," // first packet send time
<< ev.last_packet_send_time // last packet send time
<< std::endl;
for (auto s : ev) {
if (known_securities.count(s->security_id()) == 0) {
known_securities.insert(s->security_id());
securities << s->security_id().value() << "," << s->symbol() << ","
<< api::SecurityType_Name(
s->definition().security_type())
<< "," << s->definition().cme_security_type() << ","
<< s->definition().security_group() << ","
<< s->definition().asset() << ","
<< s->definition().implied_matching() << ","
<< s->definition().match_algorithm() << ","
<< s->definition().display_factor() << ","
<< s->definition().cash_per_point() << ","
<< s->definition().min_price_increment() << ","
<< s->definition().min_price_increment_ammount() << ","
<< s->definition().legs().size() << ","
<< s->last_update_time() // last update time
<< std::endl;
}
if (write_books) {
books << ev.event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< s->implied_matching_status(); // implied matching status
print_book(s->book_side(Side::Buy));
print_book(s->book_side(Side::Sell));
print_book(s->implied_book_side(Side::Buy));
print_book(s->implied_book_side(Side::Sell));
books << std::endl;
}
for (auto const &change : s->order_changes())
event_items << ev.event_id << "," // event id
<< ev.transact_time.value() // transact time
<< "," // transact time
<< s->security_id().value() << "," // security id
<< ts_buffer.str() << "," // human transact time
<< s->symbol() << "," // symbol
<< s->trading_status() << "," // trading status
<< s->implied_matching_status()
<< "," // implied matching status
<< "order_change," // type
<< OrderChange::ChangeType_Name(change.change)
<< "," // change type
<< change.order.key.value() << "," // order key
<< Side_Name(change.order.side) << "," // side
<< change.order.priority.value() << "," // priority
<< change.order.price.to_double() << "," // price
<< change.order.quantity.value() << "," // quantity
<< "0," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
for (auto const &f : s->fills())
event_items << ev.event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< ts_buffer.str() << "," // human transact time
<< s->symbol() << "," // symbol
<< s->trading_status() << "," // trading status
<< s->implied_matching_status()
<< "," // implied matching status
<< "fill," // type
<< "," // null
<< f.key.value() << "," // order key
<< "," // side
<< "0," // priority
<< f.price << "," // price
<< f.quantity << "," // quantity
<< f.trade_entry_id << "," // trade_entry_id
<< Fill::Aggressor_Name(f.aggressor) // aggressor
<< std::endl;
auto deduced_orders = s->deduced_order_changes();
for (auto &deduced_order : deduced_orders)
event_items << ev.event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< ts_buffer.str() << "," // human transact time
<< s->symbol() << "," // symbol
<< s->trading_status() << "," // trading status
<< s->implied_matching_status()
<< "," // implied matching status
<< "deduced_order," // type
<< OrderChange::ChangeType_Name(deduced_order.change)
<< "," << deduced_order.order.key.value()
<< "," // order key
<< Side_Name(deduced_order.order.side) << "," // side
<< "0," // priority
<< deduced_order.order.price.to_double() << "," // price
<< deduced_order.order.quantity.value()
<< "," // quantity
<< "0," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
void on_feed_error(SpartanError const&) override {
LOG(Spartan, Error, "Detected loss in feed. Aborting");
bts_assert(false);
}
}; // namespace bts::spartan::mdp3
int to_csv(std::string const &file) {
using namespace bts::spartan::mdp3;
CSVObs cvs{file};
IcebergFinder ice_finder;
PolygonFinder poly_finder;
DefinitionCollector def_collector;
observe_channel(file, {&cvs, &ice_finder, &poly_finder, &def_collector});
auto icebergs = std::ofstream(drop_extensions(file) + ".icebergs.csv");
for (auto &i : ice_finder.result) {
auto relevent_eids = i.second.all_eids();
for (auto eid : relevent_eids)
icebergs << i.first.value() << "," << eid << ","
<< i.second.total_qty().value() << ","
<< std::get<0>(i.second.show_qty_at(eid)) << ","
<< i.second.count_fills_by(eid) << std::endl;
}
auto polys = std::ofstream(drop_extensions(file) + ".polygons.csv");
auto poly_legs = std::ofstream(drop_extensions(file) + ".polygon_legs.csv");
auto &imap = def_collector.ptr_result;
for (auto &[pid, p] : poly_finder.result) {
polys << pid << "," << polygon_descriptor(p, imap) << std::endl;
for (auto &part : p.parts) {
poly_legs << pid << "," << part.sec_id << ","
<< Side_Name(from_ratio(part.ratio)) << ","
<< std::abs(part.ratio) << std::endl;
}
}
PolygonOpportunityFinder op_finder{
poly_finder.result, def_collector.ptr_result};
observe_channel(file, {&op_finder});
auto poly_values =
std::ofstream(drop_extensions(file) + ".polygon_values.csv");
poly_values_to_csv(
op_finder.result, poly_finder.result, def_collector.ptr_result,
poly_values);
auto poly_opportunities =
std::ofstream(drop_extensions(file) + ".polygon_opportunities.csv");
poly_opportunities_to_csv(
aggregate(op_finder.result), poly_finder.result,
def_collector.ptr_result, poly_opportunities);
return 0;
}
std::unordered_map<OrderKey, IcebergInput> find_icebergs(
std::string const &pcap) {
return IcebergFinder::find_all(pcap);
}
int to_sql(std::string const &pcap) {
if (to_csv(pcap) == 0) {
auto init_script = os::sparkd_wd() + "/data_files/spartan/md.sql";
auto db_file = drop_extensions(pcap) + ".sqlite";
if (boost::filesystem::exists(db_file))
boost::filesystem::remove(db_file);
auto out_dir = boost::filesystem::path(db_file).parent_path().string();
std::cout << "Creating database " << db_file << std::endl;
auto result = boost::process::system(
fmt::format("sqlite3 {}", db_file),
boost::process::std_in < init_script,
boost::process::start_dir = out_dir);
if (result) {
std::cerr << "Error creating database" << std::endl;
return result;
}
std::initializer_list<std::string> tables = {"event_items",
"events",
"securities",
"books",
"icebergs",
"spread_legs",
"polygons",
"polygon_legs",
"packets",
"polygon_values",
"polygon_opportunities"};
std::string import = ".mode csv\n";
for (auto const &t : tables)
import += fmt::format(
".import {0}.{1}.csv {1}\n", drop_extensions(pcap), t, t);
utils::TempFile f;
std::ofstream{f.path()} << import;
std::cout << "Importing data CSVs" << std::endl;
result = boost::process::system(
fmt::format("sqlite3 {}", db_file), boost::process::std_in < f.path(),
boost::process::start_dir = out_dir);
if (result) {
std::cerr << "Error importing CSVs" << std::endl;
return result;
} else
for (auto const &t : tables)
boost::filesystem::remove(
fmt::format("{0}.{1}.csv", drop_extensions(pcap), t));
return 0;
} else
return -1;
}
} // namespace bts::spartan::mdp3
@@ -1,84 +0,0 @@
#pragma once
#include "back_testing/common/mdp3_common.h"
#include "lib/utils/datetime.h"
#include <array>
#include <iostream>
#include <string>
#include <vector>
namespace bts::spartan::mdp3 {
class FeedObserver;
// Details the ip/port used for each of the incremental a/b, snapshot
// and instrument feeds in a pcap.
struct FeedMap {
struct Feed {
std::string ip = "";
int port = 0;
};
std::array<Feed, 2> incremental;
Feed snapshot;
Feed instrument;
operator bool() const noexcept {
return incremental[0].port && incremental[1].port && instrument.port &&
snapshot.port;
}
};
std::ostream &operator<<(std::ostream &o, FeedMap const &fm);
// Examine the passed pcap and determine, if possible, the ip/port
// associated with each of the following feeds:
// - Incremental A
// - Incremental B
// - Snapshot
// - Instrument
FeedMap guess_feed_map(std::string const &pcap);
// Examine the passed pcap and throw away redundant packets. Including:
// - Instrument and Snapshoft feed packets after the initial recovery
// - Feed A/Feed B redundant packets
int compress(std::string const &pcap_in, std::string const &pcap_out);
// Convert the pcap passed to json and output it to standard out (this
// is typically only usefull for debugging)
int to_json(std::string const &pcap, size_t first_packet = 0, size_t count = 0);
// Export the passed pcap to a set of tables in CSV format. The
// following tables are generated:
// - events.csv : one row per event in the pcap
// - event_items.csv : one row per item in an event. Event items
// include: order changes, fills and deduced order
// - securities.csv : one row per security present in the pcap
// - icebergs.csv : one row per iceberg order identified in the pcap
// - books.csv : one row per security per change in MBP
int to_csv(std::string const &pcap);
// Generate a SQL lite DB with one table per each of the CSV files
// generated by to_csv
int to_sql(std::string const &pcap);
// Stream through a pcap file, invoking the passed observer on each
// call to next()
class PCapStreamer {
public:
PCapStreamer(std::string const &file, mdp3::FeedObserver &obs);
~PCapStreamer();
// Returns false when done
bool next();
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
// Stream through an entire pcap.
void observe_channel(std::string const &pcap, FeedObserver &obs);
void observe_channel(
std::string const &pcap,
std::initializer_list<FeedObserver *> observers);
} // namespace bts::spartan::mdp3
@@ -1,16 +0,0 @@
#include "order_utils.h"
#include "back_testing/matcher/common/MatcherOrder.h"
namespace bts::spartan::mdp3 {
MDP3Order to_mdp3_order(const MatcherOrder &mo) {
MDP3Order rv;
rv.key = mo.key;
rv.side = mo.side;
rv.quantity = mo.quantity;
// MatcherOrder.priority doesn't map back to the exchange's
// MDP3Order.priority. Set it to zero so nobody thinks it does.
rv.priority = MDP3OrderPriority(0);
rv.iceberg_quantity = mo.ib;
return rv;
}
} // namespace bts::spartan::mdp3
@@ -1,108 +0,0 @@
#pragma once
#include "lib/md/IntPrice.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
#include "back_testing/common/mdp3_common.h"
#include <iostream>
#include <optional>
#include <range/v3/numeric.hpp>
#include <range/v3/view.hpp>
#include <sstream>
#include <string>
#include <vector>
namespace bts::spartan::mdp3 {
inline Price
get_price(const std::tuple<Price, Quantity> &price_level) {
return std::get<0>(price_level);
}
inline Quantity
get_qty(const std::tuple<Price, Quantity> &price_level) {
return std::get<1>(price_level);
}
inline Quantity get_total_qty(const MDP3Order &order) {
return order.iceberg_quantity.has_value()
? order.iceberg_quantity.value().total
: order.quantity;
}
// all order quantity props should contain the original values (before
// subtracting fills, or any other qty updates, etc.)
inline void set_total_qty(
MDP3Order &order, Quantity total_qty, bool reset_show_qty = false) {
bts_assert(total_qty.value() >= 0);
if (order.iceberg_quantity.has_value()) {
auto ibq = order.iceberg_quantity.value();
int ordinal0 =
ibq.total.value() + (ibq.show.value() - order.quantity.value());
int mod_offset = ordinal0 % ibq.show.value();
bts_assert(mod_offset >= 0);
int adj = mod_offset > 0 ? ibq.show.value() - mod_offset : 0;
ibq.total = total_qty;
if (reset_show_qty) {
order.quantity =
std::min(order.iceberg_quantity.value().show, total_qty);
} else {
order.quantity = order.iceberg_quantity.value().show < total_qty
? (total_qty + Quantity(adj)) %
order.iceberg_quantity.value().show
: total_qty;
if (order.quantity.value() == 0)
order.quantity =
std::min(total_qty, order.iceberg_quantity.value().show);
}
order.iceberg_quantity.value().total = total_qty;
} else
order.quantity = total_qty;
}
inline Quantity book_volume(const SmallBook &book) {
auto get_qty = [](const auto &bl) { return bl.quantity.value(); };
return Quantity(
ranges::accumulate(book | ranges::views::transform(get_qty), 0));
}
struct MatcherOrder;
MDP3Order to_mdp3_order(const MatcherOrder &);
inline bool orders_equal(const MDP3Order &lhs, const MDP3Order &rhs) {
return lhs.key == rhs.key && lhs.side == rhs.side &&
lhs.price == rhs.price && lhs.quantity == rhs.quantity &&
lhs.iceberg_quantity == rhs.iceberg_quantity;
}
inline std::string fmt_order(
OrderKey key, Side side, Price price, Quantity qty) {
std::stringstream str;
str << "[" << key.value() << "] "
<< " " << (side == Side::Buy ? "Buy" : "Sell") << " "
<< qty.value() << " @ " << price.to_double();
return str.str();
}
template <class TIter>
inline void accumulate_fills(
TIter first,
TIter last,
std::unordered_map<OrderKey, Quantity> &dest) {
while (first != last) {
auto it = dest.find(first->key);
if (it == dest.end())
dest.emplace(first->key, first->quantity);
else
it->second.value() += first->quantity.value();
first++;
}
}
inline void accumulate_fills(
const std::vector<MDP3Fill> &fills,
std::unordered_map<OrderKey, Quantity> &dest) {
accumulate_fills(fills.begin(), fills.end(), dest);
}
inline std::unordered_map<OrderKey, Quantity>
accumulate_fills(const std::vector<MDP3Fill> &fills) {
std::unordered_map<OrderKey, Quantity> rv;
accumulate_fills(fills.begin(), fills.end(), rv);
return rv;
}
} // namespace bts::spartan::mdp3
@@ -1,58 +0,0 @@
#include "server/sparkd/cmds/spartan/back_testing/common/security_comp.h"
#include "server/sparkd/cmds/spartan/back_testing/common/security_parser.h"
#include <tuple>
namespace bts::spartan::mdp3::security_comp {
// c# style IComparer: -1 means lhs < rhs, 0 means lhs == rhs, 1 means lhs > rhs
template <int I, typename... T>
int compare_part(const std::tuple<T...> &lhs, const std::tuple<T...> &rhs) {
if (std::get<I>(lhs) == std::get<I>(rhs))
return 0;
return std::get<I>(lhs) > std::get<I>(rhs) ? 1 : -1;
}
inline bool
symbol_comp_impl(const std::string &lhs_sym, const std::string &rhs_sym) {
std::vector<std::tuple<std::string, uint32_t, uint32_t, int>> lhs_legs,
rhs_legs;
bts::spartan::parse_legs(lhs_sym, lhs_legs);
bts::spartan::parse_legs(rhs_sym, rhs_legs);
auto lhs_it = lhs_legs.begin();
auto rhs_it = rhs_legs.begin();
int comp = 0;
while (lhs_it != lhs_legs.end() && rhs_it != rhs_legs.end()) {
comp = compare_part<0>(*lhs_it, *rhs_it);
if (comp != 0)
break;
comp = compare_part<2>(*lhs_it, *rhs_it);
if (comp != 0)
break;
comp = compare_part<1>(*lhs_it, *rhs_it);
if (comp != 0)
break;
comp = compare_part<3>(*lhs_it, *rhs_it);
if (comp != 0)
break;
lhs_it++;
rhs_it++;
}
return comp < 0 || (lhs_it == lhs_legs.end() && rhs_it != rhs_legs.end());
}
inline bool
symbol_comp_impl(const SecuritySnapshot &lhs, const SecuritySnapshot &rhs) {
std::string lhs_sym(lhs.symbol());
std::string rhs_sym(rhs.symbol());
return symbol_comp_impl(lhs_sym, rhs_sym);
}
bool SymbolComp::operator()(
const SecuritySnapshot &lhs, const SecuritySnapshot &rhs) {
return _reverse ? symbol_comp_impl(rhs, lhs) : symbol_comp_impl(lhs, rhs);
}
bool SymbolComp::operator()(const std::string &lhs, const std::string &rhs) {
return _reverse ? symbol_comp_impl(rhs, lhs) : symbol_comp_impl(lhs, rhs);
}
} // namespace bts::spartan::mdp3::security_comp
@@ -1,22 +0,0 @@
#pragma once
#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h"
#include "server/sparkd/cmds/spartan/back_testing/common/mdp3_common.h"
#include <string>
namespace bts::spartan::mdp3::security_comp {
struct SymbolComp {
SymbolComp() : SymbolComp(false) {}
SymbolComp(bool reverse) : _reverse(reverse) {}
bool operator()(const std::string &lhs, const std::string &rhs);
bool
operator()(const SecuritySnapshot &lhs, const SecuritySnapshot &rhs);
bool
operator()(const SecuritySnapshot *lhs, const SecuritySnapshot *rhs) {
return (*this)(*lhs, *rhs);
}
private:
bool _reverse;
};
} // namespace bts::spartan::mdp3::security_comp
@@ -1,86 +0,0 @@
#include "server/sparkd/cmds/spartan/utils/StringUtils.h"
#include <algorithm>
#include <functional>
#include <iostream>
#include <string>
#include <vector>
namespace bts::spartan {
static uint32_t _get_exp_month(const std::string &sym) {
const uint32_t no_month = 13;
if (sym.size() < 2)
return no_month;
switch (sym[sym.size() - 2]) {
case 'F':
return 0;
case 'G':
return 1;
case 'H':
return 2;
case 'J':
return 3;
case 'K':
return 4;
case 'M':
return 5;
case 'N':
return 6;
case 'Q':
return 7;
case 'U':
return 8;
case 'V':
return 9;
case 'X':
return 10;
case 'Z':
return 11;
default:
return no_month;
}
}
static uint32_t _get_exp_year(const std::string &sym) {
if (sym.size() == 0)
return 0;
std::string ystr{sym[sym.size() - 1]};
try {
uint32_t y = static_cast<uint32_t>(std::stoul(ystr));
return y + 2020;
} catch (const std::invalid_argument &ex) {
std::cout << "Error getting exp year from " << sym << ": ";
std::cout << ex.what() << std::endl;
}
return 0;
}
static void _parse_legs(
const std::string &symbol,
std::vector<std::tuple<std::string, uint32_t, uint32_t, int>> &flat_legs,
int mult = 1) {
std::vector<std::string> leg_syms;
StringUtils::split(symbol, "-", leg_syms);
if (leg_syms.size() == 0)
return;
if (leg_syms.size() == 1) {
flat_legs.push_back(std::make_tuple(leg_syms[0], 0, 0, mult));
return;
}
_parse_legs(leg_syms[0], flat_legs, mult);
for (uint32_t i = 1; i < leg_syms.size(); i++) {
_parse_legs(leg_syms[i], flat_legs, -mult);
}
}
void parse_legs(
const std::string &symbol,
std::vector<std::tuple<std::string, uint32_t, uint32_t, int>> &flat_legs) {
_parse_legs(symbol, flat_legs);
for (auto &l : flat_legs) {
std::get<1>(l) = _get_exp_month(std::get<0>(l));
std::get<2>(l) = _get_exp_year(std::get<0>(l));
if (std::get<0>(l).size() > 2)
std::get<0>(l) = std::get<0>(l).substr(0, std::get<0>(l).size() - 2);
}
}
} // namespace bts::spartan
@@ -1,15 +0,0 @@
#pragma once
#include <algorithm>
#include <functional>
#include <iostream>
#include <map>
#include <string>
#include <tuple>
namespace bts::spartan {
// symbol, exp month, exp year, qty
void parse_legs(
const std::string &symbol,
std::vector<std::tuple<std::string, uint32_t, uint32_t, int>> &flat_legs);
void parse_legs(const std::string &symbol, std::map<std::string, int> &legs);
} // namespace bts::spartan
@@ -1,160 +0,0 @@
#include "BookStatRecorder.h"
#include "BookStats.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "back_testing/matcher/validation/event_flags.h"
#include "domain/SecuritySnapshot.h"
#include "domain/Quantity.h"
#include "domain/book/book_utils.h"
#include "back_testing/common/event_ordinal.h"
#include <iostream>
namespace bts::spartan::mdp3 {
using namespace book_utils;
// consts
const size_t NO_EVENT = 0, MIN_OBSERVED_EVENT = 100;
const uint32_t DEFAULT_NUM_BLOCKS = 10, DEFAULT_BLOCK_SIZE = 4;
// aliases
template <class K, class V>
using flat_map = boost::container::flat_map<K, V>;
///////////////////////
// private classes
///////////////////////
}
namespace bts::spartan::mdp3 {
/**
* All of the stats/data in each csv row written to the output stream.
*/
struct BookStatsRow {
BookStatsRow() : BookStatsRow(SecurityID(0), NO_EVENT, BookSide(), BookSide()) {}
BookStatsRow(SecurityID security_id, size_t event_id, BookSide bid, BookSide ask)
: _security_id(security_id), _event_id(event_id), _data{bid, ask} {}
static std::ostream &
headers_to_csv(std::ostream &out, uint32_t block_size, uint32_t num_blocks) {
out << "eid,sid";
auto append_side_cols = [&](const char *side) {
orderman::Formatter::join(
out, ",", side, "price", "price_avg", "price_var", "qty_total");
for (uint32_t i = 0; i < num_blocks; i++) {
out << "," << side << "_price_" << i * block_size;
out << "," << side << "_price_avg_" << i * block_size;
out << "," << side << "_price_var_" << i * block_size;
}
};
append_side_cols("bid");
append_side_cols("ask");
return out;
}
inline std::ostream &
to_csv(std::ostream &out, uint32_t block_size, uint32_t num_blocks) {
orderman::Formatter::join(out, ",", _event_id, _security_id);
out << ",";
_data.bid.to_csv(out, block_size, num_blocks);
out << ",";
_data.ask.to_csv(out, block_size, num_blocks);
return out;
}
private:
SecurityID _security_id;
size_t _event_id = NO_EVENT;
Sided<BookStats> _data;
};
struct BookStatRecorder::PImpl {
PImpl(
std::ostream &out,
flat_map<size_t, size_t> interesting_event_ranges,
uint32_t block_size,
uint32_t num_blocks)
: out(out), interesting_event_ranges(interesting_event_ranges),
block_size(block_size), num_blocks(num_blocks) {}
std::ostream &out;
flat_map<size_t, size_t> interesting_event_ranges;
uint32_t block_size;
uint32_t num_blocks;
size_t event_id = NO_EVENT;
};
///////////////////////
// constructors
///////////////////////
BookStatRecorder::BookStatRecorder(
std::ostream &out, flat_map<size_t, size_t> interesting_event_ranges)
: BookStatRecorder(
out,
interesting_event_ranges,
DEFAULT_BLOCK_SIZE,
DEFAULT_NUM_BLOCKS) {}
BookStatRecorder::BookStatRecorder(
std::ostream &out,
flat_map<size_t, size_t> interesting_event_ranges,
uint32_t block_size,
uint32_t num_blocks)
: _me(new BookStatRecorder::PImpl(
out, interesting_event_ranges, block_size, num_blocks)) {}
BookStatRecorder::~BookStatRecorder() {}
inline BookStatsRow book_stats_row(const SecuritySnapshot *s, size_t event_id) {
bts_assert(s != nullptr);
return BookStatsRow(
s->security_id(), event_id, s->book_side(Side::Buy),
s->book_side(Side::Sell));
}
/**
* Writes a csv row of statistics for each affected security's book to an output
* stream.
*/
template <class Container>
inline void record_data(
const Container &affected_securities,
size_t event_id,
uint32_t block_size,
uint32_t num_blocks,
std::ostream &out) {
bts_assert(affected_securities.size());
for (const auto *s : affected_securities) {
out << std::endl;
book_stats_row(s, event_id).to_csv(out, block_size, num_blocks);
}
}
///////////////////////
// IMDP3FeedObserver
///////////////////////
void BookStatRecorder::on_recovery_complete(InitialSnapshot const &snap) {
BookStatsRow::headers_to_csv(_me->out, _me->block_size, _me->num_blocks);
record_data(snap, NO_EVENT, _me->block_size, _me->num_blocks, _me->out);
}
void BookStatRecorder::on_event_end(FeedEvent const &ev) {
bts_assert(ev.event_id > _me->event_id);
switch (get_event_ordinal(
_me->interesting_event_ranges, ev.event_id, MIN_OBSERVED_EVENT)) {
case bts::spartan::EventOrdinal::Waiting:
break;
case bts::spartan::EventOrdinal::Interesting:
_me->event_id = ev.event_id;
record_data(
ev.secs(), ev.event_id, _me->block_size, _me->num_blocks,
_me->out);
break;
case bts::spartan::EventOrdinal::Done:
// Nothing left to do: flush the csv output, then kill the process.
_me->out.flush();
bts_assert_msg(false, "No further events of interest.");
break;
}
}
} // namespace bts::spartan::mdp3
@@ -1,52 +0,0 @@
#pragma once
#include "domain/SecuritySnapshot.h"
#include "back_testing/feed_observers/FeedObserver.h"
#include <boost/container/flat_map.hpp>
#include <memory>
#include <unordered_map>
namespace bts::spartan::mdp3 {
struct BookStatRecorder : FeedObserver {
/**
* Initialize a BookStatRecorder that will write csv rows to the specified
* ostream. Statistics will be recorded for a series of price-quantity
* blocks on each observed book-update.
*
* @param out the output stream.
* @param interesting_event_ranges for each entry, the key indicates the
* first event in an interesting range, while the value indicates the number
* of events in the range (i.e., the range's length).
*/
BookStatRecorder(
std::ostream &out,
boost::container::flat_map<size_t, size_t> interesting_event_ranges =
{});
/**
* Initialize a BookStatRecorder that will write csv rows to the specified
* ostream. Statistics will be recorded for a series of price-quantity
* blocks on each observed book-update.
*
* @param out the output stream.
* @param interesting_event_ranges for each entry, the key indicates the
* first event in an interesting range, while the value indicates the number
* of events in the range (i.e., the range's length).
* @param block_size the number of quantity in the book that constitutes a
* 'price-block' (the unit size of the price-quantity segments in each book
* update for which stastics will be recorded).
* @param num_blocks the number of price-quantity blocks for which statistics
* will be recorded.
*/
BookStatRecorder(
std::ostream &out,
boost::container::flat_map<size_t, size_t> interesting_event_ranges,
uint32_t block_size,
uint32_t num_blocks);
~BookStatRecorder();
void on_recovery_complete(InitialSnapshot const &snap) override;
void on_event_end(FeedEvent const &ev) override;
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
};
} // namespace bts::spartan::mdp3
@@ -1,50 +0,0 @@
#include "BookStats.h"
#include "domain/book/BookSlice.h"
#include "domain/book/book_utils.h"
#include "lib/orderman/orderman_utils/Formatter.h"
namespace bts::spartan::mdp3 {
using namespace book_utils;
BookStats::BookStats() : BookStats(BookSide()) {}
BookStats::BookStats(BookSide book) : _book(book) {}
const BookSide &BookStats::book() const { return _book; }
void BookStats::book(BookSide value) { _book = value; }
Quantity BookStats::total_qty() const { return stats::total_qty(book()); }
Price BookStats::avg_price() const { return stats::avg_price(book()); }
Price BookStats::variance() const { return stats::variance(book()); }
BookSlice BookStats::get_slice(uint32_t slice_num, uint32_t block_size) {
return book_utils::get_slice(
book(), Quantity(slice_num * block_size),
Quantity(slice_num * (block_size + 1)));
}
std::ostream &BookStats::to_csv(
std::ostream &out, uint32_t block_size, uint32_t num_blocks) {
orderman::Formatter::join(
out, ",", _book[0].price, avg_price(), variance(), total_qty());
for (uint32_t i = 0; i < num_blocks; i++) {
// top price of ith block
out << ","
<< stats::best_price_after_qty(
_book.begin(), _book.end(), Quantity(i * block_size));
// avg price of ith block
out << "," << stats::avg_price(_book, get_slice(i, block_size));
// variance of ith block
out << "," << stats::variance(_book, get_slice(i, block_size));
}
return out;
}
} // namespace bts::spartan::mdp3
namespace bts::spartan {
template <>
struct Sided<mdp3::BookStats>;
}
@@ -1,38 +0,0 @@
#pragma once
#include <ostream>
#include "domain/Quantity.h"
#include "domain/book/Book.h"
namespace bts::spartan {
class BookSlice;
}
namespace bts::spartan::mdp3 {
class Price;
/**
* The per-side stats in each csv row written to the output stream.
*/
struct BookStats {
BookStats();
BookStats(BookSide);
const BookSide &book() const;
void book(BookSide);
Quantity total_qty() const;
Price avg_price() const;
Price variance() const;
BookSlice get_slice(uint32_t slice_num, uint32_t block_size);
std::ostream &
to_csv(std::ostream &out, uint32_t block_size, uint32_t num_blocks);
private:
BookSide _book;
};
}
@@ -1,46 +0,0 @@
#include "CompositeObserver.h"
namespace bts::spartan::mdp3 {
CompositeObserver::CompositeObserver() {}
CompositeObserver::CompositeObserver(
const std::initializer_list<FeedObserver*> &init)
: _observers(init) {}
void CompositeObserver::init() {
for (auto *o : _observers)
o->init();
}
void CompositeObserver::finit() {
for (auto *o : _observers)
o->finit();
}
void CompositeObserver::on_recovery_complete(InitialSnapshot const &s) {
for (auto *o : _observers)
o->on_recovery_complete(s);
}
void CompositeObserver::on_packet_begin(MDP3PacketView const &pv) {
for (auto *o : _observers)
o->on_packet_begin(pv);
}
void CompositeObserver::on_event_end(FeedEvent const &ev) {
for (auto *o : _observers)
o->on_event_end(ev);
}
void CompositeObserver::append(FeedObserver *observer) {
bts_assert(observer != nullptr);
_observers.push_back(observer);
}
void CompositeObserver::safe_append(FeedObserver *observer) {
if (observer != nullptr)
_observers.push_back(observer);
}
} // namespace bts::spartan::mdp3
@@ -1,33 +0,0 @@
#pragma once
#include "domain/SecurityDefinition.h"
#include "back_testing/common/mdp3_common.h"
#include "back_testing/feed_observers/FeedObserver.h"
#include <vector>
namespace bts::spartan::mdp3 {
/**
* A single observer that represents a group of observers
*/
struct CompositeObserver : FeedObserver {
CompositeObserver();
CompositeObserver(const std::initializer_list<FeedObserver *> &);
void init() override;
void finit() override;
void on_recovery_complete(InitialSnapshot const &) override;
void on_packet_begin(MDP3PacketView const &) override;
void on_event_end(FeedEvent const &) override;
/**
* Asserts that the observer isn't a nullptr before appending it
*/
void append(FeedObserver *);
/**
* Appends the observer if it isn't a nullptr
*/
void safe_append(FeedObserver *);
private:
std::vector<FeedObserver *> _observers;
};
} // namespace bts::spartan::mdp3
@@ -1,29 +0,0 @@
#include "DefinitionCollector.h"
#include "domain/SecuritySnapshot.h"
namespace bts::spartan::mdp3 {
void DefinitionCollector::init() { result.clear(); }
void DefinitionCollector::finit() {
for (auto const &[id, i] : result)
ptr_result[id] = &i;
}
void DefinitionCollector::on_recovery_complete(
std::vector<SecuritySnapshot const *> const &snap) {
for (auto s : snap) {
auto it = result.find(s->security_id());
if (it == result.end())
result.insert({s->security_id(), s->definition()});
}
}
void DefinitionCollector::on_event_end(FeedEvent const &ev) {
for (auto s : ev) {
auto it = result.find(s->security_id());
if (it == result.end())
result.insert({s->security_id(), s->definition()});
}
}
} // namespace bts::spartan::mdp3
@@ -1,22 +0,0 @@
#pragma once
#include "back_testing/feed_observers/FeedObserver.h"
#include "back_testing/common/mdp3_common.h"
#include "domain/SecurityDefinition.h"
namespace bts::spartan::mdp3 {
// Find all security definitions in the pcap
struct DefinitionCollector : FeedObserver {
std::unordered_map<SecurityID, SecurityDefinition> result;
std::unordered_map<SecurityID, SecurityDefinition const *> ptr_result;
void finit() override;
void init() override;
void on_recovery_complete(
std::vector<SecuritySnapshot const *> const &) override;
void on_event_end(FeedEvent const &ev) override;
};
} // namespace bts::spartan::mdp3
@@ -1,15 +0,0 @@
#include "FeedObserver.h"
#include "feed_handler/FeedHandler.h"
namespace bts::spartan::mdp3 {
void FeedObserver::bind(FeedHandler &h) {
h.on_snapshot.bind<&FeedObserver::on_recovery_complete>(*this);
h.on_packet_begin.bind<&FeedObserver::on_packet_begin>(*this);
h.on_packet_end.bind<&FeedObserver::on_packet_end>(*this);
h.on_event_begin.bind<&FeedObserver::on_event_begin>(*this);
h.on_event_end.bind<&FeedObserver::on_event_end>(*this);
h.on_error.bind<&FeedObserver::on_feed_error>(*this);
}
}
@@ -1,33 +0,0 @@
#pragma once
#include "feed_handler/InitialSnapshot.h"
#include "lib/utils/bts_assert.h"
namespace bts::spartan{
class MDP3PacketView;
class FeedHandler;
class FeedEvent;
class SpartanError;
}
namespace bts::spartan::mdp3 {
class FeedObserver {
public:
virtual void on_recovery_complete(InitialSnapshot const &) {}
virtual void on_packet_begin(MDP3PacketView const &) {}
virtual void on_packet_end(MDP3PacketView const &) {}
virtual void on_event_begin(FeedEvent const &) {}
virtual void on_event_end(FeedEvent const &) {}
virtual void on_feed_error(SpartanError const&) {bts_assert(false);}
virtual void init() {}
virtual void finit() {}
void bind(FeedHandler &h);
};
} // namespace bts::spartan::mdp3
@@ -1,48 +0,0 @@
#include "FillCounter.h"
#include "domain/SecuritySnapshot.h"
namespace bts::spartan::mdp3 {
FillCounter::FillCounter() {}
FillCounter::~FillCounter() {}
struct FillCounter::PImpl {
std::map<SecurityID, FillCount> counts;
};
void FillCounter::init() {
result.clear();
me.reset(new PImpl());
}
void FillCounter::finit() {
for (auto &p : me->counts)
result.push_back(p.second);
std::sort(result.begin(), result.end(), [](auto const &c1, auto const &c2) {
return c2.fills < c1.fills;
});
}
void FillCounter::on_recovery_complete(
std::vector<SecuritySnapshot const *> const &snaps) {
for (auto snap : snaps) {
auto &c = me->counts[snap->security_id()];
c.sec_id = snap->security_id();
c.symbol = snap->definition().symbol();
c.fills = 0;
}
}
void FillCounter::on_event_end(FeedEvent const &ev) {
for (auto snap : ev) {
auto &c = me->counts[snap->security_id()];
if (c.sec_id.value() == 0) {
c.sec_id = snap->security_id();
c.symbol = snap->definition().symbol();
c.fills = 0;
}
c.fills += snap->fills().size();
}
}
} // namespace bts::spartan::mdp3
@@ -1,39 +0,0 @@
#pragma once
#include "back_testing/feed_observers/FeedObserver.h"
#include "back_testing/common/mdp3_common.h"
namespace bts::spartan::mdp3 {
// Count all fills for each security and return them sorted by fill
// count in descending order
struct FillCount {
SecurityID sec_id;
std::string symbol;
size_t fills;
};
struct FillCounter : FeedObserver {
std::vector<FillCount> result;
std::unordered_map<SecurityID, size_t> result_map() {
std::unordered_map<SecurityID, size_t> result_idx;
for (auto const &f : result)
result_idx[f.sec_id] = f.fills;
return result_idx;
}
FillCounter();
~FillCounter();
void init() override;
void finit() override;
void on_recovery_complete(
std::vector<SecuritySnapshot const *> const &) override;
void on_event_end(FeedEvent const &ev) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
@@ -1,277 +0,0 @@
#include "IcebergFinder.h"
#include "domain/SecuritySnapshot.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "back_testing/matcher/iceberg_helper.h"
#include "back_testing/common/mdp3_utils.h"
#include "utils/ActionSummarizer.h"
#include <range/v3/algorithm.hpp>
namespace bts::spartan::mdp3 {
#define _LOG(level, ...) \
LOG(OrderMan, level, "{} {}", _log_header(), \
bts::orderman::Formatter::fmt(__VA_ARGS__))
#define _LOG_FN(level, ...) \
LOG(OrderMan, level, "{} {} {}", _log_header(), \
bts::orderman::Formatter::fmt(__FUNCTION__), \
bts::orderman::Formatter::fmt(__VA_ARGS__))
#define DIAG(...) _LOG(Diag, __VA_ARGS__)
#define DIAG_FN(...) _LOGFN(Diag, __VA_ARGS__)
#define DIAG_FN_EV(event_id, ...) \
_LOG_FN(Diag, bts::orderman::Formatter::fmt("[", event_id, "]"), __VA_ARGS__)
std::string _log_header() { return "IcebergFinder"; }
struct FillInfo {
Quantity aggressive, passive;
FillInfo() : aggressive(0), passive(0) {}
Quantity total() const { return aggressive + passive; }
void on_fill(const MDP3Fill f) {
if (f.aggressor != MDP3Fill::Aggressor::THIS_BUY &&
f.aggressor != MDP3Fill::Aggressor::THIS_SELL)
passive += f.quantity;
else
aggressive += f.quantity;
}
};
inline std::ostream &operator<<(std::ostream &out, const FillInfo &fi) {
out << "A. Fills: " << fi.aggressive << ", P. Fills: " << fi.passive;
return out;
}
Quantity _count_fills_by(
size_t event_id,
const std::vector<std::tuple<size_t, Quantity>> &fill_quantities) {
Quantity rv(0);
for (auto [ev, fqty] : fill_quantities) {
if (ev > event_id)
break;
rv += fqty;
}
return rv;
}
std::tuple<size_t, Quantity> _show_qty_at(
size_t event_id,
const std::vector<std::tuple<size_t, Quantity>> &show_quantities) {
if (show_quantities.empty())
return {0, Quantity(0)};
// get show qty from latest event at or before the specified event.
auto it = std::upper_bound(
show_quantities.begin(), show_quantities.end(),
std::make_tuple(event_id, Quantity(0)),
[](const auto &a, const auto &b) {
return std::get<0>(a) < std::get<0>(b);
});
if (it == show_quantities.begin())
return {0, Quantity(0)};
it--;
return *it;
}
IcebergInput::IcebergInput(
Quantity total_qty,
std::vector<std::tuple<size_t, Quantity>> show_quantities,
std::vector<std::tuple<size_t, Quantity>> fill_quantities)
: _total_qty(total_qty), _show_quantities(show_quantities),
_fill_quantities(fill_quantities) {}
IcebergFinder::IcebergFinder() {}
IcebergFinder::~IcebergFinder() {}
Quantity IcebergInput::count_fills_by(size_t event_id) const {
return _count_fills_by(event_id, _fill_quantities);
}
std::tuple<size_t, Quantity>
IcebergInput::show_qty_at(size_t event_id) const {
return _show_qty_at(event_id, _show_quantities);
}
void IcebergInputBuilder::on_change(
size_t event_id,
Quantity qty,
Quantity pass_fills,
Quantity aggr_fills) {
if (_show_quantities.empty() || _show_qty_realized) {
_show_quantities.emplace_back(event_id, qty);
_show_qty_realized = false;
} else if (qty != _active_qty) {
if (pass_fills.value() > 0 && aggr_fills.value() == 0) {
// The only reason for this update was that
// an aggressor order crossed it. It follows that the
// [correct] show qty must be at least as large as the
// order's new qty.
auto show_qty = std::get<1>(_show_quantities.back());
if (qty > show_qty)
std::get<1>(_show_quantities.back()) = qty;
// auto exp_qty = _active_qty - (fill_qty % show_qty);
// if (exp_qty.value() <= 0)
// exp_qty += show_qty;
} else if (pass_fills.value() == 0 && aggr_fills.value() == 0) {
// since there were no fills, this update has to have been
// caused by the trader changing the price and/or qty.
_show_quantities.emplace_back(event_id, qty);
_show_qty_realized = true;
}
}
if (pass_fills.value() > 0 && aggr_fills.value() == 0) {
if (!_is_iceberg) {
_is_iceberg = (qty != _active_qty - pass_fills);
if (_is_iceberg)
DIAG_FN_EV(
event_id, "detected iceberg (order ", _key, "). Qty ",
_active_qty, " -> ", qty, "(", total_qty(), " -> ",
(qty + _filled_qty + pass_fills + aggr_fills), ") A. Fills ",
aggr_fills, " P. Fills ", pass_fills);
}
}
_active_qty = qty;
_filled_qty += pass_fills + aggr_fills;
_fill_quantities.emplace_back(event_id, pass_fills + aggr_fills);
}
void IcebergInputBuilder::on_removed(
size_t event_id,
Quantity qty,
Quantity pass_fills,
Quantity aggr_fills) {
if (pass_fills.value() > 0 && aggr_fills.value() == 0) {
if (!_is_iceberg) {
_is_iceberg = (qty != _active_qty - pass_fills);
if (_is_iceberg)
DIAG_FN_EV(
event_id, "detected iceberg (order ", _key, "). Qty ",
_active_qty, " -> ", qty, "(", total_qty(), " -> ",
(qty + _filled_qty + pass_fills + aggr_fills), ") A. Fills ",
aggr_fills, " P. Fills ", pass_fills);
}
}
if ((pass_fills + aggr_fills).value() > 0) {
// assume order was filled to completion
_active_qty.value() = 0;
}
_filled_qty += pass_fills + aggr_fills;
}
struct IcebergFinder::PImpl {
std::unordered_map<OrderKey, FillInfo> fills;
std::unordered_map<OrderKey, IcebergInputBuilder> builder;
};
void IcebergFinder::init() {
result.clear();
me.reset(new PImpl());
}
void IcebergFinder::finit() {
for (const auto &p : me->builder) {
if (p.second.is_iceberg())
result.emplace(p.first, p.second.to_input());
}
}
void IcebergFinder::on_recovery_complete(InitialSnapshot const &snap) {
bts_assert(me->fills.empty());
for (const auto *s : snap) {
bts_assert_msg(
s->fills().empty(),
"Fills are not expected when the order book is being bootstrapped.");
for (const auto &o : s->orders())
_on_order_change(0, o.second, OrderChange::ChangeType::added);
}
}
void IcebergFinder::on_event_end(FeedEvent const &ev) {
me->fills.clear();
for (auto s : ev) {
_on_fills(s->fills());
_on_security_changed(ev.event_id, *s);
}
}
void IcebergFinder::_on_security_changed(
size_t event_id, const SecuritySnapshot &sec) {
// Fills are communicated in the snapshot of the
// security associated with the aggressor order
// that triggered the fill. Because of implied
// fills this is not always the same security as
// that of the order associated with the fill
// itself. For that reason we first collect all
// the order changes, across all changed
// snapshots, in a single flat map. This allows
// us to get the order information for each fill
// as we iterate over all fills below.
// Additionally we use this iteration to
// aggregate life time fills for each order. This
// information is used to estimate the quantity
// behind an iceberg. Since at the point an order
// is created we cannot know yet if its an
// iceberg we have to aggregate all the fills for
// all orders through the entire trading day and
// later throw away the non-iceberg
// ones. Finally, as long as we iterating over
// all fills, we also aggregate passive fills,
// which we use below to identify icebergs.
for (auto const &o : sec.order_changes())
_on_order_change(event_id, o);
}
void IcebergFinder::_on_fill(const MDP3Fill &f) {
me->fills.emplace(f.key, FillInfo()).first->second.on_fill(f);
}
void IcebergFinder::_on_order_change(size_t event_id, const OrderChange &o) {
_on_order_change(event_id, o.order, o.change);
}
void IcebergFinder::_on_order_change(
size_t event_id, const MDP3Order &order, OrderChange::ChangeType change) {
auto bd = me->builder.emplace(order.key, IcebergInputBuilder());
auto &od = bd.first->second;
od.key(order.key);
auto fit = me->fills.find(order.key);
Quantity pfills(0);
Quantity afills(0);
if (fit != me->fills.end()) {
afills += fit->second.aggressive;
pfills += fit->second.passive;
}
switch (change) {
case OrderChange::ChangeType::added:
case OrderChange::ChangeType::updated:
od.on_change(event_id, order.quantity, pfills, afills);
break;
case OrderChange::ChangeType::removed:
od.on_removed(event_id, order.quantity, pfills, afills);
break;
default:
break;
}
me->fills.erase(order.key);
}
std::unordered_map<OrderKey, IcebergInput>
IcebergFinder::find_all(const std::string &pcap) {
IcebergFinder ibf;
observe_channel(pcap, ibf);
return ibf.result;
}
std::ostream &operator<<(std::ostream &out, const IcebergInput &ib) {
out << "Total: " << ib.total_qty() << " Show: ";
if (ib.show_quantities().size() == 1) {
auto [ev, show] = ib.show_quantities().back();
out << "[" << ev << "] " << show;
} else {
for (auto [ev, show] : ib.show_quantities())
out << "\n\t[" << ev << "] " << show;
}
return out;
}
} // namespace bts::spartan::mdp3
@@ -1,132 +0,0 @@
#pragma once
#include "back_testing/common/mdp3_common.h"
#include "back_testing/feed_observers/FeedObserver.h"
#include <boost/container/flat_map.hpp>
#include <iostream>
#include <sstream>
#include <string>
#include <tuple>
#include <vector>
namespace bts::spartan::mdp3 {
// Find all the icebegers in the pcap, estimate their size.
class IcebergInput {
public:
IcebergInput(
Quantity total_qty,
std::vector<std::tuple<size_t, Quantity>> show_quantities,
std::vector<std::tuple<size_t, Quantity>> fill_quantities);
Quantity total_qty() const { return _total_qty; }
const std::vector<std::tuple<size_t, Quantity>> show_quantities() const {
return _show_quantities;
}
std::tuple<size_t, Quantity> show_qty_at(size_t event_id) const;
const std::vector<std::tuple<size_t, Quantity>> fill_quantities() const {
return _fill_quantities;
}
Quantity count_fills_by(size_t event_id) const;
std::set<size_t> all_eids() {
std::set<size_t> result;
for (auto const &p : _show_quantities)
result.insert(std::get<0>(p));
for (auto const &p : _fill_quantities)
result.insert(std::get<0>(p));
return result;
}
private:
Quantity _total_qty = Quantity(0);
std::vector<std::tuple<size_t, Quantity>> _show_quantities;
std::vector<std::tuple<size_t, Quantity>> _fill_quantities;
};
using IcebergMap = std::unordered_map<OrderKey, IcebergInput>;
std::ostream &operator<<(std::ostream &out, const IcebergInput &ib);
class IcebergInputBuilder {
public:
Quantity total_qty() const { return _filled_qty + _active_qty; }
Quantity filled_qty() const { return _filled_qty; }
Quantity active_qty() const { return _active_qty; }
bool is_iceberg() const { return _is_iceberg; }
OrderKey key() const { return _key; }
void key(OrderKey k) { _key = k; }
Quantity count_fills_by(size_t event_id) const;
void on_change(
size_t event_id,
Quantity qty,
Quantity pass_fill_qty,
Quantity aggr_fill_qty);
void on_removed(
size_t event_id,
Quantity qty,
Quantity pass_fill_qty,
Quantity aggr_fill_qty);
IcebergInput to_input() const {
IcebergInput rv(total_qty(), _show_quantities, _fill_quantities);
return rv;
}
private:
OrderKey _key = OrderKey(0);
Quantity _filled_qty = Quantity(0);
Quantity _active_qty = Quantity(0);
std::vector<std::tuple<size_t, Quantity>> _show_quantities;
std::vector<std::tuple<size_t, Quantity>> _fill_quantities;
bool _is_iceberg = false;
// when this is true, the subsequent show_qty change
// will be applied to a new show qty
// (instead of being interpretted as a potentially
// greater minimum of the current show qty).
// This will be set to true when the trader changes the qty,
// since that show qty is locked in for a span of events.
bool _show_qty_realized = false;
std::tuple<size_t, Quantity> _at_index(uint32_t i) const {
if (i < _show_quantities.size())
return _show_quantities[i];
return {0, Quantity(0)};
}
};
class IcebergFinder : public FeedObserver {
public:
std::unordered_map<OrderKey, IcebergInput> result;
IcebergFinder();
~IcebergFinder();
void on_recovery_complete(InitialSnapshot const &snap) override;
void init() override;
void finit() override;
void on_event_end(FeedEvent const &ev) override;
static std::unordered_map<OrderKey, IcebergInput>
find_all(const std::string &pcap);
private:
struct OrderData;
struct PImpl;
std::unique_ptr<PImpl> me;
void _on_security_changed(size_t event_id, const SecuritySnapshot &);
void _on_order_change(size_t event_id, const OrderChange &);
void _on_order_change(
size_t event_id, const MDP3Order &, OrderChange::ChangeType);
void _on_fill(const MDP3Fill &);
template <class Container> inline void _on_fills(const Container &fills) {
for (auto const &f : fills)
_on_fill(f);
}
};
} // namespace bts::spartan::mdp3
@@ -1,190 +0,0 @@
#include "SyntheticBookRecorder.h"
#include "domain/book/MBPUpdate.h"
#include "domain/book/book_utils.h"
#include "lib/utils/MapUtils.h"
#include "utils/sql/sql_utils.h"
#include <range/v3/view.hpp>
#include <set>
#include <unordered_set>
namespace bts::spartan::mdp3 {
using SyntheticDefinitionMap = std::unordered_map<SecurityID, LegVec>;
using EventUpdate = std::unordered_map<SecurityID, SyntheticMBPUpdate>;
using SyntheticBookHistory = std::map<EventID, EventUpdate>;
using ReverseLegMap = std::unordered_map<SecurityID, std::unordered_set<SecurityID>>;
using BookMap = std::unordered_map<SecurityID, std::tuple<size_t, Book>>;
// leg sec id, event id (last change), leg qty, leg price mult, bids, asks
using LegBookVec =
std::vector<std::tuple<size_t, SecurityID, Quantity, Price, Book>>;
ReverseLegMap build_reverse_leg_map(
const SyntheticDefinitionMap &synthetic_defs) {
ReverseLegMap rlm;
for (const auto &sd_p : synthetic_defs) {
for (auto leg_def : sd_p.second) {
rlm.emplace(leg_def.security_id, std::unordered_set<SecurityID>())
.first->second.insert(sd_p.first);
}
}
return rlm;
}
const LegVec &get_legs(
SecurityID synth_id, const SyntheticDefinitionMap &synth_defs) {
auto it = synth_defs.find(synth_id);
bts_assert(it != synth_defs.end());
return it->second;
}
// updates book entries for securities in the source sequence,
// then returns the set of affected polygons
template <class Iter>
void update_books(size_t event_id, Iter first, Iter last, BookMap &books) {
while (first != last) {
const SecuritySnapshot *s = *first;
books.insert_or_assign(
s->security_id(), std::make_tuple(event_id, s->book()));
first++;
}
}
template <class TIter>
std::unordered_set<SecurityID> synthetic_owners(
TIter first, TIter last, const ReverseLegMap &reverse_legs) {
std::unordered_set<SecurityID> all_owners;
while (first != last) {
const auto *leg_owners = MapUtils::find_value(reverse_legs, *first);
if (leg_owners != nullptr) {
for (auto id : *leg_owners)
all_owners.insert(id);
}
first++;
}
return all_owners;
}
SyntheticMBPUpdate::SyntheticMBPUpdate() {}
SyntheticMBPUpdate::SyntheticMBPUpdate(
SecurityID synthetic_id,
std::map<SecurityID, size_t> leg_book_eids,
SyntheticBook bids,
SyntheticBook asks)
: synthetic_id(synthetic_id), leg_book_eids(leg_book_eids),
bids(std::move(bids)), asks(std::move(asks)) {}
struct SyntheticBookRecorder::PImpl {
PImpl(SyntheticDefinitionMap synthetic_defs)
: _synthetic_defs(synthetic_defs) {
_reverse_legs = build_reverse_leg_map(_synthetic_defs);
}
const SyntheticDefinitionMap &synthetic_defs() const {
return _synthetic_defs;
}
const BookMap &books() const { return _books; }
BookMap &mbooks() { return _books; }
const ReverseLegMap &reverse_legs() const { return _reverse_legs; }
const std::map<size_t, std::unordered_map<SecurityID, SyntheticMBPUpdate>> &
synthetic_book_history() const {
return _synthetic_book_history;
}
std::map<size_t, std::unordered_map<SecurityID, SyntheticMBPUpdate>> &
msynthetic_book_history() {
return _synthetic_book_history;
}
const std::unordered_map<SecurityID, Sided<SyntheticBook>> &
synthetic_books() const {
return _synthetic_books;
}
std::unordered_map<SecurityID, Sided<SyntheticBook>> &synthetic_books() {
return _synthetic_books;
}
private:
SyntheticDefinitionMap _synthetic_defs;
ReverseLegMap _reverse_legs;
// map of sec id -> tuple<bids, asks>
// need to keep the latest books around for each security
// so they can be referenced to build the complete synthetic book
// whenever a particular leg's book changes for any synthetic
BookMap _books;
std::map<size_t, std::unordered_map<SecurityID, SyntheticMBPUpdate>>
_synthetic_book_history;
std::unordered_map<SecurityID, Sided<SyntheticBook>> _synthetic_books;
};
SyntheticBookRecorder::SyntheticBookRecorder(
SyntheticDefinitionMap synthetic_defs)
: _me(new SyntheticBookRecorder::PImpl(synthetic_defs)) {}
SyntheticBookRecorder::~SyntheticBookRecorder() {}
template <class Iter>
void on_securities_changed(
size_t event_id,
Iter first,
Iter last,
const SyntheticDefinitionMap &synth_defs,
const ReverseLegMap &reverse_legs,
BookMap &books,
std::unordered_map<SecurityID, Sided<SyntheticBook>> &synth_books,
SyntheticBookHistory &synth_book_history) {
update_books(event_id, first, last, books);
std::map<SecurityID, EventID> leg_book_eid_map;
auto &event_upd =
synth_book_history.emplace(event_id, EventUpdate()).first->second;
// using EventUpdate = std::unordered_map<SecurityID, SyntheticMBPUpdate>;
while (first != last) {
auto sec_id = (*first)->security_id();
auto *affected_synths = MapUtils::find_value(reverse_legs, sec_id);
if (affected_synths == nullptr)
continue;
const auto &bk = (*first)->book();
for (auto synth_id : *affected_synths) {
Sided<SyntheticBook> &sb =
MapUtils::assert_find_value(synth_books, synth_id);
// update bids
auto bid_upd = MBPUpdate::deduce(
Side::Buy, sb.bid.find_leg(sec_id).book(), bk[Side::Buy]);
sb.bid.on_update(sec_id, bid_upd);
// update asks
auto ask_upd = MBPUpdate::deduce(
Side::Sell, sb.ask.find_leg(sec_id).book(), bk[Side::Sell]);
sb.ask.on_update(sec_id, ask_upd);
event_upd[synth_id].leg_book_eids[sec_id] = event_id;
}
first++;
}
for (auto &[synth_id, synth_upd] : event_upd) {
const auto &legs = MapUtils::assert_find_value(synth_defs, synth_id);
for (auto &l : legs) {
auto it = synth_upd.leg_book_eids.find(l.security_id);
if (it == synth_upd.leg_book_eids.end())
it->second = std::get<0>(books[l.security_id]);
}
}
}
void SyntheticBookRecorder::on_recovery_complete(const InitialSnapshot &snap) {
on_securities_changed(
0, snap.begin(), snap.end(), _me->synthetic_defs(), _me->reverse_legs(),
_me->mbooks(), _me->synthetic_books(), _me->msynthetic_book_history());
}
void SyntheticBookRecorder::on_event_end(const FeedEvent &ev) {
on_securities_changed(
ev.event_id, ev.begin(), ev.end(), _me->synthetic_defs(),
_me->reverse_legs(), _me->mbooks(), _me->synthetic_books(),
_me->msynthetic_book_history());
}
const std::map<size_t, std::unordered_map<SecurityID, SyntheticMBPUpdate>> &
SyntheticBookRecorder::result() const {
return _me->synthetic_book_history();
}
} // namespace bts::spartan::mdp3
@@ -1,48 +0,0 @@
#pragma once
#include "back_testing/common/mdp3_common.h"
#include "back_testing/common/mdp3_utils.h"
#include "back_testing/feed_observers/FeedObserver.h"
#include "back_testing/feed_observers/IcebergFinder.h"
#include "back_testing/matcher/FifoMatcherContext.h"
#include "domain/LegDefinition.h"
#include "domain/SecuritySnapshot.h"
#include "domain/book/SyntheticBook.h"
#include <functional>
#include <memory>
#include <tuple>
#include <unordered_map>
namespace bts::spartan::mdp3 {
struct SyntheticMBPUpdate {
SyntheticMBPUpdate();
SyntheticMBPUpdate(
SecurityID synthetic_id,
std::map<SecurityID, size_t> leg_book_eids,
SyntheticBook bids,
SyntheticBook asks);
SecurityID synthetic_id;
// Note: a map is used (instead of other containers like
// unordered_map/vector<tuple> etc.) to ensure a consistent order of legs,
// and to reduce the risk of misuse that might accompany using a vector (like
// accidentally adding a duplicate entry for one of the legs)
std::map<SecurityID, size_t> leg_book_eids;
SyntheticBook bids;
SyntheticBook asks;
};
struct SyntheticBookRecorder : FeedObserver {
SyntheticBookRecorder(std::unordered_map<SecurityID, LegVec> synthetic_def);
~SyntheticBookRecorder();
void on_recovery_complete(InitialSnapshot const &snap) override;
void on_event_end(FeedEvent const &ev) override;
const std::map<EventID, std::unordered_map<SecurityID, SyntheticMBPUpdate>> &
result() const;
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
};
} // namespace bts::spartan::mdp3
@@ -1,94 +0,0 @@
#pragma once
#include "domain/SecuritySnapshot.h"
#include "back_testing/common/mdp3_common.h"
#include "feed_handler/IMDP3FeedObserver.h"
#include <functional>
#include <iostream>
#include <string>
namespace bts::spartan::mdp3::observers {
struct OrderTraceObserver : public IMDP3FeedObserver {
OrderTraceObserver(
std::function<bool(const FeedEvent &)> event_predicate,
std::function<bool(const SecuritySnapshot &)> security_predicate,
std::function<bool(const MDP3Order &)> order_predicate,
std::function<bool(const MDP3Fill &)> fill_predicate)
: _event_predicate(event_predicate),
_security_predicate(security_predicate),
_order_predicate(order_predicate), _fill_predicate(fill_predicate) {}
void on_recovery_complete(const InitialSnapshot &) override {}
void on_real_time_channel_loss() override {}
bool print_orders(
const bts::spartan::FeedEvent &ev,
bool has_printed_header = false) {
bool has_printed_order_header = false;
if (_event_predicate(ev)) {
std::stringstream str;
for (auto *s : ev.affected_securities) {
if (_security_predicate(*s)) {
for (auto o : s->orders()) {
if (_order_predicate(o.second)) {
if (!has_printed_header) {
str << "Order Trace (event " << ev.event_id
<< "):" << std::endl;
has_printed_header = true;
}
if (!has_printed_order_header) {
str << "\tOrders:" << std::endl;
has_printed_order_header = true;
}
str << "\t\t" << o.second << std::endl;
}
}
}
}
std::cout << str.str();
}
return has_printed_order_header;
}
bool print_fills(
const bts::spartan::FeedEvent &ev,
bool has_printed_header = false) {
bool has_printed_fill_header = false;
if (_event_predicate(ev)) {
std::stringstream str;
for (auto *s : ev.affected_securities) {
if (_security_predicate(*s)) {
for (auto f : s->fills()) {
if (_fill_predicate(f)) {
if (!has_printed_header) {
str << "Order Trace (event " << ev.event_id
<< "):" << std::endl;
has_printed_header = true;
}
if (!has_printed_fill_header) {
str << "\tFills:" << std::endl;
has_printed_fill_header = true;
}
str << "\t\t" << f << std::endl;
}
}
}
}
std::cout << str.str();
}
return has_printed_fill_header;
}
void on_real_time_event(const bts::spartan::FeedEvent &ev) override {
bool has_printed_header = print_orders(ev);
bool hph = print_fills(ev, has_printed_header);
if (has_printed_header || hph)
std::cout << std::endl;
}
private:
std::function<bool(const FeedEvent &)> _event_predicate;
std::function<bool(const SecuritySnapshot &)> _security_predicate;
std::function<bool(const MDP3Order &)> _order_predicate;
std::function<bool(const MDP3Fill &)> _fill_predicate;
};
} // namespace bts::spartan::mdp3::observers
@@ -1,59 +0,0 @@
#pragma once
#include "back_testing/common/mdp3_common.h"
#include "feed_handler/detail/MDP3Parser.h"
#include "back_testing/feed_observers/FeedObserver.h"
#include <iostream>
namespace bts::spartan::mdp3 {
struct IFeedRecorder {
virtual void init(std::ostream &) {}
virtual void finit(std::ostream &) {}
virtual void on_recovery_complete(
std::ostream &, const std::vector<const SecuritySnapshot *> &) {}
virtual void on_packet_begin(std::ostream &, MDP3PacketView const &) {}
virtual void on_packet_end(std::ostream &, MDP3PacketView const &) {}
virtual void on_event_begin(std::ostream &, FeedEvent const &) {}
virtual void on_event_end(std::ostream &, FeedEvent const &) {}
virtual void on_packet_loss(std::ostream &) {}
};
template <class Observer>
struct FeedRecorderWrapper : public FeedObserver {
FeedRecorderWrapper() {}
FeedRecorderWrapper(std::ostream &out, Observer *observer)
: _out(out), _observer(observer) {}
void init() override { _observer->init(_out); }
void finit() override { _observer->finit(_out); }
void on_recovery_complete(
const std::vector<const SecuritySnapshot *> &s) override {
_observer->on_recovery_complete(_out, s);
}
void on_packet_begin(MDP3PacketView const &mpv) override {
_observer->on_packet_begin(_out, mpv);
}
void on_packet_end(MDP3PacketView const &mpv) override {
_observer->on_packet_end(_out, mpv);
}
void on_event_begin(FeedEvent const &ev) override {
_observer->on_event_begin(_out, ev);
}
void on_event_end(FeedEvent const &ev) override {
_observer->on_event_end(_out, ev);
}
void on_feed_error(const SpartanError &) override {
_observer->on_packet_loss(_out);
}
private:
std::ostream &_out;
Observer *_observer;
};
} // namespace bts::spartan::mdp3
@@ -1,50 +0,0 @@
#include "MatcherBookRecorder.h"
#include "domain/EventID.h"
#include "domain/book/Book.h"
#include "lib/utils/bts_assert.h"
#include "back_testing/common/mdp3_common.h"
namespace bts::spartan::mdp3 {
static void _write_book(std::ostream &out, const BookSide &book) {
for (const auto &l : book)
out << "," << l.price << "," << l.quantity << "," << l.n_orders;
}
static void _write_row(
std::ostream &out,
EventID event_id,
SecurityID sec_id,
const BookSide &bids,
const BookSide &asks) {
out << event_id << "," << sec_id;
_write_book(out, bids);
_write_book(out, asks);
out << "\n";
}
MatcherBookRecorder::MatcherBookRecorder() {}
static void _write_rows(
const std::vector<std::variant<SimBookUpdate, SimOrderUpdate>> &effects,
std::ostream &out,
EventID event_id) {
for (const auto &eff : effects) {
if (std::holds_alternative<SimBookUpdate>(eff)) {
const auto &up = std::get<SimBookUpdate>(eff);
_write_row(out, event_id, up.security_id, up.mbp_buy, up.mbp_sell);
}
}
}
void MatcherBookRecorder::on_recovery_complete(RecoveryCompleteArgs &args) {
bts_assert(args.effects != nullptr);
_write_rows(*args.effects, args.out, 0);
}
void MatcherBookRecorder::on_event_end(EventEndArgs &args) {
bts_assert(args.effects != nullptr);
_write_rows(*args.effects, args.out, args.mdp3_event.event_id);
}
} // namespace bts::spartan::mdp3
@@ -1,14 +0,0 @@
#pragma once
#include "FeedRecorder.h"
#include "MatcherRecorder.h"
namespace bts::spartan::mdp3 {
struct MatcherBookRecorder : public MatcherRecorder {
MatcherBookRecorder();
void on_recovery_complete(RecoveryCompleteArgs &) override;
void on_event_end(EventEndArgs &) override;
};
} // namespace bts::spartan::mdp3
@@ -1,43 +0,0 @@
#include "MatcherFillRecorder.h"
#include <iostream>
namespace bts::spartan::mdp3 {
/**
* Writes a csv row followed by a new line to the output stream
*/
static void _write_row(
std::ostream &out,
size_t event_id,
SecurityID sec_id,
const SimOrderUpdate &upd) {
out << event_id << "," << sec_id << "," << upd.fill_type << "," << upd.key
<< "," << upd.side << "," << upd.last_price << "," << upd.last_fills
<< "\n";
}
MatcherFillRecorder::MatcherFillRecorder() {}
static void _write_rows(
const std::vector<std::variant<SimBookUpdate, SimOrderUpdate>> &effects,
std::ostream &out,
size_t event_id) {
for (const auto &eff : effects) {
if (std::holds_alternative<SimOrderUpdate>(eff)) {
const auto &up = std::get<SimOrderUpdate>(eff);
_write_row(out, event_id, up.security_id, up);
}
}
}
void MatcherFillRecorder::on_recovery_complete(RecoveryCompleteArgs &args) {
bts_assert(args.effects != nullptr);
_write_rows(*args.effects, args.out, 0);
}
void MatcherFillRecorder::on_event_end(EventEndArgs &args) {
bts_assert(args.effects != nullptr);
_write_rows(*args.effects, args.out, args.mdp3_event.event_id);
}
} // namespace bts::spartan::mdp3
@@ -1,13 +0,0 @@
#pragma once
#include "MatcherRecorder.h"
namespace bts::spartan::mdp3 {
struct MatcherFillRecorder : public MatcherRecorder {
MatcherFillRecorder();
void on_recovery_complete(RecoveryCompleteArgs &) override;
void on_event_end(EventEndArgs &) override;
};
} // namespace bts::spartan::mdp3
@@ -1,25 +0,0 @@
#include "MatcherProcTimeRecorder.h"
#include <ostream>
#include <cstdint>
namespace bts::spartan::mdp3 {
#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__)
#define DIAG(...) _LOG(Diag, __VA_ARGS__)
#define INFO(...) _LOG(Info, __VA_ARGS__)
#define WARN(...) _LOG(Warn, __VA_ARGS__)
static void _write_row(
std::ostream &out, EventID event_id, std::int64_t proc_time) {
out << event_id << "," << proc_time << "\n";
}
void MatcherProcTimeRecorder::on_recovery_complete(RecoveryCompleteArgs &args) {
_write_row(args.out, no_event(), args.matcher_proc_results.proc_time_nanos);
}
void MatcherProcTimeRecorder::on_event_end(EventEndArgs &args) {
_write_row(args.out, no_event(), args.matcher_proc_results.proc_time_nanos);
}
} // namespace bts::spartan::mdp3
@@ -1,11 +0,0 @@
#pragma once
#include "MatcherRecorder.h"
namespace bts::spartan::mdp3 {
struct MatcherProcTimeRecorder : public MatcherRecorder {
void on_recovery_complete(RecoveryCompleteArgs &) override;
void on_event_end(EventEndArgs &) override;
};
} // namespace bts::spartan::mdp3
@@ -1,277 +0,0 @@
#include "MatcherRecorder.h"
#include "back_testing/sim/PCapOrderChangeBootstrapper.h"
#include "common/Stopwatch.h"
#include "domain/SecuritySnapshot.h"
#include "back_testing/common/EventRangeMap.h"
#include "back_testing/common/EventRangeMonitor.h"
#include "feed_handler/detail/gen/src/SecurityTradingStatus.h"
#include <string>
namespace bts::spartan::mdp3 {
#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__)
#define DIAG(...) _LOG(Diag, __VA_ARGS__)
#define INFO(...) _LOG(Info, __VA_ARGS__)
#define WARN(...) _LOG(Warn, __VA_ARGS__)
static EventRangeMap event_range_map(const std::vector<EventRange> events) {
EventRangeMap erm;
for (auto ev_range : events)
erm[ev_range.start()] = ev_range.length();
return erm;
}
/**
* Determines if the matcher's mbp equals that of the SecuritySnapshot (n_orders
* at price levels is ignored)
*/
static bool mbp_equals(
const SecuritySnapshot *snap, MergedMatcher &matcher, Side side) {
auto exch_books = snap->book_side(side);
auto match_books = matcher.book(snap->security_id(), side);
for (uint32_t i = 0; i < exch_books.size(); i++) {
if (match_books[i].quantity != exch_books[i].quantity)
return false;
if (match_books[i].quantity == 0)
return true;
if (match_books[i].price != exch_books[i].price)
return false;
}
return true;
}
static SmallOrderChangeVec to_small_change_vec(
const OrderChanges &changes, const IcebergMap &icebergs, size_t event_id) {
SmallOrderChangeVec rv;
for (const auto &ch : changes) {
auto &sch = rv.emplace_back(ch, true);
iceberg_helper::add_iceberg_annotation(
sch.order, sch.change, icebergs, event_id, false);
}
return rv;
}
static bool is_security_active(
const SecuritySnapshot *security,
const SecurityCriteria &security_criteria) {
return security->trading_status() ==
::mdp3::SecurityTradingStatus::ReadyToTrade &&
security_criteria.includes_or_undefined(security->definition());
}
static std::uint16_t count_aggressor_secs(const FeedEvent::SecContainer &secs) {
std::uint16_t num_aggs = 0;
for (const auto *s : secs) {
for (const auto &f : s->fills()) {
if (f.is_aggressor()) {
num_aggs++;
break;
}
}
}
return num_aggs;
}
template <class SnapshotContainer>
static inline MatcherRecorder::MatcherProcResults process_changes(
const SnapshotContainer &snaps,
const SecurityCriteria &security_criteria,
PCapOrderChangeBootstrapper &bootstrapper,
MergedMatcher &matcher,
const IcebergMap &icebergs,
std::vector<MatchEffect> &effects,
EventID event_id) {
Stopwatch stopwatch;
for (const auto *s : snaps) {
if (!is_security_active(s, security_criteria))
continue;
auto changes = bootstrapper.deduced_order_changes(s, event_id);
auto schanges = to_small_change_vec(changes, icebergs, event_id);
stopwatch.start();
matcher.match(
MergedMatcher::ChangeSrc::SIMULATOR_INPUT, schanges, &effects,
{event_id, 0});
stopwatch.stop();
}
bool mbp_correct = true;
for (const auto *s : snaps) {
if (!is_security_active(s, security_criteria))
continue;
if (!mbp_equals(s, matcher, Side::Buy) ||
!mbp_equals(s, matcher, Side::Sell)) {
WARN(
"[event {}] Discrepancy between matcher books and actual books "
"detected for [{}] {}",
event_id, s->security_id(), s->symbol());
mbp_correct = false;
}
}
return {mbp_correct, stopwatch.elapsed<std::nano>().count()};
}
static void reset_books(
MergedMatcher &matcher,
PCapOrderChangeBootstrapper &bootstrapper,
const IcebergMap &icebergs,
const std::unordered_map<SecurityID, const SecuritySnapshot *> &securities,
EventID event_id) {
INFO(
"[event {}] MatcherRecorder initializing matcher with resting orders",
event_id);
// Reset all security's books because implied matching could have corrupted
// any of them. Clear the matcher before adding the resting orders so they
// don't inadvertantly cross a corrupt implied market.
matcher.clear();
for (const auto &sec_p : securities) {
auto resting_orders = bootstrapper.init_orders(sec_p.second, event_id);
for (auto &o : resting_orders)
iceberg_helper::add_iceberg_annotation(
o, OrderChange::ChangeType::added, icebergs, event_id, true);
matcher.reset(sec_p.second->security_id(), resting_orders);
}
}
struct MatcherRecorder::Curry::PImpl {
std::ostream &out;
MergedMatcher &matcher;
const IcebergMap &icebergs;
SecurityCriteria security_criteria;
std::unique_ptr<MatcherRecorder> recorder;
PCapOrderChangeBootstrapper bootstrapper;
EventRangeMonitor event_monitor;
std::uint16_t max_aggressors;
bool verbose;
std::unordered_map<SecurityID, const SecuritySnapshot *> securities{};
bool is_listening = true;
bool done = false;
};
MatcherRecorder::Curry::Curry(MatcherRecorder::Curry::Args args)
: _me(new MatcherRecorder::Curry::PImpl{
args.out, args.matcher, args.icebergs, args.security_criteria,
std::move(args.recorder), PCapOrderChangeBootstrapper(),
event_range_map(args.observed_events), args.max_aggressors,
args.verbose}) {}
MatcherRecorder::Curry::~Curry() {}
void MatcherRecorder::Curry::on_recovery_complete(
const std::vector<const SecuritySnapshot *> &snaps) {
if (_me->done)
return;
INFO("[event 0, on_recovery_complete] MatcherRecorder resetting all state");
_me->securities.clear();
_me->matcher.clear();
_me->bootstrapper.reset();
_me->event_monitor.event() = 0;
_me->is_listening = true;
// Add defs to the matcher to build implied matching index
for (const auto *s : snaps)
if (_me->security_criteria.includes_or_undefined(s->definition()) &&
_me->securities.emplace(s->security_id(), s).second)
_me->matcher.add_def(s->definition());
// Add resting orders
std::vector<MatchEffect> effects;
auto proc_res = process_changes(
snaps, _me->security_criteria, _me->bootstrapper, _me->matcher,
_me->icebergs, effects, 0);
MatcherRecorder::RecoveryCompleteArgs args{
snaps, _me->matcher, proc_res, _me->out, &effects};
_me->recorder->on_recovery_complete(args);
}
void MatcherRecorder::Curry::on_event_end(const FeedEvent &ev) {
if (_me->done)
return;
// Add resting orders
_me->event_monitor.event() = ev.event_id;
bool was_listening = _me->is_listening;
switch (_me->event_monitor.state()) {
case EventRangeMonitor::State::Idle:
_me->is_listening = false;
break;
case EventRangeMonitor::State::Done:
_me->is_listening = false;
if (!_me->done) {
INFO("[event {}] MatcherRecorder finished observation.", ev.event_id);
_me->done = true;
}
return;
case EventRangeMonitor::State::Listening:
case EventRangeMonitor::State::None:
_me->is_listening = true;
break;
default:
bts_assert(false);
break;
}
// 'skip_this_event_only' has a different meaning than '!_me->listening.'
// If the former is true, the matcher is reset with the current resting
// orders, so incremental changes may be processed on the next update (if it
// isn't ignored/skipped). However, if the latter is false, the matcher is
// cleared, and is not initialized with resting orders until the next event
// where '_me->is_listening' is true (this means the matcher won't process
// incremental changes on that update either, as it won't already have been
// bootstrapped).
//
// Note that 'skip_this_event_only' doesn't affect state beyond this call.
// Predicates (other than the event range filter) should flip
// 'skip_this_event_only' to true if they evaluate to false.
bool skip_this_event_only = false;
if (_me->is_listening && _me->max_aggressors > 0) {
auto num_aggs = count_aggressor_secs(ev.secs());
if (num_aggs > _me->max_aggressors) {
INFO(
"[event {}] MatcherRecorder ignoring event: too many securities "
"with aggressor orders ({}, max is {})",
ev.event_id, num_aggs, _me->max_aggressors);
skip_this_event_only = true;
}
}
if (skip_this_event_only) {
// This event falls within an observed range, but another predicate
// evaluated to false, indicating this event's incremental changes should
// not be processed. Reset the matcher with resting orders so it is
// prepared to process incremental changes on the next update.
reset_books(
_me->matcher, _me->bootstrapper, _me->icebergs, _me->securities,
ev.event_id);
} else if (!_me->is_listening) {
if (was_listening) {
// This event is ignored, but the previous event was observed.
// Clear the matcher and reset the bootstrapper (the latter is reset to
// ensure securities are bootstrapped on their next update)
INFO("[event {}] MatcherRecorder clearing all books", ev.event_id);
_me->bootstrapper.reset();
_me->matcher.clear();
}
} else if (!was_listening) {
// This event is observed, but the previous event was ignored. Bootstrap
// the matcher with resting orders
reset_books(
_me->matcher, _me->bootstrapper, _me->icebergs, _me->securities,
ev.event_id);
} else {
// This event is observed, as was the previous event. Process incremental
// changes and reconcile the matcher's results with those of the exchange.
std::vector<MatchEffect> effects;
if (_me->verbose)
DIAG(
"[event {}] MatcherRecorder processing order changes",
ev.event_id);
auto proc_res = process_changes(
ev.secs(), _me->security_criteria, _me->bootstrapper, _me->matcher,
_me->icebergs, effects, ev.event_id);
MatcherRecorder::EventEndArgs args{
ev, _me->matcher, proc_res, _me->out, &effects};
_me->recorder->on_event_end(args);
if (!proc_res.mbp_correct) {
// Reset the books to avoid compounding errors on subsequent events
reset_books(
_me->matcher, _me->bootstrapper, _me->icebergs, _me->securities,
ev.event_id);
}
}
}
} // namespace bts::spartan::mdp3
@@ -1,78 +0,0 @@
#pragma once
#include "FeedRecorder.h"
#include "back_testing/matcher/Matcher.h"
#include "back_testing/matcher/iceberg_helper.h"
#include "domain/EventRange.h"
#include "domain/SecurityCriteria.h"
#include "back_testing/common/mdp3_common.h"
#include <memory>
#include <variant>
#include <vector>
namespace bts::spartan::mdp3 {
class MatcherRecorder {
public:
struct MatcherProcResults {
bool mbp_correct;
std::int64_t proc_time_nanos;
};
struct RecoveryCompleteArgs {
const std::vector<const SecuritySnapshot *> initial_snapshot;
MergedMatcher &matcher;
MatcherProcResults matcher_proc_results;
std::ostream &out;
const std::vector<MatchEffect> *effects;
};
struct EventEndArgs {
const FeedEvent &mdp3_event;
MergedMatcher &matcher;
MatcherProcResults matcher_proc_results;
std::ostream &out;
const std::vector<std::variant<SimBookUpdate, SimOrderUpdate>> *effects;
};
struct BooksResetArgs {
const FeedEvent &mdp3_event;
MergedMatcher &matcher;
std::ostream &out;
const std::vector<std::variant<SimBookUpdate, SimOrderUpdate>> *effects;
};
virtual ~MatcherRecorder() {}
virtual void on_recovery_complete(RecoveryCompleteArgs &) {}
virtual void on_event_end(EventEndArgs &) {}
virtual void on_books_reset(BooksResetArgs &) {}
class Curry : public FeedObserver {
public:
struct Args {
std::ostream &out;
MergedMatcher &matcher;
const IcebergMap &icebergs;
std::unique_ptr<MatcherRecorder> recorder;
SecurityCriteria security_criteria{};
std::vector<EventRange> observed_events{};
std::uint16_t max_aggressors = 0;
bool verbose = false;
};
Curry();
Curry(Args);
~Curry();
void on_recovery_complete(
const std::vector<const SecuritySnapshot *> &s) override;
void on_event_end(FeedEvent const &ev) override;
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
};
};
} // namespace bts::spartan::mdp3
@@ -1,45 +0,0 @@
#pragma once
#include "lib/domain_model/Security.pb.h"
#include "domain/SecuritySnapshot.h"
#include "back_testing/feed_observers/FeedObserver.h"
#include "back_testing/common/mdp3_common.h"
#include <functional>
#include <iostream> #include <string>
namespace bts::spartan::mdp3::observers {
struct WhereObserver : public FeedObserver {
WhereObserver(
std::shared_ptr<FeedObserver> observer,
std::function<bool(const FeedEvent &)> predicate)
: _observer(observer), _predicate(predicate) {}
void on_recovery_complete(const InitialSnapshot &s) override {
_observer->on_recovery_complete(s);
}
void on_real_time_event(const bts::spartan::FeedEvent &ev) override {
if (_predicate(ev))
_observer->on_real_time_event(ev);
}
void on_real_time_channel_loss() override {
_observer->on_real_time_channel_loss();
}
private:
std::shared_ptr<FeedObserver> _observer;
std::function<bool(const FeedEvent &)> _predicate;
};
struct WhereSecurityObserver : public WhereObserver {
WhereSecurityObserver(
std::shared_ptr<FeedObserver> observer,
std::function<bool(const SecuritySnapshot &)> predicate)
: WhereObserver(observer, [&](const auto &ev) {
for (auto s : ev.affected_securities) {
if (predicate(*s))
return true;
}
return false;
}) {}
};
} // namespace bts::spartan::mdp3::observers
@@ -1,165 +0,0 @@
#include "implied_market.h"
#include "lib/utils/MapUtils.h"
#include "lib/utils/bts_assert.h"
#include "server/sparkd/cmds/spartan/domain/SecurityDefinition.h"
#include "server/sparkd/cmds/spartan/utils/Permutation.h"
#include <boost/container/flat_map.hpp>
#include <boost/container/small_vector.hpp>
#include <functional>
#include <tuple>
namespace bts::spartan::mdp3 {
/**
* Determine if two LegMaps are equal
*/
static bool are_equal(const LegMap &lhs, const LegMap &rhs) {
return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
}
/**
* Multiply the quantities of each leg by a particular value
*/
static LegMap multiply(LegMap legs, int mult) {
if (mult == 1)
return legs;
if (mult == 0)
return {};
for (auto &leg : legs) {
leg.second.ratio.value() *= mult;
if (leg.second.ratio < 0) {
leg.second.side = other_side(leg.second.side);
leg.second.ratio = -leg.second.ratio;
}
}
return legs;
}
/**
* Flip the side of each leg
*/
static LegMap negate(const LegMap &legs) {
return multiply(legs, -1);
}
/**
* Combine one collection of legs with another. Legs with new securities
* are appended to the target collection, while legs with common securities
* are merged together. The merged leg's quantity is the sum of the two legs'
* quantities if they have the same side, otherwise it is the difference. If
* the resultant quantity is zero, the leg for that security is removed; if it
* is negative, the side is flipped, and the quantity replaced with its absolute
* value.
*/
static void merge_legs(LegMap &target_legs, const LegMap &other_legs) {
for (const auto &[osec, oleg] : other_legs) {
auto res = target_legs.emplace(osec, oleg);
auto &l = res.first->second;
if (!res.second)
l.ratio += l.side == oleg.side ? oleg.ratio : -oleg.ratio;
if (l.ratio == 0)
target_legs.erase(osec);
else if (l.ratio < 0) {
l.side = other_side(l.side);
l.ratio = -l.ratio;
}
}
}
/**
* Holds per-security data needed to discover implied market candidates
*/
struct SecData {
LegMap legs;
bool is_spread() const { return legs.size() > 1; }
};
struct ImpliedMarketModel::PImpl {
std::unordered_map<SecurityID, SecData> securities;
};
ImpliedMarketModel::ImpliedMarketModel()
: _me(new ImpliedMarketModel::PImpl()) {}
ImpliedMarketModel::~ImpliedMarketModel() {}
void ImpliedMarketModel::insert(SecurityID sec, const LegMap &legs) {
_me->securities[sec] = {legs};
}
void ImpliedMarketModel::erase(SecurityID sec) { _me->securities.erase(sec); }
void ImpliedMarketModel::clear() { _me->securities.clear(); }
std::vector<LegMap> ImpliedMarketModel::find_implied_markets(SecurityID sec) {
std::vector<LegMap> dest;
std::vector<SecurityID> securities_vec;
for (auto it = _me->securities.begin(); it != _me->securities.end(); it++)
securities_vec.push_back(it->first);
std::vector<std::vector<SecurityID>> sec_sets;
if (_me->securities[sec].is_spread()) {
for (auto leg1_it = _me->securities.begin();
leg1_it != _me->securities.end(); leg1_it++) {
if (leg1_it->second.is_spread())
continue;
for (auto leg2_it = std::next(leg1_it);
leg2_it != _me->securities.end(); leg2_it++) {
if (leg2_it->second.is_spread())
continue;
auto &ss = sec_sets.emplace_back();
ss.push_back(leg1_it->first);
ss.push_back(leg2_it->first);
}
}
} else {
for (auto leg1_it = _me->securities.begin();
leg1_it != _me->securities.end(); leg1_it++) {
for (auto leg2_it = std::next(leg1_it);
leg2_it != _me->securities.end(); leg2_it++) {
if (leg1_it->second.is_spread() == leg2_it->second.is_spread())
continue;
auto &ss = sec_sets.emplace_back();
ss.push_back(leg1_it->first);
ss.push_back(leg2_it->first);
}
}
}
// Add implied market if the legs match the target
auto add_candidate_if_match = [&](const std::vector<SecurityID> &leg_secs,
const std::vector<int> &leg_qtys,
const LegMap &fut_legs) {
if (are_equal(_me->securities[sec].legs, fut_legs)) {
auto &implied_legs = dest.emplace_back();
for (uint32_t i = 0; i < leg_secs.size(); i++) {
auto &l = implied_legs[leg_secs[i]];
l.security_id = leg_secs[i];
l.ratio.value() = std::abs(leg_qtys[i]);
l.side = leg_qtys[i] > 0 ? Side::Buy : Side::Sell;
}
return true;
}
return false;
};
for (const auto &secs : sec_sets) {
// The implementation presupposes that implied markets consist of exactly
// 2 legs, and that the quantities of said legs are +/- 1. This
// simplifies implementation by eliminating the need for complicated
// factorizations of arbitrary leg quantities, as there are only 4
// possible permutations of quantities for the pair of legs:
// {1, 1}, {1, -1}, {-1, -1}, {-1, 1}.
bts_assert(secs.size() == 2);
bts_assert(secs[0] != secs[1]);
LegMap and_legs = _me->securities[secs[0]].legs;
LegMap xor_legs = _me->securities[secs[0]].legs;
merge_legs(and_legs, _me->securities[secs[1]].legs);
merge_legs(xor_legs, negate(_me->securities[secs[1]].legs));
(add_candidate_if_match(secs, {1, 1}, and_legs) ||
add_candidate_if_match(secs, {-1, -1}, negate(and_legs)) ||
add_candidate_if_match(secs, {1, -1}, xor_legs) ||
add_candidate_if_match(secs, {-1, 1}, negate(xor_legs)));
}
return dest;
}
} // namespace bts::spartan::mdp3
@@ -1,96 +0,0 @@
#pragma once
#include "domain/SecurityDefinition.h"
#include "back_testing/common/mdp3_common.h"
#include <boost/container/flat_map.hpp>
#include <iostream>
#include <map>
#include <memory>
#include <vector>
namespace bts::spartan::mdp3 {
// Implementation Notes:
// The current implied market implementation makes the following assumptions:
//
// * IMPLIED FUTURE MARKETS are 1 spread and 1 future
// * the spread has exactly 2 legs, each with qty +/- 1
// * one leg's security is the target future
// * the other leg is canceled out by the future component in the
// implied market
// * there are typically many potential implied markets for a given future
// (as many as there are spreads with a leg for the target future)
//
// * IMPLIED SPREAD MARKETS are the outrights that make up the spread
// * unlike futures, this means there is only one potential implied market
// for spreads
//
// The assumptions above naturally limit the securities for which implied
// markets can be accurately modeled. Market channels with more complex
// securities (e.g., butterfly spreads), and more sophisticated implied
// matching behavior (e.g., inter-product spreads, ratios, etc.) will require
// a more complex model. This could be implemented with a contract matrix,
// whose rows represent future contracts, and columns represent exchange
// listed securities. Given such a matrix 'A,' an implied candidate column
// space basis for an exchange listed security (represented by column 'b' in
// the contract matrix) would be the independent solution vectors 'x' for
// 'Ax=b.'
//
// A simple example may better illustrate the concept of the 'contract
// matrix.' Consider a contract matrix that represents a few ES futures and
// calendar spreads:
// __________________________________________________________
// _______|_ESU1_|_ESZ1_|_ESG2_|_ESU1-ESZ1_|_ESU1-ESG2_|_ESZ1-ESG2_|
// | ESU1 | 1 | 0 | 0 | 1 | 1 | 0 |
// | ESZ1 | 0 | 1 | 0 | -1 | 0 | 1 |
// |_ESG2_|__0___|__0___|__1___|_____0_____|____-1_____|____-1_____|
//
// Notice how the rows represent futures, and the columns represent exch
// listed securities. There are no spread rows, only spread columns.
//
// If 'A' is the contract matrix, and 'b' is the column vector for 'ESU1,'
// then the potential implied markets for 'ESU1' are the solutions 'x' to
// the equation 'Ax=b'
//
// (A) (x) (b)
// | 1 | 0 | 0 | 1 | 0 | * | ? | = | 1 | --> x = | 1 |, | 0 |, | 0 |
// | 0 | 1 | 0 |-1 | 1 | | ? | | 0 | | 0 | | 1 | | 0 |
// | 0 | 0 | 1 | 0 |-1 | | ? | | 0 | | 0 | | 0 | | 1 |
// | 0 | | 1 | | 0 |
// | 0 | | 0 | | 1 |
// | 0 | | 0 | | 0 |
// Since 'b' is itself an exchange listed security, one of these solutions
// (the first one) is simply the security itself. The other 2 solutions are
// implied markets for ESU1:
//
// 1) ESU1 = ESZ1 + ESU1-ESZ1
// 2) ESU1 = ESG2 + ESU1-ESG2
/**
* A model used to generate implied market candidates for securities
* of a particular asset (cross product implied markets are not supported)
*/
class ImpliedMarketModel {
public:
ImpliedMarketModel();
~ImpliedMarketModel();
void insert(SecurityID, const LegMap &);
void erase(SecurityID);
void clear();
/**
* Get all possible implied markets for a particular security
*
* @param security The security for which to get implied markets
* @returns The vector of implied markets
*/
std::vector<LegMap> find_implied_markets(SecurityID security);
const std::vector<LegMap> &find() const;
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
};
} // namespace bts::spartan::mdp3
@@ -1,162 +0,0 @@
#include "FifoImpliedSecurityMatcher.h"
#include "FifoSecurityMatcher.h"
#include "FillDepot.h"
#include "back_testing/common/mdp3_common.h"
#include "back_testing/common/order_utils.h"
#include "back_testing/matcher/fill_factory.h"
#include "back_testing/sim/sim_common.h"
#include "domain/LegDefinition.h"
#include "domain/SideQuantity.h"
#include "domain/book/book_utils.h"
#include "lib/domain_model/Security.h"
#include <limits>
#include <variant>
namespace bts::spartan::mdp3 {
struct FifoImpliedSecurityMatcher::PImpl {
PImpl(SecurityID security_id)
: security_id(security_id), implied_mkt() {}
SecurityID security_id;
std::map<SecurityID, std::tuple<FifoSecurityMatcher *, Quantity>>
implied_mkt;
};
///////////////////////
// constructors
///////////////////////
FifoImpliedSecurityMatcher::FifoImpliedSecurityMatcher(SecurityID security_id)
: _me(new FifoImpliedSecurityMatcher::PImpl(security_id)) {}
FifoImpliedSecurityMatcher::~FifoImpliedSecurityMatcher() {}
///////////////////////
// diagnostic
///////////////////////
SecurityID FifoImpliedSecurityMatcher::security_id() const {
return _me->security_id;
}
///////////////////////
// book related
///////////////////////
ImpliedBookSide FifoImpliedSecurityMatcher::book(Side side) const {
// this isn't the best way to do this (copying),
// but the ImpliedBook only has 2 levels in practice,
// so it should be fine.
SmallBook tuple_ib;
ImpliedBookSide ib;
_book(side, tuple_ib, ib.size());
for (uint32_t i = 0; i < tuple_ib.size() && i < ib.size(); i++) {
ib[i].price = tuple_ib[i].price;
ib[i].quantity = tuple_ib[i].quantity;
}
return ib;
}
SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(Side side) const {
SmallBook b;
_book(side, b, 1);
if (b.empty())
return {Price(), Quantity(0)};
return b[0];
}
void FifoImpliedSecurityMatcher::_book(
Side side, SmallBook &dest, uint32_t depth) const {
std::vector<std::tuple<Quantity, Price, SmallBook>> im_vec;
for (const auto &p : _me->implied_mkt) {
SmallBook bk;
auto [sec_matcher, leg_qty] = p.second;
bts_assert(leg_qty.value() != 0);
auto leg_side = leg_qty.value() > 0 ? side : other_side(side);
for (auto bl : sec_matcher->book(leg_side))
bk.emplace_back(bl.price, bl.quantity);
im_vec.emplace_back(leg_qty, Price{}, bk);
}
book_utils::compute_synthetic_book(dest, im_vec, depth);
}
void FifoImpliedSecurityMatcher::add_leg(
LegDefinition leg, FifoSecurityMatcher *m) {
_me->implied_mkt.insert(
{leg.security_id, {m, SideQuantity(leg.ratio, leg.side).value()}});
}
Quantity FifoImpliedSecurityMatcher::fill(
OrderKey key,
Side side,
Quantity qty,
std::vector<MatchEffect> *effects,
FillDepot &fills) {
SmallBook b;
auto oside = other_side(side);
_book(oside, b, ImpliedBookSide::max_depth());
auto orig_qty = qty;
// can't fill more than the volume in the book
qty = std::min(book_volume(b), qty);
Price impl_fill_price = top_of_book(oside).price;
bool has_added_aggr_fills = false;
int fill_qty = 0;
bool local_fills = effects == nullptr;
for (auto kv : _me->implied_mkt) {
auto leg_size = std::get<1>(kv.second);
// todo: support other leg sizes (keeping it simple for now)
bts_assert(leg_size.value() == 1 || leg_size.value() == -1);
Side leg_side = leg_size.value() > 0 ? side : other_side(side);
FifoSecurityMatcher &m = *std::get<0>(kv.second);
if (local_fills)
effects = new std::vector<MatchEffect>();
uint32_t leg_fills_pos = effects->size();
auto raw_leg_fill_qty = qty * abs(leg_size);
auto raw_leg_rem_qty =
m.fill(key, leg_side, Price(), raw_leg_fill_qty, effects, false);
bts_assert(raw_leg_rem_qty.value() == 0);
for (auto it = effects->begin() + leg_fills_pos; it != effects->end();
it++) {
auto &passive = *it;
bts_assert(std::holds_alternative<SimOrderUpdate>(passive));
fills.add_fill(
kv.first, key,
_to_passive_fill(std::get<SimOrderUpdate>(passive), true));
}
// the implied matchers should not produce partial leg fills (modulus
// leg_size should be 0)
bts_assert(
raw_leg_fill_qty.value() == 0 ||
raw_leg_fill_qty.value() % abs(leg_size).value() == 0);
uint32_t leg_fill_qty = (raw_leg_fill_qty / abs(leg_size)).value();
bts_assert(leg_fill_qty == static_cast<uint32_t>(qty.value()));
if (!has_added_aggr_fills) {
// This only needs to be done once.
// Any leg's passive fills can serve as a reference for
// the fill qty wrt the implied market: impl_fill_qty = leg_fill_qty /
// abs(leg_size), as the ratio raw_leg_fill_qty:abs(leg_size) should be
// the same for all legs. It might not be a bad idea to assert this is
// the case: that all legs produce the exact same aggressor fills.
for (auto it = effects->begin() + leg_fills_pos; it != effects->end();
it++) {
auto &f = *it;
bts_assert(std::holds_alternative<SimOrderUpdate>(f));
MDP3Fill aggr;
aggr.key = key;
aggr.trade_entry_id = TradeEntryID{0};
aggr.aggressor = _to_aggressor(side);
aggr.price = impl_fill_price;
aggr.quantity = std::get<SimOrderUpdate>(f).last_fills / abs(leg_size);
aggr.summary_quantity = Quantity{0};
fills.add_fill(security_id(), key, aggr);
fill_qty += aggr.quantity.value();
}
has_added_aggr_fills = true;
}
if (local_fills) {
delete effects;
effects = nullptr;
}
}
bts_assert(fill_qty <= orig_qty.value());
return Quantity(orig_qty.value() - fill_qty);
}
} // namespace bts::spartan::mdp3
@@ -1,44 +0,0 @@
#pragma once
#include "back_testing/common/MatchEffect.h"
#include "back_testing/common/mdp3_common.h"
#include "domain/Quantity.h"
#include "domain/SecurityID.h"
#include "domain/Side.h"
#include "domain/book/Book.h"
#include <memory>
#include <vector>
namespace bts::spartan {
struct LegDefinition;
}
namespace bts::spartan::mdp3 {
class FifoSecurityMatcher;
class FillDepot;
struct SmallBookLevel;
class FifoImpliedSecurityMatcher {
public:
FifoImpliedSecurityMatcher(SecurityID security_id);
~FifoImpliedSecurityMatcher();
ImpliedBookSide book(Side) const;
Quantity fill(
OrderKey, Side, Quantity, std::vector<MatchEffect> *, FillDepot &);
void add_leg(LegDefinition leg, FifoSecurityMatcher *matcher);
SecurityID security_id() const;
SmallBookLevel top_of_book(Side side) const;
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
// hide copy ctr
FifoImpliedSecurityMatcher(const FifoImpliedSecurityMatcher &);
void _book(Side side, SmallBook &dest, uint32_t depth) const;
uint32_t _fill_order(const MDP3Order &order);
};
} // namespace bts::spartan::mdp3
@@ -1,312 +0,0 @@
#include "FifoMatcherContext.h"
#include "FifoImpliedSecurityMatcher.h"
#include "FifoMergedSecurityMatcher.h"
#include "FifoSecurityMatcher.h"
#include "FillDepot.h"
#include "back_testing/common/mdp3_common.h"
#include "back_testing/common/order_utils.h"
#include "back_testing/common/security_comp.h"
#include "back_testing/implied/implied_market.h"
#include "domain/book/book_utils.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
#include "utils/ActionSummarizerMacros.h"
#include "utils/TableDisplay.h"
#include <algorithm>
#include <functional>
#include <iostream>
#include <string>
#include <vector>
using namespace bts::spartan::diagnostics;
template <typename K, typename V>
using flat_map = boost::container::flat_map<K, V>;
namespace bts::spartan::mdp3 {
///////////////////////
// stateless helpers
///////////////////////
// static inline std::string _log_header() { return "FifoMatcherContext"; }
struct FifoMatcherContext::PImpl {
const bool ignore_ops_on_deleted_orders;
const Trace trace;
// _implied_candidates_dirty should be set to true anytime a security is
// added/removed. This signals that implied market candidates must be
// re-evaluated before any susbequent order-matching (re-evaluation is
// performed lazily).
bool implied_candidates_dirty;
ImpliedMarketModel implied_market_model;
flat_map<SecurityID, FifoMergedSecurityMatcher *> matchers;
flat_map<SecurityID, std::string> symbols;
PImpl() : ignore_ops_on_deleted_orders(false) {}
PImpl(const InitArgs &args)
: ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders),
trace(args.trace) {}
~PImpl() {
for (auto &p : matchers)
delete p.second;
matchers.clear();
}
};
///////////////////////
// constructors
///////////////////////
FifoMatcherContext::FifoMatcherContext() : FifoMatcherContext({false}) {}
FifoMatcherContext::FifoMatcherContext(const InitArgs &args)
: _me(new PImpl(args)) {}
FifoMatcherContext::~FifoMatcherContext() {}
///////////////////////
// security accessors
///////////////////////
bool FifoMatcherContext::add_security(
SecurityID sec_id, const std::string &symbol, const LegVec &legs) {
auto curr = _me->matchers.find(sec_id);
if (curr == _me->matchers.end()) {
_me->symbols.insert(std::make_pair(sec_id, symbol));
LegMap ic;
// Add the legs explicitly (ImpliedMarketBuilder doesn't know about
// 'SecurityDefinition,' or its leg structure 'LegDef').
if (legs.empty()) {
// must be an outright (to normalize outrights and spreads,
// the former is treated like a 'single leg spread')
auto &l = ic[sec_id];
l.security_id = sec_id;
l.side = Side::Buy;
l.ratio = Quantity(1);
} else {
for (const auto &l : legs)
ic.emplace(l.security_id, l);
}
_me->implied_market_model.insert(sec_id, ic);
_me->matchers.insert(std::make_pair(
sec_id,
new FifoMergedSecurityMatcher(
{sec_id, _me->ignore_ops_on_deleted_orders, _me->trace})));
_me->implied_candidates_dirty = true;
} else {
// Not sure why, but [John]I've seen _me->symbols contain empty strings.
// Overwrite the existing value to ensure a valid symbol.
_me->symbols[sec_id] = symbol;
return false;
}
return true;
}
bool FifoMatcherContext::remove_security(SecurityID sec_id) {
auto curr = _me->symbols.find(sec_id);
if (curr == _me->symbols.end())
return false;
bts_assert(_me->symbols[sec_id] == curr->second);
_me->implied_market_model.erase(sec_id);
_me->symbols.erase(sec_id);
_me->implied_candidates_dirty = true;
return true;
}
bool FifoMatcherContext::implied_matching_status(SecurityID sec_id) const {
auto *m = _get_matcheri(sec_id);
return m != nullptr && m->implied_matching_status();
}
void FifoMatcherContext::implied_matching_status(
SecurityID sec_id, bool value) {
auto *m = _get_or_add_matcher(sec_id);
if (m->implied_matching_status() != value) {
m->implied_matching_status(value);
_me->implied_candidates_dirty = true;
}
}
///////////////////////
// book
///////////////////////
BookSide FifoMatcherContext::book(SecurityID sec_id, Side side) {
return _get_matcher(sec_id)->book(side);
}
ImpliedBookSide FifoMatcherContext::implied_book(SecurityID sec_id, Side side) {
return _get_matcher(sec_id)->implied_book(side);
}
///////////////////////
// matcher accessors
///////////////////////
FifoMergedSecurityMatcher *FifoMatcherContext::get_matcher(SecurityID sec_id) {
return _get_matcher(sec_id);
}
FifoMergedSecurityMatcher *FifoMatcherContext::_get_matcher(SecurityID sec_id) {
auto m = _me->matchers.find(sec_id);
return m == _me->matchers.end() ? nullptr : m->second;
}
const FifoMergedSecurityMatcher *FifoMatcherContext::_get_matcheri(
SecurityID sec_id) const {
auto m = _me->matchers.find(sec_id);
return m == _me->matchers.end() ? nullptr : m->second;
}
FifoMergedSecurityMatcher *FifoMatcherContext::_get_or_add_matcher(
SecurityID sec_id) {
auto m = _me->matchers.find(sec_id);
if (m == _me->matchers.end()) {
auto *new_m = new FifoMergedSecurityMatcher(
{sec_id, _me->ignore_ops_on_deleted_orders, _me->trace});
_me->matchers.insert(std::make_pair(sec_id, new_m));
return new_m;
}
return m->second;
}
std::tuple<FifoPriceLevelMatcher *, MatcherOrder *>
FifoMatcherContext::find_order(SecurityID sec_id, OrderKey ok) {
auto *m = _get_matcher(sec_id);
if (m != nullptr)
return m->matcher()->find_order(ok);
return {nullptr, nullptr};
}
///////////////////////
// order callbacks
///////////////////////
void FifoMatcherContext::on_order_change(
SecurityID sec_id,
const std::vector<SmallOrderChange> &changes,
std::vector<MatchEffect> *effects,
FillDepot &fills,
AlgoEventID event_id) {
//
// SUMMARIZE(_log_header());
_ensure_sync_implied_candidates();
auto *matcher = _get_or_add_matcher(sec_id);
bts_assert(matcher != nullptr);
for (auto ch : changes)
matcher->on_order_change(ch, effects, fills, event_id);
}
void FifoMatcherContext::on_order_change(
const SmallOrderChange &change,
std::vector<MatchEffect> *effects,
FillDepot &fills,
AlgoEventID event_id) {
//
on_order_change(change.order.security, {change}, effects, fills, event_id);
}
///////////////////////
// diagnostic
///////////////////////
std::optional<std::string> FifoMatcherContext::has_same_orders(
SecurityID sec, const Orders &orders) {
return _get_matcher(sec)->matcher()->has_same_orders(orders);
}
void FifoMatcherContext::clear() {
for (auto kv : _me->matchers)
kv.second->matcher()->clear();
}
template <class TIter>
void FifoMatcherContext::_reset(
SecurityID sec_id,
TIter first,
TIter last,
std::vector<MatchEffect> *effects) {
_ensure_sync_implied_candidates();
auto *matcher = _get_or_add_matcher(sec_id);
// The 'order' in which orders are added to the matcher matters.
// The matcher uses an internal order priority counter
// so it can integrate custom orders (i.e., generated by our algos)
// into its books during simulations.
// The scope of an order's priority does not extend beyond its price
// level.
struct Comp {
bool operator()(const MDP3Order &a, const MDP3Order &b) const {
if (a.side == b.side) {
return a.price.is_better(a.side, b.price) ||
(a.price == b.price && a.priority < b.priority);
}
return a.side == Side::Buy;
}
};
std::set<MDP3Order, Comp> sorted_orders;
while (first != last) {
sorted_orders.emplace(*first);
first++;
}
matcher->matcher()->reset(sorted_orders.begin(), sorted_orders.end());
if (effects != nullptr) {
effects->emplace_back(SimBookUpdate());
SimBookUpdate &mbp = std::get<SimBookUpdate>(effects->back());
mbp.security_id = sec_id;
mbp.mbp_buy = matcher->book(Side::Buy);
mbp.mbp_sell = matcher->book(Side::Sell);
mbp.implied_mbp_buy = matcher->implied_book(Side::Buy);
mbp.implied_mbp_sell = matcher->implied_book(Side::Sell);
}
}
void FifoMatcherContext::reset(
SecurityID sec,
std::vector<Order> orders,
std::vector<MatchEffect> *effects) {
_reset(sec, orders.begin(), orders.end(), effects);
}
using BBORow = std::tuple<std::string, Price, Quantity, Price, Quantity>;
void FifoMatcherContext::print_all_bbos(std::ostream &out) {
std::vector<BBORow> rows;
for (auto kv : _me->matchers) {
auto bid = kv.second->matcher()->book_level(Side::Buy, 0);
auto ask = kv.second->matcher()->book_level(Side::Sell, 0);
rows.emplace_back(std::make_tuple(
_me->symbols[kv.first], std::get<0>(bid), std::get<1>(bid),
std::get<0>(ask), std::get<1>(ask)));
}
std::sort(
rows.begin(), rows.end(), [](const BBORow &lhs, const BBORow &rhs) {
return security_comp::SymbolComp()(
{std::get<0>(lhs)}, {std::get<0>(rhs)});
});
bts::spartan::TableDisplay::print_table(
out, rows.begin(), rows.end(),
std::vector<
std::tuple<std::string, int, std::function<std::string(BBORow)>>>{
{" Security ", 12, [](auto r) { return std::get<0>(r); }},
{" Qty ", 10, [](auto r) { return TO_STRING(std::get<2>(r)); }},
{" Bid ", 10, [](auto r) { return TO_STRING(std::get<1>(r)); }},
{" Ask ", 10, [](auto r) { return TO_STRING(std::get<3>(r)); }},
{" Qty ", 10, [](auto r) { return TO_STRING(std::get<4>(r)); }}});
}
void FifoMatcherContext::_ensure_sync_implied_candidates() {
if (!_me->implied_candidates_dirty)
return;
// For each security, just remove the existing implied matchers and generate
// them again. While this isn't the most efficient implementation (it could
// be optimized to add/remove implied matchers incrementally), it is the
// simplest.
for (auto p : _me->matchers) {
std::vector<LegMap> candidates =
_me->implied_market_model.find_implied_markets(p.first);
for (const auto &can : candidates) {
auto *imatcher = p.second->add_implied_matcher(can);
if (imatcher == nullptr)
continue;
for (const auto &[sec, leg] : can)
imatcher->add_leg(leg, _get_or_add_matcher(sec)->matcher());
}
}
_me->implied_candidates_dirty = false;
}
} // namespace bts::spartan::mdp3
@@ -1,85 +0,0 @@
#pragma once
#include "back_testing/common/Trace.h"
#include "domain/LegDefinition.h"
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <tuple>
#include <vector>
namespace bts::spartan::mdp3 {
class FillDepot;
class FifoMergedSecurityMatcher;
class FifoSecurityMatcher;
class FifoPriceLevelMatcher;
class FifoImpliedSecurityMatcher;
struct MatcherOrder;
class FifoMatcherContext {
public:
struct InitArgs {
bool ignore_ops_on_deleted_orders;
Trace trace{};
};
FifoMatcherContext();
FifoMatcherContext(const InitArgs &args);
~FifoMatcherContext();
bool add_security(
SecurityID sec_id, const std::string &symbol, const LegVec &legs);
bool remove_security(SecurityID sec_id);
void implied_matching_status(SecurityID sec_id, bool value);
bool implied_matching_status(SecurityID sec_id) const;
void clear();
BookSide book(SecurityID sec_id, Side side);
ImpliedBookSide implied_book(SecurityID sec_id, Side side);
// the Fills param records individual fills (potentially more than one per
// order)
void on_order_change(
SecurityID sec_id,
const std::vector<SmallOrderChange> &changes,
std::vector<MatchEffect> *effects,
FillDepot &fills,
AlgoEventID = no_algo_event());
void on_order_change(
const SmallOrderChange &change,
std::vector<MatchEffect> *effects,
FillDepot &fills,
AlgoEventID = no_algo_event());
void print_all_bbos(std::ostream &out);
std::optional<std::string> has_same_orders(SecurityID, const Orders &);
std::tuple<FifoPriceLevelMatcher *, MatcherOrder *> find_order(
SecurityID, OrderKey);
FifoMergedSecurityMatcher *get_matcher(SecurityID);
/**
* Resets a security's order book with a particular collection of resting orders.
* It is the caller's responsibility to ensure that there are no crosses in the
* 'resting orders,' as this method does not attempt to fill them.
*/
void reset(SecurityID, std::vector<Order>, std::vector<MatchEffect> * = nullptr);
private:
struct PImpl;
std::shared_ptr<PImpl> _me;
// FifoMatcherContext(const FifoMatcherContext& other);
FifoMergedSecurityMatcher *_get_matcher(SecurityID sec_id);
const FifoMergedSecurityMatcher *_get_matcheri(SecurityID sec_id) const;
FifoMergedSecurityMatcher *_get_or_add_matcher(SecurityID sec_id);
void _ensure_sync_implied_candidates();
template <class TIter>
void _reset(
SecurityID sec_id,
TIter first,
TIter last,
std::vector<MatchEffect> *effects = nullptr);
};
} // namespace bts::spartan::mdp3
@@ -1,466 +0,0 @@
#include "FifoMergedSecurityMatcher.h"
#include "FifoImpliedSecurityMatcher.h"
#include "FifoSecurityMatcher.h"
#include "FillDepot.h"
#include "back_testing/common/OrderModification.h"
#include "back_testing/common/order_utils.h"
#include "common/MatcherOneof.h"
#include "common/MatchingSession.h"
#include "domain/book/book_utils.h"
#include "fill_factory.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
#include <boost/container/flat_map.hpp>
#include <fmt/format.h>
#include <range/v3/algorithm.hpp>
#include <range/v3/numeric.hpp>
#include <range/v3/view.hpp>
#define _LOG(level, ...) \
LOG(OrderMan, level, "{} {}", _log_header(), fmt::format(__VA_ARGS__))
#define DIAG(...) _LOG(Diag, __VA_ARGS__)
#define INFO(...) _LOG(Info, __VA_ARGS__)
#define WARN(...) _LOG(Info, __VA_ARGS__)
#define ERROR(...) _LOG(Error, __VA_ARGS__)
using namespace ranges;
using namespace bts::spartan::mdp3::book_utils;
namespace bts::spartan::mdp3 {
struct LegComp {
bool operator()(const LegDef &lhs, const LegDef &rhs) {
if (lhs.security_id != rhs.security_id)
return lhs.security_id < rhs.security_id;
if (lhs.side != rhs.side)
return lhs.side < rhs.side;
return lhs.ratio < rhs.ratio;
}
};
struct LegMapComp {
bool operator()(const LegMap &lhs, const LegMap &rhs) const {
auto lit = lhs.begin();
auto rit = rhs.begin();
while (lit != lhs.end() && rit != rhs.end()) {
if (lit->second == rit->second) {
lit++;
rit++;
}
return LegComp()(lit->second, rit->second);
}
return lhs.size() < rhs.size();
}
};
// The number of legs in an implied market is presumably very small (for ex, GC
// implieds never have more than 2 legs), so storing the implied markets in a
// flat map whose keys are themselves a flat_map (of the implied market's legs)
// should exhibit Log(n) lookups. Although technically lookups for such a
// container are O(m)Log(n). If 'm' becomes large, a tree-based
// implementation may become appropriate.
using ImpliedMatchers = boost::container::
flat_map<LegMap, FifoImpliedSecurityMatcher *, LegMapComp>;
struct FifoMergedSecurityMatcher::PImpl {
PImpl(const FifoMergedSecurityMatcher::InitArgs &args)
: security_id(args.security_id),
ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders),
trace(args.trace), implied_matching_status(true),
matcher(new FifoSecurityMatcher(
{args.security_id, 0, args.ignore_ops_on_deleted_orders, nullptr,
args.trace})),
implied_matchers() {}
~PImpl() {
for (auto p : implied_matchers)
delete p.second;
implied_matchers.clear();
}
const SecurityID security_id;
const bool ignore_ops_on_deleted_orders;
const Trace trace;
bool implied_matching_status;
// the Security matcher is kept in a shared_ptr
// so it can be passed around to one or more ImpliedMatchers.
// Note: if a signficant performance penalty is attributed
// to the use of shared_ptrs here, it should be fine if this
// class deletes them in its destructor.
// It seems inappropriate for an ImpliedMatcher to outlive
// the SecurityMatchers that make up its legs anyways.
std::shared_ptr<FifoSecurityMatcher> matcher;
ImpliedMatchers implied_matchers;
};
///////////////////////
// constructors
///////////////////////
FifoMergedSecurityMatcher::FifoMergedSecurityMatcher(
const FifoMergedSecurityMatcher::InitArgs &args)
: _me(new FifoMergedSecurityMatcher::PImpl(args)) {}
FifoMergedSecurityMatcher::~FifoMergedSecurityMatcher() {}
///////////////////////
// diagnostic related
///////////////////////
std::string FifoMergedSecurityMatcher::_log_header() const {
return fmt::format("MergedSecMatcher (sid {})", security_id());
}
SecurityID FifoMergedSecurityMatcher::security_id() const {
return _me->security_id;
}
///////////////////////
// matcher accessors
///////////////////////
FifoSecurityMatcher *FifoMergedSecurityMatcher::matcher() {
return _me->matcher.get();
}
const FifoSecurityMatcher *FifoMergedSecurityMatcher::matcheri() const {
return _me->matcher.get();
}
FifoImpliedSecurityMatcher *FifoMergedSecurityMatcher::add_implied_matcher(
const LegMap &legs) {
auto res = _me->implied_matchers.emplace(legs, nullptr);
if (res.second) {
res.first->second = new FifoImpliedSecurityMatcher(security_id());
return res.first->second;
}
return nullptr;
}
bool FifoMergedSecurityMatcher::implied_matching_status() const {
return _me->implied_matching_status;
}
void FifoMergedSecurityMatcher::implied_matching_status(bool value) {
_me->implied_matching_status = value;
}
void FifoMergedSecurityMatcher::on_order_change(
const SmallOrderChange &ch,
std::vector<MatchEffect> *effects,
FillDepot &fills,
AlgoEventID event_id) {
if (!implied_matching_status()) {
// there is no "merged matching" when implied matching is off,
// just pass the order changes the direct matcher.
matcher()->on_order_change(ch, effects, event_id);
return;
}
bts_assert_msg(
ch.affect_book, "affect-free matching is not supported in "
"markets with implied matching.");
switch (ch.change) {
case OrderChange::ChangeType::updated_price:
case OrderChange::ChangeType::updated: {
// Iceberg quantity should be null (it is only non-null when it is added).
bts_assert(!ch.order.iceberg_quantity.has_value());
auto [prev_price_lev, prev_ord] = matcher()->find_order(ch.order.key);
if (prev_ord == nullptr) {
bts_assert_msg(
_me->ignore_ops_on_deleted_orders,
"unable to process order-update event: failed to find order.");
break;
}
bts_assert(prev_price_lev != nullptr);
auto new_ch = ch;
if (new_ch.order.quantity == 0) {
// TODO: Unify price-change-only code/terminology. For now, the
// matcher just interprets 0 qty to mean price-change-only.
new_ch.order.quantity = prev_ord->quantity;
}
if (ch.order.price.is_better(ch.order.side, prev_ord->price)) {
// price improved, try to fill the order
// Iceberg quantity is null on updates, but needs to be reassigned to
// get the correct partial and total fill quantities.
if (prev_ord->ib.has_value())
new_ch.order.iceberg_quantity = prev_ord->ib;
std::optional<Quantity> new_qty = std::nullopt;
if (ch.order.quantity != 0)
new_qty = ch.order.quantity;
_fill(new_ch.order, effects, fills, false);
if (new_ch.order.quantity.value() == 0) {
matcher()->on_order_completed(ch.order.key, event_id);
} else {
auto &new_ord = new_ch.order;
if (new_ord.iceberg_quantity.has_value() && new_qty.has_value()) {
if (new_ord.quantity > prev_ord->ib.value().total)
new_ord.quantity = prev_ord->ib.value().total;
new_ord.iceberg_quantity.value().show = std::min(
new_ord.iceberg_quantity.value().total, new_qty.value());
}
MDP3Order old_ord;
old_ord.key = prev_ord->key;
old_ord.security = security_id();
old_ord.side = prev_ord->side;
old_ord.price = prev_ord->price;
old_ord.quantity = prev_ord->quantity;
old_ord.iceberg_quantity = prev_ord->ib;
auto mod = OrderModification::deduce(old_ord, new_ch.order);
matcher()->update_order(new_ch.order.key, mod, event_id);
}
} else {
// As far as we know, the only type of update that could trigger fills
// is a price improvement. It is worth noting, however, that there is
// the potential for this assumption to be invalid. In a very strange
// market, we may observe down the road: an implied market with a leg
// for this security's same side market, that also has a leg size
// greater than 1.
matcher()->on_order_change(ch, effects, event_id);
}
break;
}
case OrderChange::ChangeType::added: {
auto new_ch = ch;
_fill(new_ch.order, effects, fills, false);
if (get_total_qty(new_ch.order).value() > 0)
matcher()->on_order_change(new_ch, effects, event_id);
break;
}
case OrderChange::ChangeType::removed: {
matcher()->on_order_change(ch, effects, event_id);
break;
}
}
}
/////////////////////////
// books
/////////////////////////
BookSide FifoMergedSecurityMatcher::book(Side side) {
return matcher()->book(side);
}
ImpliedBookSide FifoMergedSecurityMatcher::implied_book(Side side) {
if (_me->implied_matchers.empty())
return ImpliedBookSide();
// In theory, computing the quantity of any price level beyond the top of the
// book should be done in a hypothetical scenario, where an order with the
// specified price and infinite qty is partially filled.
//
// This accounts for the possibility that implied markets may have securities
// in common, and therefore filling an order in one implied market may impact
// another implied market.
//
// However, in practice, the implied market candidates the exchange uses are
// independent.
//
// TODO: this should be a linked list, as
// the only functionality needed is forward iteration,
// and erase (the latter for optimization).
std::vector<std::tuple<ImpliedBookSide, int>> ibooks;
auto oside = other_side(side);
bts::orderman::SidePriceComp cmp(map_side(oside));
for (auto it = _me->implied_matchers.begin();
it != _me->implied_matchers.end(); it++) {
auto res = ibooks.emplace_back(it->second->book(oside), 0);
auto &ib = std::get<0>(res);
if (ib[0].price.is_nan())
ibooks.pop_back();
}
auto is_exhausted = [&](const std::tuple<ImpliedBookSide, int> &ib_tup) {
const auto &ib = std::get<0>(ib_tup);
auto i = std::get<1>(ib_tup);
return static_cast<uint32_t>(i) >= ib.size() || ib[i].price.is_nan();
};
auto rem_exhausted = [&]() {
ibooks.erase(std::remove_if(ibooks.begin(), ibooks.end(), is_exhausted));
};
ImpliedBookSide dest;
for (auto &curr : dest) {
// find best price
curr.price = std::get<0>(*max_element(
ibooks, cmp,
[](const auto &ibt) {
return std::get<0>(ibt)[std::get<1>(ibt)].price.to_double();
}))[0]
.price;
if (curr.price.is_nan())
break;
curr.quantity = Quantity(0);
// combine quantities of implied books with the same top price, as there
// could be multiple.
for (auto &ib_tup : ibooks) {
auto &ib = std::get<0>(ib_tup);
auto &index = std::get<1>(ib_tup);
if (ib[index].price == curr.price)
curr.quantity =
Quantity(curr.quantity.value() + ib[index++].quantity.value());
}
rem_exhausted();
}
return dest;
}
void FifoMergedSecurityMatcher::merged_book(
Side side, std::vector<BookLevel> &dest) {
//
std::vector<std::tuple<ImpliedBookSide, int>> ibooks;
auto oside = other_side(side);
bts::orderman::SidePriceComp cmp(map_side(oside));
for (auto it = _me->implied_matchers.begin();
it != _me->implied_matchers.end(); it++) {
auto res = ibooks.emplace_back(it->second->book(oside), 0);
auto &ib = std::get<0>(res);
if (ib[0].price.is_nan())
ibooks.pop_back();
}
auto dbook = matcher()->book(side);
int dindex = 0;
auto is_exhausted = [&](const std::tuple<ImpliedBookSide, int> &ib_tup) {
const auto &ib = std::get<0>(ib_tup);
auto i = std::get<1>(ib_tup);
return static_cast<uint32_t>(i) >= ib.size() || ib[i].price.is_nan();
};
while (true) {
// erase empties
ibooks.erase(
std::remove_if(ibooks.begin(), ibooks.end(), is_exhausted),
ibooks.end());
// dir top price/qty
Price dprice(
static_cast<uint32_t>(dindex) < dbook.size() ? dbook[dindex].price
: Price());
Quantity dqty(dprice.is_nan() ? 0 : dbook[dindex].quantity.value());
// init best price/qty
Price best_price(dprice);
Quantity curr_qty(0);
// find best impl price
for (auto &ib_tup : ibooks) {
auto &ib = std::get<0>(ib_tup);
auto index = std::get<1>(ib_tup);
if (cmp(ib[index].price, best_price))
best_price = ib[index].price;
}
// combine quantities of books with the same top price
for (auto &ib_tup : ibooks) {
auto &ib = std::get<0>(ib_tup);
auto &index = std::get<1>(ib_tup);
auto &bl = ib[index];
if (bl.price == best_price) {
curr_qty = Quantity(curr_qty.value() + bl.quantity.value());
index++;
}
}
if (dprice == best_price) {
curr_qty = Quantity(curr_qty.value() + dqty.value());
dindex++;
}
if (curr_qty.value() == 0)
break;
dest.push_back({best_price, curr_qty});
}
}
void FifoMergedSecurityMatcher::_fill(
MDP3Order &order,
std::vector<MatchEffect> *effects,
FillDepot &fills,
bool reset_show_qty) {
auto rem = _fill(
order.key, order.side, order.price, get_total_qty(order), effects,
fills);
set_total_qty(order, rem, reset_show_qty);
}
Quantity FifoMergedSecurityMatcher::_fill(
OrderKey key,
Side side,
Price price,
Quantity quantity,
std::vector<MatchEffect> *effects,
FillDepot &fills) {
//
std::vector<FifoImpliedSecurityMatcher *> imatcher_vec;
for (auto &p : _me->implied_matchers)
imatcher_vec.push_back(p.second);
auto oside = other_side(side);
MatchingSession match_session(*_me->matcher, imatcher_vec, oside, price);
auto match = match_session.get_next();
auto ord_desc = [&]() { return fmt_order(key, side, price, quantity); };
while (quantity.value() > 0 &&
match.matcher_case() != MatcherOneof::Case::None) {
// At the end of each iteration, assert that 'quantity' decreased. The
// fill qty is deducted from 'quantity' so this indicates that the order
// was actually filled. This guards against an infinite loop.
auto orig_qty = quantity;
// Fill one price level at a time.
// While one market may have the best 1st price level,
// another may have the best 2nd price level.
switch (match.matcher_case()) {
case MatcherOneof::Case::Direct: {
auto [top_price, top_qty] = match.direct().book_level(oside, 0);
DIAG(
"filling {} in direct market {} {} @ {}", ord_desc(), oside,
top_qty, top_price);
bool local_effects = effects == nullptr;
if (local_effects)
effects = new std::vector<MatchEffect>();
uint32_t fills_pos = effects->size();
auto target_fill_qty = std::min(quantity, top_qty);
auto rem = matcher()->fill(
key, side, top_price, target_fill_qty, effects, true);
auto fill_qty = target_fill_qty - rem;
quantity -= fill_qty;
DIAG("filled {}. Order now: {}", fill_qty, ord_desc());
MDP3Fill aggr_fill;
aggr_fill.trade_entry_id = TradeEntryID{0};
aggr_fill.quantity = Quantity{0};
aggr_fill.summary_quantity = Quantity{0};
bts_assert(aggr_fill.price.is_nan());
aggr_fill.key = key;
aggr_fill.aggressor = _to_aggressor(side);
for (auto it = effects->begin() + fills_pos; it != effects->end();
it++) {
auto &df = *it;
bts_assert(std::holds_alternative<SimOrderUpdate>(df));
auto &upd = std::get<SimOrderUpdate>(df);
if (upd.key == key) {
if (aggr_fill.price.is_nan()) {
aggr_fill.price = upd.last_price;
aggr_fill.quantity = upd.last_fills;
} else if (aggr_fill.price == upd.last_price) {
aggr_fill.quantity = Quantity(
aggr_fill.quantity.value() + upd.last_fills.value());
} else {
fills.add_fill(security_id(), key, aggr_fill);
aggr_fill.quantity = Quantity(0);
aggr_fill.price = Price();
bts_assert(aggr_fill.price.is_nan());
}
} else {
fills.add_fill(security_id(), key, _to_passive_fill(upd, false));
}
}
if (local_effects) {
delete effects;
effects = nullptr;
}
if (!aggr_fill.price.is_nan())
fills.add_fill(security_id(), key, aggr_fill);
break;
}
case MatcherOneof::Case::Implied: {
auto top = match.implied().top_of_book(oside);
DIAG("filling {} in implied market {} {}", ord_desc(), oside, top);
auto target_fill_qty = std::min(quantity, top.quantity);
auto rem =
match.implied().fill(key, side, target_fill_qty, effects, fills);
auto fill_qty = target_fill_qty - rem;
quantity -= fill_qty;
DIAG("filled {}. Order now: {}", fill_qty, ord_desc());
break;
}
default:
bts_assert_msg(false, "unexpected MatcherOneof::Case");
break;
}
// Assert fill(s) occurred (the fill qty is deducted from 'quantity').
bts_assert(quantity < orig_qty);
match = match_session.get_next();
}
bts_assert(quantity.value() >= 0);
return quantity;
}
} // namespace bts::spartan::mdp3
Loaded 100 of 614 files, more files were not shown because too many files have changed in this diff. Show more