diff --git a/.vscode/launch.json b/.vscode/launch.json index c8d1c78ce..0eaae7c1f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,7 +209,8 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - "args": ["test","Co2RelayTests"], + // "args": ["test","Co2RelayTests"], + "args": ["md", "run", "10"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/doc/user/docs/Trading.md b/doc/user/docs/Trading.md index 4b994d85c..9a7472a88 100644 --- a/doc/user/docs/Trading.md +++ b/doc/user/docs/Trading.md @@ -54,22 +54,22 @@ ![](images/image4.png) -- ![](images/image48.png)][Fills - +- ![](images/image48.png)Fills - clicking on this will launch a fills window -- ![](images/image57.png)][ Modify - +- ![](images/image57.png) Modify - clicking on this will launch an order ticket for a live order highlighted in the Order Book. -- ![](images/image73.png)][Cancel +- ![](images/image73.png)Cancel selected - clicking on this will cancel the selected orders you have highlighted in the order book.  You can choose more than one by holding hold the Ctrl key when click on each row of orders -- ![](images/image74.png)][Cancel +- ![](images/image74.png)Cancel buys - clicking on this will cancel all the buy orders you have in the order book -- ![](images/image33.png)][Cancel +- ![](images/image33.png)Cancel sells - clicking on this will cancel all the sell orders you have in the order book -- ![](images/image2.png)][Cancel +- ![](images/image2.png)Cancel all - clicking on this will cancel all working orders you have the order book @@ -168,10 +168,10 @@ - Iceberg - This is a buy or sell order where the order is divided into a - visible or “show” portion which is seen at the exchange.  ][The + visible or “show” portion which is seen at the exchange.  The rest of the order is hidden and will remain hidden until the show quantity is filled, essentially reloading the order back to the show - quantity.  You will need to ][define what type of order that will + quantity.  You will need to define what type of order that will be used for the child order to be sent. ![](images/image47.png) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index 5dda1897a..cd9cede36 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,5 +1,14 @@ # Release Notes +## 1.3.8 + +### Speed Enhancements + - Further speed enhancements +### Conditional (new quoting parameter) + - Reduces re-quoting updates by only quoting if N ticks away from the opposite inside market. + - Quotes are cancelled when it gets further away. + - "Price Type" dictates if this logic applies to the quoting market or to the synthetic spread market. + ## 1.3.7 ### Net Change pricing diff --git a/src/client/spark/Modules/Market/MarketView/MarketViewView.xaml b/src/client/spark/Modules/Market/MarketView/MarketViewView.xaml index 7031a7151..3ded0304e 100644 --- a/src/client/spark/Modules/Market/MarketView/MarketViewView.xaml +++ b/src/client/spark/Modules/Market/MarketView/MarketViewView.xaml @@ -48,14 +48,12 @@ - - - - - - - - + + + + + + diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml index f3552d905..aa2fb3998 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml @@ -49,13 +49,7 @@ - - - - - - - + diff --git a/src/client/spark/Modules/Spreader/SyntheticManagementView.xaml b/src/client/spark/Modules/Spreader/SyntheticManagementView.xaml index 0aa2c43ba..78e6570b9 100644 --- a/src/client/spark/Modules/Spreader/SyntheticManagementView.xaml +++ b/src/client/spark/Modules/Spreader/SyntheticManagementView.xaml @@ -46,14 +46,12 @@ - - - - - - - - + + + + + + diff --git a/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs b/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs index 2b162676a..7f294ab6c 100644 --- a/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs +++ b/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs @@ -2,7 +2,6 @@ using bts.shell; using bts.spark; using bts.utils; -using mbts.spark.utilities; using mbts.ui; using mbts.ui.shell; using mbts.ui.spark; @@ -110,7 +109,7 @@ namespace mbts.spark { _validate_leg_count(); }; - async Task commit_async(){ + async Task commit_async() { RequestUpdate?.Invoke(); bool valid = true; if (string.IsNullOrWhiteSpace(name)) { @@ -125,8 +124,7 @@ namespace mbts.spark { valid = false; } if (valid) { - _synthetic.Synthetic.Legs.Clear(); - _synthetic.Synthetic.Legs.AddRange(leg_tuples.Select(l => l.leg).OrderBy(l => l.Security, SecurityHelper.Comparer.Instance)); + _synthetic.Synthetic.Legs.SetItems(leg_tuples.Select(l => l.leg)); _synthetic.Symbol = _deduce_symbol(_synthetic); double tick_frac = _tick_factors.tick_unit / _tick_factors.tick_base; _synthetic.ContractSpec.LowTick = tick_frac; @@ -296,7 +294,6 @@ namespace mbts.spark { static IEnumerable _get_legs(Security synthetic, ExecutionStrategyImmutable strategy) => synthetic.Synthetic.Legs - .OrderBy(l => l.Security, SecurityHelper.Comparer.Instance) .Select(l => new LegSpreaderParamsTuple(l, strategy.LegParamsMap[l.Security.Id].Clone())); void _close_new_leg_popup() { @@ -339,4 +336,4 @@ namespace mbts.spark { else clear_error(nameof(leg_count)); } } -} +} \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/Fills/FillsView.xaml b/src/client/spark/Modules/Trading/Fills/FillsView.xaml index 983a4deba..690bdffed 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsView.xaml +++ b/src/client/spark/Modules/Trading/Fills/FillsView.xaml @@ -49,14 +49,12 @@ - - - - - - - - + + + + + + diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index dbcd9cdc7..5a5213efd 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -104,7 +104,7 @@ namespace mbts.spark { if (old_order.order_type() != order.order_type()) RaisePropertyChanged(nameof(order_type), nameof(order_metadata)); if (!had_security && _security_md.HasValue) { - RaisePropertyChanged(nameof(security)); + RaisePropertyChanged(nameof(security), nameof(security_text)); // need to raise invalidated to refresh the price cells (the security.ContractSpec determines how they are formatted - Ticks!) invalidate = true; } diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml b/src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml index 48ebb5f0e..b5698bbca 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml @@ -155,18 +155,12 @@ - - - - - - - - - - - - + + + + + + diff --git a/src/client/spark/Orders/OrderRecordTupleSource.cs b/src/client/spark/Orders/OrderRecordTupleSource.cs index 2bb3ca29a..51ffa7f78 100644 --- a/src/client/spark/Orders/OrderRecordTupleSource.cs +++ b/src/client/spark/Orders/OrderRecordTupleSource.cs @@ -18,8 +18,8 @@ namespace mbts.spark { // data cache readonly Dictionary _state = new Dictionary(); readonly SetMap _security_orders = new SetMap(); + readonly IDictionary _security_md_cache = new Dictionary(0); IReadOnlyDictionary _orders_cache = new Dictionary(0); - IReadOnlyDictionary _security_md_cache = new Dictionary(0); IDisposable _dispose; public OrderRecordTupleSource(IObservable> securities, IObservable> orders) => @@ -92,11 +92,14 @@ namespace mbts.spark { Dictionary added = new Dictionary(), removed = new Dictionary(); Dictionary added_securities = new Dictionary(), removed_securities = new Dictionary(); _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); + /// Warning: must NOT be reassigned with ; the latter is always the same object. + /// This would mean that the + /// call above would literally be calculating the difference between the keys of the same dictionary! foreach (var k in removed_keys) { if (_security_orders.try_lookup(k, out ISet order_keys)) { - removed_securities.Add(k, _security_md_cache[k]); - IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); SecurityMD old_sec_md = _security_md_cache[k]; + removed_securities.Add(k, old_sec_md); + IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); foreach (OrderRecordImmutable order in orders) { OrderRecordTuple old_ft = new OrderRecordTuple(order, old_sec_md); removed.Add(old_ft.order.Key, old_ft); @@ -104,12 +107,13 @@ namespace mbts.spark { added.Add(new_ft.order.Key, new_ft); } } + _security_md_cache.Remove(k); } foreach (var k in added_keys) { + SecurityMD new_sec_md = securities[k]; if (_security_orders.try_lookup(k, out ISet order_keys)) { - added_securities.Add(k, securities[k]); + added_securities.Add(k, new_sec_md); IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); - SecurityMD new_sec_md = securities[k]; foreach (OrderRecordImmutable order in orders) { OrderRecordTuple old_ft = new OrderRecordTuple(order, null); removed.Add(old_ft.order.Key, old_ft); @@ -117,8 +121,8 @@ namespace mbts.spark { added.Add(new_ft.order.Key, new_ft); } } + _security_md_cache.Add(k, new_sec_md); } - _security_md_cache = securities; _apply_changes(added, removed, added_securities, removed_securities); } } diff --git a/src/client/spark/core/Utility/KeyDrivenObservable.cs b/src/client/spark/core/Utility/KeyDrivenObservable.cs index 9e3b668ed..e0e3e27b8 100644 --- a/src/client/spark/core/Utility/KeyDrivenObservable.cs +++ b/src/client/spark/core/Utility/KeyDrivenObservable.cs @@ -32,8 +32,11 @@ namespace mbts.utils { // need to lock bc simultaneous callbacks on diff threads could generate 2 subscriptions at once, // only 1 of which would get disposed in the next callback. Disposable.dispose(ref _current_key_sub); - if (!_is_disposed) - _current_key_sub = _selector(key)?.Subscribe(_value_subject); + if (!_is_disposed) { + // IMPORTANT: do NOT subscribe _value_subject directly to the inner sequences. Subscribe with a callback delegate instead (e.g. _value_subject.OnNext). + // Reason: If a sequence 'completes' while _value_subject's subscription is active, it will enter a 'Terminated' state, and forever cease notifying observers of subsequent updates. + _current_key_sub = _selector(key)?.Subscribe(_value_subject.OnNext); + } } } diff --git a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml index a5ee0b0cf..09759a533 100644 --- a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml +++ b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml @@ -223,7 +223,7 @@ - + @@ -231,13 +231,37 @@ - Cross Thresh + + Cross Thresh + + - Max Chase Ticks + + Max Chase Ticks + + - On Toward - - Child + + On Toward + + + + + Child + + - Join Thresh + + Join Thresh + + - Child + + Child + + - Top Thresh + + Top Thresh + + - Child + + Child + + + - Hedge Ticks + + Hedge Ticks + + - Hedge + + Hedge + + - Child + + Child + + + - + @@ -458,7 +526,7 @@ Price - + @@ -634,8 +702,8 @@ - - + + @@ -647,9 +715,9 @@ - - - + + + @@ -660,9 +728,9 @@ - - - + + + @@ -673,9 +741,9 @@ - - - + + + @@ -686,9 +754,9 @@ - - - + + + @@ -699,11 +767,11 @@ - - - + + + - + Child +#include +#include +#include + +namespace bts { +namespace md { + +/* + * Design considerations + * + * - We want this test to be 'fair' + * - We want a manageable amount of data + * - +*/ + +struct Entry { + uint32_t seq; + MDConsumerHandle handle; + std::unique_ptr security; +}; + +std::string print_book( + PrintFormat format, + const Entry& e, + utils::TimeStamp twrite, + utils::TimeStamp tread, + uint32_t seq, + bool ambiguous, + const std::array& bid, + const std::array& ask) { + + std::stringstream ss; + if(format == PrintFormat::Human) { + ss << std::endl; + ss << e.security->glbx().security_desc() << " (" << e.handle.key() << ")" << std::endl; + ss << "Sequence: " << seq; + if (ambiguous) ss << "*"; + ss << std::endl; + ss << "TWrit: " << twrite.to_uint64() << " ns" << std::endl; + ss << "TRead: " << tread.to_uint64() << " ns" << std::endl; + char buffer[512]; + + snprintf(buffer, 512, "%-5s %-14s | %-14s %-5s\n", "sz", "bid", "ask", "sz"); + ss << buffer; + ss << "-------------------- - -------------------- \n"; + for (auto x = COCONUT_MAX_DEPTH; x != 0; --x) { + auto a = ask[x-1]; + if (!os::is_nan(a.price())) { + snprintf(buffer, 512, "%5s %14s | %14.7f %5d\n", "", "", a.price(), a.size()); + ss << buffer; + } + } + ss << "-------------------- - -------------------- \n"; + for (uint32_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + auto b = bid[x]; + if (!os::is_nan(b.price())) { + snprintf(buffer, 512, "%5d %14.7f | \n", b.size(), b.price()); + ss << buffer; + } + } + } + else if (format == PrintFormat::CSV) { + // Symbol, Id, Sequence, Ambiguous, TWrite, TRead, BidBook, AskBook + ss << e.security->glbx().security_desc() << "," + << e.security->id() << "," + << seq << "," + << (ambiguous ? "true," : "false,") + << twrite.to_uint64() << "," + << tread.to_uint64() << ","; + for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + if (!bid[x].is_nan()) { + ss << " " << std::setprecision(11) << bid[x].price() << "-"; + if (bid[x].implied_size() > 0) { ss << "*"; } + ss << bid[x].size(); + } + } + ss << ","; + for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + if (!ask[x].is_nan()) { + ss << " " << std::setprecision(11) << ask[x].price() << "-"; + if (ask[x].implied_size() > 0) { ss << "*"; } + ss << ask[x].size(); + } + } + ss << std::endl; + } + return ss.str(); +} + +datetime::Date expiration(const api::Security& sec) { + auto f = sec.future(); + return { (uint16_t)f.expiration_year(), (uint8_t)f.expiration_month(), (uint8_t)f.expiration_day() }; +} + +std::vector filter_entries(std::vector& entries) { + // Choose the front month for all symbols + auto today = datetime::today(); + std::map map; + for (auto& e : entries) { + auto type = securitytype_of(*e.security); + // Only futures because front month is complicated + if (type != api::SECURITYTYPE_FUTURE) continue; + UserSymbol key { e.security->symbol(), e.security->exchange(), type }; + auto& current_entry = map[key]; + if (current_entry == nullptr) { + current_entry = &e; + } + else { + auto cur = expiration(*current_entry->security); + auto next = expiration(*e.security); + if (next > today && next < cur) { + current_entry = &e; + } + } + } + + std::vector result; + for (const auto& pair : map) { + result.push_back(std::move(*pair.second)); + } + return result; +} + +void write_header(PrintFormat format, long seq_mod, const std::vector& entries) { + if (format == PrintFormat::Human) { + std::cout << "Dumping every sequence % " << seq_mod << " == 0 for\n"; + std::cout << "Selected securities: " << std::endl; + for (const auto& e : entries) { + std::cout << e.security->glbx().security_desc() << std::endl; + } + } + else { + // CSV header + std::cout << "Symbol,Id,Sequence,Ambiguous,TWrite,TRead,BidBook,AskBook"; + // for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + // std::cout << "," << "Bid" << x << ", DirectBidSize" << x << ", ImpliedBidSize" << x; + // } + // for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + // std::cout << "," << "Ask" << x << ", AskSize" << x << ", ImpliedAskSize" << x; + // } + std::cout << std::endl; + } +} + +void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, PrintFormat format) { + + MDConsumer consumer(symbols, nullptr); + + // read all securities into a vector + std::vector entries; + consumer.walk_security_list([&](const api::Security& sec, MDConsumerHandle handle) { + entries.push_back({ 0, handle, std::make_unique(sec) }); + }); + + entries = filter_entries(entries); + + write_header(format, seq_mod, entries); + + auto& disp = utils::Dispatcher::background(); + + // spin around on the vector detecting changes + std::array bid,ask; + while(true) { + for(auto& e : entries) { + auto sequence = e.handle.sequence(); + if (sequence != e.seq && sequence % seq_mod == 0) { + e.seq = sequence; + auto tread = TimeStamp::now(); + utils::TimeStamp twrite; + e.handle.get_book(twrite, COCONUT_MAX_DEPTH, bid.data(), ask.data()); + auto seq2 = e.handle.sequence(); + // print book to stdout (async?) + disp.post([=,&e]() { + std::cout << print_book(format, e, twrite, tread, sequence, (sequence != seq2), bid, ask); + }); + } + } + } +} + +}}//ns \ No newline at end of file diff --git a/src/lib/md/Diag/MDPeriodicDumper.h b/src/lib/md/Diag/MDPeriodicDumper.h new file mode 100644 index 000000000..e59a201f0 --- /dev/null +++ b/src/lib/md/Diag/MDPeriodicDumper.h @@ -0,0 +1,16 @@ +#pragma once +#include "lib/md/UserSymbol.h" +#include + +namespace bts { +namespace md { + +enum class PrintFormat { + Human, + CSV +}; + +void run_periodic_dumper(const std::set&, uint32_t seq_mod, PrintFormat format); + + +}}//ns \ No newline at end of file diff --git a/src/lib/md/Diag/MDPriceLantencyTester.cpp b/src/lib/md/Diag/MDPriceLantencyTester.cpp deleted file mode 100644 index c15319110..000000000 --- a/src/lib/md/Diag/MDPriceLantencyTester.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef NO_CPP0X -#include "MDPriceLantencyTester.h" -#include "../MDConsumer.h" -#include "lib/utils/Dispatcher.h" - -#include -#include -#include - -using namespace std; -namespace bts { -namespace md { - -struct HandleData { - MDConsumerHandle handle; - utils::TimeStamp time; - utils::TimeStamp last_price_change; - float bid, ask; -}; - -void run_md_price_latency_test(UserSymbol symbol) { - set symbols; - symbols.insert(symbol); - std::shared_ptr null; - MDConsumer consumer(symbols, null); - - vector handles; - - consumer.walk_security_list([&](const Security&, MDConsumerHandle handle) { - HandleData entry = { handle, TimeStamp(), TimeStamp(), 0, 0 }; - if (!entry.handle.is_initialized()) { - cout << "Please start the md publisher before running this command." << endl; - return; - } - handles.push_back(entry); - }); - - cout << "key, sequence, handover, delta, bid price, ask price" << endl; - while (true) { - for(auto& entry : handles) { - PriceSize bid, ask; - if (entry.handle.get_direct_book(entry.time, 1, &bid, &ask) == HandleReadResult::SUCCESS) { - auto bid_price = bid.price(); - auto ask_price = ask.price(); - if (bid_price != entry.bid || ask_price != entry.ask) { - auto now = TimeStamp::now(); - auto handover = (now - entry.time); - auto delta = (now - entry.last_price_change); - entry.last_price_change = entry.time; - entry.bid = bid_price; - entry.ask = ask_price; - utils::Dispatcher::background().post([=]() { - cout << - entry.handle.key() << ',' - << entry.handle.sequence() << ',' - << handover.microseconds() << ',' - << delta.microseconds() << ',' - << bid_price << ',' << ask_price - << endl; - }); - } - } - } - } -} - - -}}//namespace -#endif diff --git a/src/lib/md/Diag/MDPriceLantencyTester.h b/src/lib/md/Diag/MDPriceLantencyTester.h deleted file mode 100644 index 6f169e4d3..000000000 --- a/src/lib/md/Diag/MDPriceLantencyTester.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "lib/utils/bts_export.h" -#include "../common.h" -#include "../UserSymbol.h" -#include - -namespace bts { -namespace md { - -BTS_EXPORT void run_md_price_latency_test(UserSymbol); - - -}}//namespace - diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index a078dbef1..6f11dcece 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -305,17 +305,6 @@ CoconutSecurityStatus MDConsumerHandle::status() const { } never; } -uint16_t MDConsumerHandle::sequence() const { - switch(_type) { - case HANDLETYPE_VOID: return 0; - case HANDLETYPE_DEPTH: return book.hot->sequence; - case HANDLETYPE_BBO: return bbo.hot->sequence; - case HANDLETYPE_SYNTHETIC: - bts_assert(!synthetic.def->legs.empty()); - return sane::min_by(synthetic.def->legs, [](const MDConsumerHandle& h) { return h.last_market_update(); })->sequence(); - } - never; -} }}//namespace diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 7f1097d73..904a44e21 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -159,7 +159,17 @@ public: bool is_heart_beating() const; TimeStamp heartbeat() const; CoconutSecurityStatus status() const; - uint16_t sequence() const; + uint32_t sequence() const { + switch(_type) { + case HANDLETYPE_VOID: return 0; + case HANDLETYPE_DEPTH: return book.hot->sequence; + case HANDLETYPE_BBO: return bbo.hot->sequence; + case HANDLETYPE_SYNTHETIC: + bts_assert(!synthetic.def->legs.empty()); + return sane::min_by(synthetic.def->legs, [](const MDConsumerHandle& h) { return h.last_market_update(); })->sequence(); + } + bts_assert(false); + } security_key key() const { switch(_type) { case HANDLETYPE_VOID: return invalid_security_key; diff --git a/src/lib/md/MDProducer.cpp b/src/lib/md/MDProducer.cpp index b519f52d9..1971fd647 100644 --- a/src/lib/md/MDProducer.cpp +++ b/src/lib/md/MDProducer.cpp @@ -5,6 +5,7 @@ #include "lib/utils/WaitFreeQueue.h" #include "Shm/MDStructure.h" #include "MemorySegmentUtils.h" +#include "SecurityIDRecorder.h" using namespace std; @@ -102,12 +103,21 @@ private: // MDProducer::MDProducer(Platform platform, bool shared) : - _mutex(new Mutex()), + _mutex(new std::mutex()), _producers(), + _id_recorder(), _platform(platform), _shared(shared) -{ -} +{ } +#ifndef WIN32 +MDProducer::MDProducer(Platform platform, SecurityIDRecorder recorder, bool shared) : + _mutex(new std::mutex()), + _producers(), + _id_recorder(std::make_unique(recorder)), + _platform(platform), + _shared(shared) +{ } +#endif MDProducer::MDProducer(MDProducer&& other) : _mutex(std::move(other._mutex)), _producers(std::move(other._producers)), @@ -143,15 +153,18 @@ MDProducerImpl& MDProducer::get_producer(const char* symbol, SecurityType type, MDProducerHandle MDProducer::add_security(const FixedSecurity& security) { bts_assert_d(strlen(security.symbol()) > 0); bts_assert(security.size >= sizeof(FixedSecurity)); - Mutex::scoped_lock lock(*_mutex); + std::lock_guard lock(*_mutex); auto& impl = get_producer(security.symbol(), security.type, security.exchange); auto result = impl.add_security(security); if (security.key != invalid_security_key) { result.set_key(security.key); } + else if(_id_recorder) { + _id_recorder->ensure_key(result); + } return result; } MDProducerChannel MDProducer::get_channel(const char* symbol, SecurityType type, Venue exchange) { - Mutex::scoped_lock lock(*_mutex); + std::lock_guard lock(*_mutex); return MDProducerChannel(get_producer(symbol, type, exchange)); } diff --git a/src/lib/md/MDProducer.h b/src/lib/md/MDProducer.h index 4123b59e6..aaad1ee1a 100644 --- a/src/lib/md/MDProducer.h +++ b/src/lib/md/MDProducer.h @@ -7,19 +7,21 @@ #include "MDProducerHandle.h" #include "MDProducerChannel.h" #include "Shm/FixedSecurity.h" -#include "tbb/queuing_mutex.h" #include #include +#include namespace bts { namespace md { class MDProducerImpl; +class SecurityIDRecorder; class BTS_EXPORT MDProducer { public: MDProducer(Platform platform, bool shared=true); + MDProducer(Platform platform, SecurityIDRecorder recorder, bool shared=true); MDProducer(MDProducer&& other); ~MDProducer(); @@ -39,9 +41,9 @@ private: MDProducerImpl& get_producer(const char* symbol, SecurityType type, Venue exchange); - typedef tbb::queuing_mutex Mutex; - std::unique_ptr _mutex; + std::unique_ptr _mutex; std::vector> _producers; + std::unique_ptr _id_recorder; Platform _platform; bool _shared; }; diff --git a/src/lib/md/MDProducerHandle.h b/src/lib/md/MDProducerHandle.h index 050dec46d..5f5e5f52b 100644 --- a/src/lib/md/MDProducerHandle.h +++ b/src/lib/md/MDProducerHandle.h @@ -118,7 +118,7 @@ private: // Private default constructor allows friend classes to store this class in a concurrent_queue MDProducerHandle(); friend class tbb::concurrent_queue < MDProducerHandle > ; - friend class SecurityIDRecorder; + friend class SecurityIDRecorderState; friend struct bts::co2::Work; HotEntry& hdata() const { diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 00c66c222..192ca68e2 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -1,7 +1,6 @@ #ifndef WIN32 #include "SecurityIDRecorder.h" -#include "lib/utils/defer.h" #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" @@ -33,14 +32,30 @@ struct SecurityIDRecorderState { work(), pending_handle(), db(nullptr, sqlite3_close), - select_stmt(nullptr, sqlite3_finalize), - insert_stmt(nullptr, sqlite3_finalize) + timer(), + id((size_t)this) {} tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; sqlite3_ptr db; - sqlite3_stmt_ptr select_stmt; - sqlite3_stmt_ptr insert_stmt; + unique_ptr timer; + size_t id; + + void run(utils::Dispatcher& dispatcher) { + timer = std::make_unique( + dispatcher, 500, + [=]() { do_work(); }); + timer->start(); + } + void do_work() { + work_handles(); + work_db(); + } + void work_handles(); + void work_db(); + void assign_key(const api::Security& security, const SecurityIDRecorder::Callback& key_assigned); + security_key search(const std::string& descriptor); + security_key insert(const std::string& descriptor, SecurityType); }; SecurityIDRecorder::SecurityIDRecorder(): @@ -49,6 +64,13 @@ SecurityIDRecorder::SecurityIDRecorder(): init(); } +SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher): + _self(new SecurityIDRecorderState()) +{ + init(); + _self->run(dispatcher); +} + SecurityIDRecorder::~SecurityIDRecorder() { } void SecurityIDRecorder::init() { @@ -66,27 +88,27 @@ void SecurityIDRecorder::init() { sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0"); sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); - - // Prepare statements - _self->insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); - _self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); } void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { _self->pending_handle.push(handle); } + void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { + _self->assign_key(security, callback); +} + +void SecurityIDRecorderState::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { if (security.id() == invalid_security_key) { auto work = std::make_shared(); work->security = security; work->callback = callback; - _self->work.push(work); + this->work.push(work); } } void SecurityIDRecorder::do_work() { - work_handles(); - work_db(); + _self->do_work(); } bool has_all_leg_keys(const MDProducerHandle& handle) { @@ -102,11 +124,12 @@ bool has_all_leg_keys(const MDProducerHandle& handle) { return true; } -void SecurityIDRecorder::work_handles() { +void SecurityIDRecorderState::work_handles() { Security security; MDProducerHandle handle; vector retry; - while (_self->pending_handle.try_pop(handle)) { + int count = 0; + while (pending_handle.try_pop(handle)) { security.Clear(); // if a spread does not have all of its legs we need to wait if (!has_all_leg_keys(handle)) { @@ -114,6 +137,7 @@ void SecurityIDRecorder::work_handles() { } else { convert_security(handle.security(), &security); + ++count; assign_key(security, [handle](const api::Security& security) mutable { handle.set_key(security.id()); }); @@ -121,11 +145,11 @@ void SecurityIDRecorder::work_handles() { } for(const auto& h : retry) { - _self->pending_handle.push(h); + pending_handle.push(h); } } -void SecurityIDRecorder::work_db() { +void SecurityIDRecorderState::work_db() { // Drain the work queue // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list @@ -139,7 +163,7 @@ void SecurityIDRecorder::work_db() { vector new_descriptors; size_t count = 0; string descriptor; - while (_self->work.try_pop(temp)) { + while (work.try_pop(temp)) { descriptor = security_descriptor(temp->security); // If we're unable to create a descriptor try again later @@ -167,12 +191,12 @@ void SecurityIDRecorder::work_db() { } for (auto& d : defer) { - _self->work.push(move(d)); + work.push(move(d)); } if (!new_entries.empty()) { - sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); + sqlite3_exec_nofail(db.get(), "BEGIN TRANSACTION"); for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; @@ -181,7 +205,7 @@ void SecurityIDRecorder::work_db() { security.set_id(key); } - sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + sqlite3_exec_nofail(db.get(), "COMMIT TRANSACTION"); } // record the securities @@ -225,23 +249,23 @@ void SecurityIDRecorder::bulk_insert(const std::vector& da sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); } -security_key SecurityIDRecorder::search(const std::string& descriptor) { - sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); - DEFER(sqlite3_reset(_self->select_stmt.get())); - int result = sqlite3_step(_self->select_stmt.get()); +security_key SecurityIDRecorderState::search(const std::string& descriptor) { + auto select_stmt = prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); + sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); + int result = sqlite3_step(select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_column_int(_self->select_stmt.get(), 0); + return sqlite3_column_int(select_stmt.get(), 0); } return invalid_security_key; } -security_key SecurityIDRecorder::insert(const std::string& descriptor, SecurityType type) { - sqlite3_bind_text(_self->insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); - sqlite3_bind_int(_self->insert_stmt.get(), 2, (int)type); - DEFER(sqlite3_reset(_self->insert_stmt.get())); - int result = sqlite3_step(_self->insert_stmt.get()); +security_key SecurityIDRecorderState::insert(const std::string& descriptor, SecurityType type) { + auto insert_stmt = prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); + sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); + sqlite3_bind_int(insert_stmt.get(), 2, (int)type); + int result = sqlite3_step(insert_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_last_insert_rowid(_self->db.get()); + return sqlite3_last_insert_rowid(db.get()); } // Just in case our insert failed due to duplicate row auto key = search(descriptor); diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 34e1a2f91..f63731ff1 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -8,7 +8,6 @@ #include #include #include -#include namespace bts { namespace md { @@ -23,6 +22,8 @@ class SecurityIDRecorder { public: SecurityIDRecorder(); + // Automatically call do_work on this dispatcher + SecurityIDRecorder(utils::Dispatcher&); ~SecurityIDRecorder(); typedef std::function Callback; @@ -32,23 +33,24 @@ public: void assign_key(const api::Security& security, const Callback& key_assigned); // Someone must call work periodically to do lookups and flush the reserved ids to disk. - // returne true if there is more work to be done void do_work(); void bulk_insert(const std::vector&, const char* prefix); private: - std::unique_ptr _self; + std::shared_ptr _self; void init(); - security_key search(const std::string& descriptor); - security_key insert(const std::string& descriptor, SecurityType); - - void work_handles(); - void work_db(); - - BTS_NO_COPY(SecurityIDRecorder); }; }}//namespace +#else +#include "MDProducerHandle.h" +namespace bts { +namespace md { + class SecurityIDRecorder { + public: + void ensure_key(MDProducerHandle){}; + }; +}} #endif diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index 84be15e0e..8d6ca60aa 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -25,15 +25,12 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher MetadataPublishe struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher& fn) : - recorder(), - timer(utils::Dispatcher::background(), 200, [&]() { recorder.do_work(); }), + recorder(utils::Dispatcher::background()), synthetics(), on_security(), meta(), on_metadata(), metadata_publisher(fn) { - timer.start(); } md::SecurityIDRecorder recorder; - utils::DispatcherTimer timer; std::unordered_map synthetics; utils::Event on_security; std::unordered_map meta; diff --git a/src/lib/orderman/actors/SODPosition.cpp b/src/lib/orderman/actors/SODPosition.cpp index 4a13d4837..e5a09e0ca 100644 --- a/src/lib/orderman/actors/SODPosition.cpp +++ b/src/lib/orderman/actors/SODPosition.cpp @@ -31,6 +31,7 @@ timeval SODPosition::_to_timeval(uint16_t hh, uint16_t mm, uint16_t ss){ ptm->tm_hour=hh; ptm->tm_min=mm; ptm->tm_sec=ss; + ptm->tm_isdst=-1; secs=mktime(ptm); timeval rv; rv.tv_sec=secs; diff --git a/src/sdk/examples/md_publisher_example.cpp b/src/sdk/examples/md_publisher_example.cpp index 4d09fd4d9..c81ebddb1 100644 --- a/src/sdk/examples/md_publisher_example.cpp +++ b/src/sdk/examples/md_publisher_example.cpp @@ -11,6 +11,7 @@ using namespace bts::md; void produce() { + // For Production use you should use provide a SecurityIDRecorder so security keys are generated md::MDProducer producer(co2::PLATFORM_SIM); // Create temporary security (on the stack) @@ -20,7 +21,6 @@ void produce() fsec.exchange = api::VENUE_GLBX; fsec.low_tick = 0.0078125; fsec.high_tick = 0.0078125; - fsec.cash_per_point = 0.0128; // display price is price * pow10(display_factor) fsec.display_factor = 0; fsec.expiration_year = 2018; @@ -34,11 +34,36 @@ void produce() fsec.cme_price_display_format = 3; fsec.cme_market_segment_id = 84; + md::FixedSpreadSecurityL fspread; + fspread.numeric_id = 1200; + fspread.symbol("ZF"); + fsec.security_desc("ZFU8-ZFU9"); + fsec.cfi_code("FMDXSX"); + fspread.exchange = api::VENUE_GLBX; + fspread.low_tick = 0.001; + fspread.high_tick = 0.001; + fspread.display_factor = 0; + fspread.set_leg_count(2); + for (size_t x = 0; x < 2; ++x) { + auto& leg = fspread.legs[x]; + leg.leg_price = 0.25;//optional LegPrice id=566 + leg.leg_delta = 0.1;//optional LegOptionDelta id=1017 + leg.leg_quantity = 1; // LegRatioQty id=623 + leg.leg_side = api::SIDE_BUY; // LegSide id=624 + } + std::vector producer_handles; // Add the security to the producer (this copies fsec) and get a handle to update it // This call is thread safe and may take a symbol specific lock internally - producer_handles.push_back(producer.add_security(fsec)); + MDProducerHandle future_handle = producer.add_security(fsec); + // future_handle is small and safe to copy + producer_handles.push_back(future_handle); + producer_handles.push_back(producer.add_security(fspread)); + + // The securities added though producer.add_security will not be visible to the rest of the system + // until a security key has been generated. If you supplied SecurityIDRecorder to the MDProducer constructor + // then this will happen automatically. // initialized bid side book data std::array bids {{ diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index de7fd2d39..b788c6a2a 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -15,14 +15,12 @@ namespace md { MDP3Listener::MDP3Listener( CMESecurityIndex& index, MDProducer& producer, - SecurityIDRecorder& recorder, // co2::SettlementRecorder& , const std::vector channels) : _index(index), _producer(producer), _packet_arrived(), _latest_sequence(0), - _security_recorder(recorder), // _settlement_recorder(settles), _symbol_filter_storage(), _symbol_filter(), @@ -227,12 +225,7 @@ void MDP3Listener::add_security(const FixedSecurity& security) { auto handle = _producer.add_security(security); bts_assert(security.numeric_id != 0); // Add it to the index so it can receive messages - SecurityData* data = _index.add_security(security.numeric_id, handle); - - // Assign a security key - if (data->handle.key() == invalid_security_key) { - _security_recorder.ensure_key(data->handle); - } + _index.add_security(security.numeric_id, handle); } void MDP3Listener::flush_books() { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index b95f3ab5e..1a54c9bd0 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -31,7 +31,6 @@ class MDP3Listener : public: MDP3Listener(CMESecurityIndex& index, MDProducer&, - SecurityIDRecorder&, // co2::SettlementRecorder&, const std::vector); virtual ~MDP3Listener(); @@ -185,7 +184,6 @@ private: MDProducer& _producer; utils::TimeStamp _packet_arrived; uint32_t _latest_sequence; - SecurityIDRecorder& _security_recorder; // co2::SettlementRecorder& _settlement_recorder; std::vector _symbol_filter_storage; std::set _symbol_filter; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index a0e4bb3d5..679f92f3c 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -26,8 +26,7 @@ namespace bts { namespace md { MDP3Main::MDP3Main() : - _producer(PLATFORM_MDP3ONIXS), - _security_recorder(), + _producer(PLATFORM_MDP3ONIXS, { utils::Dispatcher::current() }), // _settlement_recorder(PLATFORM_MDP3ONIXS), _engines(), _handlers(), @@ -153,11 +152,6 @@ void MDP3Main::run(std::vector& symbols) { // Timers auto& disp = utils::Dispatcher::current(); - utils::DispatcherTimer security_timer(disp, 500, [&]() { - _security_recorder.do_work(); - }); - security_timer.start(); - utils::DispatcherTimer spread_resolve_timer(disp, 1000, [&]() { _index.resolve_spreads(); }); @@ -346,7 +340,7 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con const auto& options = channel_settings_for(channel.id); - unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, /*_settlement_recorder,*/ producer_channels)); + unique_ptr listener(new MDP3Listener(_index, _producer, /*_settlement_recorder,*/ producer_channels)); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index 1ebfdb99b..a14825fdc 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -39,7 +39,6 @@ private: CMESecurityIndex _index; MDProducer _producer; - SecurityIDRecorder _security_recorder; // co2::SettlementRecorder _settlement_recorder; std::map> _engines; std::map> _handlers; diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index e2a06f7c7..76e337de1 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -8,6 +8,7 @@ #include "lib/md/get_symbols.h" #include "lib/md/UserSymbol.h" #include "lib/md/Diag/ShmDumper.h" +#include "lib/md/Diag/MDPeriodicDumper.h" #include "platforms/mdp3_onixs/MDP3Main.h" #include "lib/md/SecurityDB.h" #include "server/sparkd/exchange_sim/ExchangeSim.h" @@ -111,51 +112,89 @@ string& uppercase(string& s) { return s; } template -T parse_enum(const string& value, const string& prefix, T2 parse, T default_value) { +bool parse_enum(const string& value, const string& prefix, T2 parse, T default_value, T* presult) { string temp = value; if (temp.find(prefix) == string::npos) { temp = prefix + "_" + temp; } - T result; - if (parse(temp, &result)) { - return result; - } - return default_value; -} -void run_dump(const vector& args) { - // either - // name, [hot|cold] - // symbol type exchange [venue|hot] [hot|cold] - - string name_or_symbol = default_arg(args, 0, ""); - string hotcold_or_type = default_arg(args, 1, "cold"); - string venue = default_arg(args, 2, "glbx"); - string platform = default_arg(args, 3, ""); - bool hot = std::find(args.begin(), args.end(), "hot") != args.end(); - if (name_or_symbol.empty()) return; - if (name_or_symbol.find("PLATFORM") == string::npos /*heuristic*/) { - Platform p = parse_enum(uppercase(platform), "PLATFORM", co2::Platform_Parse, co2::PLATFORM_UNSET); - if (p == co2::PLATFORM_UNSET) { - api::Venue v = parse_enum(uppercase(venue), "VENUE", api::Venue_Parse, api::VENUE_UNSET); - p = platform_for_exchange(v); - } - stringstream ss; - ss << uppercase(name_or_symbol) - << "_SECURITYTYPE_" << uppercase(hotcold_or_type) - << "_VENUE_" << uppercase(venue) << "_" - << co2::Platform_Name(p); - name_or_symbol = ss.str(); - } - if (name_or_symbol.find("bts_edge_") == string::npos) { - name_or_symbol = "bts_edge_" + config::get().shared_memory_namespace() + "_" + name_or_symbol; - } - ShmDumper dumper(name_or_symbol); - if (!hot || hotcold_or_type == "cold" || hotcold_or_type == "c") { - dumper.dump_cold(); + if(parse(temp, presult)) { + return true; } else { + *presult = default_value; + return false; + } +} + +bool int_of_string(const std::string& str, long* result) { + try { + size_t pos = 0; + int i = std::stol(str, &pos); + if (pos == str.size()) { + *result = i; + return true; + } + } + catch(const std::exception&) { } + return false; +} + +typedef std::string Symbol; +struct MDArgs { + api::Venue venue; + co2::Platform platform; + api::SecurityType type; + Symbol symbol; + bool hot; + long number; +}; +MDArgs parse_md_args(const vector& args) { + using namespace api; + using namespace co2; + auto venue = VENUE_UNSET; // Default to glbx. If you change this change below (used) also + auto platform = PLATFORM_UNSET; + auto type = SECURITYTYPE_UNSET; + bool hot = false; + long number = 0; + std::string symbol; + for (auto str : args) { + uppercase(str); + bool used = + (venue == VENUE_UNSET && parse_enum(str, "VENUE", Venue_Parse, venue, &venue)) || + (platform == PLATFORM_UNSET && parse_enum(str, "PLATFORM", Platform_Parse, platform, &platform)) || + parse_enum(str, "SECURITYTYPE", SecurityType_Parse, type, &type); + if (str == "HOT") { hot = true; used = true; } + else if (str == "COLD") { hot = false; used = true; } + else { + used |= int_of_string(str, &number); + } + if (!used) { + symbol = str; + } + } + if (venue == VENUE_UNSET) venue = VENUE_GLBX; + if (platform == PLATFORM_UNSET) platform = platform_for_exchange(venue); + return { venue, platform, type, symbol, hot, number }; +} + +void run_dump(const vector& args) { + using std::get; + + auto p = parse_md_args(args); + stringstream ss; + ss << "bts_edge_" << config::get().shared_memory_namespace() << "_" + << uppercase(p.symbol) << "_" + << SecurityType_Name(p.type) << "_" + << Venue_Name(p.venue) << "_" + << Platform_Name(p.platform); + + ShmDumper dumper(ss.str()); + if (p.hot) { dumper.dump_hot(); } + else { + dumper.dump_cold(); + } } @@ -178,7 +217,7 @@ void run_rectool(const vector args) { } } -void dump_db(const vector args) { +void dump_db(const vector& args) { set keys; bool all = false; for (const auto& arg : args) { @@ -208,6 +247,19 @@ void dump_db(const vector args) { }); } +void run_tail(const vector& args) { + std::set products; + auto conf = parse_md_args(args); + PlatformConfig config; + config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE)); + config.set_platform(co2::Platform_Name(conf.platform)); + get_system_md_symbols(config, products); + auto mod = conf.number > 0 ? conf.number : 1000; + PrintFormat fmt = PrintFormat::CSV; + for (const auto& a : args) { if (a == "longform") fmt = PrintFormat::Human; } + md::run_periodic_dumper(products, mod, fmt); +} + string help() { @@ -229,6 +281,7 @@ void md_tool(std::vector const &args) { else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); + else if (cmd == "tail") run_tail(args); else printhelp(); } diff --git a/src/web/ui/src/Accounts/State.fs b/src/web/ui/src/Accounts/State.fs index e1d11ae06..ccd356ae6 100644 --- a/src/web/ui/src/Accounts/State.fs +++ b/src/web/ui/src/Accounts/State.fs @@ -7,6 +7,7 @@ open Fable.PowerPack.PromiseImpl open Fable.Import open bts.co2 open Fable +open Fable.PowerPack let isNewAccount (account:BTSAccount) = account.id = 0u && account.isPresent @@ -32,6 +33,7 @@ let init (context:ServiceContext.ServiceContext) = ConfirmModal = "", (Msg.SelectTab General) SelectedClipLimits = [] SelectedPositionLimits = [] + ShowLoading = false }, Access.getAdminData admin UpdateAdminData NoAdminData let existingSelection (scratch:BTSAccount) (previousScratch:BTSAccount) (data:AdminData) = @@ -67,7 +69,6 @@ let applySIMUpdate (data:SIMAccountData) = function | CTI v -> { data with cti = v } | Origin v -> { data with origin = v } | ClearingAccount v -> { data with clearingAccount = v } - | _ -> data let saveAccount (admin:IAdminService) (account:BTSAccount) = promise { let! result = admin.update_account { completeData = account } @@ -125,6 +126,8 @@ let clearSelection model = { model with SelectedClipLimits = [] SelectedPositionLimits = [] + SelectedOrders = [] + ShowLoading = false } let orderIsEqual (order1:OrdersSummaryUnit) (order2:OrdersSummaryUnit) = @@ -168,17 +171,24 @@ let update msg (model:Model) : Model * Cmd = Global.alert Global.Info msg model, Cmd.Empty | GetOrdersData -> - model, (Access.getAccountOrdersData model.admin UpdateOrdersData (box>>ErrorOccured)) + { model with ShowLoading = false}, (Access.getAccountOrdersData model.admin UpdateOrdersData (box>>ErrorOccured)) | UpdateOrdersData summary -> {model with ordersData = List.ofArray summary.units}, Cmd.Empty | CancelOrders orders -> - model, Cmd.ofExistingPromise (cancelOrders model.admin model.scratch.name orders) id (box>>ErrorOccured) + { model with ConfirmModal = ("", Msg.SelectTab model.tab); SelectedOrders = []; ShowLoading = true }, + Cmd.batch [ + Cmd.ofExistingPromise (cancelOrders model.admin model.scratch.name orders) id (box>>ErrorOccured) + Cmd.ofExistingPromise (promise { + do! Promise.sleep 1000 + return GetOrdersData + }) id (fun _ -> GetOrdersData) + ] | SelectOrder (chckd, orders) -> - let so = match chckd, orders.Length > 1 with - | true, false -> (orders.[0]::model.SelectedOrders) - | false, false -> List.filter (fun elm -> (orderIsEqual elm orders.[0]) |> not) model.SelectedOrders - | true, true -> orders - | false, true -> [] + let so = match chckd, orders with + | true, h::m::tail -> orders + | true, h::_ -> h::model.SelectedOrders + | false, h::_ -> List.filter (fun elm -> (orderIsEqual elm h) |> not) model.SelectedOrders + | false, _ -> [] {model with SelectedOrders = so}, Cmd.Empty | SelectAccount account -> { (clearSelection model) with previousScratch = model.scratch; scratch = account }, Cmd.Empty diff --git a/src/web/ui/src/Accounts/Types.fs b/src/web/ui/src/Accounts/Types.fs index 402f009e8..ffbc6ebe0 100644 --- a/src/web/ui/src/Accounts/Types.fs +++ b/src/web/ui/src/Accounts/Types.fs @@ -43,11 +43,12 @@ type SIMExchangeField = type Msg = | UpdateAdminData of AdminData + | NoAdminData of string + // Orders | GetOrdersData | UpdateOrdersData of OrdersSummary | SelectOrder of bool * OrdersSummaryUnit list | CancelOrders of OrdersSummaryUnit list - | NoAdminData of string | SelectAccount of BTSAccount | DeleteAccount of BTSAccount | AddAccount @@ -102,4 +103,5 @@ type Model = { SelectedPositionLimits: AccountProductPositionLimitUnit list OptionsMenuOpen:string ConfirmModal: string * Msg + ShowLoading: bool } \ No newline at end of file diff --git a/src/web/ui/src/Accounts/View.fs b/src/web/ui/src/Accounts/View.fs index ed74b8a34..3d22692b1 100644 --- a/src/web/ui/src/Accounts/View.fs +++ b/src/web/ui/src/Accounts/View.fs @@ -78,7 +78,6 @@ let generalView model dispatch = ] let GetSymbolArray (model:Model) = - Array.map (fun a -> a.symbol ) model.mdRelayProductData |> Array.distinct @@ -125,10 +124,23 @@ let ordersView (model:Model) dispatch = } ] - let cancelButton = Button.button "Cancel Selected Orders" "is-danger" "times" Button.Enabled (fun _ -> model.SelectedOrders |> CancelOrders |> dispatch) + let enableCancel = + match model.SelectedOrders.Length with + | 0 -> Button.Disabled + | _ -> Button.boolToDisabledConverter model.ShowLoading + + let addLoadingString s b = + match b with + | true -> sprintf "is-loading %s" s + | _ -> s + + let cancelMessage = model.SelectedOrders |> CancelOrders + let confirmText = sprintf "Are you sure you want to cancel all orders in %s for users: %s?" model.scratch.name (String.concat ", " (List.map (fun (elm:OrdersSummaryUnit) -> elm.user) model.SelectedOrders)) + let cancelButton = Button.button "Cancel Selected Orders" (addLoadingString "is-danger" model.ShowLoading) "times" enableCancel (fun _ -> OpenConfirmModal (confirmText, cancelMessage) |> dispatch) + let reloadFun = fun _ -> dispatch (SelectTab model.tab) div [][ - TableControl.createDataTable (Some headerRow) rws cancelButton + TableControl.createDataTable (Some headerRow) rws (span [][ (Button.button "Refresh" "" "exchange" (Button.boolToDisabledConverter model.ShowLoading) reloadFun); nbsp.element; cancelButton ]) ] let limitFilter (positionLimit:AccountSecurityPositionLimitUnit option) (clipLimit:AccountSecurityClipLimitUnit option) allValueDisplay (array:ExpirationNameList []) = diff --git a/src/web/ui/src/Controls/Button.fs b/src/web/ui/src/Controls/Button.fs index 6275a64d2..7caf22245 100644 --- a/src/web/ui/src/Controls/Button.fs +++ b/src/web/ui/src/Controls/Button.fs @@ -12,6 +12,12 @@ let isDisabled = function | Disabled -> true | Enabled -> false +let boolToDisabledConverter d = + match d with + | true -> Disabled + | false -> Enabled + + let addButton disabled click = a [ ClassName "button fabutton " HTMLAttr.Disabled (isDisabled disabled) diff --git a/testing/manual/admin/order_cancel.txt b/testing/manual/admin/order_cancel.txt new file mode 100644 index 000000000..3a820bc04 --- /dev/null +++ b/testing/manual/admin/order_cancel.txt @@ -0,0 +1,18 @@ +This test is to ensure that orders for a user and account can be canceled through the Spark Admin + +This test refers to users bts and bts_2 and accounts ACC A and ACC A1, any two users and any two accounts can be used. + +1. Login to the shell as bts +2. Open at least one ladder with market data +3. Enter a few working limit orders and few of your favorite algos in ACC A +4. Logout of bts and in as bts_2 +5. Repeat steps 2 and 3 for this user + +6. Open the Spark Admin for your VM in your preferred non Microsoft web browser +7. Go to the Orders tab under the Accounts module for ACC A; the number of orders you entered for each user should be displayed in the table +8. Check the box in the header to select all rows and click the big red button + +9. Go back to the shell an make sure all of your orders have been canceled +10. Logout of bts_2 and log back in as bts +11. Repeat step 9 +12. Repeat step 3,6,7,8, and 9 for account ACC A1 diff --git a/testing/manual/algos/join.txt b/testing/manual/algos/join.txt index 4b194f7e6..82799b450 100644 --- a/testing/manual/algos/join.txt +++ b/testing/manual/algos/join.txt @@ -3,7 +3,6 @@ - Make sure you have limits to trade this future - Select Join order from the menu Label 1) Set join thresh to absolute 10 -Label 2) Set initial tick offset to 2 The goal of the join order is to watch the opposite-inside market and join when the market *flips and @@ -12,14 +11,9 @@ has at least threshold size *flip is when the best offer becomes the best bid, or the market trades away from you (the line moves up in the gui) Label 3) Put an order in to way down in the bid side of the book -- Because of the parameter in label 2, the order should instead join two levels below the best bid - Watch the opposite inside market, if it flips make sure the order moves itself to join the new best bid (former offer) -Goto Label 2 and set tick offset to 0 - { - Orders should then go in where you put them - } Goto Label l and test the ratio threshold { - Use a threshold of at least 1.0 @@ -28,7 +22,7 @@ Goto Label 3 and try a sell order { } -Now do this test again with "Enable Hedge" checked and hedge ticks set to 1, hedge type set to limit +Now do this test again but use a Cover of Join (that's a Cover order with a Join Child and a Limit Hedge, HedgeTicks = 1) This should modify the behavior such that the algorithm will send a limit order on the opposite side of the market, one tick away from fill price in a quantity equivalent to the amount of the fill diff --git a/testing/manual/algos/lean.txt b/testing/manual/algos/lean.txt index cffa08e52..2e6a0ccbf 100644 --- a/testing/manual/algos/lean.txt +++ b/testing/manual/algos/lean.txt @@ -6,11 +6,12 @@ Label 1) Set cross thresh to absolute 10 The goal of the lean order is to watch the opposite-inside market size and *cross when that size goes below the threshold (this also implies a cross if that size goes to zero and the market trades away from you) -(if the market trades toward you, we just start watching the new ask) + *cross means take an order working on the bid and 'cross' by buying into the asks. This is also called 'paying up' - Open up order book and fills +- Make sure OnToward behavior is set to No Change (not Cancel) Label 2) Put an order in to Join the best bid - Watch the opposite inside market, if it dips below the threshold (or trades away from you) make sure the order moves itself to cross - Verify that the order filled at the offer price diff --git a/testing/manual/positions_and_fills/PAndL.txt b/testing/manual/positions_and_fills/PAndL.txt index 295fb1c9c..a863aed2f 100644 --- a/testing/manual/positions_and_fills/PAndL.txt +++ b/testing/manual/positions_and_fills/PAndL.txt @@ -16,13 +16,13 @@ and you may also want to clear your orders for a clean slate: Throughout the test we'll refer to accounts ACC A and ACC A1. These specific accounts aren't necessary, but rather that we have fills in multiple distinct accounts. -Bring up the pnl form (empty) . +Bring up the pnl form (empty). Bring up a SIM:SI ladder form, pick your favorite expiration. It should be completely empty since we're controlling the md, and the ACC A account should be selected in the account drop down list. Bring up an order ticket for your corresponding ladder place a buy order with price 14.000 and qty 10 your ladder should populate now place a sell order with price 14.010 at qty 10 - this will give you a valid Midpoint for you PnL Price in the PnL + this will give you a valid Midpoint for your PnL Price in the PnL Sell a 1 lot at your bid (14.000) The Start Net Pos should be 0 Start Pnl should be 0 @@ -33,9 +33,9 @@ Sell a 1 lot at your bid (14.000) PnL Price should be at the midpoint of your bid and ask (14.000 + 14.010)/2 = 14.005. Improve the bid Under ACC A1 account (this is important for the next step), enter a 10 lot bid at 14.005 - Observe the PnL Price reflect the current Midpoint (14.0075) - Switch back to ACC A -Sell a 1 lot at the current bid + Observe the PnL Price reflect the current Midpoint (14.0075 displayed as 14.010) +Switch back to ACC A + Sell a 1 lot at the current bid You should now have rows for both the ACC A and ACC A1 account ACC A Day Long 1 @@ -44,7 +44,7 @@ Sell a 1 lot at the current bid (14.005 [fill price] - 14.0075 [driver price]) * 1 [qty] * 5000 [cash per point] = -12.50 Total Net Pos -1 Total PnL = -12.5 - PnL Price = 14.0075 + PnL Price = 14.0075 (displayed as 14.010) ACC A1 Day Long 1 Day Short 0 @@ -52,7 +52,7 @@ Sell a 1 lot at the current bid (14.0075 - 14.005) * 1 * 5000 Move the market Move your working sell order up to 14.020 - The PnL Price should update to 14.0125 ((14.020 + 14.005) /2) + The PnL Price should update to 14.0125 ((14.020 + 14.005) /2) (displayed as 14.015) The Day and Total PnLs should update to according to the same calculation as earlier ACC A: -37.5 ACC A1: 37.5 @@ -64,60 +64,57 @@ Check the Totals Footer Force the SOD process to aggregate the fills Configure your sod time to be after your last fill and before now. Note that SODTime is currently expressed in UTC, so you'll have to make the appropriate conversion. - + Here is a sample config: - + SODTime{ value { sod_hh: 15 sod_mm: 00 } } - + Restart sparkd and the SOD process should consolidate your day fills so... - - ACC A: + + ACC A: Start Net Pos -1 - Start PnL -12.5 - (14.005 - 14.0125) * 2 * 5000 = -75 - (14.0125 - 14.000) * 1 * 5000 = 62.5 - -75 + 62.5 = -12.5 + Start PnL blank Day Long 0 Day Short 0 Day PnL 0 Total Net Pos -1 Total PnL -12.5 - PnL Price 14.0125 - ACC A1: + PnL Price 14.0125 (displayed as 14.015) + ACC A1: Start Net Pos 1 - Start PnL 37.5 + Start PnL blank Day Long 0 Day Short 0 Day PnL 0 Total Net Pos 1 Total PnL 37.5 (14.0125 - 14.005) * 1 * 5000 - PnL Price 14.0125 + PnL Price 14.0125 (displayed as 14.015) Sell a 1 lot as the current bid in ACC A ACC A: Start Net Pos -1 (no change) - Start PnL -12.5 (no change) + Start PnL -37.5 Day Long 0 Day Short 1 Day PnL -37.5 Total Net Pos -2 Total PnL -50 - PnL Price 14.0125 - ACC A1: + PnL Price 14.0125 (displayed as 14.015) + ACC A1: Start Net Pos 1 (no change) - Start PnL 37.5 (no change) + Start PnL 37.5 Day Long 1 Day Short 0 Day PnL 37.5 Total Net Pos 2 Total PnL 75 - PnL Price 14.0125 + PnL Price 14.0125 (displayed as 14.015) Test sorting and filtering diff --git a/testing/manual/positions_and_fills/SOD.txt b/testing/manual/positions_and_fills/SOD.txt index 95406c323..6d8eee837 100644 --- a/testing/manual/positions_and_fills/SOD.txt +++ b/testing/manual/positions_and_fills/SOD.txt @@ -1,23 +1,22 @@ clear your fills and orders database: rm ~/sparkdata/data/databases/fills/* rm ~/sparkdata/data/databases/orders/* -and bounce your server. +and bounce your server. -0. We will force fills across for 3 securities, across 2 accounts. Lets pick 2 futures and -one exchange traded spread (make sure there are no overlaps). Choose these securities -such that there is market data (sim or cert) that you can get fills against (maybe bring up -the market view and select securities from there). Bring up the 6 ladders. Make sure the position -button the ladder displays - +0. We will force fills across for 3 securities, across 2 accounts. + a. Pick 2 futures and one exchange traded spread (make sure there are no overlaps). + b. Choose these securities such that there is market data (sim or cert) that you can get fills against (maybe bring up the market view and select securities from there). + c. Bring up the 6 ladders. Make sure the position button the ladder displays 1. Force buy and sell fills across the 6 ladders. Keep track of the long and short positions and verify that that the position button in the ladder is reflecting the net position carefully. Open up the PNL and make sure the Day Long and Day Short positions match what you expect. -The Start Net Pos should be 0. (The Total Net Pos in the PNL should match the pos button +The Start Net Pos should be 0. (The Total Net Pos in the PNL should match the pos button in the ladder) -2. Now comes the SOD part. The SOD is what demarkates a day. Find the most recent time on your fills, select 1 minute past that and convert it to UTC and override the SOD time in your spark.conf +2. Now comes the SOD part. The SOD is what demarcates a day. Find the most recent time on your fills, select 1 minute past that and convert it to UTC and override the SOD time in your spark.conf Here's an example of what the key looks like: @@ -30,10 +29,8 @@ SODTime{ 3. Now restart the server. Verify that all rows in the fills have a Fill Kind 'Sod' and the time matches the SODTime (converted to your local time zone) -4. Verify that the ladder is displaying the correct position (it should be the same as before -the server start) +4. Verify that the ladder is displaying the correct position (it should be the same as before the server start) 5. Verify that the PNL is displaying the correct Start Net Pos and the Day part is 0. -6. Now force a few more fills. Verify that the ladder position box and the PNL is -correctly counting them. \ No newline at end of file +6. Now force a few more fills. Verify that the ladder position box and the PNL is correctly counting them. \ No newline at end of file diff --git a/testing/manual/settings/GridSettings.txt b/testing/manual/settings/GridSettings.txt index 43e096259..db8cb0072 100644 --- a/testing/manual/settings/GridSettings.txt +++ b/testing/manual/settings/GridSettings.txt @@ -1,7 +1,7 @@ ----Grid Settings Test (11 minutes) --------- 1) Open the Orderbook -2) Click the Window Settings button at the bottom of the Spark menu to open the settings dialog +2) Click the Settings button at the bottom of the Spark menu to open the settings dialog ----Columns----------------------------------