From 9f406e83aa1f64b221497ff097bfe193b93ea54d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 3 May 2019 17:39:47 -0400 Subject: [PATCH] SPARK-2151 Dump shared memory on bad handle for forensic analysis --- src/client/spark_api/Co2/Co2NativeMethods.cs | 4 +++ src/client/spark_api/Co2/MDConsumer.cs | 12 +++++-- src/client/spark_api/Co2/MDConsumerHandle.cs | 13 +++++-- src/lib/md/MDConsumer.cpp | 20 ++++++++++- src/lib/md/MDConsumer.h | 2 ++ src/lib/md/MDConsumerInterop.cpp | 6 ++++ src/lib/utils/SharedList.h | 36 +++++++++++++++++++- 7 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/client/spark_api/Co2/Co2NativeMethods.cs b/src/client/spark_api/Co2/Co2NativeMethods.cs index f744b9963..b2ae76044 100644 --- a/src/client/spark_api/Co2/Co2NativeMethods.cs +++ b/src/client/spark_api/Co2/Co2NativeMethods.cs @@ -67,6 +67,10 @@ namespace mbts.co2 [DllImport("libspark")] internal unsafe static extern void md_consumer_get_system_md_symbols(raw_usersymbol_callback cb); + + [DllImport("libspark")] + internal unsafe static extern void dump_co2_memory_to_disk(IntPtr consumer, string path); + // // Relay control diff --git a/src/client/spark_api/Co2/MDConsumer.cs b/src/client/spark_api/Co2/MDConsumer.cs index 439d65af0..3d92f6e69 100644 --- a/src/client/spark_api/Co2/MDConsumer.cs +++ b/src/client/spark_api/Co2/MDConsumer.cs @@ -37,7 +37,10 @@ namespace mbts.co2 unsafe void init() { _raw_sec_cb = (key, buf, len, handle) => { var security = Co2NativeMethods.deserialize(buf, len); - var next = new SecurityAndHandle() { security = security, handle = new MDConsumerHandle(handle) }; + var next = new SecurityAndHandle() { + security = security, + handle = new MDConsumerHandle(_consumer, security, handle) + }; _securities.Add(next); _on_security?.Invoke(new List(1) { next }); }; @@ -67,7 +70,7 @@ namespace mbts.co2 public MDConsumerHandle try_get_handle(security_key key) { Co2NativeMethods.BlitPointerTuple raw; if (Co2NativeMethods.md_consumer_get_handle(_consumer, key, out raw)) { - return new MDConsumerHandle(raw); + return new MDConsumerHandle(this._consumer, null, raw); } return null; } @@ -77,7 +80,10 @@ namespace mbts.co2 List result = new List(); Co2NativeMethods.raw_security_callback cb = (key, buf, len, handle) => { var security = Co2NativeMethods.deserialize(buf, len); - result.Add(new SecurityAndHandle() { security = new Security(security), handle = new MDConsumerHandle(handle) }); + result.Add( + new SecurityAndHandle() { + security = new Security(security), + handle = new MDConsumerHandle(_consumer, security, handle) }); }; Co2NativeMethods.md_consumer_walk_security_list(_consumer, cb); return result; diff --git a/src/client/spark_api/Co2/MDConsumerHandle.cs b/src/client/spark_api/Co2/MDConsumerHandle.cs index 4a12f2f99..7b03bdaee 100644 --- a/src/client/spark_api/Co2/MDConsumerHandle.cs +++ b/src/client/spark_api/Co2/MDConsumerHandle.cs @@ -156,13 +156,22 @@ namespace mbts.co2 { // // Internal - internal MDConsumerHandle(Co2NativeMethods.BlitPointerTuple tuple) { + internal MDConsumerHandle( + IntPtr consumer, + Security security, + Co2NativeMethods.BlitPointerTuple tuple) { _header = (ColdHeader*)tuple.cold_header.ToPointer(); _cold = (ColdEntry*)tuple.cold_data.ToPointer(); if (has_depth) { _hot_depth = (HotDepthEntry*)tuple.hot.ToPointer(); - Log.assert(_hot_depth->key == _cold->security_key); + if (_hot_depth->key != _cold->security_key) { + var temp_path = Path.Combine(Path.GetTempPath(), "btsshm"); + Co2NativeMethods.dump_co2_memory_to_disk(consumer, temp_path); + Log.assert( + _hot_depth->key == _cold->security_key, + $"Please attach the contents of '${temp_path}' to the bug report you are about to file. security:{security?.Id} cold:{_cold->security_key} hot:{_hot->key}"); + } } else { _hot = (HotEntry*)tuple.hot.ToPointer(); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index a303c34ad..5969ec994 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -269,6 +269,16 @@ public: if (!_cold.is_initialized()) return true; return is_heart_beat_ok(cold_header()); } + + void dump_shm_to(const char* directory) const { + _cold.dump_to(directory); + if (_hot) { + _hot->dump_to(directory); + } + if (_hot_depth) { + _hot_depth->dump_to(directory); + } + } private: @@ -533,7 +543,7 @@ private: /////////////////////////////////////////////////////////////////////////////////////////////////// class MDConsumerImpl { typedef tbb::queuing_mutex Mutex; - Mutex _workmutex; + mutable Mutex _workmutex; vector> _instances; std::shared_ptr _sci; MBOReader* _mbo; @@ -910,6 +920,12 @@ public: return result; } + void dump_shm_to(const char* p) const { + //Mutex::scoped_lock lock(_workmutex); + for (auto& inst : _instances) { + inst->dump_shm_to(p); + } + } }; @@ -957,6 +973,8 @@ MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityTyp MDConsumerChannel MDConsumer::get_channel(const UserSymbol& u) const { return _pimpl->get_channel(u.symbol(), u.type(), u.exchange()); } +void MDConsumer::dump_shm_to(const char* p) const { _pimpl->dump_shm_to(p); } + MDConsumerChannel::MDConsumerChannel(const shared_ptr& inst) : _pimpl(inst) {} int64_t MDConsumerChannel::read_trades(google::protobuf::RepeatedPtrField& trades, int64_t cursor) { return _pimpl->read_trades(trades, cursor); } diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 103e8bb19..d197499a3 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -67,6 +67,8 @@ public: } uintptr_t id() const { return reinterpret_cast(_pimpl.get()); } + void dump_shm_to(const char* directory) const; + private: //BTS_NO_COPY(MDConsumer); COPY OK std::shared_ptr _pimpl; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 17a36b072..88972112d 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -177,5 +177,11 @@ CAPI void post_on_scan_security_thread(just_action a) { md::scan_securities_thread().post(a); } +CAPI void dump_co2_memory_to_disk(MDConsumer* consumer, const char* path) { + if (consumer) { + consumer->dump_shm_to(path); + } +} + }} //namespace #endif diff --git a/src/lib/utils/SharedList.h b/src/lib/utils/SharedList.h index 345055ebf..9a47e2701 100644 --- a/src/lib/utils/SharedList.h +++ b/src/lib/utils/SharedList.h @@ -15,8 +15,10 @@ #include #include #include +#include #include +#include namespace bts { namespace utils { @@ -238,6 +240,38 @@ public: offset = sizeof(SegmentHeader); } } + + void dump_to(const char* directory) const { + using namespace std; + boost::system::error_code _; + if (!boost::filesystem::exists(directory, _)) { + boost::filesystem::create_directories(directory, _); + } + + if (!boost::filesystem::is_directory(directory)) { + LOG(md,Error, "Error dumping memory to {}", std::string(directory)); + return; + } + + boost::filesystem::path root(directory); + + // OK to hold the lock for a while as we're typically crashing when this is called + tbb::queuing_mutex::scoped_lock lock(_mutex); + + for (size_t x = 0; x < _segments.size(); ++x) { + char segment_name[512]; + SharedList::get_segment_name(segment_name, BTS_ARRAYSIZE(segment_name), _name, x); + + // create/overwrite output file + std::ofstream writer( (root/segment_name).string(), ios::out | ios::app | ios::binary); + if (writer.is_open()) { + // writer just loves signed types + writer.write( + (const char*)_segments[x].address, + (std::streamsize)_segments[x].size); + } + } + } private: BTS_NO_COPY(SharedListReader); @@ -281,7 +315,7 @@ private: const std::string _name; THeader* _header; // null value indicates no memory segment - tbb::queuing_mutex _mutex; + mutable tbb::queuing_mutex _mutex; std::unique_ptr _shm; std::vector _segments; };