/** * @file include/ActivMiddleware/ActivFramework/RingBuffer.h * * @brief Header for the RingBuffer class. */ #if (!defined ACTIV_FRAMEWORK_RING_BUFFER_H) #define ACTIV_FRAMEWORK_RING_BUFFER_H #include "ActivMiddleware/ActivFramework/External.h" #include "ActivMiddleware/ActivBase/IEvent.h" #include "ActivMiddleware/ActivBase/IMappedMemory.h" #include "ActivMiddleware/ActivBase/Timeout.h" #include "ActivMiddleware/ActivFramework/ManagedEndPoint.h" #include "ActivMiddleware/ActivFramework/ThreadTimer.h" #include "ActivMiddleware/CxxYZ/functional.h" #include "ActivMiddleware/CxxYZ/memory.h" #include "ActivMiddleware/CxxYZ/atomic.h" //! Activ namespace. namespace Activ { class EventManagee; class RingBufferHelper; /** * @brief Ring buffer class. * * Supports reading/writing variable length messages up to ((size / 2) - sizeof(uint32_t)) bytes. */ class RingBuffer { public: //! Atomic type. typedef CxxYZ::atomic AtomicUint32; class OpenTag {}; class CreateTag{}; class Reader; //! Writer class. class Writer { public: //! Back pressure mode enumeration. enum BackPressureMode { BACK_PRESSURE_MODE_SPIN, ///< If ring fills, busy wait until space becomes available. BACK_PRESSURE_MODE_DROP ///< If ring fills, drop message. }; //! Statistics class. class Statistics { public: //! Default constructor. Statistics(); uint64_t m_byteCount; uint64_t m_messageCount; /** * @brief Get the back pressure count. * * When in BACK_PRESSURE_MODE_SPIN this is the number of times a busy wait has been required to wait for space. * * When in BACK_PRESSURE_MODE_DROP this is the number of messages dropped as a result of back pressure. */ uint64_t m_backPressureCount; uint32_t m_intervalPeak; uint32_t m_peak; DateTime m_peakDateTime; }; /** * @brief Construct a writer to write to a reader object. * * @param managedEndPoint ManagedEndPoint this Writer is associated with. * @param OpenTag the writer will be opening the ring buffer. * @param reader the reader to write to. * @param backPressureMode the back pressure mode to use. * * @throw Exception */ ACTIV_FRAMEWORK_API Writer(ManagedEndPoint &managedEndPoint, const OpenTag, const Reader &reader, const BackPressureMode backPressureMode); /** * @brief Construct a writer to write to a shared memory buffer. * * Opens the shared memory buffer. * * @param managedEndPoint ManagedEndPoint this Writer is associated with. * @param OpenTag the writer will be opening the ring buffer. * @param name name of the memory mapped file to open. * @param backPressureMode the back pressure mode to use. * @param timeoutPeriod how long to wait, in ms. * * @throw Exception */ ACTIV_FRAMEWORK_API Writer(ManagedEndPoint &managedEndPoint, const OpenTag, const String &name, const BackPressureMode backPressureMode, const TimeoutPeriod timeoutPeriod = ACTIV_WAIT_INFINITE); /** * @brief Construct a writer to write to a shared memory buffer. * * Creates the shared memory buffer. * * @param managedEndPoint ManagedEndPoint this Writer is associated with. * @param CreateTag the writer will be creating the ring buffer. * @param name name of the memory mapped file to create. * @param size size of the data portion of the ring buffer. * @param backPressureMode the back pressure mode to use. * @param isUsingTimestamps whether the writer will transmit timestamps, requires support in the reader. * * @throw Exception */ ACTIV_FRAMEWORK_API Writer(ManagedEndPoint &managedEndPoint, const CreateTag, const String &name, const size_t size, const BackPressureMode backPressureMode, const bool isUsingTimestamps); //! Destructor. ACTIV_FRAMEWORK_API virtual ~Writer(); /** * @brief Allocate a write buffer. Any Write*() function will invalidate the buffer. * * @param pBuffer * @param length * * @retval true if the write can proceed. * * @throw Exception */ bool AllocateWriteBuffer(void *&pBuffer, const size_t length); /** * @brief Commit write buffer. Previously allocated via AllocateWriteBuffer. * * @param length * * @throw Exception */ void CommitWriteBuffer(const size_t length); /** * @brief Write a buffer to the ring buffer. * * @param pData * @param length * @param pTimestamp optional timestamp to attach with the message. * * @retval true if the buffer was successfully written. * * @throw Exception */ bool WriteBuffer(const void *pData, const size_t length, const timespec *pTimestamp = ACTIV_NULLPTR); /** * @brief Write a fixed length message to the ring buffer. * * F must be a callable type accepting a signature of void (void *pData, uint32_t length). The data written into the memory * must be exactly @a length bytes long. * * @param function * @param length * @param pTimestamp optional timestamp to attach with the message. * * @retval true if the buffer was successfully written. * * @throw Exception */ template bool WriteFixedMessage(const F &function, const size_t length, const timespec *pTimestamp = ACTIV_NULLPTR); /** * @brief Write a variable length message to the ring buffer. * * F must be a callable type accepting a signature of size_t (void *pData, size_t length). The data written into * the memory must be at most @a length bytes long. The number of bytes actually written must be returned. * * @param function * @param length * @param pTimestamp optional timestamp to attach with the message. * * @retval true if the buffer was successfully written. * * @throw Exception */ template bool WriteVariableMessage(const F &function, const size_t length, const timespec *pTimestamp = ACTIV_NULLPTR); //! Flush. void Flush(); /** * @brief Set a callback function to be invoked when the interval peak has been updated. * * @param callbackFunction the callback function. */ void SetOnIntervalPeakUpdateCallback(const CxxYZ::function &callbackFunction); //! Get the statistics. const Statistics &GetStatistics() const; private: //! Initialize the writer. void Initialize(); /** * @brief Initialize a write of up to @a length bytes. * * @param length the number of bytes to make write. * * @return true if write can proceed, false otherwise (dropped). */ ACTIV_FRAMEWORK_API bool InitializeWrite(const size_t length); /** * @brief Prepare to write up to @a length bytes. * * @param length the number of bytes to make write. * * @return whether the write can be performed. */ bool PrepareWrite(const uint32_t length); /** * @brief Initialize a write of up to @a length bytes. * * @param pHeader pointer to the message header in the ring buffer to update. * @param pBuffer pointer to the message data. * @param length the number of bytes written. */ void FinalizeWrite(uint32_t *pHeader, void *pBuffer, const uint32_t length); /** * @brief Initialize message header, write timestamp and return pointers to header and message buffer. * * @param pHeader * @param pBuffer * @param pTimestamp */ void InitializeHeader(uint32_t *&pHeader, void *&pBuffer, const timespec *pTimestamp); //! Check and set event if necessary. ACTIV_FRAMEWORK_API void CheckAndSetEvent(); /** * @brief Check blocking loop timeout. * * @param loopCount * @param blockingWriteStartTickCount * * @retval Boolean */ bool CheckBlockingLoopTimeout(size_t &loopCount, Boost::optional &oBlockingWriteStartTickCount); //! Process interval peak timer. void ProcessIntervalPeakTimer(); //! Process monitor timer. void ProcessMonitorTimer(); //! Process the reset timer. void ProcessResetTimer(); //! On post end point processing callback. bool OnPostEndPointProcessing(); //! On event timer callback. void OnEventTimer(); ManagedEndPoint & m_managedEndPoint; RingBufferHelper & m_ringBufferHelper; const String m_name; const BackPressureMode m_backPressureMode; const bool m_isUsingTimestamps; CxxYZ::unique_ptr m_pRingBuffer; uint32_t m_cachedReadOffset; uint32_t m_cachedWriteOffset; uint32_t m_cachedReaderCompletionEventCount; Statistics m_statistics; uint32_t m_monitorIntervalPeak; bool m_hasAlertedForBlockingWriteDrop; bool m_hasBlockingWriteDroppedSinceLastSuccess; //! Byte count when the event was last checked. uint64_t m_lastEventCheckByteCount; //! Threshold to check ring buffer event (if not checked due to idleness). const uint32_t m_eventByteThreshold; CxxYZ::unique_ptr m_pIntervalPeakTimer; CxxYZ::function m_onIntervalPeakUpdateCallback; CxxYZ::unique_ptr m_pMonitorTimer; String m_monitorName; uint64_t m_backPressureCountAtReset; CxxYZ::unique_ptr m_pResetTimer; bool m_hasUpdatedWritePointerSinceLastPostEndPointProcessing; bool m_hasEventBeenCheckedSinceLastUpdate; uint32_t m_messagesPerWritePointerUpdate; ///< The current total number of messages before the write pointer will update. uint32_t m_messagesUntilWritePointerUpdate; ///< The remaining number of messages before the next write pointer update. uint32_t m_maxFlushFrequency; ///< The number of messages which must be received in a single message loop before throttling of the ring buffer's write pointer updates will no longer increase. friend class RingBufferHelper; }; class MessageReader; //! Reader class. class Reader { public: //! Callback function signature for event notification. typedef CxxYZ::function CallbackFunction; /** * @brief Construct a reader to read from private memory. * * @param managedEndPoint ManagedEndPoint this Reader is associated with. * @param CreateTag the reader will be creating the ring buffer. * @param callbackFunction function for notification of data. * @param size size of the data buffer to create. * * @throw Exception */ ACTIV_FRAMEWORK_API Reader(ManagedEndPoint &managedEndPoint, const CreateTag, const CallbackFunction &callbackFunction, const size_t size); /** * @brief Construct a reader to read from shared memory. * * Creates the shared memory buffer. * * @param managedEndPoint ManagedEndPoint this Reader is associated with. * @param CreateTag the reader will be creating the ring buffer. * @param callbackFunction function for notification of data. * @param name name of the memory mapped file to create. * @param size size of the data portion of the ring buffer. * * @throw Exception */ ACTIV_FRAMEWORK_API Reader(ManagedEndPoint &managedEndPoint, const CreateTag, const CallbackFunction &callbackFunction, const String &name, const size_t size); /** * @brief Construct a reader to read from shared memory. * * Opens the shared memory buffer. * * @param managedEndPoint ManagedEndPoint this Reader is associated with. * @param OpenTag the reader will be opening the ring buffer. * @param callbackFunction function for notification of data. * @param name name of the memory mapped file to open. * @param timeoutPeriod how long to wait, in ms. * * @throw Exception */ ACTIV_FRAMEWORK_API Reader(ManagedEndPoint &managedEndPoint, const OpenTag, const CallbackFunction &callbackFunction, const String &name, const TimeoutPeriod timeoutPeriod = ACTIV_WAIT_INFINITE); //! Destructor. ACTIV_FRAMEWORK_API ~Reader(); //! Process read event. void ProcessReadEvent(); //! Get the number of blocking reads. size_t GetNumberOfBlockingReads() const; //! Get the number of attempted blocking reads that were re-enabled. size_t GetNumberOfTimesEventHasReenabled() const; //! Get the current number of bytes queued. ACTIV_FRAMEWORK_API uint32_t GetCurrentBytesQueued(); private: /** * @brief Read a data block from the ring buffer. * * @param pData * @param length * @param timestamp reference to receive a timestamp, tv_sec is set to zero if a timestamp is not available. * * @return whether data was available to read. */ ACTIV_FRAMEWORK_API bool Read(const void *&pData, size_t &length, timespec ×tamp); //! Initialize the reader. void Initialize(ManagedEndPoint &managedEndPoint); void UpdateCachedBytesQueued(); //! Reset event. void ResetEvent(); //! Read from buffer using cached write offset. bool InternalCachedRead(const void *&pData, size_t &length, timespec ×tamp); //! Read from buffer using atomic write offset. bool InternalRead(const void *&pData, size_t &length, timespec ×tamp); //! Process lazy timer. void ProcessLazyTimer(); ManagedEndPoint & m_managedEndPoint; CxxYZ::unique_ptr m_pRingBuffer; uint32_t m_cachedWriteOffset; uint32_t m_currentPosition; uint32_t m_cachedBytesQueued; bool m_shouldTriggerPeakQueuedBytesTrace; uint64_t m_lastEmptyTimestamp; ScopedTracePoint m_processTracePoint; size_t m_numberOfBlockingReads; size_t m_numberOfTimesEventHasReenabled; CallbackFunction m_callbackFunction; CxxYZ::unique_ptr m_pEventManagee; ThreadTimer::ChimeTimer m_lazyTimer; const String m_monitorName; ManagedEndPoint::ScopedPostEndPointProcessingTask m_scopedPostEndPointProcessingTask; friend class MessageReader; friend class Writer; }; //! Message reader class to access messages from the buffer. Used to enforce trace processing at the beginning. class MessageReader { public: //! Constructs the reader and records a trace event. MessageReader(Reader &reader); /** * @brief Reads a message. * * @param pData * @param length * @param timestamp reference to receive a timestamp, tv_sec is set to zero if a timestamp is not available. */ bool operator()(const void *&pData, size_t &length, timespec ×tamp) const; private: Reader & m_reader; }; //! Get cache line size. uint32_t GetCacheLineSize() const; //! Get maximum ring buffer size. uint32_t GetMaximumRingBufferSize(); /** * @brief Round offset to the next cache line. * * @param offset * * @return rounded offset. */ template T RoundOffsetToNextCacheLineT(const T offset) const; private: //! Class to trace queuing. struct ProcessTraceEvent : TraceEvent { public: ACTIV_FRAMEWORK_API String Decode(const String &name) const; uint32_t m_cachedBytesQueued; }; //! Default constructor. RingBuffer(); /** * @brief Create a private memory ring buffer. * * @param size size of the data portion of the ring buffer. * * @throw Exception */ void Create(const size_t size); /** * @brief Create a shared memory ring buffer. * * @param name name of the memory mapped file to create. * @param size size of the data portion of the ring buffer. * @param version protocol version to use. * * @throw Exception */ void Create(const String &name, const size_t size, const uint32_t version = VERSION_0); /** * @brief Open a private memory ring buffer. * * @param ringBuffer the ring buffer to open. * * @throw Exception */ void Open(const RingBuffer &ringBuffer); /** * @brief Open a shared memory ring buffer. * * @param name name of the memory mapped file to open. * @param timeout object specifying how long to wait for the object to exist, in ms. * * @throw Exception */ void Open(const String &name, Timeout &timeout); /** * @brief Initial the shared data. * * @param isCreator whether the ring buffer is being created. * @param creatorVersion protocol version to set for the creator. */ void InitializeSharedData(const bool isCreator, const uint32_t creatorVersion = VERSION_0); //! Get memory map size before the version was added. size_t GetOldMemoryMapSize() const; //! Get memory map size after the version was added. size_t GetNewMemoryMapSize() const; //! Value written to indicate the next message will not fit into the end of the ring. Jump back to the beginning. static const uint32_t WRAP = Boost::integer_traits::const_max; //! Top bit of length indicates either a wrap or a timestamp is present. static const uint32_t TIMESTAMP_OR_WRAP_BIT = (1 << 31); //! Size of the timestamp. static const size_t TIMESTAMP_SIZE = sizeof(uint32_t) + sizeof(uint32_t); static const uint32_t VERSION_0 = 0; //! Version 1 adds support for transmit timestamps. static const uint32_t VERSION_1 = 1; ACTIV_BOOST_STATIC_ASSERT(sizeof(size_t) >= sizeof(uint32_t)); IEvent * m_pDataEvent; CxxYZ::unique_ptr m_pMappedMemory; CxxYZ::shared_ptr > m_pPrivateBuffer; uint8_t * m_pBuffer; uint32_t m_bufferSize; AtomicUint32 * m_pReadOffset; AtomicUint32 * m_pWriteOffset; AtomicUint32 * m_pReaderCompletionEventCount; AtomicUint32 * m_pBufferSize; AtomicUint32 * m_pVersion; AtomicUint32 * m_pReaderPeakSize; CxxYZ::shared_ptr m_pInitializationEvent; CxxYZ::shared_ptr m_pSharedDataEvent; const uint32_t m_l1CacheLineSize; const uint64_t m_cacheLineMask; const bool m_isHugePagesEnabled; }; // --------------------------------------------------------------------------------------------------------------------------------- inline uint32_t RingBuffer::GetCacheLineSize() const { return m_l1CacheLineSize; } // --------------------------------------------------------------------------------------------------------------------------------- template inline T RingBuffer::RoundOffsetToNextCacheLineT(const T offset) const { const T cacheLineMask = static_cast(m_cacheLineMask); return (offset + cacheLineMask) & ~cacheLineMask; } // --------------------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------------------- inline bool RingBuffer::Writer::AllocateWriteBuffer(void *&pBuffer, const size_t length) { // If false the message has been dropped. if (ACTIV_UNLIKELY(!InitializeWrite(length))) return false; uint32_t *pHeader; InitializeHeader(pHeader, pBuffer, ACTIV_NULLPTR); return true; } // --------------------------------------------------------------------------------------------------------------------------------- inline void RingBuffer::Writer::CommitWriteBuffer(const size_t length) { assert(length <= Boost::integer_traits::const_max); uint32_t *pHeader = reinterpret_cast(m_pRingBuffer->m_pBuffer + m_cachedWriteOffset); FinalizeWrite(pHeader, pHeader + 1, static_cast(length)); } // --------------------------------------------------------------------------------------------------------------------------------- inline bool RingBuffer::Writer::WriteBuffer(const void *pData, const size_t length, const timespec *pTimestamp) { // If false the message has been dropped. if (ACTIV_UNLIKELY(!InitializeWrite(length))) return false; uint32_t *pHeader; void *pBuffer; InitializeHeader(pHeader, pBuffer, pTimestamp); if (ACTIV_LIKELY(length > 0)) ::memcpy(pBuffer, pData, length); FinalizeWrite(pHeader, pBuffer, static_cast(length)); return true; } // --------------------------------------------------------------------------------------------------------------------------------- template inline bool RingBuffer::Writer::WriteFixedMessage(const F &function, const size_t length, const timespec *pTimestamp) { // If false the message has been dropped. if (ACTIV_UNLIKELY(!InitializeWrite(length))) return false; uint32_t *pHeader; void *pBuffer; InitializeHeader(pHeader, pBuffer, pTimestamp); function(pBuffer, length); FinalizeWrite(pHeader, pBuffer, static_cast(length)); return true; } // --------------------------------------------------------------------------------------------------------------------------------- template inline bool RingBuffer::Writer::WriteVariableMessage(const F &function, const size_t length, const timespec *pTimestamp) { // If false the message has been dropped. if (ACTIV_UNLIKELY(!InitializeWrite(length))) return false; uint32_t *pHeader; void *pBuffer; InitializeHeader(pHeader, pBuffer, pTimestamp); const size_t writtenLength = function(pBuffer); // writtenLength can't validly be >= UINT32_MAX unless their function is broken. assert(writtenLength < Boost::integer_traits::const_max); FinalizeWrite(pHeader, pBuffer, static_cast(writtenLength)); return true; } // --------------------------------------------------------------------------------------------------------------------------------- inline void RingBuffer::Writer::Flush() { CheckAndSetEvent(); } // --------------------------------------------------------------------------------------------------------------------------------- inline void RingBuffer::Writer::SetOnIntervalPeakUpdateCallback(const CxxYZ::function &callbackFunction) { m_onIntervalPeakUpdateCallback = callbackFunction; } // --------------------------------------------------------------------------------------------------------------------------------- inline const RingBuffer::Writer::Statistics &RingBuffer::Writer::GetStatistics() const { return m_statistics; } // --------------------------------------------------------------------------------------------------------------------------------- inline void RingBuffer::Writer::FinalizeWrite(uint32_t *pHeader, void *pBuffer, const uint32_t length) { // The timestamp bit may be set, just update the length. *pHeader |= length; const uint32_t headerLength = static_cast(uintptr_t(pBuffer) - uintptr_t(pHeader)); const uint32_t totalLength = m_pRingBuffer->RoundOffsetToNextCacheLineT(headerLength + length); m_cachedWriteOffset += totalLength; // Only update the write pointer after receiving a certain number of messages. The number of messages required // between updates scales directly with the current rate of messages being received. if (!(--m_messagesUntilWritePointerUpdate)) { m_pRingBuffer->m_pWriteOffset->store(m_cachedWriteOffset, CxxYZ::memory_order_release); // Each time the write pointer is updated, increase the number of messages required for the next // update up to the configured maximum. m_messagesPerWritePointerUpdate = std::min(m_messagesPerWritePointerUpdate << 1, m_maxFlushFrequency); m_messagesUntilWritePointerUpdate = m_messagesPerWritePointerUpdate; m_hasUpdatedWritePointerSinceLastPostEndPointProcessing = true; } m_statistics.m_byteCount += totalLength; ++m_statistics.m_messageCount; #if (defined ACTIV_FRAMEWORK_RING_BUFFER_DEBUG) LogMessage(LOG_TYPE_DEBUG, Fmt::Format("{}: wrote {} bytes, write-offset={}, read-offset={}", __FUNCTION__, totalLength, m_cachedWriteOffset, m_cachedReadOffset)); #endif } // --------------------------------------------------------------------------------------------------------------------------------- inline void RingBuffer::Writer::InitializeHeader(uint32_t *&pHeader, void *&pBuffer, const timespec *pTimestamp) { pHeader = reinterpret_cast(m_pRingBuffer->m_pBuffer + m_cachedWriteOffset); if (!pTimestamp) { // Set the header to zero. Later the length will be OR'ed with this value. *pHeader = 0; pBuffer = pHeader + 1; return; } else { assert(m_isUsingTimestamps); // Set the timestamp bit. Later the length will be OR'ed with this value. *pHeader = TIMESTAMP_OR_WRAP_BIT; pHeader[1] = static_cast(pTimestamp->tv_sec); pHeader[2] = static_cast(pTimestamp->tv_nsec); pBuffer = pHeader + 3; } } // --------------------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------------------- inline RingBuffer::Writer::Statistics::Statistics() : m_byteCount(0), m_messageCount(0), m_backPressureCount(0), m_intervalPeak(0), m_peak(0) { } // --------------------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------------------- inline size_t RingBuffer::Reader::GetNumberOfBlockingReads() const { return m_numberOfBlockingReads; } // --------------------------------------------------------------------------------------------------------------------------------- inline size_t RingBuffer::Reader::GetNumberOfTimesEventHasReenabled() const { return m_numberOfTimesEventHasReenabled; } // --------------------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------------------- inline RingBuffer::MessageReader::MessageReader(Reader &reader) : m_reader(reader) { // Record the current cached bytes queued. ProcessTraceEvent &event = m_reader.m_processTracePoint.Record(); event.m_cachedBytesQueued = m_reader.m_cachedBytesQueued; } // --------------------------------------------------------------------------------------------------------------------------------- inline bool RingBuffer::MessageReader::operator()(const void *&pData, size_t &length, timespec ×tamp) const { return m_reader.Read(pData, length, timestamp); } } // namespace Activ #endif // (!defined ACTIV_FRAMEWORK_RING_BUFFER_H)