From c18089d28708e029aa4cace5ec098af49cb10832 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 8 Aug 2017 19:24:03 -0400 Subject: [PATCH] Added moodycammel, fixes sqlite --- linux/bin/sqlite3 | Bin 4839872 -> 4839872 bytes linux/include/moodycamel/atomicops.h | 665 ++++ linux/include/moodycamel/concurrentqueue.h | 3634 ++++++++++++++++++ linux/include/moodycamel/readerwriterqueue.h | 815 ++++ linux/include/{ => sqlite}/sqlite3.h | 2 +- linux/lib/debug/libsqlite3.a | Bin 8514022 -> 8514022 bytes linux/lib/release/libsqlite3.a | Bin 8514022 -> 8514022 bytes src/moodycamel/build_moodycamel.sh | 17 + src/moodycamel/moodycamel.zip | Bin 0 -> 44163 bytes src/sqlite/build_sqlite.sh | 3 +- 10 files changed, 5134 insertions(+), 2 deletions(-) create mode 100644 linux/include/moodycamel/atomicops.h create mode 100644 linux/include/moodycamel/concurrentqueue.h create mode 100644 linux/include/moodycamel/readerwriterqueue.h rename linux/include/{ => sqlite}/sqlite3.h (99%) create mode 100755 src/moodycamel/build_moodycamel.sh create mode 100644 src/moodycamel/moodycamel.zip diff --git a/linux/bin/sqlite3 b/linux/bin/sqlite3 index 8472bcdd383e0a4217bf29863914d56b782cb8fa..4a4891d5bce2a340c1ca6d5c2bd8324def74007a 100755 GIT binary patch delta 384 zcmWl~IZnes06;?x&*aP<1@dlUxW~+08WCkQEL`9Phq+G!j6;fIxibxbF zXQ1UWe2S;n`|OAwK3+wkQV#3SiTU%JH{LqN)A##h^6-+0b|(7pzkz-XU=TwX#t23+ z*3Cq&W!f_K!r1nTjUd#KD;>Yu)PYvAp(5LpDslr~Do-mVW%J&-)9uvCVH|l(U=mZ9 z#taIW#T@3bfJH1}875Y+3JXQ7VI3PNp^QyzVH-Qx#UA!i!2u3o!-0eg4^{XG5F$d1 cBOK!d2~JVN8P0Km6qmTdHEz18b=&&-0|`oQP5=M^ delta 384 zcmWl~$u0w70KoD1)LvV))-q$?>s!pysokI+z|Eo486KcV>5ZgE*NGP(agm5Laq2U8 zABn%iFTc*`tLWh4N#v{c%Z;j48ov*j+s4=1d+GV7@R*8LD!TB0H!|o!FZ$4r0SscO zor-L;C{OnJ=V%~<4Af@E3~p0#t23+hH>OD zfjkPB#1y76gIUa>h diff --git a/linux/include/moodycamel/atomicops.h b/linux/include/moodycamel/atomicops.h new file mode 100644 index 000000000..fae9fc283 --- /dev/null +++ b/linux/include/moodycamel/atomicops.h @@ -0,0 +1,665 @@ +// ©2013-2016 Cameron Desrochers. +// Distributed under the simplified BSD license (see the license file that +// should have come with this header). +// Uses Jeff Preshing's semaphore implementation (under the terms of its +// separate zlib license, embedded below). + +#pragma once + +// Provides portable (VC++2010+, Intel ICC 13, GCC 4.7+, and anything C++11 compliant) implementation +// of low-level memory barriers, plus a few semi-portable utility macros (for inlining and alignment). +// Also has a basic atomic type (limited to hardware-supported atomics with no memory ordering guarantees). +// Uses the AE_* prefix for macros (historical reasons), and the "moodycamel" namespace for symbols. + +#include +#include +#include +#include +#include + +// Platform detection +#if defined(__INTEL_COMPILER) +#define AE_ICC +#elif defined(_MSC_VER) +#define AE_VCPP +#elif defined(__GNUC__) +#define AE_GCC +#endif + +#if defined(_M_IA64) || defined(__ia64__) +#define AE_ARCH_IA64 +#elif defined(_WIN64) || defined(__amd64__) || defined(_M_X64) || defined(__x86_64__) +#define AE_ARCH_X64 +#elif defined(_M_IX86) || defined(__i386__) +#define AE_ARCH_X86 +#elif defined(_M_PPC) || defined(__powerpc__) +#define AE_ARCH_PPC +#else +#define AE_ARCH_UNKNOWN +#endif + + +// AE_UNUSED +#define AE_UNUSED(x) ((void)x) + + +// AE_FORCEINLINE +#if defined(AE_VCPP) || defined(AE_ICC) +#define AE_FORCEINLINE __forceinline +#elif defined(AE_GCC) +//#define AE_FORCEINLINE __attribute__((always_inline)) +#define AE_FORCEINLINE inline +#else +#define AE_FORCEINLINE inline +#endif + + +// AE_ALIGN +#if defined(AE_VCPP) || defined(AE_ICC) +#define AE_ALIGN(x) __declspec(align(x)) +#elif defined(AE_GCC) +#define AE_ALIGN(x) __attribute__((aligned(x))) +#else +// Assume GCC compliant syntax... +#define AE_ALIGN(x) __attribute__((aligned(x))) +#endif + + +// Portable atomic fences implemented below: + +namespace moodycamel { + +enum memory_order { + memory_order_relaxed, + memory_order_acquire, + memory_order_release, + memory_order_acq_rel, + memory_order_seq_cst, + + // memory_order_sync: Forces a full sync: + // #LoadLoad, #LoadStore, #StoreStore, and most significantly, #StoreLoad + memory_order_sync = memory_order_seq_cst +}; + +} // end namespace moodycamel + +#if (defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli))) || defined(AE_ICC) +// VS2010 and ICC13 don't support std::atomic_*_fence, implement our own fences + +#include + +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) +#define AeFullSync _mm_mfence +#define AeLiteSync _mm_mfence +#elif defined(AE_ARCH_IA64) +#define AeFullSync __mf +#define AeLiteSync __mf +#elif defined(AE_ARCH_PPC) +#include +#define AeFullSync __sync +#define AeLiteSync __lwsync +#endif + + +#ifdef AE_VCPP +#pragma warning(push) +#pragma warning(disable: 4365) // Disable erroneous 'conversion from long to unsigned int, signed/unsigned mismatch' error when using `assert` +#ifdef __cplusplus_cli +#pragma managed(push, off) +#endif +#endif + +namespace moodycamel { + +AE_FORCEINLINE void compiler_fence(memory_order order) +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: _ReadBarrier(); break; + case memory_order_release: _WriteBarrier(); break; + case memory_order_acq_rel: _ReadWriteBarrier(); break; + case memory_order_seq_cst: _ReadWriteBarrier(); break; + default: assert(false); + } +} + +// x86/x64 have a strong memory model -- all loads and stores have +// acquire and release semantics automatically (so only need compiler +// barriers for those). +#if defined(AE_ARCH_X86) || defined(AE_ARCH_X64) +AE_FORCEINLINE void fence(memory_order order) +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: _ReadBarrier(); break; + case memory_order_release: _WriteBarrier(); break; + case memory_order_acq_rel: _ReadWriteBarrier(); break; + case memory_order_seq_cst: + _ReadWriteBarrier(); + AeFullSync(); + _ReadWriteBarrier(); + break; + default: assert(false); + } +} +#else +AE_FORCEINLINE void fence(memory_order order) +{ + // Non-specialized arch, use heavier memory barriers everywhere just in case :-( + switch (order) { + case memory_order_relaxed: + break; + case memory_order_acquire: + _ReadBarrier(); + AeLiteSync(); + _ReadBarrier(); + break; + case memory_order_release: + _WriteBarrier(); + AeLiteSync(); + _WriteBarrier(); + break; + case memory_order_acq_rel: + _ReadWriteBarrier(); + AeLiteSync(); + _ReadWriteBarrier(); + break; + case memory_order_seq_cst: + _ReadWriteBarrier(); + AeFullSync(); + _ReadWriteBarrier(); + break; + default: assert(false); + } +} +#endif +} // end namespace moodycamel +#else +// Use standard library of atomics +#include + +namespace moodycamel { + +AE_FORCEINLINE void compiler_fence(memory_order order) +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: std::atomic_signal_fence(std::memory_order_acquire); break; + case memory_order_release: std::atomic_signal_fence(std::memory_order_release); break; + case memory_order_acq_rel: std::atomic_signal_fence(std::memory_order_acq_rel); break; + case memory_order_seq_cst: std::atomic_signal_fence(std::memory_order_seq_cst); break; + default: assert(false); + } +} + +AE_FORCEINLINE void fence(memory_order order) +{ + switch (order) { + case memory_order_relaxed: break; + case memory_order_acquire: std::atomic_thread_fence(std::memory_order_acquire); break; + case memory_order_release: std::atomic_thread_fence(std::memory_order_release); break; + case memory_order_acq_rel: std::atomic_thread_fence(std::memory_order_acq_rel); break; + case memory_order_seq_cst: std::atomic_thread_fence(std::memory_order_seq_cst); break; + default: assert(false); + } +} + +} // end namespace moodycamel + +#endif + + +#if !defined(AE_VCPP) || (_MSC_VER >= 1700 && !defined(__cplusplus_cli)) +#define AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC +#endif + +#ifdef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC +#include +#endif +#include + +// WARNING: *NOT* A REPLACEMENT FOR std::atomic. READ CAREFULLY: +// Provides basic support for atomic variables -- no memory ordering guarantees are provided. +// The guarantee of atomicity is only made for types that already have atomic load and store guarantees +// at the hardware level -- on most platforms this generally means aligned pointers and integers (only). +namespace moodycamel { +template +class weak_atomic +{ +public: + weak_atomic() { } +#ifdef AE_VCPP +#pragma warning(push) +#pragma warning(disable: 4100) // Get rid of (erroneous) 'unreferenced formal parameter' warning +#endif + template weak_atomic(U&& x) : value(std::forward(x)) { } +#ifdef __cplusplus_cli + // Work around bug with universal reference/nullptr combination that only appears when /clr is on + weak_atomic(nullptr_t) : value(nullptr) { } +#endif + weak_atomic(weak_atomic const& other) : value(other.value) { } + weak_atomic(weak_atomic&& other) : value(std::move(other.value)) { } +#ifdef AE_VCPP +#pragma warning(pop) +#endif + + AE_FORCEINLINE operator T() const { return load(); } + + +#ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC + template AE_FORCEINLINE weak_atomic const& operator=(U&& x) { value = std::forward(x); return *this; } + AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) { value = other.value; return *this; } + + AE_FORCEINLINE T load() const { return value; } + + AE_FORCEINLINE T fetch_add_acquire(T increment) + { +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) + if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment); +#if defined(_M_AMD64) + else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment); +#endif +#else +#error Unsupported platform +#endif + assert(false && "T must be either a 32 or 64 bit type"); + return value; + } + + AE_FORCEINLINE T fetch_add_release(T increment) + { +#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86) + if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment); +#if defined(_M_AMD64) + else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment); +#endif +#else +#error Unsupported platform +#endif + assert(false && "T must be either a 32 or 64 bit type"); + return value; + } +#else + template + AE_FORCEINLINE weak_atomic const& operator=(U&& x) + { + value.store(std::forward(x), std::memory_order_relaxed); + return *this; + } + + AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) + { + value.store(other.value.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + + AE_FORCEINLINE T load() const { return value.load(std::memory_order_relaxed); } + + AE_FORCEINLINE T fetch_add_acquire(T increment) + { + return value.fetch_add(increment, std::memory_order_acquire); + } + + AE_FORCEINLINE T fetch_add_release(T increment) + { + return value.fetch_add(increment, std::memory_order_release); + } +#endif + + +private: +#ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC + // No std::atomic support, but still need to circumvent compiler optimizations. + // `volatile` will make memory access slow, but is guaranteed to be reliable. + volatile T value; +#else + std::atomic value; +#endif +}; + +} // end namespace moodycamel + + + +// Portable single-producer, single-consumer semaphore below: + +#if defined(_WIN32) +// Avoid including windows.h in a header; we only need a handful of +// items, so we'll redeclare them here (this is relatively safe since +// the API generally has to remain stable between Windows versions). +// I know this is an ugly hack but it still beats polluting the global +// namespace with thousands of generic names or adding a .cpp for nothing. +extern "C" { + struct _SECURITY_ATTRIBUTES; + __declspec(dllimport) void* __stdcall CreateSemaphoreW(_SECURITY_ATTRIBUTES* lpSemaphoreAttributes, long lInitialCount, long lMaximumCount, const wchar_t* lpName); + __declspec(dllimport) int __stdcall CloseHandle(void* hObject); + __declspec(dllimport) unsigned long __stdcall WaitForSingleObject(void* hHandle, unsigned long dwMilliseconds); + __declspec(dllimport) int __stdcall ReleaseSemaphore(void* hSemaphore, long lReleaseCount, long* lpPreviousCount); +} +#elif defined(__MACH__) +#include +#elif defined(__unix__) +#include +#endif + +namespace moodycamel +{ + // Code in the spsc_sema namespace below is an adaptation of Jeff Preshing's + // portable + lightweight semaphore implementations, originally from + // https://github.com/preshing/cpp11-on-multicore/blob/master/common/sema.h + // LICENSE: + // Copyright (c) 2015 Jeff Preshing + // + // This software is provided 'as-is', without any express or implied + // warranty. In no event will the authors be held liable for any damages + // arising from the use of this software. + // + // Permission is granted to anyone to use this software for any purpose, + // including commercial applications, and to alter it and redistribute it + // freely, subject to the following restrictions: + // + // 1. The origin of this software must not be misrepresented; you must not + // claim that you wrote the original software. If you use this software + // in a product, an acknowledgement in the product documentation would be + // appreciated but is not required. + // 2. Altered source versions must be plainly marked as such, and must not be + // misrepresented as being the original software. + // 3. This notice may not be removed or altered from any source distribution. + namespace spsc_sema + { +#if defined(_WIN32) + class Semaphore + { + private: + void* m_hSema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + Semaphore(int initialCount = 0) + { + assert(initialCount >= 0); + const long maxLong = 0x7fffffff; + m_hSema = CreateSemaphoreW(nullptr, initialCount, maxLong, nullptr); + } + + ~Semaphore() + { + CloseHandle(m_hSema); + } + + void wait() + { + const unsigned long infinite = 0xffffffff; + WaitForSingleObject(m_hSema, infinite); + } + + bool try_wait() + { + const unsigned long RC_WAIT_TIMEOUT = 0x00000102; + return WaitForSingleObject(m_hSema, 0) != RC_WAIT_TIMEOUT; + } + + bool timed_wait(std::uint64_t usecs) + { + const unsigned long RC_WAIT_TIMEOUT = 0x00000102; + return WaitForSingleObject(m_hSema, (unsigned long)(usecs / 1000)) != RC_WAIT_TIMEOUT; + } + + void signal(int count = 1) + { + ReleaseSemaphore(m_hSema, count, nullptr); + } + }; +#elif defined(__MACH__) + //--------------------------------------------------------- + // Semaphore (Apple iOS and OSX) + // Can't use POSIX semaphores due to http://lists.apple.com/archives/darwin-kernel/2009/Apr/msg00010.html + //--------------------------------------------------------- + class Semaphore + { + private: + semaphore_t m_sema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + Semaphore(int initialCount = 0) + { + assert(initialCount >= 0); + semaphore_create(mach_task_self(), &m_sema, SYNC_POLICY_FIFO, initialCount); + } + + ~Semaphore() + { + semaphore_destroy(mach_task_self(), m_sema); + } + + void wait() + { + semaphore_wait(m_sema); + } + + bool try_wait() + { + return timed_wait(0); + } + + bool timed_wait(std::int64_t timeout_usecs) + { + mach_timespec_t ts; + ts.tv_sec = timeout_usecs / 1000000; + ts.tv_nsec = (timeout_usecs % 1000000) * 1000; + + // added in OSX 10.10: https://developer.apple.com/library/prerelease/mac/documentation/General/Reference/APIDiffsMacOSX10_10SeedDiff/modules/Darwin.html + kern_return_t rc = semaphore_timedwait(m_sema, ts); + + return rc != KERN_OPERATION_TIMED_OUT; + } + + void signal() + { + semaphore_signal(m_sema); + } + + void signal(int count) + { + while (count-- > 0) + { + semaphore_signal(m_sema); + } + } + }; +#elif defined(__unix__) + //--------------------------------------------------------- + // Semaphore (POSIX, Linux) + //--------------------------------------------------------- + class Semaphore + { + private: + sem_t m_sema; + + Semaphore(const Semaphore& other); + Semaphore& operator=(const Semaphore& other); + + public: + Semaphore(int initialCount = 0) + { + assert(initialCount >= 0); + sem_init(&m_sema, 0, initialCount); + } + + ~Semaphore() + { + sem_destroy(&m_sema); + } + + void wait() + { + // http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error + int rc; + do + { + rc = sem_wait(&m_sema); + } + while (rc == -1 && errno == EINTR); + } + + bool try_wait() + { + int rc; + do { + rc = sem_trywait(&m_sema); + } while (rc == -1 && errno == EINTR); + return !(rc == -1 && errno == EAGAIN); + } + + bool timed_wait(std::uint64_t usecs) + { + struct timespec ts; + const int usecs_in_1_sec = 1000000; + const int nsecs_in_1_sec = 1000000000; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += usecs / usecs_in_1_sec; + ts.tv_nsec += (usecs % usecs_in_1_sec) * 1000; + // sem_timedwait bombs if you have more than 1e9 in tv_nsec + // so we have to clean things up before passing it in + if (ts.tv_nsec > nsecs_in_1_sec) { + ts.tv_nsec -= nsecs_in_1_sec; + ++ts.tv_sec; + } + + int rc; + do { + rc = sem_timedwait(&m_sema, &ts); + } while (rc == -1 && errno == EINTR); + return !(rc == -1 && errno == ETIMEDOUT); + } + + void signal() + { + sem_post(&m_sema); + } + + void signal(int count) + { + while (count-- > 0) + { + sem_post(&m_sema); + } + } + }; +#else +#error Unsupported platform! (No semaphore wrapper available) +#endif + + //--------------------------------------------------------- + // LightweightSemaphore + //--------------------------------------------------------- + class LightweightSemaphore + { + public: + typedef std::make_signed::type ssize_t; + + private: + weak_atomic m_count; + Semaphore m_sema; + + bool waitWithPartialSpinning(std::int64_t timeout_usecs = -1) + { + ssize_t oldCount; + // Is there a better way to set the initial spin count? + // If we lower it to 1000, testBenaphore becomes 15x slower on my Core i7-5930K Windows PC, + // as threads start hitting the kernel semaphore. + int spin = 10000; + while (--spin >= 0) + { + if (m_count.load() > 0) + { + m_count.fetch_add_acquire(-1); + return true; + } + compiler_fence(memory_order_acquire); // Prevent the compiler from collapsing the loop. + } + oldCount = m_count.fetch_add_acquire(-1); + if (oldCount > 0) + return true; + if (timeout_usecs < 0) + { + m_sema.wait(); + return true; + } + if (m_sema.timed_wait(timeout_usecs)) + return true; + // At this point, we've timed out waiting for the semaphore, but the + // count is still decremented indicating we may still be waiting on + // it. So we have to re-adjust the count, but only if the semaphore + // wasn't signaled enough times for us too since then. If it was, we + // need to release the semaphore too. + while (true) + { + oldCount = m_count.fetch_add_release(1); + if (oldCount < 0) + return false; // successfully restored things to the way they were + // Oh, the producer thread just signaled the semaphore after all. Try again: + oldCount = m_count.fetch_add_acquire(-1); + if (oldCount > 0 && m_sema.try_wait()) + return true; + } + } + + public: + LightweightSemaphore(ssize_t initialCount = 0) : m_count(initialCount) + { + assert(initialCount >= 0); + } + + bool tryWait() + { + if (m_count.load() > 0) + { + m_count.fetch_add_acquire(-1); + return true; + } + return false; + } + + void wait() + { + if (!tryWait()) + waitWithPartialSpinning(); + } + + bool wait(std::int64_t timeout_usecs) + { + return tryWait() || waitWithPartialSpinning(timeout_usecs); + } + + void signal(ssize_t count = 1) + { + assert(count >= 0); + ssize_t oldCount = m_count.fetch_add_release(count); + assert(oldCount >= -1); + if (oldCount < 0) + { + m_sema.signal(1); + } + } + + ssize_t availableApprox() const + { + ssize_t count = m_count.load(); + return count > 0 ? count : 0; + } + }; + } // end namespace spsc_sema +} // end namespace moodycamel + +#if defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli)) +#pragma warning(pop) +#ifdef __cplusplus_cli +#pragma managed(pop) +#endif +#endif diff --git a/linux/include/moodycamel/concurrentqueue.h b/linux/include/moodycamel/concurrentqueue.h new file mode 100644 index 000000000..d5b303770 --- /dev/null +++ b/linux/include/moodycamel/concurrentqueue.h @@ -0,0 +1,3634 @@ +// Provides a C++11 implementation of a multi-producer, multi-consumer lock-free queue. +// An overview, including benchmark results, is provided here: +// http://moodycamel.com/blog/2014/a-fast-general-purpose-lock-free-queue-for-c++ +// The full design is also described in excruciating detail at: +// http://moodycamel.com/blog/2014/detailed-design-of-a-lock-free-queue + +// Simplified BSD license: +// Copyright (c) 2013-2016, Cameron Desrochers. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// - Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, this list of +// conditions and the following disclaimer in the documentation and/or other materials +// provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +// OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +#pragma once + +#if defined(__GNUC__) +// Disable -Wconversion warnings (spuriously triggered when Traits::size_t and +// Traits::index_t are set to < 32 bits, causing integer promotion, causing warnings +// upon assigning any computed values) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" + +#ifdef MCDBGQ_USE_RELACY +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#endif +#endif + +#if defined(__APPLE__) +#include "TargetConditionals.h" +#endif + +#ifdef MCDBGQ_USE_RELACY +#include "relacy/relacy_std.hpp" +#include "relacy_shims.h" +// We only use malloc/free anyway, and the delete macro messes up `= delete` method declarations. +// We'll override the default trait malloc ourselves without a macro. +#undef new +#undef delete +#undef malloc +#undef free +#else +#include // Requires C++11. Sorry VS2010. +#include +#endif +#include // for max_align_t +#include +#include +#include +#include +#include +#include +#include // for CHAR_BIT +#include +#include // partly for __WINPTHREADS_VERSION if on MinGW-w64 w/ POSIX threading + +// Platform-specific definitions of a numeric thread ID type and an invalid value +namespace moodycamel { namespace details { + template struct thread_id_converter { + typedef thread_id_t thread_id_numeric_size_t; + typedef thread_id_t thread_id_hash_t; + static thread_id_hash_t prehash(thread_id_t const& x) { return x; } + }; +} } +#if defined(MCDBGQ_USE_RELACY) +namespace moodycamel { namespace details { + typedef std::uint32_t thread_id_t; + static const thread_id_t invalid_thread_id = 0xFFFFFFFFU; + static const thread_id_t invalid_thread_id2 = 0xFFFFFFFEU; + static inline thread_id_t thread_id() { return rl::thread_index(); } +} } +#elif defined(_WIN32) || defined(__WINDOWS__) || defined(__WIN32__) +// No sense pulling in windows.h in a header, we'll manually declare the function +// we use and rely on backwards-compatibility for this not to break +extern "C" __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(void); +namespace moodycamel { namespace details { + static_assert(sizeof(unsigned long) == sizeof(std::uint32_t), "Expected size of unsigned long to be 32 bits on Windows"); + typedef std::uint32_t thread_id_t; + static const thread_id_t invalid_thread_id = 0; // See http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx + static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU; // Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4. + static inline thread_id_t thread_id() { return static_cast(::GetCurrentThreadId()); } +} } +#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) +namespace moodycamel { namespace details { + static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes"); + + typedef std::thread::id thread_id_t; + static const thread_id_t invalid_thread_id; // Default ctor creates invalid ID + + // Note we don't define a invalid_thread_id2 since std::thread::id doesn't have one; it's + // only used if MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is defined anyway, which it won't + // be. + static inline thread_id_t thread_id() { return std::this_thread::get_id(); } + + template struct thread_id_size { }; + template<> struct thread_id_size<4> { typedef std::uint32_t numeric_t; }; + template<> struct thread_id_size<8> { typedef std::uint64_t numeric_t; }; + + template<> struct thread_id_converter { + typedef thread_id_size::numeric_t thread_id_numeric_size_t; +#ifndef __APPLE__ + typedef std::size_t thread_id_hash_t; +#else + typedef thread_id_numeric_size_t thread_id_hash_t; +#endif + + static thread_id_hash_t prehash(thread_id_t const& x) + { +#ifndef __APPLE__ + return std::hash()(x); +#else + return *reinterpret_cast(&x); +#endif + } + }; +} } +#else +// Use a nice trick from this answer: http://stackoverflow.com/a/8438730/21475 +// In order to get a numeric thread ID in a platform-independent way, we use a thread-local +// static variable's address as a thread identifier :-) +#if defined(__GNUC__) || defined(__INTEL_COMPILER) +#define MOODYCAMEL_THREADLOCAL __thread +#elif defined(_MSC_VER) +#define MOODYCAMEL_THREADLOCAL __declspec(thread) +#else +// Assume C++11 compliant compiler +#define MOODYCAMEL_THREADLOCAL thread_local +#endif +namespace moodycamel { namespace details { + typedef std::uintptr_t thread_id_t; + static const thread_id_t invalid_thread_id = 0; // Address can't be nullptr + static const thread_id_t invalid_thread_id2 = 1; // Member accesses off a null pointer are also generally invalid. Plus it's not aligned. + static inline thread_id_t thread_id() { static MOODYCAMEL_THREADLOCAL int x; return reinterpret_cast(&x); } +} } +#endif + +// Exceptions +#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED +#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__)) +#define MOODYCAMEL_EXCEPTIONS_ENABLED +#endif +#endif +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED +#define MOODYCAMEL_TRY try +#define MOODYCAMEL_CATCH(...) catch(__VA_ARGS__) +#define MOODYCAMEL_RETHROW throw +#define MOODYCAMEL_THROW(expr) throw (expr) +#else +#define MOODYCAMEL_TRY if (true) +#define MOODYCAMEL_CATCH(...) else if (false) +#define MOODYCAMEL_RETHROW +#define MOODYCAMEL_THROW(expr) +#endif + +#ifndef MOODYCAMEL_NOEXCEPT +#if !defined(MOODYCAMEL_EXCEPTIONS_ENABLED) +#define MOODYCAMEL_NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) true +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) true +#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1800 +// VS2012's std::is_nothrow_[move_]constructible is broken and returns true when it shouldn't :-( +// We have to assume *all* non-trivial constructors may throw on VS2012! +#define MOODYCAMEL_NOEXCEPT _NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference::value && std::is_move_constructible::value ? std::is_trivially_move_constructible::value : std::is_trivially_copy_constructible::value) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference::value && std::is_move_assignable::value ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) +#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1900 +#define MOODYCAMEL_NOEXCEPT _NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference::value && std::is_move_constructible::value ? std::is_trivially_move_constructible::value || std::is_nothrow_move_constructible::value : std::is_trivially_copy_constructible::value || std::is_nothrow_copy_constructible::value) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference::value && std::is_move_assignable::value ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) +#else +#define MOODYCAMEL_NOEXCEPT noexcept +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) noexcept(expr) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) noexcept(expr) +#endif +#endif + +#ifndef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#ifdef MCDBGQ_USE_RELACY +#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#else +// VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445 +// g++ <=4.7 doesn't support thread_local either. +// Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work +#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) +// Assume `thread_local` is fully supported in all other C++11 compilers/platforms +//#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // always disabled for now since several users report having problems with it on +#endif +#endif +#endif + +// VS2012 doesn't support deleted functions. +// In this case, we declare the function normally but don't define it. A link error will be generated if the function is called. +#ifndef MOODYCAMEL_DELETE_FUNCTION +#if defined(_MSC_VER) && _MSC_VER < 1800 +#define MOODYCAMEL_DELETE_FUNCTION +#else +#define MOODYCAMEL_DELETE_FUNCTION = delete +#endif +#endif + +// Compiler-specific likely/unlikely hints +namespace moodycamel { namespace details { +#if defined(__GNUC__) + inline bool likely(bool x) { return __builtin_expect((x), true); } + inline bool unlikely(bool x) { return __builtin_expect((x), false); } +#else + inline bool likely(bool x) { return x; } + inline bool unlikely(bool x) { return x; } +#endif +} } + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG +#include "internal/concurrentqueue_internal_debug.h" +#endif + +namespace moodycamel { +namespace details { + template + struct const_numeric_max { + static_assert(std::is_integral::value, "const_numeric_max can only be used with integers"); + static const T value = std::numeric_limits::is_signed + ? (static_cast(1) << (sizeof(T) * CHAR_BIT - 1)) - static_cast(1) + : static_cast(-1); + }; + +#if defined(__GNUC__) && !defined( __clang__ ) + typedef ::max_align_t std_max_align_t; // GCC forgot to add it to std:: for a while +#else + typedef std::max_align_t std_max_align_t; // Others (e.g. MSVC) insist it can *only* be accessed via std:: +#endif + + // Some platforms have incorrectly set max_align_t to a type with <8 bytes alignment even while supporting + // 8-byte aligned scalar values (*cough* 32-bit iOS). Work around this with our own union. See issue #64. + typedef union { + std_max_align_t x; + long long y; + void* z; + } max_align_t; +} + +// Default traits for the ConcurrentQueue. To change some of the +// traits without re-implementing all of them, inherit from this +// struct and shadow the declarations you wish to be different; +// since the traits are used as a template type parameter, the +// shadowed declarations will be used where defined, and the defaults +// otherwise. +struct ConcurrentQueueDefaultTraits +{ + // General-purpose size type. std::size_t is strongly recommended. + typedef std::size_t size_t; + + // The type used for the enqueue and dequeue indices. Must be at least as + // large as size_t. Should be significantly larger than the number of elements + // you expect to hold at once, especially if you have a high turnover rate; + // for example, on 32-bit x86, if you expect to have over a hundred million + // elements or pump several million elements through your queue in a very + // short space of time, using a 32-bit type *may* trigger a race condition. + // A 64-bit int type is recommended in that case, and in practice will + // prevent a race condition no matter the usage of the queue. Note that + // whether the queue is lock-free with a 64-int type depends on the whether + // std::atomic is lock-free, which is platform-specific. + typedef std::size_t index_t; + + // Internally, all elements are enqueued and dequeued from multi-element + // blocks; this is the smallest controllable unit. If you expect few elements + // but many producers, a smaller block size should be favoured. For few producers + // and/or many elements, a larger block size is preferred. A sane default + // is provided. Must be a power of 2. + static const size_t BLOCK_SIZE = 32; + + // For explicit producers (i.e. when using a producer token), the block is + // checked for being empty by iterating through a list of flags, one per element. + // For large block sizes, this is too inefficient, and switching to an atomic + // counter-based approach is faster. The switch is made for block sizes strictly + // larger than this threshold. + static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = 32; + + // How many full blocks can be expected for a single explicit producer? This should + // reflect that number's maximum for optimal performance. Must be a power of 2. + static const size_t EXPLICIT_INITIAL_INDEX_SIZE = 32; + + // How many full blocks can be expected for a single implicit producer? This should + // reflect that number's maximum for optimal performance. Must be a power of 2. + static const size_t IMPLICIT_INITIAL_INDEX_SIZE = 32; + + // The initial size of the hash table mapping thread IDs to implicit producers. + // Note that the hash is resized every time it becomes half full. + // Must be a power of two, and either 0 or at least 1. If 0, implicit production + // (using the enqueue methods without an explicit producer token) is disabled. + static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = 32; + + // Controls the number of items that an explicit consumer (i.e. one with a token) + // must consume before it causes all consumers to rotate and move on to the next + // internal queue. + static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = 256; + + // The maximum number of elements (inclusive) that can be enqueued to a sub-queue. + // Enqueue operations that would cause this limit to be surpassed will fail. Note + // that this limit is enforced at the block level (for performance reasons), i.e. + // it's rounded up to the nearest block size. + static const size_t MAX_SUBQUEUE_SIZE = details::const_numeric_max::value; + + +#ifndef MCDBGQ_USE_RELACY + // Memory allocation can be customized if needed. + // malloc should return nullptr on failure, and handle alignment like std::malloc. +#if defined(malloc) || defined(free) + // Gah, this is 2015, stop defining macros that break standard code already! + // Work around malloc/free being special macros: + static inline void* WORKAROUND_malloc(size_t size) { return malloc(size); } + static inline void WORKAROUND_free(void* ptr) { return free(ptr); } + static inline void* (malloc)(size_t size) { return WORKAROUND_malloc(size); } + static inline void (free)(void* ptr) { return WORKAROUND_free(ptr); } +#else + static inline void* malloc(size_t size) { return std::malloc(size); } + static inline void free(void* ptr) { return std::free(ptr); } +#endif +#else + // Debug versions when running under the Relacy race detector (ignore + // these in user code) + static inline void* malloc(size_t size) { return rl::rl_malloc(size, $); } + static inline void free(void* ptr) { return rl::rl_free(ptr, $); } +#endif +}; + + +// When producing or consuming many elements, the most efficient way is to: +// 1) Use one of the bulk-operation methods of the queue with a token +// 2) Failing that, use the bulk-operation methods without a token +// 3) Failing that, create a token and use that with the single-item methods +// 4) Failing that, use the single-parameter methods of the queue +// Having said that, don't create tokens willy-nilly -- ideally there should be +// a maximum of one token per thread (of each kind). +struct ProducerToken; +struct ConsumerToken; + +template class ConcurrentQueue; +template class BlockingConcurrentQueue; +class ConcurrentQueueTests; + + +namespace details +{ + struct ConcurrentQueueProducerTypelessBase + { + ConcurrentQueueProducerTypelessBase* next; + std::atomic inactive; + ProducerToken* token; + + ConcurrentQueueProducerTypelessBase() + : next(nullptr), inactive(false), token(nullptr) + { + } + }; + + template struct _hash_32_or_64 { + static inline std::uint32_t hash(std::uint32_t h) + { + // MurmurHash3 finalizer -- see https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp + // Since the thread ID is already unique, all we really want to do is propagate that + // uniqueness evenly across all the bits, so that we can use a subset of the bits while + // reducing collisions significantly + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + return h ^ (h >> 16); + } + }; + template<> struct _hash_32_or_64<1> { + static inline std::uint64_t hash(std::uint64_t h) + { + h ^= h >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + return h ^ (h >> 33); + } + }; + template struct hash_32_or_64 : public _hash_32_or_64<(size > 4)> { }; + + static inline size_t hash_thread_id(thread_id_t id) + { + static_assert(sizeof(thread_id_t) <= 8, "Expected a platform where thread IDs are at most 64-bit values"); + return static_cast(hash_32_or_64::thread_id_hash_t)>::hash( + thread_id_converter::prehash(id))); + } + + template + static inline bool circular_less_than(T a, T b) + { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4554) +#endif + static_assert(std::is_integral::value && !std::numeric_limits::is_signed, "circular_less_than is intended to be used only with unsigned integer types"); + return static_cast(a - b) > static_cast(static_cast(1) << static_cast(sizeof(T) * CHAR_BIT - 1)); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + } + + template + static inline char* align_for(char* ptr) + { + const std::size_t alignment = std::alignment_of::value; + return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; + } + + template + static inline T ceil_to_pow_2(T x) + { + static_assert(std::is_integral::value && !std::numeric_limits::is_signed, "ceil_to_pow_2 is intended to be used only with unsigned integer types"); + + // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + --x; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + for (std::size_t i = 1; i < sizeof(T); i <<= 1) { + x |= x >> (i << 3); + } + ++x; + return x; + } + + template + static inline void swap_relaxed(std::atomic& left, std::atomic& right) + { + T temp = std::move(left.load(std::memory_order_relaxed)); + left.store(std::move(right.load(std::memory_order_relaxed)), std::memory_order_relaxed); + right.store(std::move(temp), std::memory_order_relaxed); + } + + template + static inline T const& nomove(T const& x) + { + return x; + } + + template + struct nomove_if + { + template + static inline T const& eval(T const& x) + { + return x; + } + }; + + template<> + struct nomove_if + { + template + static inline auto eval(U&& x) + -> decltype(std::forward(x)) + { + return std::forward(x); + } + }; + + template + static inline auto deref_noexcept(It& it) MOODYCAMEL_NOEXCEPT -> decltype(*it) + { + return *it; + } + +#if defined(__clang__) || !defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) + template struct is_trivially_destructible : std::is_trivially_destructible { }; +#else + template struct is_trivially_destructible : std::has_trivial_destructor { }; +#endif + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#ifdef MCDBGQ_USE_RELACY + typedef RelacyThreadExitListener ThreadExitListener; + typedef RelacyThreadExitNotifier ThreadExitNotifier; +#else + struct ThreadExitListener + { + typedef void (*callback_t)(void*); + callback_t callback; + void* userData; + + ThreadExitListener* next; // reserved for use by the ThreadExitNotifier + }; + + + class ThreadExitNotifier + { + public: + static void subscribe(ThreadExitListener* listener) + { + auto& tlsInst = instance(); + listener->next = tlsInst.tail; + tlsInst.tail = listener; + } + + static void unsubscribe(ThreadExitListener* listener) + { + auto& tlsInst = instance(); + ThreadExitListener** prev = &tlsInst.tail; + for (auto ptr = tlsInst.tail; ptr != nullptr; ptr = ptr->next) { + if (ptr == listener) { + *prev = ptr->next; + break; + } + prev = &ptr->next; + } + } + + private: + ThreadExitNotifier() : tail(nullptr) { } + ThreadExitNotifier(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; + ThreadExitNotifier& operator=(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; + + ~ThreadExitNotifier() + { + // This thread is about to exit, let everyone know! + assert(this == &instance() && "If this assert fails, you likely have a buggy compiler! Change the preprocessor conditions such that MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is no longer defined."); + for (auto ptr = tail; ptr != nullptr; ptr = ptr->next) { + ptr->callback(ptr->userData); + } + } + + // Thread-local + static inline ThreadExitNotifier& instance() + { + static thread_local ThreadExitNotifier notifier; + return notifier; + } + + private: + ThreadExitListener* tail; + }; +#endif +#endif + + template struct static_is_lock_free_num { enum { value = 0 }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_CHAR_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_SHORT_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_INT_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_LONG_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_LLONG_LOCK_FREE }; }; + template struct static_is_lock_free : static_is_lock_free_num::type> { }; + template<> struct static_is_lock_free { enum { value = ATOMIC_BOOL_LOCK_FREE }; }; + template struct static_is_lock_free { enum { value = ATOMIC_POINTER_LOCK_FREE }; }; +} + + +struct ProducerToken +{ + template + explicit ProducerToken(ConcurrentQueue& queue); + + template + explicit ProducerToken(BlockingConcurrentQueue& queue); + + ProducerToken(ProducerToken&& other) MOODYCAMEL_NOEXCEPT + : producer(other.producer) + { + other.producer = nullptr; + if (producer != nullptr) { + producer->token = this; + } + } + + inline ProducerToken& operator=(ProducerToken&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + void swap(ProducerToken& other) MOODYCAMEL_NOEXCEPT + { + std::swap(producer, other.producer); + if (producer != nullptr) { + producer->token = this; + } + if (other.producer != nullptr) { + other.producer->token = &other; + } + } + + // A token is always valid unless: + // 1) Memory allocation failed during construction + // 2) It was moved via the move constructor + // (Note: assignment does a swap, leaving both potentially valid) + // 3) The associated queue was destroyed + // Note that if valid() returns true, that only indicates + // that the token is valid for use with a specific queue, + // but not which one; that's up to the user to track. + inline bool valid() const { return producer != nullptr; } + + ~ProducerToken() + { + if (producer != nullptr) { + producer->token = nullptr; + producer->inactive.store(true, std::memory_order_release); + } + } + + // Disable copying and assignment + ProducerToken(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; + ProducerToken& operator=(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; + +private: + template friend class ConcurrentQueue; + friend class ConcurrentQueueTests; + +protected: + details::ConcurrentQueueProducerTypelessBase* producer; +}; + + +struct ConsumerToken +{ + template + explicit ConsumerToken(ConcurrentQueue& q); + + template + explicit ConsumerToken(BlockingConcurrentQueue& q); + + ConsumerToken(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT + : initialOffset(other.initialOffset), lastKnownGlobalOffset(other.lastKnownGlobalOffset), itemsConsumedFromCurrent(other.itemsConsumedFromCurrent), currentProducer(other.currentProducer), desiredProducer(other.desiredProducer) + { + } + + inline ConsumerToken& operator=(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + void swap(ConsumerToken& other) MOODYCAMEL_NOEXCEPT + { + std::swap(initialOffset, other.initialOffset); + std::swap(lastKnownGlobalOffset, other.lastKnownGlobalOffset); + std::swap(itemsConsumedFromCurrent, other.itemsConsumedFromCurrent); + std::swap(currentProducer, other.currentProducer); + std::swap(desiredProducer, other.desiredProducer); + } + + // Disable copying and assignment + ConsumerToken(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; + ConsumerToken& operator=(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; + +private: + template friend class ConcurrentQueue; + friend class ConcurrentQueueTests; + +private: // but shared with ConcurrentQueue + std::uint32_t initialOffset; + std::uint32_t lastKnownGlobalOffset; + std::uint32_t itemsConsumedFromCurrent; + details::ConcurrentQueueProducerTypelessBase* currentProducer; + details::ConcurrentQueueProducerTypelessBase* desiredProducer; +}; + +// Need to forward-declare this swap because it's in a namespace. +// See http://stackoverflow.com/questions/4492062/why-does-a-c-friend-class-need-a-forward-declaration-only-in-other-namespaces +template +inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT; + + +template +class ConcurrentQueue +{ +public: + typedef ::moodycamel::ProducerToken producer_token_t; + typedef ::moodycamel::ConsumerToken consumer_token_t; + + typedef typename Traits::index_t index_t; + typedef typename Traits::size_t size_t; + + static const size_t BLOCK_SIZE = static_cast(Traits::BLOCK_SIZE); + static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = static_cast(Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD); + static const size_t EXPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::EXPLICIT_INITIAL_INDEX_SIZE); + static const size_t IMPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::IMPLICIT_INITIAL_INDEX_SIZE); + static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = static_cast(Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE); + static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = static_cast(Traits::EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE); +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4307) // + integral constant overflow (that's what the ternary expression is for!) +#pragma warning(disable: 4309) // static_cast: Truncation of constant value +#endif + static const size_t MAX_SUBQUEUE_SIZE = (details::const_numeric_max::value - static_cast(Traits::MAX_SUBQUEUE_SIZE) < BLOCK_SIZE) ? details::const_numeric_max::value : ((static_cast(Traits::MAX_SUBQUEUE_SIZE) + (BLOCK_SIZE - 1)) / BLOCK_SIZE * BLOCK_SIZE); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::size_t must be an unsigned integral type"); + static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::index_t must be an unsigned integral type"); + static_assert(sizeof(index_t) >= sizeof(size_t), "Traits::index_t must be at least as wide as Traits::size_t"); + static_assert((BLOCK_SIZE > 1) && !(BLOCK_SIZE & (BLOCK_SIZE - 1)), "Traits::BLOCK_SIZE must be a power of 2 (and at least 2)"); + static_assert((EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD > 1) && !(EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD & (EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD - 1)), "Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD must be a power of 2 (and greater than 1)"); + static_assert((EXPLICIT_INITIAL_INDEX_SIZE > 1) && !(EXPLICIT_INITIAL_INDEX_SIZE & (EXPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::EXPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); + static_assert((IMPLICIT_INITIAL_INDEX_SIZE > 1) && !(IMPLICIT_INITIAL_INDEX_SIZE & (IMPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::IMPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); + static_assert((INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) || !(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE & (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - 1)), "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be a power of 2"); + static_assert(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0 || INITIAL_IMPLICIT_PRODUCER_HASH_SIZE >= 1, "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be at least 1 (or 0 to disable implicit enqueueing)"); + +public: + // Creates a queue with at least `capacity` element slots; note that the + // actual number of elements that can be inserted without additional memory + // allocation depends on the number of producers and the block size (e.g. if + // the block size is equal to `capacity`, only a single block will be allocated + // up-front, which means only a single producer will be able to enqueue elements + // without an extra allocation -- blocks aren't shared between producers). + // This method is not thread safe -- it is up to the user to ensure that the + // queue is fully constructed before it starts being used by other threads (this + // includes making the memory effects of construction visible, possibly with a + // memory barrier). + explicit ConcurrentQueue(size_t capacity = 6 * BLOCK_SIZE) + : producerListTail(nullptr), + producerCount(0), + initialBlockPoolIndex(0), + nextExplicitConsumerId(0), + globalExplicitConsumerOffset(0) + { + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + populate_initial_block_list(capacity / BLOCK_SIZE + ((capacity & (BLOCK_SIZE - 1)) == 0 ? 0 : 1)); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + // Track all the producers using a fully-resolved typed list for + // each kind; this makes it possible to debug them starting from + // the root queue object (otherwise wacky casts are needed that + // don't compile in the debugger's expression evaluator). + explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + } + + // Computes the correct amount of pre-allocated blocks for you based + // on the minimum number of elements you want available at any given + // time, and the maximum concurrent number of each type of producer. + ConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers) + : producerListTail(nullptr), + producerCount(0), + initialBlockPoolIndex(0), + nextExplicitConsumerId(0), + globalExplicitConsumerOffset(0) + { + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + size_t blocks = (((minCapacity + BLOCK_SIZE - 1) / BLOCK_SIZE) - 1) * (maxExplicitProducers + 1) + 2 * (maxExplicitProducers + maxImplicitProducers); + populate_initial_block_list(blocks); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + } + + // Note: The queue should not be accessed concurrently while it's + // being deleted. It's up to the user to synchronize this. + // This method is not thread safe. + ~ConcurrentQueue() + { + // Destroy producers + auto ptr = producerListTail.load(std::memory_order_relaxed); + while (ptr != nullptr) { + auto next = ptr->next_prod(); + if (ptr->token != nullptr) { + ptr->token->producer = nullptr; + } + destroy(ptr); + ptr = next; + } + + // Destroy implicit producer hash tables + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE != 0) { + auto hash = implicitProducerHash.load(std::memory_order_relaxed); + while (hash != nullptr) { + auto prev = hash->prev; + if (prev != nullptr) { // The last hash is part of this object and was not allocated dynamically + for (size_t i = 0; i != hash->capacity; ++i) { + hash->entries[i].~ImplicitProducerKVP(); + } + hash->~ImplicitProducerHash(); + (Traits::free)(hash); + } + hash = prev; + } + } + + // Destroy global free list + auto block = freeList.head_unsafe(); + while (block != nullptr) { + auto next = block->freeListNext.load(std::memory_order_relaxed); + if (block->dynamicallyAllocated) { + destroy(block); + } + block = next; + } + + // Destroy initial free list + destroy_array(initialBlockPool, initialBlockPoolSize); + } + + // Disable copying and copy assignment + ConcurrentQueue(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; + ConcurrentQueue& operator=(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; + + // Moving is supported, but note that it is *not* a thread-safe operation. + // Nobody can use the queue while it's being moved, and the memory effects + // of that move must be propagated to other threads before they can use it. + // Note: When a queue is moved, its tokens are still valid but can only be + // used with the destination queue (i.e. semantically they are moved along + // with the queue itself). + ConcurrentQueue(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT + : producerListTail(other.producerListTail.load(std::memory_order_relaxed)), + producerCount(other.producerCount.load(std::memory_order_relaxed)), + initialBlockPoolIndex(other.initialBlockPoolIndex.load(std::memory_order_relaxed)), + initialBlockPool(other.initialBlockPool), + initialBlockPoolSize(other.initialBlockPoolSize), + freeList(std::move(other.freeList)), + nextExplicitConsumerId(other.nextExplicitConsumerId.load(std::memory_order_relaxed)), + globalExplicitConsumerOffset(other.globalExplicitConsumerOffset.load(std::memory_order_relaxed)) + { + // Move the other one into this, and leave the other one as an empty queue + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + swap_implicit_producer_hashes(other); + + other.producerListTail.store(nullptr, std::memory_order_relaxed); + other.producerCount.store(0, std::memory_order_relaxed); + other.nextExplicitConsumerId.store(0, std::memory_order_relaxed); + other.globalExplicitConsumerOffset.store(0, std::memory_order_relaxed); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + explicitProducers.store(other.explicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(other.implicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + + other.initialBlockPoolIndex.store(0, std::memory_order_relaxed); + other.initialBlockPoolSize = 0; + other.initialBlockPool = nullptr; + + reown_producers(); + } + + inline ConcurrentQueue& operator=(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT + { + return swap_internal(other); + } + + // Swaps this queue's state with the other's. Not thread-safe. + // Swapping two queues does not invalidate their tokens, however + // the tokens that were created for one queue must be used with + // only the swapped queue (i.e. the tokens are tied to the + // queue's movable state, not the object itself). + inline void swap(ConcurrentQueue& other) MOODYCAMEL_NOEXCEPT + { + swap_internal(other); + } + +private: + ConcurrentQueue& swap_internal(ConcurrentQueue& other) + { + if (this == &other) { + return *this; + } + + details::swap_relaxed(producerListTail, other.producerListTail); + details::swap_relaxed(producerCount, other.producerCount); + details::swap_relaxed(initialBlockPoolIndex, other.initialBlockPoolIndex); + std::swap(initialBlockPool, other.initialBlockPool); + std::swap(initialBlockPoolSize, other.initialBlockPoolSize); + freeList.swap(other.freeList); + details::swap_relaxed(nextExplicitConsumerId, other.nextExplicitConsumerId); + details::swap_relaxed(globalExplicitConsumerOffset, other.globalExplicitConsumerOffset); + + swap_implicit_producer_hashes(other); + + reown_producers(); + other.reown_producers(); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + details::swap_relaxed(explicitProducers, other.explicitProducers); + details::swap_relaxed(implicitProducers, other.implicitProducers); +#endif + + return *this; + } + +public: + // Enqueues a single item (by copying it). + // Allocates memory if required. Only fails if memory allocation fails (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, + // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(T const& item) + { + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + return inner_enqueue(item); + } + + // Enqueues a single item (by moving it, if possible). + // Allocates memory if required. Only fails if memory allocation fails (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, + // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(T&& item) + { + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + return inner_enqueue(std::move(item)); + } + + // Enqueues a single item (by copying it) using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails (or + // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(producer_token_t const& token, T const& item) + { + return inner_enqueue(token, item); + } + + // Enqueues a single item (by moving it, if possible) using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails (or + // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(producer_token_t const& token, T&& item) + { + return inner_enqueue(token, std::move(item)); + } + + // Enqueues several items. + // Allocates memory if required. Only fails if memory allocation fails (or + // implicit production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE + // is 0, or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Note: Use std::make_move_iterator if the elements should be moved instead of copied. + // Thread-safe. + template + bool enqueue_bulk(It itemFirst, size_t count) + { + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + return inner_enqueue_bulk(itemFirst, count); + } + + // Enqueues several items using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails + // (or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return inner_enqueue_bulk(token, itemFirst, count); + } + + // Enqueues a single item (by copying it). + // Does not allocate memory. Fails if not enough room to enqueue (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE + // is 0). + // Thread-safe. + inline bool try_enqueue(T const& item) + { + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + return inner_enqueue(item); + } + + // Enqueues a single item (by moving it, if possible). + // Does not allocate memory (except for one-time implicit producer). + // Fails if not enough room to enqueue (or implicit production is + // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). + // Thread-safe. + inline bool try_enqueue(T&& item) + { + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + return inner_enqueue(std::move(item)); + } + + // Enqueues a single item (by copying it) using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Thread-safe. + inline bool try_enqueue(producer_token_t const& token, T const& item) + { + return inner_enqueue(token, item); + } + + // Enqueues a single item (by moving it, if possible) using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Thread-safe. + inline bool try_enqueue(producer_token_t const& token, T&& item) + { + return inner_enqueue(token, std::move(item)); + } + + // Enqueues several items. + // Does not allocate memory (except for one-time implicit producer). + // Fails if not enough room to enqueue (or implicit production is + // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool try_enqueue_bulk(It itemFirst, size_t count) + { + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + return inner_enqueue_bulk(itemFirst, count); + } + + // Enqueues several items using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool try_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return inner_enqueue_bulk(token, itemFirst, count); + } + + + + // Attempts to dequeue from the queue. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + bool try_dequeue(U& item) + { + // Instead of simply trying each producer in turn (which could cause needless contention on the first + // producer), we score them heuristically. + size_t nonEmptyCount = 0; + ProducerBase* best = nullptr; + size_t bestSize = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); nonEmptyCount < 3 && ptr != nullptr; ptr = ptr->next_prod()) { + auto size = ptr->size_approx(); + if (size > 0) { + if (size > bestSize) { + bestSize = size; + best = ptr; + } + ++nonEmptyCount; + } + } + + // If there was at least one non-empty queue but it appears empty at the time + // we try to dequeue from it, we need to make sure every queue's been tried + if (nonEmptyCount > 0) { + if (details::likely(best->dequeue(item))) { + return true; + } + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr != best && ptr->dequeue(item)) { + return true; + } + } + } + return false; + } + + // Attempts to dequeue from the queue. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // This differs from the try_dequeue(item) method in that this one does + // not attempt to reduce contention by interleaving the order that producer + // streams are dequeued from. So, using this method can reduce overall throughput + // under contention, but will give more predictable results in single-threaded + // consumer scenarios. This is mostly only useful for internal unit tests. + // Never allocates. Thread-safe. + template + bool try_dequeue_non_interleaved(U& item) + { + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr->dequeue(item)) { + return true; + } + } + return false; + } + + // Attempts to dequeue from the queue using an explicit consumer token. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + bool try_dequeue(consumer_token_t& token, U& item) + { + // The idea is roughly as follows: + // Every 256 items from one producer, make everyone rotate (increase the global offset) -> this means the highest efficiency consumer dictates the rotation speed of everyone else, more or less + // If you see that the global offset has changed, you must reset your consumption counter and move to your designated place + // If there's no items where you're supposed to be, keep moving until you find a producer with some items + // If the global offset has not changed but you've run out of items to consume, move over from your current position until you find an producer with something in it + + if (token.desiredProducer == nullptr || token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { + if (!update_current_producer_after_rotation(token)) { + return false; + } + } + + // If there was at least one non-empty queue but it appears empty at the time + // we try to dequeue from it, we need to make sure every queue's been tried + if (static_cast(token.currentProducer)->dequeue(item)) { + if (++token.itemsConsumedFromCurrent == EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { + globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); + } + return true; + } + + auto tail = producerListTail.load(std::memory_order_acquire); + auto ptr = static_cast(token.currentProducer)->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + while (ptr != static_cast(token.currentProducer)) { + if (ptr->dequeue(item)) { + token.currentProducer = ptr; + token.itemsConsumedFromCurrent = 1; + return true; + } + ptr = ptr->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + } + return false; + } + + // Attempts to dequeue several elements from the queue. + // Returns the number of items actually dequeued. + // Returns 0 if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + size_t try_dequeue_bulk(It itemFirst, size_t max) + { + size_t count = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + count += ptr->dequeue_bulk(itemFirst, max - count); + if (count == max) { + break; + } + } + return count; + } + + // Attempts to dequeue several elements from the queue using an explicit consumer token. + // Returns the number of items actually dequeued. + // Returns 0 if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + size_t try_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max) + { + if (token.desiredProducer == nullptr || token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { + if (!update_current_producer_after_rotation(token)) { + return 0; + } + } + + size_t count = static_cast(token.currentProducer)->dequeue_bulk(itemFirst, max); + if (count == max) { + if ((token.itemsConsumedFromCurrent += static_cast(max)) >= EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { + globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); + } + return max; + } + token.itemsConsumedFromCurrent += static_cast(count); + max -= count; + + auto tail = producerListTail.load(std::memory_order_acquire); + auto ptr = static_cast(token.currentProducer)->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + while (ptr != static_cast(token.currentProducer)) { + auto dequeued = ptr->dequeue_bulk(itemFirst, max); + count += dequeued; + if (dequeued != 0) { + token.currentProducer = ptr; + token.itemsConsumedFromCurrent = static_cast(dequeued); + } + if (dequeued == max) { + break; + } + max -= dequeued; + ptr = ptr->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + } + return count; + } + + + + // Attempts to dequeue from a specific producer's inner queue. + // If you happen to know which producer you want to dequeue from, this + // is significantly faster than using the general-case try_dequeue methods. + // Returns false if the producer's queue appeared empty at the time it + // was checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + inline bool try_dequeue_from_producer(producer_token_t const& producer, U& item) + { + return static_cast(producer.producer)->dequeue(item); + } + + // Attempts to dequeue several elements from a specific producer's inner queue. + // Returns the number of items actually dequeued. + // If you happen to know which producer you want to dequeue from, this + // is significantly faster than using the general-case try_dequeue methods. + // Returns 0 if the producer's queue appeared empty at the time it + // was checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + inline size_t try_dequeue_bulk_from_producer(producer_token_t const& producer, It itemFirst, size_t max) + { + return static_cast(producer.producer)->dequeue_bulk(itemFirst, max); + } + + + // Returns an estimate of the total number of elements currently in the queue. This + // estimate is only accurate if the queue has completely stabilized before it is called + // (i.e. all enqueue and dequeue operations have completed and their memory effects are + // visible on the calling thread, and no further operations start while this method is + // being called). + // Thread-safe. + size_t size_approx() const + { + size_t size = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + size += ptr->size_approx(); + } + return size; + } + + + // Returns true if the underlying atomic variables used by + // the queue are lock-free (they should be on most platforms). + // Thread-safe. + static bool is_lock_free() + { + return + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::thread_id_numeric_size_t>::value == 2; + } + + +private: + friend struct ProducerToken; + friend struct ConsumerToken; + friend struct ExplicitProducer; + friend class ConcurrentQueueTests; + + enum AllocationMode { CanAlloc, CannotAlloc }; + + + /////////////////////////////// + // Queue methods + /////////////////////////////// + + template + inline bool inner_enqueue(producer_token_t const& token, U&& element) + { + return static_cast(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue(std::forward(element)); + } + + template + inline bool inner_enqueue(U&& element) + { + auto producer = get_or_add_implicit_producer(); + return producer == nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue(std::forward(element)); + } + + template + inline bool inner_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return static_cast(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue_bulk(itemFirst, count); + } + + template + inline bool inner_enqueue_bulk(It itemFirst, size_t count) + { + auto producer = get_or_add_implicit_producer(); + return producer == nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue_bulk(itemFirst, count); + } + + inline bool update_current_producer_after_rotation(consumer_token_t& token) + { + // Ah, there's been a rotation, figure out where we should be! + auto tail = producerListTail.load(std::memory_order_acquire); + if (token.desiredProducer == nullptr && tail == nullptr) { + return false; + } + auto prodCount = producerCount.load(std::memory_order_relaxed); + auto globalOffset = globalExplicitConsumerOffset.load(std::memory_order_relaxed); + if (details::unlikely(token.desiredProducer == nullptr)) { + // Aha, first time we're dequeueing anything. + // Figure out our local position + // Note: offset is from start, not end, but we're traversing from end -- subtract from count first + std::uint32_t offset = prodCount - 1 - (token.initialOffset % prodCount); + token.desiredProducer = tail; + for (std::uint32_t i = 0; i != offset; ++i) { + token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); + if (token.desiredProducer == nullptr) { + token.desiredProducer = tail; + } + } + } + + std::uint32_t delta = globalOffset - token.lastKnownGlobalOffset; + if (delta >= prodCount) { + delta = delta % prodCount; + } + for (std::uint32_t i = 0; i != delta; ++i) { + token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); + if (token.desiredProducer == nullptr) { + token.desiredProducer = tail; + } + } + + token.lastKnownGlobalOffset = globalOffset; + token.currentProducer = token.desiredProducer; + token.itemsConsumedFromCurrent = 0; + return true; + } + + + /////////////////////////// + // Free list + /////////////////////////// + + template + struct FreeListNode + { + FreeListNode() : freeListRefs(0), freeListNext(nullptr) { } + + std::atomic freeListRefs; + std::atomic freeListNext; + }; + + // A simple CAS-based lock-free free list. Not the fastest thing in the world under heavy contention, but + // simple and correct (assuming nodes are never freed until after the free list is destroyed), and fairly + // speedy under low contention. + template // N must inherit FreeListNode or have the same fields (and initialization of them) + struct FreeList + { + FreeList() : freeListHead(nullptr) { } + FreeList(FreeList&& other) : freeListHead(other.freeListHead.load(std::memory_order_relaxed)) { other.freeListHead.store(nullptr, std::memory_order_relaxed); } + void swap(FreeList& other) { details::swap_relaxed(freeListHead, other.freeListHead); } + + FreeList(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; + FreeList& operator=(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; + + inline void add(N* node) + { +#if MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugLock lock(mutex); +#endif + // We know that the should-be-on-freelist bit is 0 at this point, so it's safe to + // set it using a fetch_add + if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST, std::memory_order_acq_rel) == 0) { + // Oh look! We were the last ones referencing this node, and we know + // we want to add it to the free list, so let's do it! + add_knowing_refcount_is_zero(node); + } + } + + inline N* try_get() + { +#if MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugLock lock(mutex); +#endif + auto head = freeListHead.load(std::memory_order_acquire); + while (head != nullptr) { + auto prevHead = head; + auto refs = head->freeListRefs.load(std::memory_order_relaxed); + if ((refs & REFS_MASK) == 0 || !head->freeListRefs.compare_exchange_strong(refs, refs + 1, std::memory_order_acquire, std::memory_order_relaxed)) { + head = freeListHead.load(std::memory_order_acquire); + continue; + } + + // Good, reference count has been incremented (it wasn't at zero), which means we can read the + // next and not worry about it changing between now and the time we do the CAS + auto next = head->freeListNext.load(std::memory_order_relaxed); + if (freeListHead.compare_exchange_strong(head, next, std::memory_order_acquire, std::memory_order_relaxed)) { + // Yay, got the node. This means it was on the list, which means shouldBeOnFreeList must be false no + // matter the refcount (because nobody else knows it's been taken off yet, it can't have been put back on). + assert((head->freeListRefs.load(std::memory_order_relaxed) & SHOULD_BE_ON_FREELIST) == 0); + + // Decrease refcount twice, once for our ref, and once for the list's ref + head->freeListRefs.fetch_sub(2, std::memory_order_release); + return head; + } + + // OK, the head must have changed on us, but we still need to decrease the refcount we increased. + // Note that we don't need to release any memory effects, but we do need to ensure that the reference + // count decrement happens-after the CAS on the head. + refs = prevHead->freeListRefs.fetch_sub(1, std::memory_order_acq_rel); + if (refs == SHOULD_BE_ON_FREELIST + 1) { + add_knowing_refcount_is_zero(prevHead); + } + } + + return nullptr; + } + + // Useful for traversing the list when there's no contention (e.g. to destroy remaining nodes) + N* head_unsafe() const { return freeListHead.load(std::memory_order_relaxed); } + + private: + inline void add_knowing_refcount_is_zero(N* node) + { + // Since the refcount is zero, and nobody can increase it once it's zero (except us, and we run + // only one copy of this method per node at a time, i.e. the single thread case), then we know + // we can safely change the next pointer of the node; however, once the refcount is back above + // zero, then other threads could increase it (happens under heavy contention, when the refcount + // goes to zero in between a load and a refcount increment of a node in try_get, then back up to + // something non-zero, then the refcount increment is done by the other thread) -- so, if the CAS + // to add the node to the actual list fails, decrease the refcount and leave the add operation to + // the next thread who puts the refcount back at zero (which could be us, hence the loop). + auto head = freeListHead.load(std::memory_order_relaxed); + while (true) { + node->freeListNext.store(head, std::memory_order_relaxed); + node->freeListRefs.store(1, std::memory_order_release); + if (!freeListHead.compare_exchange_strong(head, node, std::memory_order_release, std::memory_order_relaxed)) { + // Hmm, the add failed, but we can only try again when the refcount goes back to zero + if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST - 1, std::memory_order_release) == 1) { + continue; + } + } + return; + } + } + + private: + // Implemented like a stack, but where node order doesn't matter (nodes are inserted out of order under contention) + std::atomic freeListHead; + + static const std::uint32_t REFS_MASK = 0x7FFFFFFF; + static const std::uint32_t SHOULD_BE_ON_FREELIST = 0x80000000; + +#if MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugMutex mutex; +#endif + }; + + + /////////////////////////// + // Block + /////////////////////////// + + enum InnerQueueContext { implicit_context = 0, explicit_context = 1 }; + + struct Block + { + Block() + : next(nullptr), elementsCompletelyDequeued(0), freeListRefs(0), freeListNext(nullptr), shouldBeOnFreeList(false), dynamicallyAllocated(true) + { +#if MCDBGQ_TRACKMEM + owner = nullptr; +#endif + } + + template + inline bool is_empty() const + { + if (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Check flags + for (size_t i = 0; i < BLOCK_SIZE; ++i) { + if (!emptyFlags[i].load(std::memory_order_relaxed)) { + return false; + } + } + + // Aha, empty; make sure we have all other memory effects that happened before the empty flags were set + std::atomic_thread_fence(std::memory_order_acquire); + return true; + } + else { + // Check counter + if (elementsCompletelyDequeued.load(std::memory_order_relaxed) == BLOCK_SIZE) { + std::atomic_thread_fence(std::memory_order_acquire); + return true; + } + assert(elementsCompletelyDequeued.load(std::memory_order_relaxed) <= BLOCK_SIZE); + return false; + } + } + + // Returns true if the block is now empty (does not apply in explicit context) + template + inline bool set_empty(index_t i) + { + if (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set flag + assert(!emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].load(std::memory_order_relaxed)); + emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].store(true, std::memory_order_release); + return false; + } + else { + // Increment counter + auto prevVal = elementsCompletelyDequeued.fetch_add(1, std::memory_order_release); + assert(prevVal < BLOCK_SIZE); + return prevVal == BLOCK_SIZE - 1; + } + } + + // Sets multiple contiguous item statuses to 'empty' (assumes no wrapping and count > 0). + // Returns true if the block is now empty (does not apply in explicit context). + template + inline bool set_many_empty(index_t i, size_t count) + { + if (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set flags + std::atomic_thread_fence(std::memory_order_release); + i = BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1)) - count + 1; + for (size_t j = 0; j != count; ++j) { + assert(!emptyFlags[i + j].load(std::memory_order_relaxed)); + emptyFlags[i + j].store(true, std::memory_order_relaxed); + } + return false; + } + else { + // Increment counter + auto prevVal = elementsCompletelyDequeued.fetch_add(count, std::memory_order_release); + assert(prevVal + count <= BLOCK_SIZE); + return prevVal + count == BLOCK_SIZE; + } + } + + template + inline void set_all_empty() + { + if (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set all flags + for (size_t i = 0; i != BLOCK_SIZE; ++i) { + emptyFlags[i].store(true, std::memory_order_relaxed); + } + } + else { + // Reset counter + elementsCompletelyDequeued.store(BLOCK_SIZE, std::memory_order_relaxed); + } + } + + template + inline void reset_empty() + { + if (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Reset flags + for (size_t i = 0; i != BLOCK_SIZE; ++i) { + emptyFlags[i].store(false, std::memory_order_relaxed); + } + } + else { + // Reset counter + elementsCompletelyDequeued.store(0, std::memory_order_relaxed); + } + } + + inline T* operator[](index_t idx) MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } + inline T const* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } + + private: + // IMPORTANT: This must be the first member in Block, so that if T depends on the alignment of + // addresses returned by malloc, that alignment will be preserved. Apparently clang actually + // generates code that uses this assumption for AVX instructions in some cases. Ideally, we + // should also align Block to the alignment of T in case it's higher than malloc's 16-byte + // alignment, but this is hard to do in a cross-platform way. Assert for this case: + static_assert(std::alignment_of::value <= std::alignment_of::value, "The queue does not support super-aligned types at this time"); + // Additionally, we need the alignment of Block itself to be a multiple of max_align_t since + // otherwise the appropriate padding will not be added at the end of Block in order to make + // arrays of Blocks all be properly aligned (not just the first one). We use a union to force + // this. + union { + char elements[sizeof(T) * BLOCK_SIZE]; + details::max_align_t dummy; + }; + public: + Block* next; + std::atomic elementsCompletelyDequeued; + std::atomic emptyFlags[BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE : 1]; + public: + std::atomic freeListRefs; + std::atomic freeListNext; + std::atomic shouldBeOnFreeList; + bool dynamicallyAllocated; // Perhaps a better name for this would be 'isNotPartOfInitialBlockPool' + +#if MCDBGQ_TRACKMEM + void* owner; +#endif + }; + static_assert(std::alignment_of::value >= std::alignment_of::value, "Internal error: Blocks must be at least as aligned as the type they are wrapping"); + + +#if MCDBGQ_TRACKMEM +public: + struct MemStats; +private: +#endif + + /////////////////////////// + // Producer base + /////////////////////////// + + struct ProducerBase : public details::ConcurrentQueueProducerTypelessBase + { + ProducerBase(ConcurrentQueue* parent_, bool isExplicit_) : + tailIndex(0), + headIndex(0), + dequeueOptimisticCount(0), + dequeueOvercommit(0), + tailBlock(nullptr), + isExplicit(isExplicit_), + parent(parent_) + { + } + + virtual ~ProducerBase() { }; + + template + inline bool dequeue(U& element) + { + if (isExplicit) { + return static_cast(this)->dequeue(element); + } + else { + return static_cast(this)->dequeue(element); + } + } + + template + inline size_t dequeue_bulk(It& itemFirst, size_t max) + { + if (isExplicit) { + return static_cast(this)->dequeue_bulk(itemFirst, max); + } + else { + return static_cast(this)->dequeue_bulk(itemFirst, max); + } + } + + inline ProducerBase* next_prod() const { return static_cast(next); } + + inline size_t size_approx() const + { + auto tail = tailIndex.load(std::memory_order_relaxed); + auto head = headIndex.load(std::memory_order_relaxed); + return details::circular_less_than(head, tail) ? static_cast(tail - head) : 0; + } + + inline index_t getTail() const { return tailIndex.load(std::memory_order_relaxed); } + protected: + std::atomic tailIndex; // Where to enqueue to next + std::atomic headIndex; // Where to dequeue from next + + std::atomic dequeueOptimisticCount; + std::atomic dequeueOvercommit; + + Block* tailBlock; + + public: + bool isExplicit; + ConcurrentQueue* parent; + + protected: +#if MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + /////////////////////////// + // Explicit queue + /////////////////////////// + + struct ExplicitProducer : public ProducerBase + { + explicit ExplicitProducer(ConcurrentQueue* parent) : + ProducerBase(parent, true), + blockIndex(nullptr), + pr_blockIndexSlotsUsed(0), + pr_blockIndexSize(EXPLICIT_INITIAL_INDEX_SIZE >> 1), + pr_blockIndexFront(0), + pr_blockIndexEntries(nullptr), + pr_blockIndexRaw(nullptr) + { + size_t poolBasedIndexSize = details::ceil_to_pow_2(parent->initialBlockPoolSize) >> 1; + if (poolBasedIndexSize > pr_blockIndexSize) { + pr_blockIndexSize = poolBasedIndexSize; + } + + new_block_index(0); // This creates an index with double the number of current entries, i.e. EXPLICIT_INITIAL_INDEX_SIZE + } + + ~ExplicitProducer() + { + // Destruct any elements not yet dequeued. + // Since we're in the destructor, we can assume all elements + // are either completely dequeued or completely not (no halfways). + if (this->tailBlock != nullptr) { // Note this means there must be a block index too + // First find the block that's partially dequeued, if any + Block* halfDequeuedBlock = nullptr; + if ((this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) != 0) { + // The head's not on a block boundary, meaning a block somewhere is partially dequeued + // (or the head block is the tail block and was fully dequeued, but the head/tail are still not on a boundary) + size_t i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & (pr_blockIndexSize - 1); + while (details::circular_less_than(pr_blockIndexEntries[i].base + BLOCK_SIZE, this->headIndex.load(std::memory_order_relaxed))) { + i = (i + 1) & (pr_blockIndexSize - 1); + } + assert(details::circular_less_than(pr_blockIndexEntries[i].base, this->headIndex.load(std::memory_order_relaxed))); + halfDequeuedBlock = pr_blockIndexEntries[i].block; + } + + // Start at the head block (note the first line in the loop gives us the head from the tail on the first iteration) + auto block = this->tailBlock; + do { + block = block->next; + if (block->ConcurrentQueue::Block::template is_empty()) { + continue; + } + + size_t i = 0; // Offset into block + if (block == halfDequeuedBlock) { + i = static_cast(this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); + } + + // Walk through all the items in the block; if this is the tail block, we need to stop when we reach the tail index + auto lastValidIndex = (this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) == 0 ? BLOCK_SIZE : static_cast(this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); + while (i != BLOCK_SIZE && (block != this->tailBlock || i != lastValidIndex)) { + (*block)[i++]->~T(); + } + } while (block != this->tailBlock); + } + + // Destroy all blocks that we own + if (this->tailBlock != nullptr) { + auto block = this->tailBlock; + do { + auto nextBlock = block->next; + if (block->dynamicallyAllocated) { + destroy(block); + } + else { + this->parent->add_block_to_free_list(block); + } + block = nextBlock; + } while (block != this->tailBlock); + } + + // Destroy the block indices + auto header = static_cast(pr_blockIndexRaw); + while (header != nullptr) { + auto prev = static_cast(header->prev); + header->~BlockIndexHeader(); + (Traits::free)(header); + header = prev; + } + } + + template + inline bool enqueue(U&& element) + { + index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); + index_t newTailIndex = 1 + currentTailIndex; + if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + // We reached the end of a block, start a new one + auto startBlock = this->tailBlock; + auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; + if (this->tailBlock != nullptr && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { + // We can re-use the block ahead of us, it's empty! + this->tailBlock = this->tailBlock->next; + this->tailBlock->ConcurrentQueue::Block::template reset_empty(); + + // We'll put the block on the block index (guaranteed to be room since we're conceptually removing the + // last block from it first -- except instead of removing then adding, we can just overwrite). + // Note that there must be a valid block index here, since even if allocation failed in the ctor, + // it would have been re-attempted when adding the first block to the queue; since there is such + // a block, a block index must have been successfully allocated. + } + else { + // Whatever head value we see here is >= the last value we saw here (relatively), + // and <= its current value. Since we have the most recent tail, the head must be + // <= to it. + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) + || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { + // We can't enqueue in another block because there's not enough leeway -- the + // tail could surpass the head by the time the block fills up! (Or we'll exceed + // the size limit, if the second part of the condition was true.) + return false; + } + // We're going to need a new block; check that the block index has room + if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize) { + // Hmm, the circular block index is already full -- we'll need + // to allocate a new index. Note pr_blockIndexRaw can only be nullptr if + // the initial allocation failed in the constructor. + + if (allocMode == CannotAlloc || !new_block_index(pr_blockIndexSlotsUsed)) { + return false; + } + } + + // Insert a new block in the circular linked list + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + return false; + } +#if MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + if (this->tailBlock == nullptr) { + newBlock->next = newBlock; + } + else { + newBlock->next = this->tailBlock->next; + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + ++pr_blockIndexSlotsUsed; + } + + if (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (nullptr) T(std::forward(element)))) { + // The constructor may throw. We want the element not to appear in the queue in + // that case (without corrupting the queue): + MOODYCAMEL_TRY { + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + } + MOODYCAMEL_CATCH (...) { + // Revert change to the current block, but leave the new block available + // for next time + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? this->tailBlock : startBlock; + MOODYCAMEL_RETHROW; + } + } + else { + (void)startBlock; + (void)originalBlockIndexSlotsUsed; + } + + // Add block to block index + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + blockIndex.load(std::memory_order_relaxed)->front.store(pr_blockIndexFront, std::memory_order_release); + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + + if (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (nullptr) T(std::forward(element)))) { + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + } + + // Enqueue + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + bool dequeue(U& element) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + if (details::circular_less_than(this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { + // Might be something to dequeue, let's give it a try + + // Note that this if is purely for performance purposes in the common case when the queue is + // empty and the values are eventually consistent -- we may enter here spuriously. + + // Note that whatever the values of overcommit and tail are, they are not going to change (unless we + // change them) and must be the same value at this point (inside the if) as when the if condition was + // evaluated. + + // We insert an acquire fence here to synchronize-with the release upon incrementing dequeueOvercommit below. + // This ensures that whatever the value we got loaded into overcommit, the load of dequeueOptisticCount in + // the fetch_add below will result in a value at least as recent as that (and therefore at least as large). + // Note that I believe a compiler (signal) fence here would be sufficient due to the nature of fetch_add (all + // read-modify-write operations are guaranteed to work on the latest value in the modification order), but + // unfortunately that can't be shown to be correct using only the C++11 standard. + // See http://stackoverflow.com/questions/18223161/what-are-the-c11-memory-ordering-guarantees-in-this-corner-case + std::atomic_thread_fence(std::memory_order_acquire); + + // Increment optimistic counter, then check if it went over the boundary + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); + + // Note that since dequeueOvercommit must be <= dequeueOptimisticCount (because dequeueOvercommit is only ever + // incremented after dequeueOptimisticCount -- this is enforced in the `else` block below), and since we now + // have a version of dequeueOptimisticCount that is at least as recent as overcommit (due to the release upon + // incrementing dequeueOvercommit and the acquire above that synchronizes with it), overcommit <= myDequeueCount. + assert(overcommit <= myDequeueCount); + + // Note that we reload tail here in case it changed; it will be the same value as before or greater, since + // this load is sequenced after (happens after) the earlier load above. This is supported by read-read + // coherency (as defined in the standard), explained here: http://en.cppreference.com/w/cpp/atomic/memory_order + tail = this->tailIndex.load(std::memory_order_acquire); + if (details::likely(details::circular_less_than(myDequeueCount - overcommit, tail))) { + // Guaranteed to be at least one element to dequeue! + + // Get the index. Note that since there's guaranteed to be at least one element, this + // will never exceed tail. We need to do an acquire-release fence here since it's possible + // that whatever condition got us to this point was for an earlier enqueued element (that + // we already see the memory effects for), but that by the time we increment somebody else + // has incremented it, and we need to see the memory effects for *that* element, which is + // in such a case is necessarily visible on the thread that incremented it in the first + // place with the more current condition (they must have acquired a tail that is at least + // as recent). + auto index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); + + + // Determine which block the element is in + + auto localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); + + // We need to be careful here about subtracting and dividing because of index wrap-around. + // When an index wraps, we need to preserve the sign of the offset when dividing it by the + // block size (in order to get a correct signed block count offset in all cases): + auto headBase = localBlockIndex->entries[localBlockIndexHead].base; + auto blockBaseIndex = index & ~static_cast(BLOCK_SIZE - 1); + auto offset = static_cast(static_cast::type>(blockBaseIndex - headBase) / BLOCK_SIZE); + auto block = localBlockIndex->entries[(localBlockIndexHead + offset) & (localBlockIndex->size - 1)].block; + + // Dequeue + auto& el = *((*block)[index]); + if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { + // Make sure the element is still fully dequeued and destroyed even if the assignment + // throws + struct Guard { + Block* block; + index_t index; + + ~Guard() + { + (*block)[index]->~T(); + block->ConcurrentQueue::Block::template set_empty(index); + } + } guard = { block, index }; + + element = std::move(el); + } + else { + element = std::move(el); + el.~T(); + block->ConcurrentQueue::Block::template set_empty(index); + } + + return true; + } + else { + // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent + this->dequeueOvercommit.fetch_add(1, std::memory_order_release); // Release so that the fetch_add on dequeueOptimisticCount is guaranteed to happen before this write + } + } + + return false; + } + + template + bool enqueue_bulk(It itemFirst, size_t count) + { + // First, we need to make sure we have enough room to enqueue all of the elements; + // this means pre-allocating blocks and putting them in the block index (but only if + // all the allocations succeeded). + index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); + auto startBlock = this->tailBlock; + auto originalBlockIndexFront = pr_blockIndexFront; + auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; + + Block* firstAllocatedBlock = nullptr; + + // Figure out how many blocks we'll need to allocate, and do so + size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); + index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + if (blockBaseDiff > 0) { + // Allocate as many blocks as possible from ahead + while (blockBaseDiff > 0 && this->tailBlock != nullptr && this->tailBlock->next != firstAllocatedBlock && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + this->tailBlock = this->tailBlock->next; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; + + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + } + + // Now allocate as many blocks as necessary from the block pool + while (blockBaseDiff > 0) { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); + if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize || full) { + if (allocMode == CannotAlloc || full || !new_block_index(originalBlockIndexSlotsUsed)) { + // Failed to allocate, undo changes (but keep injected blocks) + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + return false; + } + + // pr_blockIndexFront is updated inside new_block_index, so we need to + // update our fallback value too (since we keep the new index even if we + // later fail) + originalBlockIndexFront = originalBlockIndexSlotsUsed; + } + + // Insert a new block in the circular linked list + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + return false; + } + +#if MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template set_all_empty(); + if (this->tailBlock == nullptr) { + newBlock->next = newBlock; + } + else { + newBlock->next = this->tailBlock->next; + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; + + ++pr_blockIndexSlotsUsed; + + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + } + + // Excellent, all allocations succeeded. Reset each block's emptiness before we fill them up, and + // publish the new block index front + auto block = firstAllocatedBlock; + while (true) { + block->ConcurrentQueue::Block::template reset_empty(); + if (block == this->tailBlock) { + break; + } + block = block->next; + } + + if (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (nullptr) T(details::deref_noexcept(itemFirst)))) { + blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); + } + } + + // Enqueue, one block at a time + index_t newTailIndex = startTailIndex + static_cast(count); + currentTailIndex = startTailIndex; + auto endBlock = this->tailBlock; + this->tailBlock = startBlock; + assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { + this->tailBlock = firstAllocatedBlock; + } + while (true) { + auto stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(newTailIndex, stopIndex)) { + stopIndex = newTailIndex; + } + if (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (nullptr) T(details::deref_noexcept(itemFirst)))) { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); + } + } + else { + MOODYCAMEL_TRY { + while (currentTailIndex != stopIndex) { + // Must use copy constructor even if move constructor is available + // because we may have to revert if there's an exception. + // Sorry about the horrible templated next line, but it was the only way + // to disable moving *at compile time*, which is important because a type + // may only define a (noexcept) move constructor, and so calls to the + // cctor will not compile, even if they are in an if branch that will never + // be executed + new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if<(bool)!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (nullptr) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); + ++currentTailIndex; + ++itemFirst; + } + } + MOODYCAMEL_CATCH (...) { + // Oh dear, an exception's been thrown -- destroy the elements that + // were enqueued so far and revert the entire bulk operation (we'll keep + // any allocated blocks in our linked list for later, though). + auto constructedStopIndex = currentTailIndex; + auto lastBlockEnqueued = this->tailBlock; + + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + + if (!details::is_trivially_destructible::value) { + auto block = startBlock; + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + block = firstAllocatedBlock; + } + currentTailIndex = startTailIndex; + while (true) { + stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(constructedStopIndex, stopIndex)) { + stopIndex = constructedStopIndex; + } + while (currentTailIndex != stopIndex) { + (*block)[currentTailIndex++]->~T(); + } + if (block == lastBlockEnqueued) { + break; + } + block = block->next; + } + } + MOODYCAMEL_RETHROW; + } + } + + if (this->tailBlock == endBlock) { + assert(currentTailIndex == newTailIndex); + break; + } + this->tailBlock = this->tailBlock->next; + } + + if (!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (nullptr) T(details::deref_noexcept(itemFirst))) && firstAllocatedBlock != nullptr) { + blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); + } + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + size_t dequeue_bulk(It& itemFirst, size_t max) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + auto desiredCount = static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); + if (details::circular_less_than(0, desiredCount)) { + desiredCount = desiredCount < max ? desiredCount : max; + std::atomic_thread_fence(std::memory_order_acquire); + + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); + assert(overcommit <= myDequeueCount); + + tail = this->tailIndex.load(std::memory_order_acquire); + auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); + if (details::circular_less_than(0, actualCount)) { + actualCount = desiredCount < actualCount ? desiredCount : actualCount; + if (actualCount < desiredCount) { + this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); + } + + // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this + // will never exceed tail. + auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); + + // Determine which block the first element is in + auto localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); + + auto headBase = localBlockIndex->entries[localBlockIndexHead].base; + auto firstBlockBaseIndex = firstIndex & ~static_cast(BLOCK_SIZE - 1); + auto offset = static_cast(static_cast::type>(firstBlockBaseIndex - headBase) / BLOCK_SIZE); + auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1); + + // Iterate the blocks and dequeue + auto index = firstIndex; + do { + auto firstIndexInBlock = index; + auto endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + auto block = localBlockIndex->entries[indexIndex].block; + if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst++ = std::move(el); + el.~T(); + ++index; + } + } + else { + MOODYCAMEL_TRY { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst = std::move(el); + ++itemFirst; + el.~T(); + ++index; + } + } + MOODYCAMEL_CATCH (...) { + // It's too late to revert the dequeue, but we can make sure that all + // the dequeued objects are properly destroyed and the block index + // (and empty count) are properly updated before we propagate the exception + do { + block = localBlockIndex->entries[indexIndex].block; + while (index != endIndex) { + (*block)[index++]->~T(); + } + block->ConcurrentQueue::Block::template set_many_empty(firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); + indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); + + firstIndexInBlock = index; + endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + } while (index != firstIndex + actualCount); + + MOODYCAMEL_RETHROW; + } + } + block->ConcurrentQueue::Block::template set_many_empty(firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); + indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); + } while (index != firstIndex + actualCount); + + return actualCount; + } + else { + // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent + this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); + } + } + + return 0; + } + + private: + struct BlockIndexEntry + { + index_t base; + Block* block; + }; + + struct BlockIndexHeader + { + size_t size; + std::atomic front; // Current slot (not next, like pr_blockIndexFront) + BlockIndexEntry* entries; + void* prev; + }; + + + bool new_block_index(size_t numberOfFilledSlotsToExpose) + { + auto prevBlockSizeMask = pr_blockIndexSize - 1; + + // Create the new block + pr_blockIndexSize <<= 1; + auto newRawPtr = static_cast((Traits::malloc)(sizeof(BlockIndexHeader) + std::alignment_of::value - 1 + sizeof(BlockIndexEntry) * pr_blockIndexSize)); + if (newRawPtr == nullptr) { + pr_blockIndexSize >>= 1; // Reset to allow graceful retry + return false; + } + + auto newBlockIndexEntries = reinterpret_cast(details::align_for(newRawPtr + sizeof(BlockIndexHeader))); + + // Copy in all the old indices, if any + size_t j = 0; + if (pr_blockIndexSlotsUsed != 0) { + auto i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & prevBlockSizeMask; + do { + newBlockIndexEntries[j++] = pr_blockIndexEntries[i]; + i = (i + 1) & prevBlockSizeMask; + } while (i != pr_blockIndexFront); + } + + // Update everything + auto header = new (newRawPtr) BlockIndexHeader; + header->size = pr_blockIndexSize; + header->front.store(numberOfFilledSlotsToExpose - 1, std::memory_order_relaxed); + header->entries = newBlockIndexEntries; + header->prev = pr_blockIndexRaw; // we link the new block to the old one so we can free it later + + pr_blockIndexFront = j; + pr_blockIndexEntries = newBlockIndexEntries; + pr_blockIndexRaw = newRawPtr; + blockIndex.store(header, std::memory_order_release); + + return true; + } + + private: + std::atomic blockIndex; + + // To be used by producer only -- consumer must use the ones in referenced by blockIndex + size_t pr_blockIndexSlotsUsed; + size_t pr_blockIndexSize; + size_t pr_blockIndexFront; // Next slot (not current) + BlockIndexEntry* pr_blockIndexEntries; + void* pr_blockIndexRaw; + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + public: + ExplicitProducer* nextExplicitProducer; + private: +#endif + +#if MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + ////////////////////////////////// + // Implicit queue + ////////////////////////////////// + + struct ImplicitProducer : public ProducerBase + { + ImplicitProducer(ConcurrentQueue* parent) : + ProducerBase(parent, false), + nextBlockIndexCapacity(IMPLICIT_INITIAL_INDEX_SIZE), + blockIndex(nullptr) + { + new_block_index(); + } + + ~ImplicitProducer() + { + // Note that since we're in the destructor we can assume that all enqueue/dequeue operations + // completed already; this means that all undequeued elements are placed contiguously across + // contiguous blocks, and that only the first and last remaining blocks can be only partially + // empty (all other remaining blocks must be completely full). + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + // Unregister ourselves for thread termination notification + if (!this->inactive.load(std::memory_order_relaxed)) { + details::ThreadExitNotifier::unsubscribe(&threadExitListener); + } +#endif + + // Destroy all remaining elements! + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto index = this->headIndex.load(std::memory_order_relaxed); + Block* block = nullptr; + assert(index == tail || details::circular_less_than(index, tail)); + bool forceFreeLastBlock = index != tail; // If we enter the loop, then the last (tail) block will not be freed + while (index != tail) { + if ((index & static_cast(BLOCK_SIZE - 1)) == 0 || block == nullptr) { + if (block != nullptr) { + // Free the old block + this->parent->add_block_to_free_list(block); + } + + block = get_block_index_entry_for_index(index)->value.load(std::memory_order_relaxed); + } + + ((*block)[index])->~T(); + ++index; + } + // Even if the queue is empty, there's still one block that's not on the free list + // (unless the head index reached the end of it, in which case the tail will be poised + // to create a new block). + if (this->tailBlock != nullptr && (forceFreeLastBlock || (tail & static_cast(BLOCK_SIZE - 1)) != 0)) { + this->parent->add_block_to_free_list(this->tailBlock); + } + + // Destroy block index + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); + if (localBlockIndex != nullptr) { + for (size_t i = 0; i != localBlockIndex->capacity; ++i) { + localBlockIndex->index[i]->~BlockIndexEntry(); + } + do { + auto prev = localBlockIndex->prev; + localBlockIndex->~BlockIndexHeader(); + (Traits::free)(localBlockIndex); + localBlockIndex = prev; + } while (localBlockIndex != nullptr); + } + } + + template + inline bool enqueue(U&& element) + { + index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); + index_t newTailIndex = 1 + currentTailIndex; + if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + // We reached the end of a block, start a new one + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { + return false; + } +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Find out where we'll be inserting this block in the block index + BlockIndexEntry* idxEntry; + if (!insert_block_index_entry(idxEntry, currentTailIndex)) { + return false; + } + + // Get ahold of a new block + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + return false; + } +#if MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + + if (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (nullptr) T(std::forward(element)))) { + // May throw, try to insert now before we publish the fact that we have this new block + MOODYCAMEL_TRY { + new ((*newBlock)[currentTailIndex]) T(std::forward(element)); + } + MOODYCAMEL_CATCH (...) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + this->parent->add_block_to_free_list(newBlock); + MOODYCAMEL_RETHROW; + } + } + + // Insert the new block into the index + idxEntry->value.store(newBlock, std::memory_order_relaxed); + + this->tailBlock = newBlock; + + if (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (nullptr) T(std::forward(element)))) { + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + } + + // Enqueue + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + bool dequeue(U& element) + { + // See ExplicitProducer::dequeue for rationale and explanation + index_t tail = this->tailIndex.load(std::memory_order_relaxed); + index_t overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + if (details::circular_less_than(this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { + std::atomic_thread_fence(std::memory_order_acquire); + + index_t myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); + assert(overcommit <= myDequeueCount); + tail = this->tailIndex.load(std::memory_order_acquire); + if (details::likely(details::circular_less_than(myDequeueCount - overcommit, tail))) { + index_t index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); + + // Determine which block the element is in + auto entry = get_block_index_entry_for_index(index); + + // Dequeue + auto block = entry->value.load(std::memory_order_relaxed); + auto& el = *((*block)[index]); + + if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + // Note: Acquiring the mutex with every dequeue instead of only when a block + // is released is very sub-optimal, but it is, after all, purely debug code. + debug::DebugLock lock(producer->mutex); +#endif + struct Guard { + Block* block; + index_t index; + BlockIndexEntry* entry; + ConcurrentQueue* parent; + + ~Guard() + { + (*block)[index]->~T(); + if (block->ConcurrentQueue::Block::template set_empty(index)) { + entry->value.store(nullptr, std::memory_order_relaxed); + parent->add_block_to_free_list(block); + } + } + } guard = { block, index, entry, this->parent }; + + element = std::move(el); + } + else { + element = std::move(el); + el.~T(); + + if (block->ConcurrentQueue::Block::template set_empty(index)) { + { +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Add the block back into the global free pool (and remove from block index) + entry->value.store(nullptr, std::memory_order_relaxed); + } + this->parent->add_block_to_free_list(block); // releases the above store + } + } + + return true; + } + else { + this->dequeueOvercommit.fetch_add(1, std::memory_order_release); + } + } + + return false; + } + + template + bool enqueue_bulk(It itemFirst, size_t count) + { + // First, we need to make sure we have enough room to enqueue all of the elements; + // this means pre-allocating blocks and putting them in the block index (but only if + // all the allocations succeeded). + + // Note that the tailBlock we start off with may not be owned by us any more; + // this happens if it was filled up exactly to the top (setting tailIndex to + // the first index of the next block which is not yet allocated), then dequeued + // completely (putting it on the free list) before we enqueue again. + + index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); + auto startBlock = this->tailBlock; + Block* firstAllocatedBlock = nullptr; + auto endBlock = this->tailBlock; + + // Figure out how many blocks we'll need to allocate, and do so + size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); + index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + if (blockBaseDiff > 0) { +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + do { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + // Find out where we'll be inserting this block in the block index + BlockIndexEntry* idxEntry = nullptr; // initialization here unnecessary but compiler can't always tell + Block* newBlock; + bool indexInserted = false; + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); + if (full || !(indexInserted = insert_block_index_entry(idxEntry, currentTailIndex)) || (newBlock = this->parent->ConcurrentQueue::template requisition_block()) == nullptr) { + // Index allocation or block allocation failed; revert any other allocations + // and index insertions done so far for this operation + if (indexInserted) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + } + currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { + currentTailIndex += static_cast(BLOCK_SIZE); + idxEntry = get_block_index_entry_for_index(currentTailIndex); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + rewind_block_index_tail(); + } + this->parent->add_blocks_to_free_list(firstAllocatedBlock); + this->tailBlock = startBlock; + + return false; + } + +#if MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + newBlock->next = nullptr; + + // Insert the new block into the index + idxEntry->value.store(newBlock, std::memory_order_relaxed); + + // Store the chain of blocks so that we can undo if later allocations fail, + // and so that we can find the blocks when we do the actual enqueueing + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr) { + assert(this->tailBlock != nullptr); + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + endBlock = newBlock; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? newBlock : firstAllocatedBlock; + } while (blockBaseDiff > 0); + } + + // Enqueue, one block at a time + index_t newTailIndex = startTailIndex + static_cast(count); + currentTailIndex = startTailIndex; + this->tailBlock = startBlock; + assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { + this->tailBlock = firstAllocatedBlock; + } + while (true) { + auto stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(newTailIndex, stopIndex)) { + stopIndex = newTailIndex; + } + if (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (nullptr) T(details::deref_noexcept(itemFirst)))) { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); + } + } + else { + MOODYCAMEL_TRY { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if<(bool)!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (nullptr) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); + ++currentTailIndex; + ++itemFirst; + } + } + MOODYCAMEL_CATCH (...) { + auto constructedStopIndex = currentTailIndex; + auto lastBlockEnqueued = this->tailBlock; + + if (!details::is_trivially_destructible::value) { + auto block = startBlock; + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + block = firstAllocatedBlock; + } + currentTailIndex = startTailIndex; + while (true) { + stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(constructedStopIndex, stopIndex)) { + stopIndex = constructedStopIndex; + } + while (currentTailIndex != stopIndex) { + (*block)[currentTailIndex++]->~T(); + } + if (block == lastBlockEnqueued) { + break; + } + block = block->next; + } + } + + currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { + currentTailIndex += static_cast(BLOCK_SIZE); + auto idxEntry = get_block_index_entry_for_index(currentTailIndex); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + rewind_block_index_tail(); + } + this->parent->add_blocks_to_free_list(firstAllocatedBlock); + this->tailBlock = startBlock; + MOODYCAMEL_RETHROW; + } + } + + if (this->tailBlock == endBlock) { + assert(currentTailIndex == newTailIndex); + break; + } + this->tailBlock = this->tailBlock->next; + } + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + size_t dequeue_bulk(It& itemFirst, size_t max) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + auto desiredCount = static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); + if (details::circular_less_than(0, desiredCount)) { + desiredCount = desiredCount < max ? desiredCount : max; + std::atomic_thread_fence(std::memory_order_acquire); + + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); + assert(overcommit <= myDequeueCount); + + tail = this->tailIndex.load(std::memory_order_acquire); + auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); + if (details::circular_less_than(0, actualCount)) { + actualCount = desiredCount < actualCount ? desiredCount : actualCount; + if (actualCount < desiredCount) { + this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); + } + + // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this + // will never exceed tail. + auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); + + // Iterate the blocks and dequeue + auto index = firstIndex; + BlockIndexHeader* localBlockIndex; + auto indexIndex = get_block_index_index_for_index(index, localBlockIndex); + do { + auto blockStartIndex = index; + auto endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + + auto entry = localBlockIndex->index[indexIndex]; + auto block = entry->value.load(std::memory_order_relaxed); + if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst++ = std::move(el); + el.~T(); + ++index; + } + } + else { + MOODYCAMEL_TRY { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst = std::move(el); + ++itemFirst; + el.~T(); + ++index; + } + } + MOODYCAMEL_CATCH (...) { + do { + entry = localBlockIndex->index[indexIndex]; + block = entry->value.load(std::memory_order_relaxed); + while (index != endIndex) { + (*block)[index++]->~T(); + } + + if (block->ConcurrentQueue::Block::template set_many_empty(blockStartIndex, static_cast(endIndex - blockStartIndex))) { +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + entry->value.store(nullptr, std::memory_order_relaxed); + this->parent->add_block_to_free_list(block); + } + indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); + + blockStartIndex = index; + endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + } while (index != firstIndex + actualCount); + + MOODYCAMEL_RETHROW; + } + } + if (block->ConcurrentQueue::Block::template set_many_empty(blockStartIndex, static_cast(endIndex - blockStartIndex))) { + { +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Note that the set_many_empty above did a release, meaning that anybody who acquires the block + // we're about to free can use it safely since our writes (and reads!) will have happened-before then. + entry->value.store(nullptr, std::memory_order_relaxed); + } + this->parent->add_block_to_free_list(block); // releases the above store + } + indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); + } while (index != firstIndex + actualCount); + + return actualCount; + } + else { + this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); + } + } + + return 0; + } + + private: + // The block size must be > 1, so any number with the low bit set is an invalid block base index + static const index_t INVALID_BLOCK_BASE = 1; + + struct BlockIndexEntry + { + std::atomic key; + std::atomic value; + }; + + struct BlockIndexHeader + { + size_t capacity; + std::atomic tail; + BlockIndexEntry* entries; + BlockIndexEntry** index; + BlockIndexHeader* prev; + }; + + template + inline bool insert_block_index_entry(BlockIndexEntry*& idxEntry, index_t blockStartIndex) + { + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); // We're the only writer thread, relaxed is OK + if (localBlockIndex == nullptr) { + return false; // this can happen if new_block_index failed in the constructor + } + auto newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); + idxEntry = localBlockIndex->index[newTail]; + if (idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE || + idxEntry->value.load(std::memory_order_relaxed) == nullptr) { + + idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); + localBlockIndex->tail.store(newTail, std::memory_order_release); + return true; + } + + // No room in the old block index, try to allocate another one! + if (allocMode == CannotAlloc || !new_block_index()) { + return false; + } + localBlockIndex = blockIndex.load(std::memory_order_relaxed); + newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); + idxEntry = localBlockIndex->index[newTail]; + assert(idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE); + idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); + localBlockIndex->tail.store(newTail, std::memory_order_release); + return true; + } + + inline void rewind_block_index_tail() + { + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); + localBlockIndex->tail.store((localBlockIndex->tail.load(std::memory_order_relaxed) - 1) & (localBlockIndex->capacity - 1), std::memory_order_relaxed); + } + + inline BlockIndexEntry* get_block_index_entry_for_index(index_t index) const + { + BlockIndexHeader* localBlockIndex; + auto idx = get_block_index_index_for_index(index, localBlockIndex); + return localBlockIndex->index[idx]; + } + + inline size_t get_block_index_index_for_index(index_t index, BlockIndexHeader*& localBlockIndex) const + { +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + index &= ~static_cast(BLOCK_SIZE - 1); + localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto tail = localBlockIndex->tail.load(std::memory_order_acquire); + auto tailBase = localBlockIndex->index[tail]->key.load(std::memory_order_relaxed); + assert(tailBase != INVALID_BLOCK_BASE); + // Note: Must use division instead of shift because the index may wrap around, causing a negative + // offset, whose negativity we want to preserve + auto offset = static_cast(static_cast::type>(index - tailBase) / BLOCK_SIZE); + size_t idx = (tail + offset) & (localBlockIndex->capacity - 1); + assert(localBlockIndex->index[idx]->key.load(std::memory_order_relaxed) == index && localBlockIndex->index[idx]->value.load(std::memory_order_relaxed) != nullptr); + return idx; + } + + bool new_block_index() + { + auto prev = blockIndex.load(std::memory_order_relaxed); + size_t prevCapacity = prev == nullptr ? 0 : prev->capacity; + auto entryCount = prev == nullptr ? nextBlockIndexCapacity : prevCapacity; + auto raw = static_cast((Traits::malloc)( + sizeof(BlockIndexHeader) + + std::alignment_of::value - 1 + sizeof(BlockIndexEntry) * entryCount + + std::alignment_of::value - 1 + sizeof(BlockIndexEntry*) * nextBlockIndexCapacity)); + if (raw == nullptr) { + return false; + } + + auto header = new (raw) BlockIndexHeader; + auto entries = reinterpret_cast(details::align_for(raw + sizeof(BlockIndexHeader))); + auto index = reinterpret_cast(details::align_for(reinterpret_cast(entries) + sizeof(BlockIndexEntry) * entryCount)); + if (prev != nullptr) { + auto prevTail = prev->tail.load(std::memory_order_relaxed); + auto prevPos = prevTail; + size_t i = 0; + do { + prevPos = (prevPos + 1) & (prev->capacity - 1); + index[i++] = prev->index[prevPos]; + } while (prevPos != prevTail); + assert(i == prevCapacity); + } + for (size_t i = 0; i != entryCount; ++i) { + new (entries + i) BlockIndexEntry; + entries[i].key.store(INVALID_BLOCK_BASE, std::memory_order_relaxed); + index[prevCapacity + i] = entries + i; + } + header->prev = prev; + header->entries = entries; + header->index = index; + header->capacity = nextBlockIndexCapacity; + header->tail.store((prevCapacity - 1) & (nextBlockIndexCapacity - 1), std::memory_order_relaxed); + + blockIndex.store(header, std::memory_order_release); + + nextBlockIndexCapacity <<= 1; + + return true; + } + + private: + size_t nextBlockIndexCapacity; + std::atomic blockIndex; + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + public: + details::ThreadExitListener threadExitListener; + private: +#endif + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + public: + ImplicitProducer* nextImplicitProducer; + private: +#endif + +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + mutable debug::DebugMutex mutex; +#endif +#if MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + ////////////////////////////////// + // Block pool manipulation + ////////////////////////////////// + + void populate_initial_block_list(size_t blockCount) + { + initialBlockPoolSize = blockCount; + if (initialBlockPoolSize == 0) { + initialBlockPool = nullptr; + return; + } + + initialBlockPool = create_array(blockCount); + if (initialBlockPool == nullptr) { + initialBlockPoolSize = 0; + } + for (size_t i = 0; i < initialBlockPoolSize; ++i) { + initialBlockPool[i].dynamicallyAllocated = false; + } + } + + inline Block* try_get_block_from_initial_pool() + { + if (initialBlockPoolIndex.load(std::memory_order_relaxed) >= initialBlockPoolSize) { + return nullptr; + } + + auto index = initialBlockPoolIndex.fetch_add(1, std::memory_order_relaxed); + + return index < initialBlockPoolSize ? (initialBlockPool + index) : nullptr; + } + + inline void add_block_to_free_list(Block* block) + { +#if MCDBGQ_TRACKMEM + block->owner = nullptr; +#endif + freeList.add(block); + } + + inline void add_blocks_to_free_list(Block* block) + { + while (block != nullptr) { + auto next = block->next; + add_block_to_free_list(block); + block = next; + } + } + + inline Block* try_get_block_from_free_list() + { + return freeList.try_get(); + } + + // Gets a free block from one of the memory pools, or allocates a new one (if applicable) + template + Block* requisition_block() + { + auto block = try_get_block_from_initial_pool(); + if (block != nullptr) { + return block; + } + + block = try_get_block_from_free_list(); + if (block != nullptr) { + return block; + } + + if (canAlloc == CanAlloc) { + return create(); + } + + return nullptr; + } + + +#if MCDBGQ_TRACKMEM + public: + struct MemStats { + size_t allocatedBlocks; + size_t usedBlocks; + size_t freeBlocks; + size_t ownedBlocksExplicit; + size_t ownedBlocksImplicit; + size_t implicitProducers; + size_t explicitProducers; + size_t elementsEnqueued; + size_t blockClassBytes; + size_t queueClassBytes; + size_t implicitBlockIndexBytes; + size_t explicitBlockIndexBytes; + + friend class ConcurrentQueue; + + private: + static MemStats getFor(ConcurrentQueue* q) + { + MemStats stats = { 0 }; + + stats.elementsEnqueued = q->size_approx(); + + auto block = q->freeList.head_unsafe(); + while (block != nullptr) { + ++stats.allocatedBlocks; + ++stats.freeBlocks; + block = block->freeListNext.load(std::memory_order_relaxed); + } + + for (auto ptr = q->producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + bool implicit = dynamic_cast(ptr) != nullptr; + stats.implicitProducers += implicit ? 1 : 0; + stats.explicitProducers += implicit ? 0 : 1; + + if (implicit) { + auto prod = static_cast(ptr); + stats.queueClassBytes += sizeof(ImplicitProducer); + auto head = prod->headIndex.load(std::memory_order_relaxed); + auto tail = prod->tailIndex.load(std::memory_order_relaxed); + auto hash = prod->blockIndex.load(std::memory_order_relaxed); + if (hash != nullptr) { + for (size_t i = 0; i != hash->capacity; ++i) { + if (hash->index[i]->key.load(std::memory_order_relaxed) != ImplicitProducer::INVALID_BLOCK_BASE && hash->index[i]->value.load(std::memory_order_relaxed) != nullptr) { + ++stats.allocatedBlocks; + ++stats.ownedBlocksImplicit; + } + } + stats.implicitBlockIndexBytes += hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry); + for (; hash != nullptr; hash = hash->prev) { + stats.implicitBlockIndexBytes += sizeof(typename ImplicitProducer::BlockIndexHeader) + hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry*); + } + } + for (; details::circular_less_than(head, tail); head += BLOCK_SIZE) { + //auto block = prod->get_block_index_entry_for_index(head); + ++stats.usedBlocks; + } + } + else { + auto prod = static_cast(ptr); + stats.queueClassBytes += sizeof(ExplicitProducer); + auto tailBlock = prod->tailBlock; + bool wasNonEmpty = false; + if (tailBlock != nullptr) { + auto block = tailBlock; + do { + ++stats.allocatedBlocks; + if (!block->ConcurrentQueue::Block::template is_empty() || wasNonEmpty) { + ++stats.usedBlocks; + wasNonEmpty = wasNonEmpty || block != tailBlock; + } + ++stats.ownedBlocksExplicit; + block = block->next; + } while (block != tailBlock); + } + auto index = prod->blockIndex.load(std::memory_order_relaxed); + while (index != nullptr) { + stats.explicitBlockIndexBytes += sizeof(typename ExplicitProducer::BlockIndexHeader) + index->size * sizeof(typename ExplicitProducer::BlockIndexEntry); + index = static_cast(index->prev); + } + } + } + + auto freeOnInitialPool = q->initialBlockPoolIndex.load(std::memory_order_relaxed) >= q->initialBlockPoolSize ? 0 : q->initialBlockPoolSize - q->initialBlockPoolIndex.load(std::memory_order_relaxed); + stats.allocatedBlocks += freeOnInitialPool; + stats.freeBlocks += freeOnInitialPool; + + stats.blockClassBytes = sizeof(Block) * stats.allocatedBlocks; + stats.queueClassBytes += sizeof(ConcurrentQueue); + + return stats; + } + }; + + // For debugging only. Not thread-safe. + MemStats getMemStats() + { + return MemStats::getFor(this); + } + private: + friend struct MemStats; +#endif + + + ////////////////////////////////// + // Producer list manipulation + ////////////////////////////////// + + ProducerBase* recycle_or_create_producer(bool isExplicit) + { + bool recycled; + return recycle_or_create_producer(isExplicit, recycled); + } + + ProducerBase* recycle_or_create_producer(bool isExplicit, bool& recycled) + { +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + // Try to re-use one first + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr->inactive.load(std::memory_order_relaxed) && ptr->isExplicit == isExplicit) { + bool expected = true; + if (ptr->inactive.compare_exchange_strong(expected, /* desired */ false, std::memory_order_acquire, std::memory_order_relaxed)) { + // We caught one! It's been marked as activated, the caller can have it + recycled = true; + return ptr; + } + } + } + + recycled = false; + return add_producer(isExplicit ? static_cast(create(this)) : create(this)); + } + + ProducerBase* add_producer(ProducerBase* producer) + { + // Handle failed memory allocation + if (producer == nullptr) { + return nullptr; + } + + producerCount.fetch_add(1, std::memory_order_relaxed); + + // Add it to the lock-free list + auto prevTail = producerListTail.load(std::memory_order_relaxed); + do { + producer->next = prevTail; + } while (!producerListTail.compare_exchange_weak(prevTail, producer, std::memory_order_release, std::memory_order_relaxed)); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + if (producer->isExplicit) { + auto prevTailExplicit = explicitProducers.load(std::memory_order_relaxed); + do { + static_cast(producer)->nextExplicitProducer = prevTailExplicit; + } while (!explicitProducers.compare_exchange_weak(prevTailExplicit, static_cast(producer), std::memory_order_release, std::memory_order_relaxed)); + } + else { + auto prevTailImplicit = implicitProducers.load(std::memory_order_relaxed); + do { + static_cast(producer)->nextImplicitProducer = prevTailImplicit; + } while (!implicitProducers.compare_exchange_weak(prevTailImplicit, static_cast(producer), std::memory_order_release, std::memory_order_relaxed)); + } +#endif + + return producer; + } + + void reown_producers() + { + // After another instance is moved-into/swapped-with this one, all the + // producers we stole still think their parents are the other queue. + // So fix them up! + for (auto ptr = producerListTail.load(std::memory_order_relaxed); ptr != nullptr; ptr = ptr->next_prod()) { + ptr->parent = this; + } + } + + + ////////////////////////////////// + // Implicit producer hash + ////////////////////////////////// + + struct ImplicitProducerKVP + { + std::atomic key; + ImplicitProducer* value; // No need for atomicity since it's only read by the thread that sets it in the first place + + ImplicitProducerKVP() : value(nullptr) { } + + ImplicitProducerKVP(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT + { + key.store(other.key.load(std::memory_order_relaxed), std::memory_order_relaxed); + value = other.value; + } + + inline ImplicitProducerKVP& operator=(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + inline void swap(ImplicitProducerKVP& other) MOODYCAMEL_NOEXCEPT + { + if (this != &other) { + details::swap_relaxed(key, other.key); + std::swap(value, other.value); + } + } + }; + + template + friend void moodycamel::swap(typename ConcurrentQueue::ImplicitProducerKVP&, typename ConcurrentQueue::ImplicitProducerKVP&) MOODYCAMEL_NOEXCEPT; + + struct ImplicitProducerHash + { + size_t capacity; + ImplicitProducerKVP* entries; + ImplicitProducerHash* prev; + }; + + inline void populate_initial_implicit_producer_hash() + { + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return; + + implicitProducerHashCount.store(0, std::memory_order_relaxed); + auto hash = &initialImplicitProducerHash; + hash->capacity = INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; + hash->entries = &initialImplicitProducerHashEntries[0]; + for (size_t i = 0; i != INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; ++i) { + initialImplicitProducerHashEntries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); + } + hash->prev = nullptr; + implicitProducerHash.store(hash, std::memory_order_relaxed); + } + + void swap_implicit_producer_hashes(ConcurrentQueue& other) + { + if (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return; + + // Swap (assumes our implicit producer hash is initialized) + initialImplicitProducerHashEntries.swap(other.initialImplicitProducerHashEntries); + initialImplicitProducerHash.entries = &initialImplicitProducerHashEntries[0]; + other.initialImplicitProducerHash.entries = &other.initialImplicitProducerHashEntries[0]; + + details::swap_relaxed(implicitProducerHashCount, other.implicitProducerHashCount); + + details::swap_relaxed(implicitProducerHash, other.implicitProducerHash); + if (implicitProducerHash.load(std::memory_order_relaxed) == &other.initialImplicitProducerHash) { + implicitProducerHash.store(&initialImplicitProducerHash, std::memory_order_relaxed); + } + else { + ImplicitProducerHash* hash; + for (hash = implicitProducerHash.load(std::memory_order_relaxed); hash->prev != &other.initialImplicitProducerHash; hash = hash->prev) { + continue; + } + hash->prev = &initialImplicitProducerHash; + } + if (other.implicitProducerHash.load(std::memory_order_relaxed) == &initialImplicitProducerHash) { + other.implicitProducerHash.store(&other.initialImplicitProducerHash, std::memory_order_relaxed); + } + else { + ImplicitProducerHash* hash; + for (hash = other.implicitProducerHash.load(std::memory_order_relaxed); hash->prev != &initialImplicitProducerHash; hash = hash->prev) { + continue; + } + hash->prev = &other.initialImplicitProducerHash; + } + } + + // Only fails (returns nullptr) if memory allocation fails + ImplicitProducer* get_or_add_implicit_producer() + { + // Note that since the data is essentially thread-local (key is thread ID), + // there's a reduced need for fences (memory ordering is already consistent + // for any individual thread), except for the current table itself. + + // Start by looking for the thread ID in the current and all previous hash tables. + // If it's not found, it must not be in there yet, since this same thread would + // have added it previously to one of the tables that we traversed. + + // Code and algorithm adapted from http://preshing.com/20130605/the-worlds-simplest-lock-free-hash-table + +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + + auto id = details::thread_id(); + auto hashedId = details::hash_thread_id(id); + + auto mainHash = implicitProducerHash.load(std::memory_order_acquire); + for (auto hash = mainHash; hash != nullptr; hash = hash->prev) { + // Look for the id in this hash + auto index = hashedId; + while (true) { // Not an infinite loop because at least one slot is free in the hash table + index &= hash->capacity - 1; + + auto probedKey = hash->entries[index].key.load(std::memory_order_relaxed); + if (probedKey == id) { + // Found it! If we had to search several hashes deep, though, we should lazily add it + // to the current main hash table to avoid the extended search next time. + // Note there's guaranteed to be room in the current hash table since every subsequent + // table implicitly reserves space for all previous tables (there's only one + // implicitProducerHashCount). + auto value = hash->entries[index].value; + if (hash != mainHash) { + index = hashedId; + while (true) { + index &= mainHash->capacity - 1; + probedKey = mainHash->entries[index].key.load(std::memory_order_relaxed); + auto empty = details::invalid_thread_id; +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + auto reusable = details::invalid_thread_id2; + if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_relaxed, std::memory_order_relaxed)) || + (probedKey == reusable && mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_acquire, std::memory_order_acquire))) { +#else + if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_relaxed, std::memory_order_relaxed))) { +#endif + mainHash->entries[index].value = value; + break; + } + ++index; + } + } + + return value; + } + if (probedKey == details::invalid_thread_id) { + break; // Not in this hash table + } + ++index; + } + } + + // Insert! + auto newCount = 1 + implicitProducerHashCount.fetch_add(1, std::memory_order_relaxed); + while (true) { + if (newCount >= (mainHash->capacity >> 1) && !implicitProducerHashResizeInProgress.test_and_set(std::memory_order_acquire)) { + // We've acquired the resize lock, try to allocate a bigger hash table. + // Note the acquire fence synchronizes with the release fence at the end of this block, and hence when + // we reload implicitProducerHash it must be the most recent version (it only gets changed within this + // locked block). + mainHash = implicitProducerHash.load(std::memory_order_acquire); + if (newCount >= (mainHash->capacity >> 1)) { + auto newCapacity = mainHash->capacity << 1; + while (newCount >= (newCapacity >> 1)) { + newCapacity <<= 1; + } + auto raw = static_cast((Traits::malloc)(sizeof(ImplicitProducerHash) + std::alignment_of::value - 1 + sizeof(ImplicitProducerKVP) * newCapacity)); + if (raw == nullptr) { + // Allocation failed + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + return nullptr; + } + + auto newHash = new (raw) ImplicitProducerHash; + newHash->capacity = newCapacity; + newHash->entries = reinterpret_cast(details::align_for(raw + sizeof(ImplicitProducerHash))); + for (size_t i = 0; i != newCapacity; ++i) { + new (newHash->entries + i) ImplicitProducerKVP; + newHash->entries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); + } + newHash->prev = mainHash; + implicitProducerHash.store(newHash, std::memory_order_release); + implicitProducerHashResizeInProgress.clear(std::memory_order_release); + mainHash = newHash; + } + else { + implicitProducerHashResizeInProgress.clear(std::memory_order_release); + } + } + + // If it's < three-quarters full, add to the old one anyway so that we don't have to wait for the next table + // to finish being allocated by another thread (and if we just finished allocating above, the condition will + // always be true) + if (newCount < (mainHash->capacity >> 1) + (mainHash->capacity >> 2)) { + bool recycled; + auto producer = static_cast(recycle_or_create_producer(false, recycled)); + if (producer == nullptr) { + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); + return nullptr; + } + if (recycled) { + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); + } + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + producer->threadExitListener.callback = &ConcurrentQueue::implicit_producer_thread_exited_callback; + producer->threadExitListener.userData = producer; + details::ThreadExitNotifier::subscribe(&producer->threadExitListener); +#endif + + auto index = hashedId; + while (true) { + index &= mainHash->capacity - 1; + auto probedKey = mainHash->entries[index].key.load(std::memory_order_relaxed); + + auto empty = details::invalid_thread_id; +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + auto reusable = details::invalid_thread_id2; + if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_relaxed, std::memory_order_relaxed)) || + (probedKey == reusable && mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_acquire, std::memory_order_acquire))) { +#else + if ((probedKey == empty && mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_relaxed, std::memory_order_relaxed))) { +#endif + mainHash->entries[index].value = producer; + break; + } + ++index; + } + return producer; + } + + // Hmm, the old hash is quite full and somebody else is busy allocating a new one. + // We need to wait for the allocating thread to finish (if it succeeds, we add, if not, + // we try to allocate ourselves). + mainHash = implicitProducerHash.load(std::memory_order_acquire); + } + } + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + void implicit_producer_thread_exited(ImplicitProducer* producer) + { + // Remove from thread exit listeners + details::ThreadExitNotifier::unsubscribe(&producer->threadExitListener); + + // Remove from hash +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + auto hash = implicitProducerHash.load(std::memory_order_acquire); + assert(hash != nullptr); // The thread exit listener is only registered if we were added to a hash in the first place + auto id = details::thread_id(); + auto hashedId = details::hash_thread_id(id); + details::thread_id_t probedKey; + + // We need to traverse all the hashes just in case other threads aren't on the current one yet and are + // trying to add an entry thinking there's a free slot (because they reused a producer) + for (; hash != nullptr; hash = hash->prev) { + auto index = hashedId; + do { + index &= hash->capacity - 1; + probedKey = hash->entries[index].key.load(std::memory_order_relaxed); + if (probedKey == id) { + hash->entries[index].key.store(details::invalid_thread_id2, std::memory_order_release); + break; + } + ++index; + } while (probedKey != details::invalid_thread_id); // Can happen if the hash has changed but we weren't put back in it yet, or if we weren't added to this hash in the first place + } + + // Mark the queue as being recyclable + producer->inactive.store(true, std::memory_order_release); + } + + static void implicit_producer_thread_exited_callback(void* userData) + { + auto producer = static_cast(userData); + auto queue = producer->parent; + queue->implicit_producer_thread_exited(producer); + } +#endif + + ////////////////////////////////// + // Utility functions + ////////////////////////////////// + + template + static inline U* create_array(size_t count) + { + assert(count > 0); + auto p = static_cast((Traits::malloc)(sizeof(U) * count)); + if (p == nullptr) { + return nullptr; + } + + for (size_t i = 0; i != count; ++i) { + new (p + i) U(); + } + return p; + } + + template + static inline void destroy_array(U* p, size_t count) + { + if (p != nullptr) { + assert(count > 0); + for (size_t i = count; i != 0; ) { + (p + --i)->~U(); + } + (Traits::free)(p); + } + } + + template + static inline U* create() + { + auto p = (Traits::malloc)(sizeof(U)); + return p != nullptr ? new (p) U : nullptr; + } + + template + static inline U* create(A1&& a1) + { + auto p = (Traits::malloc)(sizeof(U)); + return p != nullptr ? new (p) U(std::forward(a1)) : nullptr; + } + + template + static inline void destroy(U* p) + { + if (p != nullptr) { + p->~U(); + } + (Traits::free)(p); + } + +private: + std::atomic producerListTail; + std::atomic producerCount; + + std::atomic initialBlockPoolIndex; + Block* initialBlockPool; + size_t initialBlockPoolSize; + +#if !MCDBGQ_USEDEBUGFREELIST + FreeList freeList; +#else + debug::DebugFreeList freeList; +#endif + + std::atomic implicitProducerHash; + std::atomic implicitProducerHashCount; // Number of slots logically used + ImplicitProducerHash initialImplicitProducerHash; + std::array initialImplicitProducerHashEntries; + std::atomic_flag implicitProducerHashResizeInProgress; + + std::atomic nextExplicitConsumerId; + std::atomic globalExplicitConsumerOffset; + +#if MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugMutex implicitProdMutex; +#endif + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + std::atomic explicitProducers; + std::atomic implicitProducers; +#endif +}; + + +template +ProducerToken::ProducerToken(ConcurrentQueue& queue) + : producer(queue.recycle_or_create_producer(true)) +{ + if (producer != nullptr) { + producer->token = this; + } +} + +template +ProducerToken::ProducerToken(BlockingConcurrentQueue& queue) + : producer(reinterpret_cast*>(&queue)->recycle_or_create_producer(true)) +{ + if (producer != nullptr) { + producer->token = this; + } +} + +template +ConsumerToken::ConsumerToken(ConcurrentQueue& queue) + : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) +{ + initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release); + lastKnownGlobalOffset = -1; +} + +template +ConsumerToken::ConsumerToken(BlockingConcurrentQueue& queue) + : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) +{ + initialOffset = reinterpret_cast*>(&queue)->nextExplicitConsumerId.fetch_add(1, std::memory_order_release); + lastKnownGlobalOffset = -1; +} + +template +inline void swap(ConcurrentQueue& a, ConcurrentQueue& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +inline void swap(ProducerToken& a, ProducerToken& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +inline void swap(ConsumerToken& a, ConsumerToken& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +template +inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +} + +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif diff --git a/linux/include/moodycamel/readerwriterqueue.h b/linux/include/moodycamel/readerwriterqueue.h new file mode 100644 index 000000000..ec465d694 --- /dev/null +++ b/linux/include/moodycamel/readerwriterqueue.h @@ -0,0 +1,815 @@ +// ©2013-2016 Cameron Desrochers. +// Distributed under the simplified BSD license (see the license file that +// should have come with this header). + +#pragma once + +#include "atomicops.h" +#include +#include +#include +#include +#include +#include +#include // For malloc/free/abort & size_t +#if __cplusplus > 199711L || _MSC_VER >= 1700 // C++11 or VS2012 +#include +#endif + + +// A lock-free queue for a single-consumer, single-producer architecture. +// The queue is also wait-free in the common path (except if more memory +// needs to be allocated, in which case malloc is called). +// Allocates memory sparingly (O(lg(n) times, amortized), and only once if +// the original maximum size estimate is never exceeded. +// Tested on x86/x64 processors, but semantics should be correct for all +// architectures (given the right implementations in atomicops.h), provided +// that aligned integer and pointer accesses are naturally atomic. +// Note that there should only be one consumer thread and producer thread; +// Switching roles of the threads, or using multiple consecutive threads for +// one role, is not safe unless properly synchronized. +// Using the queue exclusively from one thread is fine, though a bit silly. + +#ifndef MOODYCAMEL_CACHE_LINE_SIZE +#define MOODYCAMEL_CACHE_LINE_SIZE 64 +#endif + +#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED +#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__)) +#define MOODYCAMEL_EXCEPTIONS_ENABLED +#endif +#endif + +#ifdef AE_VCPP +#pragma warning(push) +#pragma warning(disable: 4324) // structure was padded due to __declspec(align()) +#pragma warning(disable: 4820) // padding was added +#pragma warning(disable: 4127) // conditional expression is constant +#endif + +namespace moodycamel { + +template +class ReaderWriterQueue +{ + // Design: Based on a queue-of-queues. The low-level queues are just + // circular buffers with front and tail indices indicating where the + // next element to dequeue is and where the next element can be enqueued, + // respectively. Each low-level queue is called a "block". Each block + // wastes exactly one element's worth of space to keep the design simple + // (if front == tail then the queue is empty, and can't be full). + // The high-level queue is a circular linked list of blocks; again there + // is a front and tail, but this time they are pointers to the blocks. + // The front block is where the next element to be dequeued is, provided + // the block is not empty. The back block is where elements are to be + // enqueued, provided the block is not full. + // The producer thread owns all the tail indices/pointers. The consumer + // thread owns all the front indices/pointers. Both threads read each + // other's variables, but only the owning thread updates them. E.g. After + // the consumer reads the producer's tail, the tail may change before the + // consumer is done dequeuing an object, but the consumer knows the tail + // will never go backwards, only forwards. + // If there is no room to enqueue an object, an additional block (of + // equal size to the last block) is added. Blocks are never removed. + +public: + // Constructs a queue that can hold maxSize elements without further + // allocations. If more than MAX_BLOCK_SIZE elements are requested, + // then several blocks of MAX_BLOCK_SIZE each are reserved (including + // at least one extra buffer block). + explicit ReaderWriterQueue(size_t maxSize = 15) +#ifndef NDEBUG + : enqueuing(false) + ,dequeuing(false) +#endif + { + assert(maxSize > 0); + assert(MAX_BLOCK_SIZE == ceilToPow2(MAX_BLOCK_SIZE) && "MAX_BLOCK_SIZE must be a power of 2"); + assert(MAX_BLOCK_SIZE >= 2 && "MAX_BLOCK_SIZE must be at least 2"); + + Block* firstBlock = nullptr; + + largestBlockSize = ceilToPow2(maxSize + 1); // We need a spare slot to fit maxSize elements in the block + if (largestBlockSize > MAX_BLOCK_SIZE * 2) { + // We need a spare block in case the producer is writing to a different block the consumer is reading from, and + // wants to enqueue the maximum number of elements. We also need a spare element in each block to avoid the ambiguity + // between front == tail meaning "empty" and "full". + // So the effective number of slots that are guaranteed to be usable at any time is the block size - 1 times the + // number of blocks - 1. Solving for maxSize and applying a ceiling to the division gives us (after simplifying): + size_t initialBlockCount = (maxSize + MAX_BLOCK_SIZE * 2 - 3) / (MAX_BLOCK_SIZE - 1); + largestBlockSize = MAX_BLOCK_SIZE; + Block* lastBlock = nullptr; + for (size_t i = 0; i != initialBlockCount; ++i) { + auto block = make_block(largestBlockSize); + if (block == nullptr) { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED + throw std::bad_alloc(); +#else + abort(); +#endif + } + if (firstBlock == nullptr) { + firstBlock = block; + } + else { + lastBlock->next = block; + } + lastBlock = block; + block->next = firstBlock; + } + } + else { + firstBlock = make_block(largestBlockSize); + if (firstBlock == nullptr) { +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED + throw std::bad_alloc(); +#else + abort(); +#endif + } + firstBlock->next = firstBlock; + } + frontBlock = firstBlock; + tailBlock = firstBlock; + + // Make sure the reader/writer threads will have the initialized memory setup above: + fence(memory_order_sync); + } + + // Note: The queue should not be accessed concurrently while it's + // being deleted. It's up to the user to synchronize this. + ~ReaderWriterQueue() + { + // Make sure we get the latest version of all variables from other CPUs: + fence(memory_order_sync); + + // Destroy any remaining objects in queue and free memory + Block* frontBlock_ = frontBlock; + Block* block = frontBlock_; + do { + Block* nextBlock = block->next; + size_t blockFront = block->front; + size_t blockTail = block->tail; + + for (size_t i = blockFront; i != blockTail; i = (i + 1) & block->sizeMask) { + auto element = reinterpret_cast(block->data + i * sizeof(T)); + element->~T(); + (void)element; + } + + auto rawBlock = block->rawThis; + block->~Block(); + std::free(rawBlock); + block = nextBlock; + } while (block != frontBlock_); + } + + + // Enqueues a copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T const& element) + { + return inner_enqueue(element); + } + + // Enqueues a moved copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T&& element) + { + return inner_enqueue(std::forward(element)); + } + + + // Enqueues a copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T const& element) + { + return inner_enqueue(element); + } + + // Enqueues a moved copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T&& element) + { + return inner_enqueue(std::forward(element)); + } + + + // Attempts to dequeue an element; if the queue is empty, + // returns false instead. If the queue has at least one element, + // moves front to result using operator=, then returns true. + template + bool try_dequeue(U& result) + { +#ifndef NDEBUG + ReentrantGuard guard(this->dequeuing); +#endif + + // High-level pseudocode: + // Remember where the tail block is + // If the front block has an element in it, dequeue it + // Else + // If front block was the tail block when we entered the function, return false + // Else advance to next block and dequeue the item there + + // Note that we have to use the value of the tail block from before we check if the front + // block is full or not, in case the front block is empty and then, before we check if the + // tail block is at the front block or not, the producer fills up the front block *and + // moves on*, which would make us skip a filled block. Seems unlikely, but was consistently + // reproducible in practice. + // In order to avoid overhead in the common case, though, we do a double-checked pattern + // where we have the fast path if the front block is not empty, then read the tail block, + // then re-read the front block and check if it's not empty again, then check if the tail + // block has advanced. + + Block* frontBlock_ = frontBlock.load(); + size_t blockTail = frontBlock_->localTail; + size_t blockFront = frontBlock_->front.load(); + + if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { + fence(memory_order_acquire); + + non_empty_front_block: + // Front block not empty, dequeue from here + auto element = reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); + result = std::move(*element); + element->~T(); + + blockFront = (blockFront + 1) & frontBlock_->sizeMask; + + fence(memory_order_release); + frontBlock_->front = blockFront; + } + else if (frontBlock_ != tailBlock.load()) { + fence(memory_order_acquire); + + frontBlock_ = frontBlock.load(); + blockTail = frontBlock_->localTail = frontBlock_->tail.load(); + blockFront = frontBlock_->front.load(); + fence(memory_order_acquire); + + if (blockFront != blockTail) { + // Oh look, the front block isn't empty after all + goto non_empty_front_block; + } + + // Front block is empty but there's another block ahead, advance to it + Block* nextBlock = frontBlock_->next; + // Don't need an acquire fence here since next can only ever be set on the tailBlock, + // and we're not the tailBlock, and we did an acquire earlier after reading tailBlock which + // ensures next is up-to-date on this CPU in case we recently were at tailBlock. + + size_t nextBlockFront = nextBlock->front.load(); + size_t nextBlockTail = nextBlock->localTail = nextBlock->tail.load(); + fence(memory_order_acquire); + + // Since the tailBlock is only ever advanced after being written to, + // we know there's for sure an element to dequeue on it + assert(nextBlockFront != nextBlockTail); + AE_UNUSED(nextBlockTail); + + // We're done with this block, let the producer use it if it needs + fence(memory_order_release); // Expose possibly pending changes to frontBlock->front from last dequeue + frontBlock = frontBlock_ = nextBlock; + + compiler_fence(memory_order_release); // Not strictly needed + + auto element = reinterpret_cast(frontBlock_->data + nextBlockFront * sizeof(T)); + + result = std::move(*element); + element->~T(); + + nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask; + + fence(memory_order_release); + frontBlock_->front = nextBlockFront; + } + else { + // No elements in current block and no other block to advance to + return false; + } + + return true; + } + + + // Returns a pointer to the front element in the queue (the one that + // would be removed next by a call to `try_dequeue` or `pop`). If the + // queue appears empty at the time the method is called, nullptr is + // returned instead. + // Must be called only from the consumer thread. + T* peek() + { +#ifndef NDEBUG + ReentrantGuard guard(this->dequeuing); +#endif + // See try_dequeue() for reasoning + + Block* frontBlock_ = frontBlock.load(); + size_t blockTail = frontBlock_->localTail; + size_t blockFront = frontBlock_->front.load(); + + if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { + fence(memory_order_acquire); + non_empty_front_block: + return reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); + } + else if (frontBlock_ != tailBlock.load()) { + fence(memory_order_acquire); + frontBlock_ = frontBlock.load(); + blockTail = frontBlock_->localTail = frontBlock_->tail.load(); + blockFront = frontBlock_->front.load(); + fence(memory_order_acquire); + + if (blockFront != blockTail) { + goto non_empty_front_block; + } + + Block* nextBlock = frontBlock_->next; + + size_t nextBlockFront = nextBlock->front.load(); + fence(memory_order_acquire); + + assert(nextBlockFront != nextBlock->tail.load()); + return reinterpret_cast(nextBlock->data + nextBlockFront * sizeof(T)); + } + + return nullptr; + } + + // Removes the front element from the queue, if any, without returning it. + // Returns true on success, or false if the queue appeared empty at the time + // `pop` was called. + bool pop() + { +#ifndef NDEBUG + ReentrantGuard guard(this->dequeuing); +#endif + // See try_dequeue() for reasoning + + Block* frontBlock_ = frontBlock.load(); + size_t blockTail = frontBlock_->localTail; + size_t blockFront = frontBlock_->front.load(); + + if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) { + fence(memory_order_acquire); + + non_empty_front_block: + auto element = reinterpret_cast(frontBlock_->data + blockFront * sizeof(T)); + element->~T(); + + blockFront = (blockFront + 1) & frontBlock_->sizeMask; + + fence(memory_order_release); + frontBlock_->front = blockFront; + } + else if (frontBlock_ != tailBlock.load()) { + fence(memory_order_acquire); + frontBlock_ = frontBlock.load(); + blockTail = frontBlock_->localTail = frontBlock_->tail.load(); + blockFront = frontBlock_->front.load(); + fence(memory_order_acquire); + + if (blockFront != blockTail) { + goto non_empty_front_block; + } + + // Front block is empty but there's another block ahead, advance to it + Block* nextBlock = frontBlock_->next; + + size_t nextBlockFront = nextBlock->front.load(); + size_t nextBlockTail = nextBlock->localTail = nextBlock->tail.load(); + fence(memory_order_acquire); + + assert(nextBlockFront != nextBlockTail); + AE_UNUSED(nextBlockTail); + + fence(memory_order_release); + frontBlock = frontBlock_ = nextBlock; + + compiler_fence(memory_order_release); + + auto element = reinterpret_cast(frontBlock_->data + nextBlockFront * sizeof(T)); + element->~T(); + + nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask; + + fence(memory_order_release); + frontBlock_->front = nextBlockFront; + } + else { + // No elements in current block and no other block to advance to + return false; + } + + return true; + } + + // Returns the approximate number of items currently in the queue. + // Safe to call from both the producer and consumer threads. + inline size_t size_approx() const + { + size_t result = 0; + Block* frontBlock_ = frontBlock.load(); + Block* block = frontBlock_; + do { + fence(memory_order_acquire); + size_t blockFront = block->front.load(); + size_t blockTail = block->tail.load(); + result += (blockTail - blockFront) & block->sizeMask; + block = block->next.load(); + } while (block != frontBlock_); + return result; + } + + +private: + enum AllocationMode { CanAlloc, CannotAlloc }; + + template + bool inner_enqueue(U&& element) + { +#ifndef NDEBUG + ReentrantGuard guard(this->enqueuing); +#endif + + // High-level pseudocode (assuming we're allowed to alloc a new block): + // If room in tail block, add to tail + // Else check next block + // If next block is not the head block, enqueue on next block + // Else create a new block and enqueue there + // Advance tail to the block we just enqueued to + + Block* tailBlock_ = tailBlock.load(); + size_t blockFront = tailBlock_->localFront; + size_t blockTail = tailBlock_->tail.load(); + + size_t nextBlockTail = (blockTail + 1) & tailBlock_->sizeMask; + if (nextBlockTail != blockFront || nextBlockTail != (tailBlock_->localFront = tailBlock_->front.load())) { + fence(memory_order_acquire); + // This block has room for at least one more element + char* location = tailBlock_->data + blockTail * sizeof(T); + new (location) T(std::forward(element)); + + fence(memory_order_release); + tailBlock_->tail = nextBlockTail; + } + else { + fence(memory_order_acquire); + if (tailBlock_->next.load() != frontBlock) { + // Note that the reason we can't advance to the frontBlock and start adding new entries there + // is because if we did, then dequeue would stay in that block, eventually reading the new values, + // instead of advancing to the next full block (whose values were enqueued first and so should be + // consumed first). + + fence(memory_order_acquire); // Ensure we get latest writes if we got the latest frontBlock + + // tailBlock is full, but there's a free block ahead, use it + Block* tailBlockNext = tailBlock_->next.load(); + size_t nextBlockFront = tailBlockNext->localFront = tailBlockNext->front.load(); + nextBlockTail = tailBlockNext->tail.load(); + fence(memory_order_acquire); + + // This block must be empty since it's not the head block and we + // go through the blocks in a circle + assert(nextBlockFront == nextBlockTail); + tailBlockNext->localFront = nextBlockFront; + + char* location = tailBlockNext->data + nextBlockTail * sizeof(T); + new (location) T(std::forward(element)); + + tailBlockNext->tail = (nextBlockTail + 1) & tailBlockNext->sizeMask; + + fence(memory_order_release); + tailBlock = tailBlockNext; + } + else if (canAlloc == CanAlloc) { + // tailBlock is full and there's no free block ahead; create a new block + auto newBlockSize = largestBlockSize >= MAX_BLOCK_SIZE ? largestBlockSize : largestBlockSize * 2; + auto newBlock = make_block(newBlockSize); + if (newBlock == nullptr) { + // Could not allocate a block! + return false; + } + largestBlockSize = newBlockSize; + + new (newBlock->data) T(std::forward(element)); + + assert(newBlock->front == 0); + newBlock->tail = newBlock->localTail = 1; + + newBlock->next = tailBlock_->next.load(); + tailBlock_->next = newBlock; + + // Might be possible for the dequeue thread to see the new tailBlock->next + // *without* seeing the new tailBlock value, but this is OK since it can't + // advance to the next block until tailBlock is set anyway (because the only + // case where it could try to read the next is if it's already at the tailBlock, + // and it won't advance past tailBlock in any circumstance). + + fence(memory_order_release); + tailBlock = newBlock; + } + else if (canAlloc == CannotAlloc) { + // Would have had to allocate a new block to enqueue, but not allowed + return false; + } + else { + assert(false && "Should be unreachable code"); + return false; + } + } + + return true; + } + + + // Disable copying + ReaderWriterQueue(ReaderWriterQueue const&) { } + + // Disable assignment + ReaderWriterQueue& operator=(ReaderWriterQueue const&) { } + + + + AE_FORCEINLINE static size_t ceilToPow2(size_t x) + { + // From http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + --x; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + for (size_t i = 1; i < sizeof(size_t); i <<= 1) { + x |= x >> (i << 3); + } + ++x; + return x; + } + + template + static AE_FORCEINLINE char* align_for(char* ptr) + { + const std::size_t alignment = std::alignment_of::value; + return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; + } +private: +#ifndef NDEBUG + struct ReentrantGuard + { + ReentrantGuard(bool& _inSection) + : inSection(_inSection) + { + assert(!inSection && "ReaderWriterQueue does not support enqueuing or dequeuing elements from other elements' ctors and dtors"); + inSection = true; + } + + ~ReentrantGuard() { inSection = false; } + + private: + ReentrantGuard& operator=(ReentrantGuard const&); + + private: + bool& inSection; + }; +#endif + + struct Block + { + // Avoid false-sharing by putting highly contended variables on their own cache lines + weak_atomic front; // (Atomic) Elements are read from here + size_t localTail; // An uncontended shadow copy of tail, owned by the consumer + + char cachelineFiller0[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic) - sizeof(size_t)]; + weak_atomic tail; // (Atomic) Elements are enqueued here + size_t localFront; + + char cachelineFiller1[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic) - sizeof(size_t)]; // next isn't very contended, but we don't want it on the same cache line as tail (which is) + weak_atomic next; // (Atomic) + + char* data; // Contents (on heap) are aligned to T's alignment + + const size_t sizeMask; + + + // size must be a power of two (and greater than 0) + Block(size_t const& _size, char* _rawThis, char* _data) + : front(0), localTail(0), tail(0), localFront(0), next(nullptr), data(_data), sizeMask(_size - 1), rawThis(_rawThis) + { + } + + private: + // C4512 - Assignment operator could not be generated + Block& operator=(Block const&); + + public: + char* rawThis; + }; + + + static Block* make_block(size_t capacity) + { + // Allocate enough memory for the block itself, as well as all the elements it will contain + auto size = sizeof(Block) + std::alignment_of::value - 1; + size += sizeof(T) * capacity + std::alignment_of::value - 1; + auto newBlockRaw = static_cast(std::malloc(size)); + if (newBlockRaw == nullptr) { + return nullptr; + } + + auto newBlockAligned = align_for(newBlockRaw); + auto newBlockData = align_for(newBlockAligned + sizeof(Block)); + return new (newBlockAligned) Block(capacity, newBlockRaw, newBlockData); + } + +private: + weak_atomic frontBlock; // (Atomic) Elements are enqueued to this block + + char cachelineFiller[MOODYCAMEL_CACHE_LINE_SIZE - sizeof(weak_atomic)]; + weak_atomic tailBlock; // (Atomic) Elements are dequeued from this block + + size_t largestBlockSize; + +#ifndef NDEBUG + bool enqueuing; + bool dequeuing; +#endif +}; + +// Like ReaderWriterQueue, but also providees blocking operations +template +class BlockingReaderWriterQueue +{ +private: + typedef ::moodycamel::ReaderWriterQueue ReaderWriterQueue; + +public: + explicit BlockingReaderWriterQueue(size_t maxSize = 15) + : inner(maxSize) + { } + + + // Enqueues a copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T const& element) + { + if (inner.try_enqueue(element)) { + sema.signal(); + return true; + } + return false; + } + + // Enqueues a moved copy of element if there is room in the queue. + // Returns true if the element was enqueued, false otherwise. + // Does not allocate memory. + AE_FORCEINLINE bool try_enqueue(T&& element) + { + if (inner.try_enqueue(std::forward(element))) { + sema.signal(); + return true; + } + return false; + } + + + // Enqueues a copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T const& element) + { + if (inner.enqueue(element)) { + sema.signal(); + return true; + } + return false; + } + + // Enqueues a moved copy of element on the queue. + // Allocates an additional block of memory if needed. + // Only fails (returns false) if memory allocation fails. + AE_FORCEINLINE bool enqueue(T&& element) + { + if (inner.enqueue(std::forward(element))) { + sema.signal(); + return true; + } + return false; + } + + + // Attempts to dequeue an element; if the queue is empty, + // returns false instead. If the queue has at least one element, + // moves front to result using operator=, then returns true. + template + bool try_dequeue(U& result) + { + if (sema.tryWait()) { + bool success = inner.try_dequeue(result); + assert(success); + AE_UNUSED(success); + return true; + } + return false; + } + + + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available, then dequeues it. + template + void wait_dequeue(U& result) + { + sema.wait(); + bool success = inner.try_dequeue(result); + AE_UNUSED(result); + assert(success); + AE_UNUSED(success); + } + + + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available up to the specified timeout, + // then dequeues it and returns true, or returns false if the timeout + // expires before an element can be dequeued. + // Using a negative timeout indicates an indefinite timeout, + // and is thus functionally equivalent to calling wait_dequeue. + template + bool wait_dequeue_timed(U& result, std::int64_t timeout_usecs) + { + if (!sema.wait(timeout_usecs)) { + return false; + } + bool success = inner.try_dequeue(result); + AE_UNUSED(result); + assert(success); + AE_UNUSED(success); + return true; + } + + +#if __cplusplus > 199711L || _MSC_VER >= 1700 + // Attempts to dequeue an element; if the queue is empty, + // waits until an element is available up to the specified timeout, + // then dequeues it and returns true, or returns false if the timeout + // expires before an element can be dequeued. + // Using a negative timeout indicates an indefinite timeout, + // and is thus functionally equivalent to calling wait_dequeue. + template + inline bool wait_dequeue_timed(U& result, std::chrono::duration const& timeout) + { + return wait_dequeue_timed(result, std::chrono::duration_cast(timeout).count()); + } +#endif + + + // Returns a pointer to the front element in the queue (the one that + // would be removed next by a call to `try_dequeue` or `pop`). If the + // queue appears empty at the time the method is called, nullptr is + // returned instead. + // Must be called only from the consumer thread. + AE_FORCEINLINE T* peek() + { + return inner.peek(); + } + + // Removes the front element from the queue, if any, without returning it. + // Returns true on success, or false if the queue appeared empty at the time + // `pop` was called. + AE_FORCEINLINE bool pop() + { + if (sema.tryWait()) { + bool result = inner.pop(); + assert(result); + AE_UNUSED(result); + return true; + } + return false; + } + + // Returns the approximate number of items currently in the queue. + // Safe to call from both the producer and consumer threads. + AE_FORCEINLINE size_t size_approx() const + { + return sema.availableApprox(); + } + + +private: + // Disable copying & assignment + BlockingReaderWriterQueue(ReaderWriterQueue const&) { } + BlockingReaderWriterQueue& operator=(ReaderWriterQueue const&) { } + +private: + ReaderWriterQueue inner; + spsc_sema::LightweightSemaphore sema; +}; + +} // end namespace moodycamel + +#ifdef AE_VCPP +#pragma warning(pop) +#endif diff --git a/linux/include/sqlite3.h b/linux/include/sqlite/sqlite3.h similarity index 99% rename from linux/include/sqlite3.h rename to linux/include/sqlite/sqlite3.h index 4193d370a..4c109b2b8 100644 --- a/linux/include/sqlite3.h +++ b/linux/include/sqlite/sqlite3.h @@ -123,7 +123,7 @@ extern "C" { */ #define SQLITE_VERSION "3.20.0" #define SQLITE_VERSION_NUMBER 3020000 -#define SQLITE_SOURCE_ID "2017-08-08 21:31:02 628dcd380a1bd35e22f52375d89a0bc2166ca07a" +#define SQLITE_SOURCE_ID "2017-08-08 22:48:04 a78d932165bd7dc2ac9042c93762c4dcc22bf1ed" /* ** CAPI3REF: Run-Time Library Version Numbers diff --git a/linux/lib/debug/libsqlite3.a b/linux/lib/debug/libsqlite3.a index 3a37b29b685f42fafadeb39a86154f289887a325..1502eb458b9c10c4dccdc59c5dac0f0892685b9e 100644 GIT binary patch delta 574 zcmWm6S5Crk0Kj2XR9v|C!o3y#EtKNkdy9KP3+}x)CQV4ZfEO|8L44r^oWq|F-|H9u zjauV~7zswAkz`nnWFy5${jq(SlJLo1pHp@O6tAvI!sT%Xbw$@iKvSKv&{RqB2u;>C zO^9IF71Avh%Xh?YE04xZ8`6-D3}hk;c4Q+5xyVC43Q&k56r%*CD8v6(fl5@N8V=N; z7Immc0~*nUX0)IcZD>aaI?)9uTo8~T!wnA%A zF@0+j#{w3ygk`M2k5#N;9UIuh7Phg2UF=~W0R*8zM+jjY;1EYR#tBYwhI3rt g5?8p!4Q_FVdpzI~Pk6=)Uh#%^d>~@R%ul=c1IijX0ssI2 delta 574 zcmWm6WlqBY06^gZ1BMNE8}3k{T{qm_8E##>;qE??CQG;g7a{3E__Gsm4j#$-{=VOz zsPh*QE6$3y5-g{cXeC+6->y$v7X2hh9#zsz!<02a7lWp(gd{1fNU~QkHJ>g74M|j0 zLl?Zd!{PXf1YD)jnC(IeQjvyqWWbF~WFZ?l$VDFVQGh}ep%^9j|CON}6{v&~(2oHOVhF<+!6?QsjtNX+ z3e%XuEaote1uS9-%Lrfvt60N2Hn52;Y-0zz*n^HB444QZjC~y75Jx!12~Kf_b6nsO eSGdLvZgGcuJm3*ec*YA}@rHLq?3n%Gmi_?YAUKQw diff --git a/linux/lib/release/libsqlite3.a b/linux/lib/release/libsqlite3.a index 3a37b29b685f42fafadeb39a86154f289887a325..1502eb458b9c10c4dccdc59c5dac0f0892685b9e 100644 GIT binary patch delta 574 zcmWm6S5Crk0Kj2XR9v|C!o3y#EtKNkdy9KP3+}x)CQV4ZfEO|8L44r^oWq|F-|H9u zjauV~7zswAkz`nnWFy5${jq(SlJLo1pHp@O6tAvI!sT%Xbw$@iKvSKv&{RqB2u;>C zO^9IF71Avh%Xh?YE04xZ8`6-D3}hk;c4Q+5xyVC43Q&k56r%*CD8v6(fl5@N8V=N; z7Immc0~*nUX0)IcZD>aaI?)9uTo8~T!wnA%A zF@0+j#{w3ygk`M2k5#N;9UIuh7Phg2UF=~W0R*8zM+jjY;1EYR#tBYwhI3rt g5?8p!4Q_FVdpzI~Pk6=)Uh#%^d>~@R%ul=c1IijX0ssI2 delta 574 zcmWm6WlqBY06^gZ1BMNE8}3k{T{qm_8E##>;qE??CQG;g7a{3E__Gsm4j#$-{=VOz zsPh*QE6$3y5-g{cXeC+6->y$v7X2hh9#zsz!<02a7lWp(gd{1fNU~QkHJ>g74M|j0 zLl?Zd!{PXf1YD)jnC(IeQjvyqWWbF~WFZ?l$VDFVQGh}ep%^9j|CON}6{v&~(2oHOVhF<+!6?QsjtNX+ z3e%XuEaote1uS9-%Lrfvt60N2Hn52;Y-0zz*n^HB444QZjC~y75Jx!12~Kf_b6nsO eSGdLvZgGcuJm3*ec*YA}@rHLq?3n%Gmi_?YAUKQw diff --git a/src/moodycamel/build_moodycamel.sh b/src/moodycamel/build_moodycamel.sh new file mode 100755 index 000000000..b317da549 --- /dev/null +++ b/src/moodycamel/build_moodycamel.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +moody="moodycamel" +root=`realpath ../..` +work=$root/work +src=$root/src +target=$root/linux + +if [ -d "$work" ]; then rm -Rf $work; fi +mkdir $work + +rm -f $target/include/moodycamel + +# expand +unzip -q $src/moodycamel/$moody -d $work +mkdir -p $target/include/moodycamel +cp -R $work/$moody/* $target/include/moodycamel diff --git a/src/moodycamel/moodycamel.zip b/src/moodycamel/moodycamel.zip new file mode 100644 index 0000000000000000000000000000000000000000..16ac0d26801858c62de8e3a4b6b4da3e002dc135 GIT binary patch literal 44163 zcmaI6Q;aTLuxQ)1ZQHhO+qU~_+gfeASG!l+wr$(CegC_2bCZ1@&U~oM%%nzAFEf=< zr6daq1_Si}p3f*q+5frxe*qQ<56Irp(ag)#*xuZZQC$rh2sCOfQd?9sTHDPN76=&Z z3=|0He}d9~5VZf&p#BFUg_l?Diw*>oC<6q9@xLL)07rXkQ%5IP2CM&{hxQrufAX&N zF5PiClYZm{!13~%rL!^T-E+hm*dqQ~BXV}&M@1(Y;@9D#`D|^Uv2}=3W8Z#W*FL(3B@s z3D>UC0)*OR+>*$^{pfCp>10Tbnj&M%w1Y=SC(OoD=HJhA| ztVwG95Z5BjlODjU{yrWa9m^&x&P)Es;>Nh9%~=sF#D{wZS#ZGGYhcLVo7JKQNS*VzF3dhD|G&A9V5= zMeWH*{Hp|y2Vg2<#?<4va}PkJg?3bl1v z2n~Tq$<|LKORQ=tUwSN>(Bga+buM@n+@qetVNom|DEaojVBFtah!0(L&RM_r^*)nR zh*PkPSZs9otnd$=gy^;sL>F%VoPQsBuV1+y9Y443&VRgrGdbW&^(TzdJ9?jVPmLh8 z;xKvgP_F>MS32kGEZ}E& zQ?1LxslRfrC%b#O19;paRG6VyT)*GH^3+*JZmd8TXckb@qJy`L1JyZ!;pr}HvFU!* z&j4H+<>6Sl=O{!#+WJ@G$yaWgZ=TRmRKk?kD~ih<4z7cOLrMW(VzX5G^W5Yoh3biKmSA-db6vzTYh@y`< z*iSfg>KAARSzo=n>xyYxDJ>DBg^g&ucP2*$?n2dS+Me9hX68qFvpzv!ZCi!!yp``% z4b>tL8P;w)^t=3vcOdmh_Y-m>YfJ_QF5g?1(&A~)C7pa{Cr)Ufv zcQZ>VwrlOX3B1RvLfCj7Gy`s$NdzBRs7E3+ikMLjR>0yRo>uYphLZ3*wEiwQF8!Oy z@W-}T>N>UwsZN|=y$d1i(mn@#xxg#Ws5*b06lx*ZA?%sW0r!pW>=SSxSKWbI zomYD|b<@CEyX(Jg-`u1qJBrl>f(;Grzv8MXvAXB}SF&Bs3@Q%WlXDN%JduP6NJ`E} zE`Z=9_HV67UPtAr7>)5|YNS@_9P5WTFJ-S*sjW_WBlbh2VGvZ}J3(9MB%aGvL#wbf zeB-O1HLSANr$z z7kj76uP6i)!BS8docYFyihnK1pOEhsh%Le=S_qV1A4nB1nCws8St2q>7`t*b0mm$m zG!YxnELU5Ykcl_>@dweAvV{FGGV`{}eI6gb}*x?tD zKDoh`IK)M!>n=WTRU!O1^CiiE3COZzPPjD`Oe3-oc3~+$c;6*ZS%`TH7g!ZGE1iMElXt^; z4i>uC4$XF1zaXhp5QqCA)1Y1{t`nd%4ql~(w~phz{6VW!d@i;#J|?`mkWyyE&|KF{gn>x`$r3tM)<0(bi2J>rQEcZ@}0Dy z7vGmxYm)nlmkzE~VVNZh!YbcAGeG&wekU^?Tv4b_3-yVLlh-inzzcXcL*^y*g+uh_ z96)=bj81;jUnWl8&Yo9m0qVCkFibx!C1NN39<*krnL6`r7$rq^ z^c)^68J+nA8$&*%cUHeA_I3}`3rJ9Q@@mD(Pz#kZP@=&4ta}&Ugcs6Z#{sO(!vdv5 zHHz<-N7pGYP)fb}7WZI6piGx+3*M`;EBf|t#RT1W61X;nah z)4jHrSfy4sw-?8RJecrcI*B-R*kLx2j!g+eGzg9QcsK`I(`jkc*LagfR~bwkp<4Z! z5?YG6k~#fs92?EZP{BD$e$QG<4TJw zGq`{7mZ8(at&xrenUQR=Bs^kR(LhcuPDy9R zzCn^`URAYMGuSP+8qC(x3-FEhQ=FU+|77QptOy#=_N2JB9;>u(r+}rmLRkRY?w7Bl zlzTe#Q`|IAEk_Nkw+-KZ5Z&;s2x$<>LLK>`oZZfvd>tsJvzGv>TmZfCX5YNuyq|>L zD^{}yY~0HQdEWiJ58SFQJz#i!WNLSZ^Kp~Y;-(YJC|hq04s96<;BZMoGZBo@|5&s^ z^9gEI-<#ElR5Tf8G%%WLk<;xD9v z!8kg5QHuBgoZF?lxbQ%9IT`<%E}sKwti#(p(Yon;U>M?mBQOG|`cQqfpc~K)Nr0vi z@kKOuzNyBts<=hhHPVqV?H;I5;(zulA&`QD)Ngpz{sYd(ghlY6$Ps6{?1-h93bLyZ=LA9N;Kyl-A3 z<=q0j%mizkIXsQ8i|eP;8Fu%9p1L`bf<5;aeT2nU@Zm@YUL&LM=p=fMAZJF!0xyf0 zN6P}3Yh7v(vRt_@B-CLR6rfkX_eU8jXE;H5rBPHF@+)E^KR5<<2+eRoMQ_4m9z1PP zo0_Hnl9k9W!shHN9evBk%Z+;P#?qulrxSxA%;v}5~pbx zUMn@ykNq76m4-&1h0}cI9pK{3k^Kv&!!MZa7r>mKH#kp~dfrWRC%SaOK<;2*)Xnf* zC)H1Zgjss9<#20W7PSG3G31qAU)?Z>VeQHu^R;x^4+avy)PjGHD>x;xSh{Zq?~X(Q zQS1&EbM0V`axlaa6Qp)fOH?BSVEe309^MVjcA!jHeTmErULt>8L}cNnyj~Hbljaw$ z4V{WR*IWt6FMeobUlLCkSR5lcA?5y6w~`GAYOuV_1|eT0kA>*j#6t{ zi{8|Pzagqy*=`dZtK_L3+eD)@y%98t{ja0iwGc`{yuUcT!=p5Wf$E&bo29hEevB)FN z$(b)_Eb`9)ueb%7-Um@2m6Ga*#u3?7k`T>tsv|LpXdHif^X;(Jz}=-7QW?E&(Bv(u zt(@#rSaObQD>z~)!Grc~X~%iVsqCrEdp+xu6@nqyW^oRITOcm0OdlYnVZws%yAGTt znSz97VK(Dc+TjC*L5gRr!$UK=$upCLCMUYMPcVe0*UEcSYc0Ltj2p&5u@libaZP9p zph=q3RAZFn71N0PgEW_#Sg_EJvONFZ<(xK@#86FSb{ZI8hz_0Bz?6veR#I5Em(m`dKWIPx%E1?;*fs)ANfXwekM$ZUl4xseBVF)Mx&)k4_) z2@V&Kv4GA|vR!@P0?P3+EQ3u}vy`(= zt>qB#MUMFwt#)9JR?=vH9S%dgy+>UD^=@!1#;j@0UcM?J191(0EsN~k{+zc>pFDMy z0RFxnpAv;EU>0WLUhW}}GndC{mEH;c>5u*Qa7v>GuWlGae%bm_yPnykLCv-fKzOBb7OxbofiVhA)c;1A&eS30q=vBBoG$Zo*>8 zY6*lHikV|i4v&+@qVw_L9IW;JRjX6ryJXF(I#= z9%mIXBF%5nw=1Cy*3?EPITnbjm--- zW1ZgL5=I>T9NiV{kVMc;Hy4FIFPER>yg+#_16HwYCvO*l&k838v_}?}>a*U!Pmmq+ ze-Ku*;O|VH@1O_R>`$H`FEzi=eE!IAhSyiVoOq+-Olq1#!eE8={s8i;SUtox!Hi@eoD1+k7^6bp=sb zT0Gt-8gSvr6r-WU%XJ6w@PZe<19xW8xpn%p1OD#ud#?PPZ`0{h{P*-|9 zZp1b_pN9b*s@E5F+hWe$13Bsud9R;W1zdlh^AIz4O+N6~97wkWF*l*xeXi!>8?prz zKL09x{vw;lh-2aJCg#3TC~?%}@+q*}-%7hs{b(!nb3I3v-)Sy@#xR{6ro zM8H8_H~v6P@;$+FfAQqv&(0FSyo0A#r%`4)*d0&xCBXadAqhDB;MM2Zopb+i<>Bwz znzVehQ9#>N%R1;btsE{`fl(E(vXqvZ=tTR^r}>L(R-JNdk_?aYBo8K!BhIV?d?V{% z{XvWY%T?W5mMeMc?Q2Z7bB(6g+Re7cD}U6B`0Fu(>CM`dw4#U5!1QmS$;~Ich9nMn zC+N?(AMP_WSM}@5f!z@n5%*4L(}t098J&IMC~*gU2+fO{&7jj-jou0)JJpo^Exic>vCH_k>So+N6(S>|{h&*aQYfvk zW=#bPE2TI2?ZPKE%VV;DDL@@nEc({EvA;WT1BQZ^8RbT6-sw$(P(D`j>!;<0-#S5x z@6vI?$VH?Adw0Y`1OG-U?Ue>^e;J5*RUrKe1)3DqI+h0R=fw<~RUncu1IrW2GEF~c{K2nFFx1}W;RGGi@aF479`e=h|+HF{sf=k>` zuhl#A(=Luqk{3yh>oC;m9lKA!xB0+ zR?PT>LyPxTL+`(6n~B~;aZxaoNQ=uySo0b-qAZ-{|Co>8QI-kXzfx0rO;;bG%VpPm zYSd&A>Yr$q}TH)9lCBkWbT5JdsKu#UrqX>CaL zph@bcFnKQjC)KWTq$|0a{qiTT^3RRNo@1V4g#=lBl@T7Usu)R*Zj`Q8JTg4Px!E%9 zvf2F=gh>2v9RNijnhrlDu%(bx1Sbg zVLJutT79W8X`5+h+NKQKtGMv)X|!>Ox}HPKz|H8Y%k4^tup9153UhU#SUfC2;x*-a z$Hj?JHo0T;C_$dsq^6n0Fm2+d#lMBMn=u{xwXA?vOz3U;M8E0zwnw$jRHfcxsj!!1 zQPoy>Hz2D8INwZB(=?8a3&_Jxug0&_n`L9nYEHrV|Jp#0 z^%;Yx#97ZE`^PK|<-M*`1kFRjd(W#~`85~xpP*h3M*VC~$6xsW?Q+SJtDaXpfq-!N zLH;wj|4Rs)Iy#uTxwx1+0G!>--OT^*M)ykh(sffJ>DTTX9krc?*j|86+qo;L3Qb%D z#g<%Ue=z#?0tOpI6q%f_%`-!#;eGdZl^+zzSE?mvyQ8Dy5fT>6!rS{}o1cHWv@Nb^ zHm?$eSp4((vYUS&+|>}AM$Y^~{BLCG8;Y6##qz21kwsjz?MW+Ym^4PY)?1>hX8wpw zRDcCE>s6MDfdtoBmyv`?{YXWy}>NNa@V{D@rvQp>J z6V;dG7xPUFtI8Y1TB0OPoX>S<=XIpMv&zPWoV#%+i1i-4vf{%PFoAx$qXD-zg|XnK zOeIB0TrNUYVv9Ju2lvp*=wg}--DQfP$KKM>-R4vB*c~LYX!fj84Qt9@76)N0{zyb> zK>l=H0E)tlUkPoR;4$zlgc#9SqY163_GnN>`5@o^P5_1K7bJ{i4&DlvOY&a?cP+LPa$H74Ht(`A}8Qm(P7T2;u=m9h$4@`h9~SAsUWL&Ef3x*UgNhxem8o6hNX+ zAEeIf-4}68c1yMQ9odUwzkh@bwL@OOcbXJB)3@g-4W6rbFXmk=wQ9+0bVHO|`Frf( zhn_v1wh48DIabdRHq)zbQkSiPfaz$e;x!z}!ZE8SW#wb0uNM$BIDBk?msie9N-!_| zK(>OG3Fb@F6zvm`s0+!}OaW@@*il=042-IKNTxR9CKWb7;SV*!ShX zcpB*&xGv&GNM&dl&EO#pFO^$&D^c!Ze&D6Rc#DDcyC39XCxaQZ zvfjXlo=m0xWS$M_&hw*I&x|q5931-8U#F&KOdjZZ7SY^Zw8iQZ>lcwY(CXTb$R$2K9it_;#^vlqBu;o)IGgEegZV;PHH zeR(Mx<9V+V0V{JAXXabE;O-z}42s*h{qQ7NM+-DGkv=1;`dgp8WIuKCxE%}Z&hiKn z4v(`-B?Z$a=g*SBw62hBii_k!OlT9v?U7omf^#s0tf!kq6ghEn(2966>3DQ&krJzjW7#+DSHCpL?rvZ~e9f;G^QFv~ z^}B%dU>GVmc0y^>{%o2l5pvcc@hpC~aAvRFlV=?tI}IN@Vw0y4gvf4_BpO zX*Ku{!Pp#p2*<|zIP8@3dAJj_-YO~V;)jL}&5A)XaJ5H{da9h#^;jvgB($3 z1=Di6aULZkmdI^=p4$SWWR~?6Op&OSdXka{KhweyA!&#wm^>~QFH801%puTEMdST1 zGY&A zaNT~PaBREfkou$z9EHX?VtS%T*x|GcF@lkgAU0c)TwYIc&WhwmP+*Fxm#Rd(2vRCa zLEe6eTlXdYBM!9AYyEQCYLN>K=O6>7_=6^zhJ-Kbtu#Vl>JTHgmGwp>UZZq3KZQ_#1($rM)%7Z~jV}YtMt!4`MY9Axnn_P$lWv$^f_lgIn zU@O<6sBT!gU;JR7r(pR#q@Ovnfs_x5SR!Z9Txd?x-7sAc!hy}wKe?H&+QjVsu5-SP zyITR9U+3SOz29pI<6l47-ag}aS;mV~V` zT$4FFgeFGm=3jwIo(G>kW1ky7CfO6}LaypwIT(aWu?O85e8n=dpIKe$M!uCpgsF!a zaZz(QXqP-#L2j^!Th~`tScz(8^;=Y5#O!*3@LLbU|QSYyu($<7HS0u-$!~f)h9GbqHLG zB@H1jzBQDTlHHUP2WbWh(dGTgBo4x2pVGrM6%i0)*z=)^kmj7|46Pij6H#JY!8Ca8 zcEcAgaCFNq)Gl#NS*JctX%84DRKyUr55byEeT4 z>|KB6@AO#T+~R*dkp^9c*GWa%jmZgPwn*IK0Zl))<%=+d?lQpj%yn3s;IoEYj4V*D zg@5FmUn$2PKpLHOI7z={qYZ{hCX^TB#SD78`IMM0^C~$*Q4|NbZFe_8WSGcljQ3bg z4upf#Ezwr<7F8N&C*{{>7!DcYD7lTtYKTNYpCAPv@a%@Lo>Iix4?PdQB_nyv5miG+>>{%qY@Du#5b^n<~r#TCgS%&SpGN9xJdhk;@UtTIWG zI^jW>fJ$4Y)v4|iWhT4u(7)L{I*#xUfLAG6YvtMV)l^oj%#yUsTZGg2zQ$#_Lg-<| zE$FP&+Q-mlif&#gcef0x7_29VMcYKDJ0%_xm@_Uv6H#eKKb$R)SbTZt_6M{ATU35I zih7m6rKQ*BqI^gisBgp()I$iU+UqxllJ)QPt43bG%tH(^?rGG(*GwwZ)}BwG zs50G3H#iQPL7(0`y8A2Aw0zsTs^-y2ub3G zcRo2CrIy>eS`lUYI1m%@=s@aMerADsW_}o3WYO+D)i5G7bB5+~VdQBPaPpW~I$+`= zSu(}3lLWQe2fc?=RJfDEBvUrJdV|@$!52|i_FV*h7BGitlyf451hDQ!Xe#eNZ>UwG zdS_x~dOkA**Xy%rijCrU7{WZ5Djx&+sqMi*#wwR zrJ=uriPV24R-TDNzdrl~If7IM*x!9zzKX=WM?e%~E$zKZk<@>)1b*N@-U1}X2oGO- zH5dtwKYlA0enHeX2$a!2kR+muh8#5_@Pr81d|7u_f$eh~Yaodg)ot^*=C)Ay`Jt&4 z|E-1f&`-!nbCe~uzE$VYf0xFy<@3}8W0tWkMs>gERDL91xo6_Jt|Fjp=q^L4Z{%=- z!f7LwQ;d*Lhk(w+I~w+4j1nT?2-+8@j*O(dzU4ek!Aeuu4qPH;SNAX5c+us{s|SD) zdgtxHrXAb^pgIZw5hO^-(ibL7>qqB&=UHuvi3+T*=vm|FFM1<9cudQ0S9G`*LlW#-s{Nx#LJ zt8(SA2bmj27p)71V-cYnC1VQhbR5JW@~V*~MZi*qX-4ce?cpfI?GFnEh1LVAKK~F8k&u{m{QunCk10P72_@Zlpk0uOv(;PkRg0hf(9!5wKp_ zf(}O%W*Ucd*^?qqvbyees{V+w!aj#&co}E$l|40f6Q6}ugvu3;aoVcAwfi2}Z6;4W zsxme`V~dx#7*;zg43)&9V{r!?!itv-NQ(DAR3dc*j5NGTPENq08@Hg5_+#su#>9k% z7{t5*cmx>PXPw+2DLSBeFU~?}ZycK+5nC$vfG7BQH{Tl`Q{`cbP-)bhk@F7Q3Ip-S zd}=ys@x?Q>!m=w1wnvNjSSq5n<$>bho~b9r&OvDt|WMf0=ph$xaqbH z2V#t!{d8o>ukxwk3<-W>6=;0aoqg#ZNLk4sYNIXClrepn*4*lu!-Vi_dTVZxy#Yq` z-50tob4XlRnRgfte>>=W2?)|U09K;z=da4na1!{2N0-mgodEqq{Y(%$HFA9Bcdy=g z8{=nKS*1(T%wI7zwZj(TnsCBRbtj^~JvvpQc;4c4-r{^L*>{mWo`LTWymLDl|F!nx$SR z&0Z|x@GN16Iv*1Ip|~)hj3}>UovMpra*!Jpvf7yt&<+@&pJ1qA?HGX_qukOt`b=p| z1MSx#`p=Ug(G=q#GYf>EeshJoI_Rufft(X8wXv`Un9Dm^#f3J!1pEAeWj3nEc+9A|$Jtdd$Req%m63=Yrb$Xu?VmeY(TBjXtZqDW)COZlmC zA_IgdaVyd;71Y_n0kiripUEX1 z2vw#_J_8Ef%iYJF$poo(2sz#teK8QhAA}OJwk<$1d5TWaB zDflj+xAPT)Ow#c$FLWs^jzzscj`c^ANTu)1{e3zI+d&*8e>0EC&7C>vv2F&XuD4`5m7-vUJV0@Sa}gg#%7Tf!-hU* zQgYJHlYt5)M~7;?mJrnDOypMy(Pv>L9c$YVAPN+sl*00^M%6$oQMHpjsyQXYMJU6! zR^irgXnhn&+HrbFM3p35 zg>QL6rSuQ-KcBJz8eNBw_A-z2sT5bst4*5u}SJ=`7@5zLZr#xckVTI0ahV7yL-MHt=QYe7{A7A_1 z0B;}P_zT(=Nt9vyV=Y=oBJ$q+Au^>5}Xg=l94wf49O!6{nc&fqeg4M$WG}X zM{hy=c;c6+F0#=ijsX)-%S0`9tomX)esYexxGtUIclDzhkMj2T6>J_W==Ky7#<4x# z${k$D4ihEJ6kBAL|3u;wdCOcBUM_`uSsg^J8FU6?w-r}R2(b!@m}5)HeWQJv zHtrJXSsYYJ$B5o(3bCOtgl$r0A{#zu#)Mvq?@4}Z$d^K~r0gUk`{_()u?e&1^7!rR{*!-Lm2)zMIH z8~>g3$7u9zA-me7nI(J2(XdBjDM-4eN?!4K^!q2CCmBw_0itD;X$I^v&$bB2vSnsh zf#E~gT=b{@?kpmuxTB4h$mSo4&iN@5%QILrEeTQdf!;)6RYb{!StvXJk!CYy1`35N z6);`LOYNw`>hKFJhhdRgmWsJ}wkZ#AarmW`PV^rcYP@sAq08|qgXJDyihMj&5chxv zh1`DVrz44$0D6QA%N;2*p@!6w~@?Znrx(x#GI^d&d=-CwdUUm4MuwbGFIF zPsTdd_*%$n#fH5U;ta2luo6szu&_X(4tO3c13DbGm-+cafTgKrGz6mPjV(TO^z~N_ znemP81T&;divR+J#RmWozMBV(mQr-5+*9!dP+P*SBY!L+ahhjEMnk{fv($FMe7N$k zxP`uy3VD9AL8E5oZ49^~QT_~a_3J&*w~ErlZ83~G3!;1A)wrk?0iG16HWr$ekD-y3 zHnC`utWmX2t~C)R3=YD>gt-Oh4nA3~-g+`**Pa}b>po<1_7n6Iup|?+8J%$CUHl1q zGl4{91b$E0goX~y4CbSQDN1ib@A2Q(_ojC2%(f=t@a+)$ru#22)sh%Q3RvPRPyM-y zdp1z5;Fy6rBH9fRCmQLl;9#fTnE6I0^?EVqj-0M*T?4yY!)hMyRvgZ!f^0t4tKgbj zz`Sgv{c6i83b7llVOX`2Z%>!GG2&Tp!ju=v*@qLY?B~`F;ibfIz80zqATPsAzNUy} zT5=2Q-QLrPi;jg^|E28iws<$#)nr^~)r7?@$* zqD%K`_zOu4cZI!gQ1eU??Vy0|xQq>-s83dNLyEzuc3#oSm3|4r08q?4?MxZc`4+_~ zd~FbL%`ec+UTsC8yV%@1>_Fv#NkKK@Q)u%0Wy@DEKX4N#U1r?N1PhMOmWHG?(~<{IaPYZzEMSDvX^-hhg8l zGC`Wv$U8K{u0g1uJ<9WVv6uH@m!Dj(%Stmi1+T%X6QO$Y)1cH(k6RGJEnZp}S0N@b zfA`j|!0<-y2&gUoT+B+APTO!UKOfEJcFvsXwzAckq|4wF+4i>n@CRB?N1+}4zkOi{ zK2T_!Nz3I%u)}oxvek(&n4ND=VijE~4Wmt@?ez5Rf%1|x#RzgS}ZZxVN4hCxg` zY;HvnAz68rQ)&^zW=V_F2il_h-H%7Vf^sv-MMXyfQp0zH3}#K=qpR!ZOy7&=!gmXr zA0_)Ci;&hY%d3zy$OB#3fHvVLIc9yeMRi{tPtV`-C_E*qz~*_4QCYi@<&)91LImknE$ z6Y$-?u)Zh;GGKB*M(}aL2=(8g@LnI}k0J)+F% z5w}Y`X^1O~4$Q7GnvJ!NG@m?til?2$*YI#r9%r`zzkX-s5%3_{`Yhtk`D?@wldrqj z1#|Ntq3NzwZ8mD+!nz0_HeQ#d1PAYI#4!QYgvFqQ&k0B8r%z@{0m3#r+!-H!5B=*5 zCT4Pg4|vBz_IkWgix0QIkvpoZEI;wdSqZ178_9hkxOtK!EKAirJ4Mv%wnplM*fH9!G{CF23+K5{qly7jQyd zVK@?U4r#8S*!pL=1DxYOQ>pY5CX-)B_qNOGALMY9KFg+$@Dt#=fx5<-PY14!3*7tt z4P`t=n(Uw=Cf>|gNDhkkH9BzzummlrW`cc(o;#1JuRfm>NL1$~C9E#*>$WAX?kKg> zu(EiT)FiYR!&SzNL7DbrlxRS%!Jiyb$F3@RwO99V?dsBNCijXr_5wgnm>G7bP@}>RyktE<`7^G}T#xUb6-&tdX`cNoq zlX=B4DeA{Il2rkX2I5L-XH1~-S7`L4iO^%x>pLX0B3JONz$%b0+t3t0j!nE^zgI)y ztowrNtgw*wl-hC=rbuQ8#9a2kBfkhU{<1r_g#BVuuKuowRTyvKKmC09ZPkO9bXd9V zi;e)_L=Jamtf8(BRcu;=?*#EjpLeY zNysN&oZItR!n<7-vEN}t_j!vGv#jLqTjkU%|1w(y=0EEjDtl8ShOs$V9qJwM$di!s zE$;vI7E_3|J5~S9S?jT>FQ=l~epEwcr4{>yVA7M`3oOMlz~9aLZQk5ER}A7xS-x}ya5vg5H(UM?87Fx46`!~_8C9bN6f>J1I)R^4m4s`N%h)u zN|VQTIe+MI12g1c;VjuIM^4MUZVSXjfu$iv~Vvuk(Vflz3;@f3i zHGtock;g^U76dN2E8o~FN=~1RCqz_I9)3%)tj!bX7NJCP)ZG2LyoFQOFY^netFr%o z8*bi!AgOiyib!Yd>2|mr&th14nvXL|`rcq(gIL>t8`ihjHBy4Fd63Dprv=Ci3ZRj>973+fF35wYN`<^DaP7JBm zD&e?RoQxwWuW$J?y0bgw7yNf#S=|0DDYNjb1u|1D^NsL^o^AC+&62wf^kIdvYnUpPR9Cp z0T5O`<9evEKd)p6XV0{Np~5iYv842X$*@JL4#m~-cy@4p#5qdI4o=E0Jg^E&=#==|r*u)n|dgrCZiWmcl!r@t0(@;rS}8_$ECO}90|X}e~7 znZ3LT+ivg;&vPdb=@99QwM*X{)KOiZyK0}iD_3Xy;MNxZBiVm5F1cSy&~#c1A5nz= zYAeZDz3~Cbta8D+kK4xfuf_}oa@WrB9hs9L6F*e3s7cMMGEN79!~7C>-|ov#NEE;u zNH=sgb#-&b+XX81mq`p-8x*TEHgPdS+A^dU2~A8`QPY?^_Di)e^ZbVJ_bngs=KH*V zd->g*@Tk*&gYd-~_OmU_56YUO_k&b;yb&=;OOW$7>~XI3JN;T@yG8WXWVA;)p!e#Q z(A7PpYW+3f-5IiwxaQ~HI_0#VecOUS0?Ss$h(9{wDsysYV_n^*UH)Zv_tl&vkr09k z$Ryn0tC0pYu(x7%E23Xa4;~{OTbbHHrrgul}w`^P}zQj3?OeWSgJ{|cCzpWE$jrxi?Z zvXmCAJT?eZ7&0aP3ah4uR~R2@(I%)=T01dT*a*{(?sV=qrCTY6>2A*AF~CX|bz`Oi zELy8=(_#UjQ?>vmgh!mJhI31Sh@N~#5#W&DoP0%^vn7$cEzJ{6jtm&npDuAnjYqXd z`9cTzk`zt9LbLly3U6;@R{D|i87Bm3@gZ&xS|rMXrh@l;CfqW%#Y1%cXa;q;4I@qB zJ){vmV4`p+gjGgRc>y|Ix4#%QX27a<4t0Y4p%Ld<>I?u|VS{2`6ueP`7K;_;?}#M( zOM##AXl{hqQIFizHM(`C<8LIf^DynCK`&UDe~(RJ>lFO_eWESt&^V?>>UjPxPOz## z^KjGrUpb7OnQYzW77I^YIns-rtJ%`ovcF92lFo9~1`!p_p4BYF7`sEIIGO63()~Fb zU9f9mn)ns+QwlubWVWJb*tigO(Hy5YS(9L|W7OaY1k=l^DvXSBI^Va(HYwG<&hNWP zDQPLm1LdnmGw$Q3q4q%;vGqA7xso}G#?adEqgVBbXlQa{{&KP0tD=F*2OAz#=peww zcdSrNFknbBC~$S`4Ti>57WG?jK4wp?dmv9iM2Po+Yq51X*x$`SlSr9Qj16cLLteYO zvYqxI9N;FxT)G>exc5nin2&0;LcNSuP!jx-WTFjbITkVMrB5vlovrpEN^N_8)%FS# zQdsaUGq%H_m(@>2E!f(<>S{5h!-YtyDIp<=+HX$+r=}HDdl*JU@(oyg$=viQu=WPl zZ=iG_ncy73=>lH}9`G9X3!5Iucr%kcyr6*t`CXZ0-%um8%TmDPck@agz4E4wk(;cw zZHJP6sIwi8W(40KGETw5fZ|Z`E0;Fi^Abh_i5J=(qCmI+m*4j4({5r`bBOjl=7?P8BE^TdkrOrirges{~dBC53_Ezr&qg8lp0EgFZm z?lH}9$mV*qV@*I1q?%}vI;OC;;v&h(wKM;qfC(c)$xH&mAkh+0kV#|DaC9Ps{A@rz z@-R>2m{0o883P#?>Jv;QoCtfm!R9DOm&z|7srBfUr`ln>; zIzZHQ8!$wu)*(la+8L1ei;H3fUKbRxzFna?A>r~8zmlFn5WtyFs~o*b>lmF=2eFo^rXP#zc`D4_UY;cg>IdDexHE!O3`6iH$GZgMg4eWwwcY zHjV$4Gr_RdjmpjQ&l|{pDh|fZA&Pz)zK2z4l@3%6JsCCqC`i)R?-K#f-eJk5`-_-` zU0`Iy--f(=7wm_f-Bg*nv#r6eFP{R`sK*Y5PJUcDc*nqTIu>&=+YFF(7B#wxVsaCc zUy}|!uSC4K@$+!op@LB1ee6l&GCced5ebv23dM|`!o!s}RftRXGj-A{L1krGFW;Oi z@IZs)U7OnAmBo2ZGVBow(y*drKy`17f2jy4#g&)v(nQ{{HlK*QM)#K(7SuHgfQ0F?6}*Os)#1Z6;hySeK6rx(frHJ;5=^eLVJp=l2s|*V6*gg3_l8 zb%b**Cq6g=_GvNS_{H9o?&;cXB+?h3g8wC`eDNH&wLi7^B+R9w7Mw6#62f8JSUV^Wt^(5h5~6x?N#@=m*S9G(Lw;~ zxe)1UHG_et3=&>?x@vC*#(=#I<+Nmk4)BVt<4CdAIh2hMj6iUUmjw@SyjEK!1b29z z-Xz^C2N^u-v!_M+;fLl)cH&%mbDu`+tSKe_7lN@Su5_#rGhY&>rxeNjBC)smpJ7W` zJGOQr+m5Ypg#$)Q4jT>?I&G{aZ4j~ccO@Oe^tr$gy~|mmF5!c^OJqrajodm00XRg@ zhPWv?9TfNKN;#u+Pk*W|G*=jl1kg6Zl6ty`o8qB1${;g>xfsKsiK@{B%2-kQK&T{#Z(#e_cE?_FK4pzE<=eCb+WEBKnh#GxhT~t1m2b8U{p%BY1lX>5HM6R0P%}OdTz*OLRR_S|X z4LQOV@U(o48VTXnexP*ROj{}KhtN22CZ9o)ED7|oOd8%!!+XdH2qPx0h*F_S)uvYZ z3M%oIy1lU~a1;4luLPB2D=lYY(K2khZj(AgH!D{WG4oYT7Xj*v6|yXK6;*s$yK4Ur z07F2$zwYF&PI5uXontXP@Q;wQhv_x)J06hcCXlg*-{X%BA4d;t5QOYM3}cBaDoBk` z)BM?MgraPaZw{1vv*!Y{ooG`($N2Rny~Uq4G}m;KNuy2g$ko1q6cA%*%nYNC+Fz9M zDOPSWnOEgO6KRE4i{0OdWl~=ykvMzp$z>4cQo*AGnr7$p(yF!jf<1(yjhCXenNvH@ zeqH8RZQvzvG|V%H-!G!cU?uvSWyoB|LXf~UjJFLU-o(9azQ0GQQ>K&JSSu`rcnTg+ z&O|4ZkS&8E0MphjsxJ?nU&S6-hgt(Gr}$a>C(R6+0VxP-1&4{>2yo%vE{C+lPPY@T6Fvy-Rf<2sD7fI2t&)| zpy6M8N$lXQS#SRG`X}gr^}6p|VaeN2x|C?bB$%6Dz)1Cp>+&cms$lX6r6H)2K z&&M0MhuHAAqwqAFVo9la0rd=BpxsB5Y%JGU%aIrJ%fY9YGtH{pu&7zL6-kGn4#j5D zr_w{A*P2_|bldzb!zQjRkZ!hRpF)&$yY;E0$!i^~P1f%Q1wgk-pIQd|Y*wqC*j8Dn zD@tzva!39%xs}z9|Kp`I|EBmT0@@86pU-vYKB`O=Wq8pY5~!yI3?4%wY(Zfz@|-1V zHD59C5mE@O9K=@Johuh+In3sQR|E46$VV)Rq;S?DD1w3@&}+hVKRad73yRPZQl;P^ zoh{ROI$dNIJsn$?!K3&Kr+I-i0!=cF#*-F_-W<)+U9BU^{7u z#)h5fP-4eWRj`9NZa`JQQnCO=1$nO&zdVM`pnfdZ>p|s`a>(&47s+|HoaZHnlkB-z z*l96+3e!iIgn0DC222PJry1hMDsP^|(V7R^5g0;nwha|Yho~lX4}#G=Oum9#vo#o= z93ZJ<$VwU%zJg=L8msSB6eU;wxTm3M`8S!B4aZZ{gjWvSuhfSQpu2O&n?fXr4gpnx zULknUraFO}&wSZefe=WqhL_O+D68~O?X95H`DNv-fsC;ts)1*~suIkup~{@%`a@k3 z!VE{u-~AnMD+JH1M}ZDh2bQ3JBs5Q#x~v+r6u{?l(RJ4+kd0`;0uGxmaJz_@!Xe8( zheM_@%kBl&*@u4xW{0uSqkKGu_;fpJ&B$Qe_L_y^&|K z4J3(sPV2yFY-YfiD_t4jG%`Z7)nMr`O2vXiAB@bw!x$iffx0L)ga7__KCn@cGhjg1 zidA*o>DF=vky!(tn+xVA%Xs#TqJdr!5$YIK0O;Gomns_PIe#8o5)u^3s)Op}oABRvq8 zB!CHXTZ%~Pe{%1iR-m97Ql!yqiL%}%c#+ydcgmJ@b5?+jWKHV(&Y`E-(csm{&5)D6 zGglF<@vskZLQH9{-b_GbL)Pc7L1Y`ssb8p2HnLW{T|=J#LNdp_P~HfR-qkPn*jcgH zQsjuh&z&__z@0|mLTC%2&^|8_!3;~T*=tSDaT(+~hn97G5j?m!A=`A;^K9BA%i`rr z%fW3^DzE$ox5R=F4|K|Ae5 zo|Z&)1xe?#ro(&xoHU5CmtXKc6u~y8#Wj-IwTZLtw)kKu_e4dY@Fv7&7s8?`Ov0=PFc-{22pwPR>oEX+D*3)nU@&C`5h)SnHInq%N_+v z*aOjBqEW!qDAZs-a*^m()nO@wetja+Q=IZlAWEk9|dUDtv~Z5s!hKRaGG0xhjoWH{>t)5`xaO1 z%tr%5%kKb9bp9~%5C-V}CYaQb6+^nxB}%xuNUUPuLHN`wg;04Ka#Z(1W9bq@;#lDD zkqu9OUW~Hjw?qQ0dx;7QN~GfQ-R6IgcK=J?oh?35w^cBjp&d)DDCLDSt&5R@lI?sA zY;;ADE}`zHU{Uw?!;cJI@SKFzKOFjbT&e=R49Q+?H?|(t2{T%j)tAn)h2gKDNLLeL zJG%Cg`^H&S{>(zT!;y(BO^8+X$(BY-Jk_FaMb_qU5ifX-B zdh#tTcQtlO=Y4;4-m{Kj^v|W%x7|y|`5Acez&A_2;#KCvfm=6VJ8ESPHEe;p2#pS* z^Yeq=UWyX^K<6a}{Q_Xrgs}n&R`c=mi!Xb6QVXbgW=_L*gvZX+q zINjm}xV}hae`6?Ti0J)rKjLeQ({X|nPX{3Sby!vAI z$ZHwa;=mSj#vt}AdHm>gAK|o>9&3Ry1vDum+ggc%qZAYbKdy@TWW9 z?2<^ zXu+ogj|RRo`enG~_C@)FgUbp&^s##2rxxDKB=S{zbTS;F1CX*qlo9@x#9?c6cfH6E zxaHXN+aZ~5si9chu??F57Ox<(zytYW7m3tts@cSUGko*;<0s#I`(J|>F9BEl*{i3f z1@o^Tj$fb9C2+bt+uwfz|M>xsXyI6OE|v>Kcp^9+><0+%r;Lo>HolRB#r|p5FQ#}F zk?2p!)_5h5S%K)ur3O26or$8D{3IVRAKut8wbk^j6?ZbUw`-V%@0oo z-#i_>d|~GmqX#3E!q#>E)XA^to94ILjgzPw4iuWTrfBsQkw_ZgCHccCIi$Ce>GRP+hqpb_IIN- z<~HuKw~}>xc#iBFdb{ranU(mz>18iDV-W+6=AMB4!}8Jcira%uyk1^nmfvJArk3%6qIW4Po{v=Z#1Fg5@6_Z#Dc!PAfW)? zn3iJ5SMj!6>Yq2Q^y?oy(|L>~`$OKUpuoCyyweTihsegmHX3M5ULaJ~IHf z1W1oEm6Twoy2=vCRv7I`FE1=&IUW?)Drtp>i?v^TSDWp2RM@wJQdW*7NKZ5f92yoj zBZqd`cVeqKphCScX&NK5W8v4eD_+MR@z!;RmYO?^Qw^0g;q%sbFUoLqEdl(OvKy)N zQ5P})#^pyTdxy{$n8BtxkwPVuQH3?!UBcxz>zL+Ks};ZxxMVF3zRju07jy)Xy}+v$0byBMA!xo7dlR~+ zA^8AH2}tz{cH$Zl7srIvjJSf4g^Xp`3NDiZp$}a=d{ZrDk&p!ggvEu@Y1uxlTxpE9HM(scL))oPSNOtwev;|y>C3?bo) z=2LrjrU{S)u4}4BDOWhmE2T9Ag_cA=9xKiR3)F~IZ8KsZ6GY9*Ujf@TD5|h^r`4;8-SER~B0&UapKtmB>J~>eX;WAEF(ave5u;CU$$7I#%Ze++n5P z6m8^!sZ=MkBY<(wY>(`}k@shG^KB~R#tOlJ)kb)hud1LlLy9U%A?t*ih^9lI*F}}8 z+YG~fH<5@?kEw07twi+Q#YN9<9_)2yP0Fqa1*8+&n%I)b!O9Kd?48 zEfrim?t%&AYQFwIY&+Q?P`nf;hPW^=6R~Iw27d@aTbPOP{L!B>C6f%W2gqE8+npu5 zj@zuG?s$5x)4|$-~anF`sbkWoA@Hb_y2K^{y`(eTD?=N zozFqpOi<-imAlq^^wD63SkP+?o)}AT4BkWZ(LIJWd=IywS(f zV7nN13v`2jM16Z}AK8*CYiEPH9@{vqCmc!WSxuYGsJ)1u&_VscJc<<bU+D_@Bo6%JJBbu7Y7e1(Fn|+hOeU%_LrFrjg8@hWjNz7BYf*@%$)P$K zEHft|+Z(2y!q+N_h8lEbLtfJ>PoV}oLsZ#o4D_KlP;bHN(r{zJc~sX6Eru)J!j7%f zV0M`zK6P3a9FXpnPNN|RK8_r#mgL<}cOqYB3#3lH{`!ddB`(eXy<`W9PwV8#j`zC+ z-BpU}u0OA|Wgl?>B$UBsub0cBloOn_V=IO0^JdMYe@P+yqJg5S@dio54{M3t zc6}HWDUBQeLjp$8TTT}FY(gk3`Ps5qmIOoy%epK{=edpha+_UhcpQ?|oQs-}&rM)} z_HO)e?OpkFU8K{?@Q76ckUsT+D>uCb0+k^NclNqRLlR!Wj&o4CoqwU*`4?~rP{NS8 znSXKio+{_aO+&x5J@8xA57!=4rF?z(2OQTSm1^1W$ll>aZn&tzSG_Y_iGkSUM35)d zoCqVwhzR&;DIWx=HR9i`!&>>z7yULSkO(=vLi`p@3JrTt9lF7_N*BHIJ|VQsr{5)7 zk(=EkJdvMtlh)QLi@0P@cC9b}Z$G=6V)Q;*p~~5XHFrJPJ@SB4D2>2Mv|XtA9p%!a z_igz!nU-bDq`}pXoGb(F$ItCmXJH@Q{9G!=ndd*ge0B2Z#mPQ<=-AO_!7D-gIwkx| zRO!(onU_wuDsEDYsRP5lp(rGV2*46xwvAEuIH(yKavve=8r*1T?B48djk_5}{x=C_52M6r8O`fB7pyyv_+d zm@wo3vk4680_wf_(vR%%y(gwAF z3*!&S2tel_jWF_7)|uUM;bPJ0@!MWjpnR*6?ES1yJGvqTo410DRbEEnn= z71OM{2hlU&nN9&~5RLb+i86XvE)c!x8S>})TNI3pbGx>bwiky;oTWi73%|ewwIr@aw!RgUKkLhMKe_+2H{dHqO42+l z(UOtjS8a*#9;!7lTjBxk*N@r!9LiLt;F$trH3lErQ)!ET+|J7v#p1_wzIZu44vO{N z-u4Z{${vXqT7pJN-vYeum}BGis2^S7s2>X?W;UM}^L?Rrxg9Km8lb5$&8OsOg0qjf zHE6OIjTO%Sc66F{kJ*I$JiB;prf0U&UQ{ym^=5AyY7CLDYt7$59v?(nLkbHH$OStF zi7Ynj&&(4r4Zp#j;H>I)!cv&K3Eg6Yo|uIthhor8fWWt)H>#R_4~s)%*oOB5Cpo+{ z_t^#FMN`S{;OV<;J}fRSa{B-pCS$y{oTGGWN6^uO!Dlqq4i8xbVwT|E<#Y5}{_f2e zLtGC6`P<8P^GpQ_joW3qIkDS~D8SlTm$GG!okB^FrlSgKq{T2rsYQ#S%Hd~e1*!W1 ze%DTI9{x?QL_L7ZO{_|D>%QRci;ZNwqRrNYJaIx-lyNJQ(9{deAP}fGB9fr4i<#o#+ znOU(YWVs~$>?DtXUBlxV-p*gTwTs{Cej>QL?lyJt%7{oLU-Q1Z?LuDRZajU;=ti@4 z;qarn>y7U+?`EJUJv2+x^@2uey4t)eX*M6YN3SQvqI^@5OSt;f9F2~ZEN$hJ7f+u4 zl_bccBct+?DA)c{&9^*0NzluhNx$Dm;a`&dd-AOm<`khv{N+=(Bt>SQLX< zu^K$!_2?f(t2X1f4}|$gnmtM?>tK0rl!KTWA0x6}v}S znw-lt;s%O&PhxGzqe6%-+#-JuB5RhT(^+A}SOl@c`xypd6g{J`PsT=5sJu zQRHZFUilDC3!n+KzUo@LJv0`aK%PS{w`og@sh#?%VdkTBe%ZqXqKp@M0tOoj8p>mn zwrzJf92)O!>jy&>1@2S&1w%6dWPRx`92sQjLX_M|v_XBtO;+(8IK`ZTkk<{K+(qJ=x%M2c zFl-)3jPh%H5=kHoS$U1fOX(t{EHL1KvV|92K@A4!?%~rSxE)KOAiSwSvunO({SIiCTIXIz5g$-8Er>Tb09b?*Ov| zIJ<7OOKPRLq0{|7Ej@a>)Ho_FSLic=+(; z!cfMFqqpZ+Z_2)nL*ugp`wyvhe@Q1f%@YowqlBAf(wG4jnB(e~{>JUbbawj1eD^EOu=L&lPr~D zHEkskY<$RVV%RrrPJ7IL>6V0(H0b4k?v-6&Ym3Gt(ko{CHq3gNw|PC}r{BZZ zg$O%RwG!5|%T0J6bAC~+^(g=kLbua%3}CM}#l=yl*aF3N1->m#%^9sj zH+3JvFM=Md4%Dgiy$K&v&i0>}#j?AJ-94$WWx`ZUVQk>4ar=!AzG*s`_*yZ~&+;kj zbH0QZ3L5r-lep{o1ET);PwAlF)HO~^L;0-zC40_TIzH#Zh>#b8UQKN7mbw(ffmfEu z<4TohYp0+hGh(wi2ruaDHbCsN^iH!Vrar?kB-E*(v|>@wQmKzaNQ0U&(Cs|q0zi`O z3w>eoJGIKiUs!zW_Y={mk!TsAHnk&MC)R41mMYF{`qkpu32A^KUl z9G=@=>{;{mTbDA59h*-KM=wbu6R2Fyv7{SCUrT5`G%viP#IjLBJ}<~L&9X#>adhb5 z3fv3nieBgdlUf)7eRA1#YXYj_!$a^vTk8gXx@WBn4nPFaKo?#0 z*PEwro)V9~r43|YthPv;7wP*DzZ9gtY7+ugP<_((BD1Q)WMjIL3@sPDy|6}X2)~$8 z3IZD?IIrC0z1pJ0lPp`MmvHngi$IS)i1L(PS1#u>fcA8n|CvCP?|35&DA^2>=g2`zcp;xZD78|RE(0$2DU`aHLGQ%Hdp}- z=L)DCp~1GsrsUQsw$HNBp?Z6a33hs;?TBTCjokPa^Y{qX&gvp7|N0j-p>n^gi~ZMS zPAGgdgNJtB?q0|2M%#83EG|yOHQ8Ng?Pj?{GQD85D$UD#=Z0jHn0ObPitCsmKjeo7 zoviXrlkioQz8X8KN_N3}ckVPvwaRXWw@M@|a`5=%<*Uv~FL~2LigR#TC-soB-Ozrl z4pVZGUZQzsg@INCGjwiiz9YYe^NxuB9*(^Ei)|E{8V2%pz~KSOrhsp@oGnEB!B4vT z^o5$($*cdXx5g+hyP;0|x9~3dxpB7smZ+{CKRS8*UDDax+w)lBD-eVhHsPN{GZ9}{ z`U2awOD1;Y^>n07Th5|{Aw%U~jlnvk9S8qydYB^p# zJ^Aj{%b$E>PDNPkKtxry(h&W;cKAW^;%MXqV0G_Ujc^B@VV7c^X(iJ2kJy3sx2gky zqWre7Q8MQr#+|q~UcjCvF zthI9Y)qpvyM;W1>=V#}rz&qFx7xvO)&^xSqo+J9p{L*7O+8hAhg)#a-mveAO1FxQC za|jYoK|eJ=LmlJN37r=g1xKh`cnPUuBOP!J9QH4x@kLEuI~nHq21 z3;|_Ooi0t!d9f@fmwPpXUx~)9IsrTfx6?QRcDwdmI4Ks`5v7DLxz2J5dq^?+1mI-h zZZ5i17Fs1WAl4^ol^*IMRLCy#5xta;yAaQ6m&q)&r%Bo6f`-UEq0Edqi_u1Xq*Uw# z^9&P-aw;#ULnEY24QK0PNCg3X5Qx=sRwy(&m}-UE%~U2v#E~8`R)bKj%DVN1Z2~+t zpkCBNyo1$aKP6eJ6yaO47x{=a9GjU(L!qF0)BSR?pri%6z&1>nts^N9vBNYrr}TaO zra6^oVgvKY<`{a*Gr$JGI1HmwVZPLvxwX4`jcwp{x#arg5D9PPp=r86Nbs?m2WVZ= zBNlPFD2z&Q*++w%uDt_f=vxn0#oQVhfk|2$6CXTmDQ86xxkZ;jbg8@L)GWbb36;tL z1BRqeutS_8Zs=;tCQ1R>B>+$Yi~*FPJ9qBi2k8i65gcCHYcPK;7PI}kcM)v`c6$tD zV^;32+29hVbNBv#Jb3Wc{XgBm3pCR=lk1x{`@{S9`^4V*IAGIZ-%h&h=hHs&NHZd% zMqvdm?Dona1u|%tFG)>lj4*S zrbd>}>R}b;^YE}HWYOUjRzH?nR&bi7OAS>?m~=JmqA`k`(AgBzOsoaszdGMzRebF9dzbJ#W=*KT=7iBx{+#$ z+%)C1;f+tGVjMo&KrGT`q2!UioH=!8(kRf$XT(Mg%Dor6$<2fj3|f(I*|=Z13!MgAC@k2s|_COKcW zWC%8H4jAxbkrFR}MTkAeZ3GplbB;9LXRz7h(Fz%}b8-oT?rMKj?)mtgLu$

4TVV z1@!+UGo0xH!VYK!C3w}H9bipU{S^g1LB$qllxby82_TSS?J%Zx@T*um+O@0PC6I9HFKsI|AcAOVi&cZ z&2Pw7Rat22GK3q+B>nBHvOBmbr2n-Z?gfdfGTtd z?;Lf4!S(I5yUE=u^oSqoRl6*m7{T7*;iH|b@@XlyGB1KvA3=+7)!GF{-p9yvc4F4S(K7sf>-<<5F)7i??mcWj~~9@8z`)>g1yUMS%f zn^#+Sked8Q*s^PrT>x7jP(lFK^_v*<=_q`VSNT6Rd(z-dYW~7J-qW+Z@i}_l$QsnT zF!X~_`nl$pP9Y@)mKisBuf%gsTR}woqd1l_7CUkTZM}N-?d| zr}9u7%hMfo;r|3vtz5cyZYh{RM&~s@3H5MJx`A<#sOn zk^+&k5!Fz3tODIf*Nud17g<;nSSUV}stcv*tCr!2+m2PDBx*y3tru8j3)#K4HTKSn zRRS1rvL7zwLPuJV{B3lS5>uO)hU_)^Bp;8#N7M<{T2Q+BgA-)kiiyrS#S|b*dUgAY!~Gex*aJb*qtR(l)WC@J@^xK6Y%XKmPvV zy2xK!2&NL3*7{S;_S=TF#t%yL*JdAge9|lpo zFI{dG4HdFr8;`|5p5Q*OI8eOwF>3BGxw&}IdYL+b*Xg@t2k){=ChynGuJ6L8t;gLsrj;9^o1}2`wE8i;HcMFE~l_O0OJ zOYjxRq@v)iRP+_xlXDCmm7S@V1=la~KYy_2Rmk$wM60zW%KDh=PZqT5LDK$j3|swV z!di2D>=%iSKx~HDWP*MbP{E_>_a1}NVtzEX%ZX_D6yko^7_!PRb%K=s-99 z%JU%Jl^8GD`?oD~JFTaO11&=8YDTS1VeJFPPlW)(&_H{JPUl$~kiLxlG(1B=EOt}m zgr zpsl;6_5L7bwPKiv3j0GBH4w2I_-#1Tjp6G}Mxmx#i5lS|mBAP{!DvBIG{O;Yd6KY& zOuy&s7+38TC3qv;H}`5Q*H_k{?!4k&N6su8;EI{G;zaU{TN;>ZcE6(PwGKcHBX)U| z?nvo@>zDO${|B6OUgye6-a1s1*f>Mt&K(SYcEh}LCqQkexCUx+GBz6@`mDgE4*prJ z)TmQUL|_LmiT429>_H|4x>(uK%ki?5ItsN*F=2yTTNnTWaTB=4RNUS+@LClV#vDTP zcQpOU^)jMV4an@LSP8DPi+*6RcCXS))gJgt@)AawaAxu@L>^LHCGx*r7wiBP!HW3; z5^7{3DS8T2gD@@ZhXRJo3mqYsZe^J`+RSLBDdam+ESPF+h-+!Vnt50~?FhW&qHyRK zA19~tbcz5f=|Tm>cspPgHXAM%HvMn2_JXN0EZlN#b4T zBCv#+jb1C(TDPPYkO#@-sm!KMein1XKfnH}tXR%0n49Zsp1(sFwt>Ly0cUb>^NOi@ zQg*~_b6p`qb!7$OdQHulgjz~y@J~zy*zzysPasXmVzsmPMhHAAS%edbedzkPi?`|3 zsv;;N*L@gH@xhz0)r~4A(Ca`eNM0MYYLa?O{@W_W(283Tdx;41VfCgewfT~qwZyww ze5et9>y0N?(OH`?cXPK?dm;Wa(hKcEeZmYn3a_{tGWh3}Sck0{#t#vvl4;R^V|{b^ zbzP@-HgQ4e|6b1X|*++7XjFCx0L8wShkICqO%E834+5T~283B5P}Xp3iEPigSFzqx4L8mNn4X zHWfeGa{BM)2dXVq1nChGjb5qa8zmCB^3o9pYK6`AqW6?#f&D6D?&>l6H;nq5__^2K zDe#Z`)7oob%j3@S8Bsh#p<U*L?PB*}ntPs>l^PskvOAD1fjJ@^f*-sCkE$O&e0a#8 zN}%GaboJvR#Dj;Ls=G%W3%5??dAl91t{QF)an@*!O}C0RK)H`_=9Q-KB`Dikg*0_W zEH&VYwX{0hqa&EDs@lyVzpLbIo(?es*$_12p;RIdEzx^1op!8-9?YuEvmBsT%(g8; zq)krj*?>^ZpGIjb1`*OQT8{p+emOn<*5hjeJG@!QU2 zrfUz8`c@Og)>NpcLcqjEoPZTu$iMw!B%qMeU9}mpDvcl>H@9G10~OKCH+R2f@i?^Q0* z5C-xe2-*aAhJ#>02BQHwgGew6@CKQ5E13&tGys#mAgTN@H zoPb$Ubjw>z82s2ZzwIjN)rQPQUtq$EJ$vSENhtfQu1J>b;A*Peg|Y2}9^_sOC~BJ` zw@2AHQ9>r=pBzJl>lcrH7(99U&6{s^P032VR=l;tUk}t)*nVrzVrwf*5|M_lU!}GrUTcKiZ>!AU9z9NHscHMNbNsxv6k%;qN)cT`si&*_dcRkO zualrd`+H4g!k{prm=`2Y=B|jx4u+1~muiF~0nj$yD85q^qZ1w9mKmPdg)ydN*awSgZ*%@dR zbBGfvvk8=DqpIx$&V%j+iaIjD!GYT6$%o_ z*(=GdQ(a~VEp959j4um97hREEfHet# zgb@&1;wC-oc4NvONW}fJmHPzZ zZ&u_b5er4zQaN3naMz|{EM~k$Xd7~AvAd0GulfP>YlxzCJ+{D(ePwTJ1+F=4q)Pk1 zw#5uV#U{A~U}z8^Fr}7$3!YE~4nKqOSJ08^r#rRm`s5ED*wMDwBtpgVp zf(yRv#v4pjZ8)bVHmsoC&M#HuK7JLs>!kMIU}pc#7I^xc6Z-8Dujbr(6=Ghzgyx>T zdTJCl))9Vu_3{aJf;s}rZl}w${rxBKpC5qP!FG2pmJ2+NDis&eaDXV5h`NXVx!ZIf z5G*(W6Xm6!avgdCD|%@@qCcr^r1q+pvd*oR&#F7wyg>G1=E$P-94#YbeS10L{zw_& z>k`7}*~(1YTL7Rsmq^Kz7okM#QP|4*h`Du?p@7Of-1fN8PPzGszNJ2ncDMMUP}cnF88DP~Q<99gB0S}A888uJTp; zaB~OX5GFOVCDZYenz?Y_EuqfWrPY5-V*FB<<%a9BaCv~x;0O^=q)UNRO_rwA2$2|$ z0#M&mX9i&z>>6eTY2sg}zRUGu!DT%@V$p*3G2qt;|NbO^?6O*QcB!jCRc{JX%~6{K z-O|6ENhdZJItS~ugrW4LY!XWL$Y+?cD0#L|K$PP@!Ww85gui;Jcs;>(r?MtmnDEtEQd}AQb{HCt6M^=5( zdR7@*_=@qEE-V1OVBZiZ{utM=ED@IgYIS&1k{Uf+yb9{1U`$N1oEiDr=+2W%i9=Zw zv!r8=IZe?Tw+L)dimVY1j^=nc0FKq8-OKC+h0JfuhzKL#{9W6&hE#vy*i(a-|;7e@x3AQ zZ@v9j(Q}eu@gb}=pZ^bqD`1z)sY60OU0U#*IhPeoC#&?*+-;d8u<+>#a1!xOz#);L z2~kHha^)Au;?D-M*un;P!VSB&lK>Xr#_jvvD3mDLG+~DBf`)Ql;U|(l#1!NAjFxOzAZp~WQQ!1o-a|^$Ib7gC4G>g(#Eh6h0_z4=&9PQQe zIe-VZ@=HY^ft3VSMW9~mz__*t!*%(3QE>gK;dz>4nKYK^q&5$SVj^O?SyjT|)*1tF zpPp@yQS9`m<6NhAausRwY((>-R4S33IUxUfxL55UJKZ7Wd$&I5sE$0Zi@mHQ@0&qo zwl`A)9J7susfweOh^EgA4)zPw>ldikFHo4RhP5%OLqhAm-zaVISLD1a#f}r^YLG!ak(7Zmvq4Q5MSU{Ni zbP%>8lL>Eo`gaf~*9)7IDTK(1*wSmbN2UF|qJ4b9RSfa8{M8{TY*Ld6XZP;sgUZ5R zEllwXF5{NwKVNVe^~imw!p(_u%V}R%ATd~{-&%mV;QcFy z@!~#h5*~O1A)r@CV-GgLyMu2i=p5s3n&++p+a6HgY6qf5pGWs** z(AH5tN>Yh=?P1_0MJc5iTX9;9Ao3pc5mZr1CpPMi2-=3rlNA&%h>p|};j_wg43UK- zL_>9^)!bYvr9=X!qw-dli;D#%AR?7Vb=19UjazML>?N{(Or z<;n)!kH8-$KOCwK3!&07PA&S%`tTyOX$hGuf4pQF>OoNd@QP)c$3y zT*sgdx90*x2WdpWdIasMm+-qlP%pn1YgnbzieuUf$H7=k3?c)bN8Ag-M*`?D4qdZ_ z)?GbBdvYY9{)Eg&RaSwff^{~&4H%8G#Mg+~Jer@asnFTo4f(Gfi&?tJsr&oi$u1R` z@LMz%>_gBDjEJs;$Zg$xHN=e)ua4fUP7?E%AqdL^RcVnY-4OOhix@aXGuCYaz5llm={0= z1hAj86fiEi6RM{KWuXDRnIFQrTxH1$ibE3=ni)5J^0A6W<*Z=K+QiC42`C(%Lvoch z7lf)_IlKc6C$ROk!o*!USbk&eWS}rkml=g=LNSCWUcY|@FlgO|eBWX%XYT}Em!aqU z(A;}~`)htx(0jILFlc8g}aUS5kt%Y)L=pt*^ z@Y*(lHnI(Mdu&{upr&tE#-&!_(KQUX22Jf+Y*6H2V?1$(=^sJ&7Vz$*isyY$hwZ;% zC4mrqp6-S4*GO-hO7TWqPfsUVqJ{bAh}(mLyp`iWNqGVaMMNZ#i*%aLmJ?3fY*(iM zwd7fWpJfBipk_Gjj)%{K962I+ zPrIwUPiWGEbUsfniLBW_Z!i+g4I-a&#D-4NrvmkJ=yJ6i*O|_q+i* z+qRSAvB~%f8{xRRrFXSU%gi+n0=^{MQZ!8%nMos?zRWB zwSM^eQIu}{DH?NlSYHh<8GF`Ta4rUvp8{amy|>(o?p8^hUlIZk-|d1 zBGwpb6ZYj2{|!)a_XG-#m3Q;|UbsHy#b2o?+Q1ymdBN3zRB*ab|F2LMjhO_<$b;n+ zf+8elteM+Ea2*XGMqln}KuncXBpCrz#M-z>hf5}!VNJzfLAfCsf;Laum`m|fooGZ& znST@i9zcuEZ^bAyQjY2{VG)9K^osO*)WIdY6+#Zs;#5urq%GOD|D4=67wA3ji^@?6 zz5pB7eg1+KW{^Vog&=bhW)|3~Y9_!f@*skcqA8KeG=yr2x9?vpPO>CAR zV@Hve=eCIrO^v*s*hCD+pw?mlU(}WSP>t$xA8qWoV1d5~F_R;{?VX(@Y<^?Qu8uZs zKKuHOx_e>E9cBFo|6By@7YNN?$YfV2&*flM)+4l!9eA5{kOaGVAQVDffsMjXd0jKY zO$OxZxn6%>yVZCy`7cj&0}v_%H$I>fYv!Z@sO(bQz3W{kbfDHbGZ87u4pgWl5JcS2 zV_>FrysJ{ga5LT&eqLof-*_{vWQya>uu97p#q=pgGy3Rclpz_zHmS)JuXNzYqOU!J zn6l2-?AlP%ysRx8j*wewUQtd^-9ht@zZm*&wzAl**gRE~bcBz1;wP=VFH`vG%}FT8 zI5dTT*n9>0sux5UBU#;L|0-|6LaMmjesx83PfF@>&LRo?oj5P z(G#tNI7%-wV&Kc^G3j<}Q2G_Ft1FsMBHyyL0wkomr~NDUd7vcn;6TELR;H)lcOr7b ztLptg>q!nhM-aGw8W_VmE*LNhoj=N&X-Sq+-b4w{1e{58%b+#(3|#dP*ofJRY;5j> zZUGU~S_zUrJ3laYA`kZWSyh5iMp2WMX0^_ayMYbYR)+=ce`7N)_M}U}f{9~zIh4OH(uzNQ7ut*NVBa=OmC^FIpqFE>``;w#wV}2w^kozgB~r{^My%t`pn9A_yt}J zjzcp^Tv(rpe~5fbv4Ab;aa zQ0WIW;bG5SZk)u4|Lr%VT@kJ+f~*SlsGOHLz^Q7ux0e#UTxH?l_Qa5n^D*B!HX$%9J|EnCsy4|#nmZMi(KpdYm;Hl!)$q#fY`nC-CNG@ zJjMxAZ0%P%w*}Z;dYh`^M;&Xi(YMM$TLTm8AFWHAo@{dGz>hdFKBaeO9wSTw2_CZ-f;(_-5b5Nn7D6FQsOjv_{=!R5JQuJ4(slGkU!e%nmtP zwav;l)*A*WM$|U=T74$Y(%jph2U2U?1d%4OVu;?!@#>skIVwU_$H>1y(xLRhtC<@0 z7Y>aBAz>$-2SguB)79mJC)UWK;MzzFaOTyGb0#TIn8|!%5PKk=IFb9nFhA1#$^wOM zv%jn**mTH1K8vJ$!Oji4x@=}1t3gsJv>F#X^`9}{w?8X;Spnf9b!8A4cndlKtZ5Ct zU)GS$FndA>eM7Od_XLN7F13g-qzbqFhLn~5aFTIwLE%wStte2q%-g0Z(Cx&ieiagW zt}fF4;Zs;K@h27BDZWL*bx6hlWBAy`Dp#qE?kyl6StxX*IQIR0rK4<`^OL*dI}DQ4 z3*SDpS6XFzRw7|n%%rv{=S@jr&1^V&hqYG6n!a$a09Ae^Rot&A+F_H)$Yn=|Ri#T<5p&UQ1C8<#_CxeT@kPPSJxd*%ZjX=(ijko`gK zo1;>GrkwoOVXrr+=5V})do+H}mrg%O4em0jJ}yx^o*JkWxbNPgB3F{ZlazY`o)N}K z!bSQDQM(Low0Wy=+MzHGFv9^`&$f04@bHQNlOpOViV278seD7EQ(C81fHj4oN_jFJm5f_Q>EMX z0bh>EO)Qy>FNy6PP+3e{PGQ6n(}1p$cw=%crjUZ3nbZyVM6L$NY{us?K?k6@j*Qhm z#t2?fB~6r2q__DJ+1A^M&n>R5SL!BFuV^Bz_L~dgwcjm@t{Ua|WU_~#V*hN6u;%Lt z)NnKkkX;0NLmy27<8+-~>x^gy+S9)B66I!pW`ratT4l>PXI7rZu+2?7a#~7~ zc@=8sQKwmZvR|+}!iENPmTN*93o4_If|p$(B?p_16BPX-l$}0bJqWc zm#iX99$%pV2rB$i{x-(4Q+NZ*)`n!t;+yhidvP+FM^b7e1dB=xx@FkMSuJEE%AT_0 zF9Xqd00TB&?*vAuVpD_4{(1aU880f4^hKE-sY9z)HiGpaCdBnLbuc8>byofE&hG23 z1?R(cpG2-O1>%+0?DM1$AKz4C%|nJLgn(6n%8b270xKu#pFNbB>8pytWN06bLEGzX z_(sY_`2;4RJM&^^9}uY7G_j~mX;V{sXCV9=lyEK5L!E{%(Jo?puA)97cih01^sB7e z6GC%s#1G(R%i2b@eKv0s8UXxfQ*CRI{qN&y7hp!s#mVt(WTi%pt6OM!8=uUuGZa(L zc%Pd)KSys4Zjat2Lkxg}d0-`f5(|kntctKdFNjzg<#;61w33=b!l*Q9;m^7JIdj{9 z?|j;Z*gSA-N9Ny$X^nHlrXknF>zn*RHdPMB#i3B536^tDp1C0?OQq7Ut^*s?FUmFz z+f&1vkud(?p9yz*2>Js*E>CrY_S^FJLq0m^zzz$Nj23fDOn=*RR0mk1L&$|E_nT<; zv@qVw;f|QE8q1V-_wm*HVe36vx}`nJRk#-(hc~R7(6&s47xou?Z{RKIEop&46?j=R zWgWt8YIvBbgq^nQZft(j$Ivo|I#08hE7~E$P^itl%+If3q8Z!-%2GK3AP<53e6jA7 z9bKe8LKZ413=^`J1;IY46r1y!rLPwcb?;rsv?W;6U_m6LAv{ioP+YrHI5(RutAs02 zrkI@}Bh!-{WEOnVfR)okF@_XY0p-x2Qo^UJ~$91DU|sBhzxWqAf` z(g-c}#D7(3czaAvP;(KhKk?>&`wGKc!c88*syTl7o$3V}ZN}|&FB0L`pI#>+JuUQ1-ExX`G>>h+ z@>p+YLzk;xB@5@y{SoT0KqoywQJN|=g~~&pE0O?@`eWyXYR@`_C7!7k~+h`YVB`oU%yr&f^&~m4j@1onkH~Fuetal4H_t}H`>+594pQ#U9)U5 zuMPiID4Ec=_c09^T~SS=95Lu8t9PGnrl?!ez7_7 z&Pw2}RZIHP=ERR01c9HGv3)jRH7p#6PD9O3P5!}@P#4ob6fSsLJul_szfKYmn&!k8 z5S=D|TjSSk#II&q$Y56G8R916A`EU4z*3~Kd6|B}g} zuIl?f7GS>$KDD5Kj8>1#^qY-sjKechK#f}<-D8!t=Y5JauMZdZuew%QdZMM^;urTe zmz0M+kIlnW4U8<+NOl4EY_iReS-JdWdvhif!t0Ozjb~{3ATgNMs8QDt@fzNRdFD_Y ziRk$ld1*(WT}q^;sEIum%n6FzGIn~GphdSu3unCnMV1og4SkDO>ew~8lc}5pwZm%!00T6niHZ%66EbZR&=qW zfzU992&!x#cYuFG_d|?IHO}v;;2VNS!LozJAL(H^r#upSF!6;Awry?=c<#{DkJhS* zbbw>7d39^EW43{x;(dSRT!h}ec%}er3|J&Q7XPQU&>kWV_nSovj^oqKJsBUY96KxY zn)z5IGv3DA`u37E>{Z}!Z@A+$<*+p`T(8eJuTN1Cd$%mwK6{BAm6UnTObwK-tWMmp zecQr**moX`a1@yJ^>vNhqB|FI1}Q;7nk|ckcmJzeW3)c8vI)8Eiv(&@?ZqB*Sb}jF zyE?q0?%GcK=X6VIWbbTM9b+8EoU6gd2>eX!TwgLP2kzb|x~dOfvd(j5g!iHuQ;~iC z>tL@y`}R3yh_-zcshPv!Vbj+iuC?pvIjUG|l<<#`90j+Kr3%2q5)4=pw5YrfEN2|j zC#A_tKOqLq1;<3HmBTklmaa_9^_Wn>{&D?8ZlU;+8ggMy2HUkt^4F8m(i;q!Ml%`; zm>dshvs6I=!k|saUcGkncsDcf*Yp+!LuU?Kbn?xP;RQ0|GOf?Hiz-3Xjhc4JsXD2g z{@?4^j)W-Cn-_uEQwxpJ>@SZuduA!6mJ5ZDLEf5^`#;_S|LK)%kTue6jt>ANTLA!s z|MW_BF*i0dcky(wb~AVRU$`Y-2-tcqN~Yy)_7)ObirBAldPuYGjcl35j5$FgMqikD zJe{AEm7F@n{z@v0Slp9b`Z!YmX5gQ4RcwsgPQl_9~>b1es{K|fE6{A-|hmR34lr@GJu%&+qleSMZ63 zJ>*W%fehG}oX|NtX5l14YMf_32r12Ntq$-gF)boVrNjCy*!kizqpwnj5^1Hzn0=(L z-gS|(1SuLd);UVp(BSkDMA+pzU@EvSDYJzGSqwoUtVEhqrUNt?TdY&fIyTQ|@L&`= z5n;4y9KWKDcOlMVP)BUOKp{Y#<%gMYs{@2ioTRyd(s!bL<)nA6;(uhl9lN|PDa14+ z<@94xP`2fWow-Bvl2KQn@JRmZlVRa+6z;4DDFP6S^d%>3WNZuFz=(}df&6pW!U*4^ zkllZ}fv)g&$|r>U+udId(4p87+*FEr5D{GR;k)mXS>fRaXVu6Rav@kUU!7w>dhpoe z*;3e9W3bu`ASYsBpYr5+CNgb`;`qT`W;&clQt&D-&Hy;4B5&|AFs@S`MiO~Gxf{3u zWK;>9(THYgKOFSBHnd3g+Px+-8F65K;%`N_5V&GE#o%@*oqghXevmOv0GFgP6>%$b zt69x0dpOGi?o_lcAQ1$Od7G`H>5SXY4%dJzOwDA+Mve(S81!W_a)}J;_LU7O3mSmU z?eoL6udYot^4A|)BRwh$uao3G-nj@i!re$F+aGc zSl&fWZ5fJ_6;1(paK~&qZloO;N^w5OQx4UO?+Ox;f|^p9V}c#27Thc57fOgoxk#}{ zeS*S(bjv^ZNd3C09hCxx=cOq21LvN-&OHl8;o&)jBXm;=}RhWC#&-BpQxM z#%+xcG<-5f(o9!R;xy-Rgh_If$M+ge6J;}&KnsHFrwVE~S=0j&%r~C0npMOYUVMGj z#A0$Vrgv)SRJ7oJ#%D)mX=(m>G-4x2in;WX`Kmf6V^RHecz_8$7d~`AaEAodA~*00 zH8De#rA%Q*d7K|JTs7{Z1=tV2z$mQCo~B|LaER#Wa8bio5t$0ZY-6GUU_Up#GSdw& z*uasFWvG#~x;aev(P0odtH9%925^?q*$p8dLQdD9z9dLSfZkcBkv6X1q59j69!SS= zW}tSANNS8no*LQ{{!7s}4AX-d|CSmaGO?Vj%0bPD4RZE?mFK7>?Gzzg-Jvw&B^(wU zpJ=DrujDO@4f$pC1PhInX^R8GPzp^(OWCj29UP!Ob58Q#kRz-?VOwD_7}ax*nS`k| zX!r6aoSew(4!EMssv|LQkDER6=?+JhH0?!pxnP|sv#JVThoXEHS- zw8p;Tn-%orOwc{_LC#!9*{4RDIx#@G3DeYRJ7ZJ>H@=CIz>?swyvZOPpEO=o>wdty zXbQkUVhEC)Dd&S=f6^(Q?bti11|A`=1TLn)OYANs4O5+B)#W=0qHKH@CzIu_=xjyM z4YV!789l;z9HLMmEmrfm+(il7VZ;@kmlR=U9TTzPtoRI348~6ig8K}o3xA(>mIO(dxcrv0D7icCx zl^szsZA4~uVZgOHFXa$=r>OT>- zX0QfPt0_`VwX(77ydh^u)#T^WEW&wA>$XLjdo!E5^n1^aco-${YkNm%3M(^I`40@< zK3Z@HYgZ93BTz@l_&UFR<)Q8SJbtw_~p7rQm?yuVVG={NBCFe~PJNmA0;yFMoG3Yk-%PC># z7Y%tKc=2Is-rzZ^QcEaid-zYD0GDAbE?S_j1g))@y({^6N5)811ZqKqWHv~~PlUXL zAB+uR_ssIEpF~zxhFn@Cf66f_=@Giv2>7*1Zv4_9xqjWU6E+McNj#dii4f%nk31cD zipxsVgdAw~NDL}h>1+YIfgSuZ5oDoeLy;8f(E;iZ+#?v&qyBp{X%m-E3LZf$+iRtI6Z1zq$NP=Kb-sA zNmvmyUrpZQ9OX%QccFesa{ETvvOI~Zn;5{XW)mP8JNIcWZ1RQ&n;#{sxA&d?WGPC% z+#PyMps>V)zH z=RSDO&Y71=`~axVV`JwusEIK7LlX5jqBd9IJgi7%+A($aP#;VJDuXI!WZAE&|DF~JmSTqS)sD~L~**2!9$B`8YFaE*5`$ytO;mY_#~&Vdpv9%9ZuvsA0t ztiL3L8rE_!m+Y&|oYKi;xJ43=}&?lQAEOM>Q4HKa@vPJBN2 zG=p9vbh97?2$LPE8EOuyj6l!S^ z`Lr_^a0lU+dpG~>*C89Uje*?jhu>DW zyHlNmMd_m%K=lRouzYlJVPc=s@5(+#;pk%%a3p=~U?PbvN+|D;&;?b`J(g?Byh9{l zVd9@=p_(cqR`(~?3dww|l53d(-Fvgxm7-d6SHhX%kFa&?_)$?JI{jP|VZlJU zxp3I0@(>vZA0!3ABo@J3x`1`{Ha0CkfE+L-Bl`e$#EjAShvXP;kE8h;x$em%LIQ|+ z8-V6gzuwX>`zII`9_9vY;GZV7|et{TitdS%Y zOfEW6F-N($Y_j;aEmisFln7c58W&>BxJr&JWBcA0zR+v*4<8;|kZ|fEo>N$;u5lVJ zV!j!sr3M=DL~Lkbmmh~Z5CZ3YK#7}AK7b-bEMhQUe43Cxc;B);usgA3ei!}6vAxO} zf>3yXg;H>x0fjmosvD*|XVpqL% z6c;%D^`uuU#sH$8;Uk@H^)iou5f8t@l2pTRC#fM88A6Of^d0qJv4=RW|0$wz{n7+@ z>r0_4fuVB}8aB4O5u|81GLzU0#~zxoI&$$(7osIj;TK{-2v2sD)#blJ0i$FltL@r) z`&e>zT@YEZvNUdDQ`W8|k19glgr72|_dtH;U#aunAC&>?PrlQJ)@;Jo+w%wpO`I{4 z>m#J~fiUrPM2q*??OL$A@F0+5-IpU5TSiiAQ!FX}iyu8T&N*CrizutsXE)Y42KaAf z3#KkwFz*H~MH~X$voMMq(*$}bI8Th(d0ij%>x0BlW+?^FV*)CXM1F=6?U1e4d@{*! zi5{YT+*n(w-=`5QHNFpXzy}XyJvm$+CUIS3j&5Yyj*4O1){iE#!DqEzX8ehRaeTLF zL}pB+D75-*v4 zxQD#yu)p0|_e%lj5X^mF;Pk{MK=fdRzF~opS$a>jlxkG#9fegHZ=1cN6YXE!obaBn zCnWAVsnU5&pq@>(%$%fh8J&WEUw^|Y0U614Yi=vf|o>mNw)8-IA4@NlwLHt zRpkeMTl^?af8}7So^AGm^B4n%24JqoylYbY67}q1|FSrAU_8kL{lGw#Dsdrc|Ky&-Z@V$SDFT@?pT^) zwD8ko*22+?@T8N6pNGr+hHYR|C+SbTCIp>y%!Xm8nJznCj^i2$3SBKm?M{WadRw2& z5~#~gY$z`>En2a3TMCRpL5FAyht9pEj_`2>%e(rx7$Te!l z#cLNTIQevIqENYlP zMKDpyU#Gue^@?$QcG6Gv;Ab(X^+~ILH_vCc*CX-KtWYE_hI!(tnsAl0paT^O;xnkl zlERGNKQ`Nq$SRKhYF^U2@e=FzK*cm za5t`biV8~?3iEv{_aEM_uk_L?y_&9B8}*@X<;7~%eV`PIb2Ozb-QJX6@Wmnz`sye% z=sVZ>z5cjDr!rMqB_@IC9dcJdjc$l|ZePk==>y20U07JnM;n~>y&#*`>>XaVuE+=u z@5FpMB%x*_#g3@!<6v=zs5}@&3G)w`$>+Ae)n`P<2Zzc1c7WCE$?iZW+dCE9;77sD z5>bJ=BX#o<`yfI?y|>SpW!GsI|Hdedy1x9vCcIF8h!hA?LSY|HzMXhO2B>V&gIoVQ z&LU;V%TH1~^uvU{uWi1D(b0U|6Q4uKBTC~kAFkw}X`x4btc}rFrVWua?b(3M4 zxng|G*TNzV2pV@gvaL?{7mKe+TFn%tTcX*yN)}3T5pA>a$wr z=3b&p1;U~)25slT+9}2g<7W5D$WoK%(pvuqYWkF>>9)0yrguI~B?ZL+^;?+l1s$W= z!Ck80&lB0xVe=;}(Ewd|RY%#K8Xe_Hh=p+sMbWZF#L=o|EMx5(8RC0pX;TTp`K61JFNCy?Qp1t7#?`YE{ zdCSyk2VuJtK3AxzQ}u}(xdcWt!cZ#OJ$F|3Rr@qgXpvGXag3kgjDP&}aqDeJc6o=`p+*65t1Hzhmb@sxtQ&9ls#IQ_u^X2R zn19_eb#PvAcKoJoS0>P3TGX5VP}KaJ}3HWbB>dVG67{|aixm9VR2$5(9^;$h)pM5b#Rs0JWu`|Dyg89Bd38M$Lq*}3Tk2GJ+wKr)efIvsl8jo7p_Tm z{8*|#wY8U6Je_U1X!Sj3NHtR2cA_7|4`_)guqHPVNz2cF-HlJBLun*QgQVx<#AAHv1(|Fl!nE~B=pz(6sEXELR2F+Z@Gm?Jt1EMy zJyQ;EU_9-P++;S}eA_~h39~xsxF5WrFE%p3Pizs9xS`{V45fdgJ$ zy;I+Z+XB|gD6{5ncaB$)B{BA!d$fG#lPj?K^_zVS*fMK(T%#@p1#ceXp6O9X z%rL#7$X{n=?kNn+QIB;Ij~>Q$qfl&3L9~{G!{Iv_BWB zc=?k8Gsu};X+`Ps-d8ytZX4chw!DV%IykehnDV=8)$L(};Okv*#p*cg`sP~JQ9w&% zun9UY^gipxqH*(sT7iETgpDnH!{MGeM6?TT;3H#o(lxzrw3Ns8qz7PaWP~E=M)kd| zl=0Vx;OUnq=hByFNTPL1vz8m`Ni|cgm%)TBflV1h4I>+6u$jZ1)lkK4&U45LCr zS?>>>vc*CIUvRg(pOAHT&$vi{g&3=uW|UyA=F~}|?p7ws6GV)MOU4so7?urzz~Zis zJ!q_jR+n(dis1BFAJt1=vyhJxLi-i3!tynsG~Z|6=)MLULjRq0&mD97jj%%DzPK6( z2O+ZIy2j(GZNO{)*ydKFp){b=4nHe|Od{3X%QQ{F|M&D)|M;LO;^|W|i^EMZOpx+5 z5;@5S`Mh3yx8;7C`Ag{%q>9bP2>l27cX#-5iPRtLoUA;gQF|7iKMDz;u}5Vtf7&=@ z7(?EGPBY-LV@*V~jEqI3G*jJD@E!4jK#1XVV2F@n4N9B(f#xhb5a})`z}NJBYUYwLyma+ zm-kWT_Vr7KYO*S$Y1=6!uxbTs;i9aNKGDypUf`(6S=k@hgd)c)*EE=^Mua7)wbKU; zMuO6-oW34=af;^ZTE`%s(RrQ579^o)k@3BKA?(VBt^N&?h7?J%i1(4F^=Gxfq;Z&U zY{N7bbBX+%1o{+wZy{kH81npxn&d(IcER#RbvpM(BV3$ztK2n2@^mc@pD&E&he-*` zX;9>>!3F52o&ND4a?1>hz>WuRH6Kn!d{&A(mnt4uX%i8vtyg$BSvq^s$WNE^8jEwm zO;yV!?Qa`}0W^?oV-)+#f?;p{ubK%YXR{H-NWKmOa|YQIsbW|YLCsjz%z^yJUS`u^ z(#%(;-p2mKXAHlgF^z6%d6zfw16L4dOXyts)3EeTL&Cjk=cEkPp5%=l^tH5@xWI?o zjXCPtfsUT73r93Dt3bP6K8i&}P4CstxSHXj&uOwpZ4tvPd==$yiP2BK{E~4U5f+^% zoGq&)OF~{jCzM$@LM@%G4heFnWtd++7304@Eu5{f4GYDCdPnz8IKckelf2oZpF>-` zQQ%yB{wnaKWK2ZB^FNa+VGt?KRZXjWRA;z=P6FZbogi3mF|ycp^-RcCeM^18wGC=@ z!cKDaK$(GfU;-hqoI73~V(DdXC`roF6aTb5k!IKX0yY|gQe>b&ePoUz;S2W)xAKTw z?7}ELplkI4?MvUQSUKO`Jx4-ep00;gGJlyCI%$W?)wgrTz|;sEk3Jsc2Sd<~vFsIB z1$ySSoeZAw@1e*x&cyfrS|98%$%(c&@u+Kti+`ohWP6z;;|3;{Q3z2_?_KuEb z-loR(=5|aP>QDfPZ!3{s-MtV1V35ZU0KmVe{{UkAeHr%OfYQL+YClW>AW;?o!1^yB z*x#qcK@$H2GEWja%Hso-(piFS3N9w0RZ>{;D4FM{}lTl>@QgVDAv@`!PMQw w#oWQ||L~6gyXybRX8zw91mgdZ;s42P{{M17LI3@90Oa4{`nTNIihtex4^WVuVE_OC literal 0 HcmV?d00001 diff --git a/src/sqlite/build_sqlite.sh b/src/sqlite/build_sqlite.sh index b0a800ddb..6cebd0a40 100755 --- a/src/sqlite/build_sqlite.sh +++ b/src/sqlite/build_sqlite.sh @@ -29,7 +29,8 @@ cd build make -j $(nproc) make -j $(nproc) sqlite3.c -cp sqlite3.h $target/include/ +mkdir -p $target/include/sqlite +cp sqlite3.h $target/include/sqlite cp sqlite3 $target/bin/ cp .libs/libsqlite3.a $target/lib/debug/ cp .libs/libsqlite3.a $target/lib/release/