416 lines
13 KiB
C++
Executable File
416 lines
13 KiB
C++
Executable File
// Protocol Buffers - Google's data interchange format
|
|
// Copyright 2008 Google Inc. All rights reserved.
|
|
// https://developers.google.com/protocol-buffers/
|
|
//
|
|
// Redistribution and use in source and binary forms, with or without
|
|
// modification, are permitted provided that the following conditions are
|
|
// met:
|
|
//
|
|
// * Redistributions of source code must retain the above copyright
|
|
// notice, this list of conditions and the following disclaimer.
|
|
// * Redistributions in binary form must reproduce the above
|
|
// copyright notice, this list of conditions and the following disclaimer
|
|
// in the documentation and/or other materials provided with the
|
|
// distribution.
|
|
// * Neither the name of Google Inc. nor the names of its
|
|
// contributors may be used to endorse or promote products derived from
|
|
// this software without specific prior written permission.
|
|
//
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
#include <google/protobuf/arena.h>
|
|
|
|
#include <algorithm>
|
|
#include <limits>
|
|
|
|
|
|
#ifdef ADDRESS_SANITIZER
|
|
#include <sanitizer/asan_interface.h>
|
|
#endif // ADDRESS_SANITIZER
|
|
|
|
#include <google/protobuf/stubs/port.h>
|
|
|
|
namespace google {
|
|
static const size_t kMinCleanupListElements = 8;
|
|
static const size_t kMaxCleanupListElements = 64; // 1kB on 64-bit.
|
|
|
|
namespace protobuf {
|
|
namespace internal {
|
|
|
|
|
|
std::atomic<int64> ArenaImpl::lifecycle_id_generator_;
|
|
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
|
|
ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
|
|
static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ =
|
|
new internal::ThreadLocalStorage<ThreadCache>();
|
|
return *thread_cache_->Get();
|
|
}
|
|
#elif defined(PROTOBUF_USE_DLLS)
|
|
ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
|
|
static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL };
|
|
return thread_cache_;
|
|
}
|
|
#else
|
|
GOOGLE_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1, NULL};
|
|
#endif
|
|
|
|
void ArenaImpl::Init() {
|
|
lifecycle_id_ =
|
|
lifecycle_id_generator_.fetch_add(1, std::memory_order_relaxed);
|
|
hint_.store(nullptr, std::memory_order_relaxed);
|
|
threads_.store(nullptr, std::memory_order_relaxed);
|
|
|
|
if (initial_block_) {
|
|
// Thread which calls Init() owns the first block. This allows the
|
|
// single-threaded case to allocate on the first block without having to
|
|
// perform atomic operations.
|
|
new (initial_block_) Block(options_.initial_block_size, NULL);
|
|
SerialArena* serial =
|
|
SerialArena::New(initial_block_, &thread_cache(), this);
|
|
serial->set_next(NULL);
|
|
threads_.store(serial, std::memory_order_relaxed);
|
|
space_allocated_.store(options_.initial_block_size,
|
|
std::memory_order_relaxed);
|
|
CacheSerialArena(serial);
|
|
} else {
|
|
space_allocated_.store(0, std::memory_order_relaxed);
|
|
}
|
|
}
|
|
|
|
ArenaImpl::~ArenaImpl() {
|
|
// Have to do this in a first pass, because some of the destructors might
|
|
// refer to memory in other blocks.
|
|
CleanupList();
|
|
FreeBlocks();
|
|
}
|
|
|
|
uint64 ArenaImpl::Reset() {
|
|
// Have to do this in a first pass, because some of the destructors might
|
|
// refer to memory in other blocks.
|
|
CleanupList();
|
|
uint64 space_allocated = FreeBlocks();
|
|
Init();
|
|
|
|
return space_allocated;
|
|
}
|
|
|
|
ArenaImpl::Block* ArenaImpl::NewBlock(Block* last_block, size_t min_bytes) {
|
|
size_t size;
|
|
if (last_block) {
|
|
// Double the current block size, up to a limit.
|
|
size = std::min(2 * last_block->size(), options_.max_block_size);
|
|
} else {
|
|
size = options_.start_block_size;
|
|
}
|
|
// Verify that min_bytes + kBlockHeaderSize won't overflow.
|
|
GOOGLE_CHECK_LE(min_bytes, std::numeric_limits<size_t>::max() - kBlockHeaderSize);
|
|
size = std::max(size, kBlockHeaderSize + min_bytes);
|
|
|
|
void* mem = options_.block_alloc(size);
|
|
Block* b = new (mem) Block(size, last_block);
|
|
space_allocated_.fetch_add(size, std::memory_order_relaxed);
|
|
return b;
|
|
}
|
|
|
|
ArenaImpl::Block::Block(size_t size, Block* next)
|
|
: next_(next), pos_(kBlockHeaderSize), size_(size) {}
|
|
|
|
GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
|
|
void ArenaImpl::SerialArena::AddCleanupFallback(void* elem,
|
|
void (*cleanup)(void*)) {
|
|
size_t size = cleanup_ ? cleanup_->size * 2 : kMinCleanupListElements;
|
|
size = std::min(size, kMaxCleanupListElements);
|
|
size_t bytes = internal::AlignUpTo8(CleanupChunk::SizeOf(size));
|
|
CleanupChunk* list = reinterpret_cast<CleanupChunk*>(AllocateAligned(bytes));
|
|
list->next = cleanup_;
|
|
list->size = size;
|
|
|
|
cleanup_ = list;
|
|
cleanup_ptr_ = &list->nodes[0];
|
|
cleanup_limit_ = &list->nodes[size];
|
|
|
|
AddCleanup(elem, cleanup);
|
|
}
|
|
|
|
GOOGLE_PROTOBUF_ATTRIBUTE_FUNC_ALIGN(32)
|
|
void* ArenaImpl::AllocateAligned(size_t n) {
|
|
SerialArena* arena;
|
|
if (GOOGLE_PREDICT_TRUE(GetSerialArenaFast(&arena))) {
|
|
return arena->AllocateAligned(n);
|
|
} else {
|
|
return AllocateAlignedFallback(n);
|
|
}
|
|
}
|
|
|
|
void* ArenaImpl::AllocateAlignedAndAddCleanup(size_t n,
|
|
void (*cleanup)(void*)) {
|
|
SerialArena* arena;
|
|
if (GOOGLE_PREDICT_TRUE(GetSerialArenaFast(&arena))) {
|
|
return arena->AllocateAlignedAndAddCleanup(n, cleanup);
|
|
} else {
|
|
return AllocateAlignedAndAddCleanupFallback(n, cleanup);
|
|
}
|
|
}
|
|
|
|
void ArenaImpl::AddCleanup(void* elem, void (*cleanup)(void*)) {
|
|
SerialArena* arena;
|
|
if (GOOGLE_PREDICT_TRUE(GetSerialArenaFast(&arena))) {
|
|
arena->AddCleanup(elem, cleanup);
|
|
} else {
|
|
return AddCleanupFallback(elem, cleanup);
|
|
}
|
|
}
|
|
|
|
GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
|
|
void* ArenaImpl::AllocateAlignedFallback(size_t n) {
|
|
return GetSerialArena()->AllocateAligned(n);
|
|
}
|
|
|
|
GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
|
|
void* ArenaImpl::AllocateAlignedAndAddCleanupFallback(size_t n,
|
|
void (*cleanup)(void*)) {
|
|
return GetSerialArena()->AllocateAlignedAndAddCleanup(n, cleanup);
|
|
}
|
|
|
|
GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
|
|
void ArenaImpl::AddCleanupFallback(void* elem, void (*cleanup)(void*)) {
|
|
GetSerialArena()->AddCleanup(elem, cleanup);
|
|
}
|
|
|
|
inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
|
|
bool ArenaImpl::GetSerialArenaFast(ArenaImpl::SerialArena** arena) {
|
|
// If this thread already owns a block in this arena then try to use that.
|
|
// This fast path optimizes the case where multiple threads allocate from the
|
|
// same arena.
|
|
ThreadCache* tc = &thread_cache();
|
|
if (GOOGLE_PREDICT_TRUE(tc->last_lifecycle_id_seen == lifecycle_id_)) {
|
|
*arena = tc->last_serial_arena;
|
|
return true;
|
|
}
|
|
|
|
// Check whether we own the last accessed SerialArena on this arena. This
|
|
// fast path optimizes the case where a single thread uses multiple arenas.
|
|
SerialArena* serial = hint_.load(std::memory_order_acquire);
|
|
if (GOOGLE_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) {
|
|
*arena = serial;
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
ArenaImpl::SerialArena* ArenaImpl::GetSerialArena() {
|
|
SerialArena* arena;
|
|
if (GOOGLE_PREDICT_TRUE(GetSerialArenaFast(&arena))) {
|
|
return arena;
|
|
} else {
|
|
return GetSerialArenaFallback(&thread_cache());
|
|
}
|
|
}
|
|
|
|
GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
|
|
void* ArenaImpl::SerialArena::AllocateAlignedFallback(size_t n) {
|
|
// Sync back to current's pos.
|
|
head_->set_pos(head_->size() - (limit_ - ptr_));
|
|
|
|
head_ = arena_->NewBlock(head_, n);
|
|
ptr_ = head_->Pointer(head_->pos());
|
|
limit_ = head_->Pointer(head_->size());
|
|
|
|
#ifdef ADDRESS_SANITIZER
|
|
ASAN_POISON_MEMORY_REGION(ptr_, limit_ - ptr_);
|
|
#endif // ADDRESS_SANITIZER
|
|
|
|
return AllocateAligned(n);
|
|
}
|
|
|
|
uint64 ArenaImpl::SpaceAllocated() const {
|
|
return space_allocated_.load(std::memory_order_relaxed);
|
|
}
|
|
|
|
uint64 ArenaImpl::SpaceUsed() const {
|
|
SerialArena* serial = threads_.load(std::memory_order_acquire);
|
|
uint64 space_used = 0;
|
|
for ( ; serial; serial = serial->next()) {
|
|
space_used += serial->SpaceUsed();
|
|
}
|
|
return space_used;
|
|
}
|
|
|
|
uint64 ArenaImpl::SerialArena::SpaceUsed() const {
|
|
// Get current block's size from ptr_ (since we can't trust head_->pos().
|
|
uint64 space_used = ptr_ - head_->Pointer(kBlockHeaderSize);
|
|
// Get subsequent block size from b->pos().
|
|
for (Block* b = head_->next(); b; b = b->next()) {
|
|
space_used += (b->pos() - kBlockHeaderSize);
|
|
}
|
|
// Remove the overhead of the SerialArena itself.
|
|
space_used -= kSerialArenaSize;
|
|
return space_used;
|
|
}
|
|
|
|
uint64 ArenaImpl::FreeBlocks() {
|
|
uint64 space_allocated = 0;
|
|
// By omitting an Acquire barrier we ensure that any user code that doesn't
|
|
// properly synchronize Reset() or the destructor will throw a TSAN warning.
|
|
SerialArena* serial = threads_.load(std::memory_order_relaxed);
|
|
|
|
while (serial) {
|
|
// This is inside a block we are freeing, so we need to read it now.
|
|
SerialArena* next = serial->next();
|
|
space_allocated += ArenaImpl::SerialArena::Free(serial, initial_block_,
|
|
options_.block_dealloc);
|
|
// serial is dead now.
|
|
serial = next;
|
|
}
|
|
|
|
return space_allocated;
|
|
}
|
|
|
|
uint64 ArenaImpl::SerialArena::Free(ArenaImpl::SerialArena* serial,
|
|
Block* initial_block,
|
|
void (*block_dealloc)(void*, size_t)) {
|
|
uint64 space_allocated = 0;
|
|
|
|
// We have to be careful in this function, since we will be freeing the Block
|
|
// that contains this SerialArena. Be careful about accessing |serial|.
|
|
|
|
for (Block* b = serial->head_; b; ) {
|
|
// This is inside the block we are freeing, so we need to read it now.
|
|
Block* next_block = b->next();
|
|
space_allocated += (b->size());
|
|
|
|
#ifdef ADDRESS_SANITIZER
|
|
// This memory was provided by the underlying allocator as unpoisoned, so
|
|
// return it in an unpoisoned state.
|
|
ASAN_UNPOISON_MEMORY_REGION(b->Pointer(0), b->size());
|
|
#endif // ADDRESS_SANITIZER
|
|
|
|
if (b != initial_block) {
|
|
block_dealloc(b, b->size());
|
|
}
|
|
|
|
b = next_block;
|
|
}
|
|
|
|
return space_allocated;
|
|
}
|
|
|
|
void ArenaImpl::CleanupList() {
|
|
// By omitting an Acquire barrier we ensure that any user code that doesn't
|
|
// properly synchronize Reset() or the destructor will throw a TSAN warning.
|
|
SerialArena* serial = threads_.load(std::memory_order_relaxed);
|
|
|
|
for ( ; serial; serial = serial->next()) {
|
|
serial->CleanupList();
|
|
}
|
|
}
|
|
|
|
void ArenaImpl::SerialArena::CleanupList() {
|
|
if (cleanup_ != NULL) {
|
|
CleanupListFallback();
|
|
}
|
|
}
|
|
|
|
void ArenaImpl::SerialArena::CleanupListFallback() {
|
|
// Cleanup newest chunk: ptrs give us length.
|
|
size_t n = cleanup_ptr_ - &cleanup_->nodes[0];
|
|
CleanupNode* node = cleanup_ptr_;
|
|
for (size_t i = 0; i < n; i++) {
|
|
--node;
|
|
node->cleanup(node->elem);
|
|
}
|
|
|
|
// Cleanup older chunks, which are known to be full.
|
|
CleanupChunk* list = cleanup_->next;
|
|
while (list) {
|
|
size_t n = list->size;
|
|
CleanupNode* node = &list->nodes[list->size];
|
|
for (size_t i = 0; i < n; i++) {
|
|
--node;
|
|
node->cleanup(node->elem);
|
|
}
|
|
list = list->next;
|
|
}
|
|
}
|
|
|
|
ArenaImpl::SerialArena* ArenaImpl::SerialArena::New(Block* b, void* owner,
|
|
ArenaImpl* arena) {
|
|
GOOGLE_DCHECK_EQ(b->pos(), kBlockHeaderSize); // Should be a fresh block
|
|
GOOGLE_DCHECK_LE(kBlockHeaderSize + kSerialArenaSize, b->size());
|
|
SerialArena* serial =
|
|
reinterpret_cast<SerialArena*>(b->Pointer(kBlockHeaderSize));
|
|
b->set_pos(kBlockHeaderSize + kSerialArenaSize);
|
|
serial->arena_ = arena;
|
|
serial->owner_ = owner;
|
|
serial->head_ = b;
|
|
serial->ptr_ = b->Pointer(b->pos());
|
|
serial->limit_ = b->Pointer(b->size());
|
|
serial->cleanup_ = NULL;
|
|
serial->cleanup_ptr_ = NULL;
|
|
serial->cleanup_limit_ = NULL;
|
|
return serial;
|
|
}
|
|
|
|
GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE
|
|
ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) {
|
|
// Look for this SerialArena in our linked list.
|
|
SerialArena* serial = threads_.load(std::memory_order_acquire);
|
|
for ( ; serial; serial = serial->next()) {
|
|
if (serial->owner() == me) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!serial) {
|
|
// This thread doesn't have any SerialArena, which also means it doesn't
|
|
// have any blocks yet. So we'll allocate its first block now.
|
|
Block* b = NewBlock(NULL, kSerialArenaSize);
|
|
serial = SerialArena::New(b, me, this);
|
|
|
|
SerialArena* head = threads_.load(std::memory_order_relaxed);
|
|
do {
|
|
serial->set_next(head);
|
|
} while (!threads_.compare_exchange_weak(
|
|
head, serial, std::memory_order_release, std::memory_order_relaxed));
|
|
}
|
|
|
|
CacheSerialArena(serial);
|
|
return serial;
|
|
}
|
|
|
|
} // namespace internal
|
|
|
|
void Arena::CallDestructorHooks() {
|
|
uint64 space_allocated = impl_.SpaceAllocated();
|
|
// Call the reset hook
|
|
if (on_arena_reset_ != NULL) {
|
|
on_arena_reset_(this, hooks_cookie_, space_allocated);
|
|
}
|
|
|
|
// Call the destruction hook
|
|
if (on_arena_destruction_ != NULL) {
|
|
on_arena_destruction_(this, hooks_cookie_, space_allocated);
|
|
}
|
|
}
|
|
|
|
void Arena::OnArenaAllocation(const std::type_info* allocated_type,
|
|
size_t n) const {
|
|
if (on_arena_allocation_ != NULL) {
|
|
on_arena_allocation_(allocated_type, n, hooks_cookie_);
|
|
}
|
|
}
|
|
|
|
} // namespace protobuf
|
|
} // namespace google
|