283 lines
9.1 KiB
C++
283 lines
9.1 KiB
C++
#ifndef _WIN32
|
|
#include "Puid.h"
|
|
|
|
#include "../os/os.h"
|
|
#include "../utils/bts_assert.h"
|
|
#include "Uuid.h"
|
|
|
|
#include <fstream>
|
|
#include <vector>
|
|
|
|
#include <boost/algorithm/string/classification.hpp>
|
|
#include <boost/algorithm/string/predicate.hpp>
|
|
#include <boost/algorithm/string/split.hpp>
|
|
#include <boost/filesystem.hpp>
|
|
#include <boost/interprocess/sync/file_lock.hpp>
|
|
#include <boost/interprocess/sync/scoped_lock.hpp>
|
|
|
|
#include <iomanip>
|
|
#include <stdint.h>
|
|
#include <tbb/mutex.h>
|
|
#include <time.h>
|
|
|
|
namespace bts {
|
|
namespace utils {
|
|
|
|
// use a single global lock for simplicity. we can do a per domain
|
|
// lock if we ever need it. we need a lock because posix locks files
|
|
// per process not per thread.
|
|
tbb::mutex lock;
|
|
|
|
const std::string lock_file_name = "lock_file";
|
|
const std::string counter_file_name = "counter_file";
|
|
|
|
// we use the algorithm in
|
|
// http://blogs.msdn.com/b/adioltean/archive/2005/12/28/507866.aspx
|
|
// relying on the fact that metadata changes (like renaming) are
|
|
// atomic to provide transactional updates to a counter file which is
|
|
// used to generate uuids.
|
|
//
|
|
// Write process (on Foo.txt):
|
|
// - Step W1: Acquire "write lock" on the existing file.
|
|
// - Step W2: Copy the old file in a new unique temporary file. (copy Foo.txt
|
|
// Foo.TmpNNNN.txt)
|
|
// - Step W3: Apply the writes to the new file (Foo.TmpNNNN.txt).
|
|
// - Step W4: Flush all the writes (for example those being remaining in the
|
|
// cache manager).
|
|
// - Step W5: Rename the old file in a new unique Alternate form (ren Foo.txt
|
|
// Foo.AltNNNN.txt)
|
|
// - Step W6: Rename the new file into the old file (ren Foo.TmpNNNN.txt
|
|
// Foo.txt)
|
|
// - Step W7: Delete the old Alternate file (del Foo.AltNNNN.txt). If this
|
|
// fails, simply ignore. The file will be deleted later during the next
|
|
// recovery.
|
|
// - Step W8: Release "write lock" on the existing file.
|
|
|
|
// Recovery from a crash during write (on Foo.txt):
|
|
// - Step R1: If Foo.txt is missing but we have both Foo.AltNNNN.txt and
|
|
// Foo.TmpNNNN.txt present, then we crashed between Step W5 and Step W6. Retry
|
|
// from Step W6.
|
|
// - Step R2: If Foo.txt is present but Foo.TmpNNNN.txt is also present, then we
|
|
// crashed before Step W5. Try to delete all Foo.TmpNNNN.txt files and ignore
|
|
// failures.
|
|
// - Step R3: If Foo.txt is present but Foo.AltNNNN.txt is also present, then we
|
|
// crashed between Step W6 and Step W7. Try to delete all Foo.AltNNNN.txt files
|
|
// and ignore failures.
|
|
|
|
void init_store(std::string const &path, std::string const &lock_path,
|
|
std::string const &counter_path) {
|
|
if (!boost::filesystem::is_directory(path)) {
|
|
boost::filesystem::create_directories(path);
|
|
std::ofstream fs(lock_path.c_str());
|
|
std::ofstream fc(counter_path.c_str());
|
|
fc << 0;
|
|
}
|
|
}
|
|
|
|
std::string find_file(std::string const &path, std::string const &ending) {
|
|
for (boost::filesystem::directory_iterator p =
|
|
boost::filesystem::directory_iterator(path);
|
|
p != boost::filesystem::directory_iterator(); p++) {
|
|
std::string fn(p->path().string());
|
|
if (boost::algorithm::ends_with(fn, ending))
|
|
return fn;
|
|
}
|
|
return std::string();
|
|
}
|
|
|
|
// boost::filesystem::copy_file is broken for C++ 11 unless one compiles boost
|
|
// with c++ 11. (See
|
|
// http://boost.2283326.n4.nabble.com/Filesystem-problems-with-g-std-c-0x-td2639716.html)
|
|
// so we provide our own.
|
|
void copy_file(std::string const &in, std::string const &out) {
|
|
std::fstream f(in.c_str(), std::fstream::in | std::fstream::binary);
|
|
f << std::noskipws;
|
|
std::istream_iterator<unsigned char> begin(f);
|
|
std::istream_iterator<unsigned char> end;
|
|
|
|
std::fstream f2(out.c_str(), std::fstream::out | std::fstream::trunc |
|
|
std::fstream::binary);
|
|
std::ostream_iterator<char> begin2(f2);
|
|
std::copy(begin, end, begin2);
|
|
}
|
|
|
|
uint32_t get_host_ip_uint32(std::string const &ip) {
|
|
std::vector<std::string> split_vec;
|
|
boost::algorithm::split(split_vec, ip, boost::algorithm::is_any_of("."));
|
|
bts_assert(split_vec.size() == 4);
|
|
uint32_t result = 0;
|
|
int idx = 0;
|
|
for (std::string &byte : split_vec) {
|
|
uint32_t this_byte = atoi(byte.c_str());
|
|
this_byte = (this_byte << (idx++ * 8));
|
|
result += this_byte;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::string &get_host_ip_string(std::string const &ip) {
|
|
static std::string result;
|
|
if (result.size() == 0) {
|
|
std::ostringstream result_stream;
|
|
|
|
bts_assert(ip.size());
|
|
bts_assert(ip != "127.0.0.1");
|
|
std::istringstream ip_str(ip);
|
|
|
|
for (int i = 0; i < 4; i++) {
|
|
int b;
|
|
ip_str >> b;
|
|
ip_str.get();
|
|
|
|
char high_nibble = (b & 0xf0) >> 4;
|
|
char low_nibble = b & 0x0f;
|
|
if (high_nibble >= 10)
|
|
high_nibble = (high_nibble - 10) + 'a';
|
|
else
|
|
high_nibble += '0';
|
|
if (low_nibble >= 10)
|
|
low_nibble = (low_nibble - 10) + 'a';
|
|
else
|
|
low_nibble += '0';
|
|
result_stream << high_nibble << low_nibble;
|
|
}
|
|
result = result_stream.str();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
uint64_t reserve_ids(std::string const &scope, int n_ids) {
|
|
tbb::mutex::scoped_lock l(lock);
|
|
|
|
// we risk race conditions if the directory is not there.
|
|
// but that's OK, this is rare and easily manageable.
|
|
std::string path = os::bts_data_path() + "/puids/" + scope;
|
|
std::string lock_path = path + "/" + lock_file_name;
|
|
std::string counter_path = path + "/" + counter_file_name;
|
|
|
|
init_store(path, lock_path, counter_path);
|
|
|
|
// W1
|
|
boost::interprocess::file_lock f_lock(lock_path.c_str());
|
|
boost::interprocess::scoped_lock<boost::interprocess::file_lock> scope_lock(
|
|
f_lock);
|
|
|
|
std::string tmp_file = find_file(path, ".tmp");
|
|
std::string alt_file = find_file(path, ".alt");
|
|
std::string counter_file = find_file(path, counter_file_name);
|
|
|
|
// r1?
|
|
if (counter_file.size() == 0 && tmp_file.size() && alt_file.size()) {
|
|
boost::filesystem::rename(tmp_file, counter_path);
|
|
boost::filesystem::remove(alt_file);
|
|
f_lock.unlock();
|
|
lock.unlock();
|
|
return reserve_ids(scope, n_ids);
|
|
|
|
}
|
|
// r2
|
|
else if (counter_file.size() && tmp_file.size()) {
|
|
boost::filesystem::remove(tmp_file);
|
|
f_lock.unlock();
|
|
lock.unlock();
|
|
return reserve_ids(scope, n_ids);
|
|
}
|
|
// r3
|
|
else if (counter_file.size() && alt_file.size()) {
|
|
boost::filesystem::remove(alt_file);
|
|
f_lock.unlock();
|
|
lock.unlock();
|
|
return reserve_ids(scope, n_ids);
|
|
} else {
|
|
|
|
// W2, copy counter file to a temp file
|
|
tmp_file = path + "/" + Uuid().to_text_string() + ".tmp";
|
|
copy_file(counter_path, tmp_file);
|
|
|
|
// W3, W4 update temp file, close and flush
|
|
uint64_t counter_value;
|
|
std::ifstream(tmp_file.c_str()) >> counter_value;
|
|
std::ofstream(tmp_file.c_str()) << counter_value + n_ids;
|
|
|
|
// W5, rename counter file to an alt file (renames are atomic)
|
|
alt_file = path + "/" + Uuid().to_text_string() + ".alt";
|
|
boost::filesystem::rename(counter_path, alt_file);
|
|
|
|
// W6, rename temp file to counter file
|
|
boost::filesystem::rename(tmp_file, counter_path);
|
|
|
|
// W7 delete alt file
|
|
boost::filesystem::remove(alt_file);
|
|
|
|
return counter_value;
|
|
}
|
|
}
|
|
|
|
std::vector<uint64_t> gen_ids(std::string const &scope, int n_ids) {
|
|
auto counter_value = reserve_ids(scope, n_ids);
|
|
std::vector<uint64_t> result;
|
|
for (int i = 0; i < n_ids; i++)
|
|
result.push_back(counter_value + i);
|
|
return result;
|
|
}
|
|
|
|
static char base_62_alphabet[] = {
|
|
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c',
|
|
'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
|
|
'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C',
|
|
'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
|
|
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'};
|
|
|
|
std::string to_base_62(uint32_t src) {
|
|
std::string out(6, ' ');
|
|
for (int i = 5; i >= 0; i--) {
|
|
int r = src % 62;
|
|
out[i] = base_62_alphabet[r];
|
|
src = src / 62;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
void to_base_62(std::vector<uint32_t> const &id_vec, std::vector<char> *out) {
|
|
for (auto src : id_vec)
|
|
for (int i = 5; i >= 0; i--) {
|
|
int r = src % 62;
|
|
out->push_back(base_62_alphabet[r]);
|
|
src = src / 62;
|
|
}
|
|
}
|
|
|
|
uint64_t gen_id(std::string const &scope) {
|
|
std::vector<uint64_t> result = gen_ids(scope, 1);
|
|
return result[0];
|
|
}
|
|
|
|
std::vector<Puid> gen_puids(std::string const &scope, int n_puids) {
|
|
// first generate a series of unique uint32 numbers (For this machine)
|
|
std::vector<uint64_t> ids = gen_ids(scope, n_puids);
|
|
|
|
// now add the host ip and a timer string to the generated id to
|
|
// guarantee uniqueness across machines
|
|
uint32_t ip = get_host_ip_uint32(os::default_host_ip());
|
|
std::string encoded_ip(to_base_62(ip));
|
|
|
|
uint32_t seconds = time(NULL);
|
|
std::string encoded_time(to_base_62(seconds));
|
|
|
|
std::vector<Puid> result;
|
|
for (uint32_t id : ids) {
|
|
std::ostringstream s;
|
|
s << encoded_ip << "-" << encoded_time << "-" << to_base_62(id);
|
|
result.push_back(s.str());
|
|
}
|
|
return result;
|
|
}
|
|
|
|
Puid gen_puid(std::string const &scope) {
|
|
return *(gen_puids(scope, 1).begin());
|
|
}
|
|
|
|
|
|
} // namespace utils
|
|
} // namespace bts
|
|
#endif |