409 lines
14 KiB
C++
409 lines
14 KiB
C++
/**
|
|
* @file include/ActivMiddleware/Misc/AmazonS3Helper.h
|
|
*
|
|
* @brief Header file for the curl FTP helper class which uses cUrl and ftpparser.
|
|
*
|
|
* When using this file you will need to link to the curl library (CurlPackage_x86_win32_vc71_mdid.lib etc.).
|
|
*/
|
|
|
|
#if (!defined ACTIV_MISC_AMAZON_S3_HELPER_H)
|
|
#define ACTIV_MISC_AMAZON_S3_HELPER_H
|
|
|
|
#include "ActivMiddleware/ActivBase/FieldTypes/DateTime.h"
|
|
#include "ActivMiddleware/ActivBase/File.h"
|
|
#include "ActivMiddleware/ActivBase/MessageRateThrottler.h"
|
|
#include "ActivMiddleware/ActivBase/StatusCodeException.h"
|
|
#include "ActivMiddleware/ActivBase/StringException.h"
|
|
#include "ActivMiddleware/Misc/Fmt.h"
|
|
#include "ActivMiddleware/Misc/IFtpHelper.h"
|
|
|
|
#include "s3/libs3.h"
|
|
|
|
//! Activ namespace.
|
|
namespace Activ
|
|
{
|
|
|
|
/**
|
|
* @brief AmazonS3Helper class.
|
|
*/
|
|
class AmazonS3Helper
|
|
{
|
|
public:
|
|
/**
|
|
* @brief Constuctor
|
|
*
|
|
* @param amazonEndpoint the amazon enpoint for AWS S3.
|
|
* @param agentContext the agent context for reporting.
|
|
* @param maxTransferRate the maximum rate in kB at which to transfer files, 0 is unlimited.
|
|
* @param s3Protocol the protocol for S3 to use.
|
|
* @param s3UriStyle the uri style.
|
|
* @param logLevel the log level.
|
|
*/
|
|
AmazonS3Helper(const String &amazonEndpoint = "s3.amazonaws.com", const String &agentContext = "", const size_t maxTransferRate = 0, const String &s3Protocol = "S3ProtocolHTTP", const String &s3UriStyle = "S3UriStylePath", const bool logLevel = false);
|
|
|
|
virtual ~AmazonS3Helper();
|
|
|
|
/**
|
|
* @brief Put an object
|
|
*
|
|
* @param localFilename the local filename.
|
|
* @param bucketName the name of the bucket to push to.
|
|
* @param objectKey the remote object key.
|
|
* @param accessKey the access key (the amazon identity of the uploader, basically an AWS username).
|
|
* @param secretKey the secret key (the amazon identity confirmation, basically an AWS password).
|
|
*
|
|
* @return StatusCodeHelper containing the status and explaining text.
|
|
*/
|
|
virtual StatusCodeHelper PutObject(const String &localFilename, const String &bucketName, const String &objectKey, const String &accessKey, const String &secretKey);
|
|
|
|
/**
|
|
* @brief Set functions
|
|
*/
|
|
void SetAmazonEndpoint(const String &amazonEndpoint);
|
|
void SetAgentContext(const String &agentContext);
|
|
StatusCodeHelper SetProtocol(const String &s3Protocol);
|
|
StatusCodeHelper SetUriStyle(const String &s3UriStyle);
|
|
void SetMaxTransferRate(const size_t &rate);
|
|
|
|
/**
|
|
* @brief Get functions
|
|
*/
|
|
String GetAmazonEndpoint() const;
|
|
String GetAgentContext() const;
|
|
String GetProtocol() const;
|
|
String GetUriStyle() const;
|
|
size_t GetMaxTransferRate() const;
|
|
|
|
private:
|
|
/**
|
|
* @brief Initialize libS3 for use
|
|
*/
|
|
void Initialize();
|
|
|
|
/**
|
|
* @brief CleanUp libS3
|
|
*/
|
|
void CleanUp();
|
|
|
|
class CallbackData
|
|
{
|
|
public:
|
|
CallbackData(const size_t rate = 0);
|
|
virtual ~CallbackData();
|
|
virtual int AppendData(void *ptr, int size);
|
|
virtual void Throttle(const size_t size);
|
|
|
|
S3Status m_status;
|
|
//S3ResponseProperties m_responseProperties;
|
|
private:
|
|
|
|
MessageRateThrottler m_messageRateThrottler;
|
|
size_t m_totalBytes;
|
|
size_t m_lastSuspendByteCount;
|
|
};
|
|
|
|
class PutObjectCallbackData : public CallbackData
|
|
{
|
|
public:
|
|
PutObjectCallbackData(const size_t rateLimit = 0);
|
|
StatusCode Initialize(const String &filename, uint64_t &size);
|
|
virtual int AppendData(void *ptr, int size) ACTIV_OVERRIDE;
|
|
|
|
private:
|
|
File m_file;
|
|
};
|
|
|
|
static S3Status ResponsePropertiesCallbackFunction(const S3ResponseProperties *properties, void *callbackData);
|
|
static void ResponseCompleteCallbackFunction(S3Status status, const S3ErrorDetails *error, void *callbackData);
|
|
static int PutObjectDataCallbackFunction(int bufferSize, char *buffer, void *callbackData);
|
|
|
|
String m_amazonEndpoint; ///< the amazon enpoint for AWS S3.
|
|
String m_agentContext; ///< the agent context for reporting.
|
|
size_t m_maxTransferRate; ///< The max transfer rate in bytes.
|
|
S3Protocol m_s3Protocol; ///< the protocol for S3 to use.
|
|
S3UriStyle m_s3UriStyle; ///< the uri style.
|
|
bool m_logLevel; ///< the log level.
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline AmazonS3Helper::AmazonS3Helper(const String &amazonEndpoint, const String &agentContext, const size_t maxTransferRate, const String &s3Protocol, const String &s3UriStyle, const bool logLevel) :
|
|
m_amazonEndpoint(amazonEndpoint),
|
|
m_agentContext(agentContext),
|
|
m_maxTransferRate(maxTransferRate),
|
|
m_logLevel(logLevel)
|
|
{
|
|
StatusCodeHelper statusCode;
|
|
statusCode = SetProtocol(s3Protocol);
|
|
if (STATUS_CODE_SUCCESS != statusCode)
|
|
ACTIV_BOOST_THROW_EXCEPTION(StringException(Fmt::Format("Invalid S3 protocol {}, {}.", s3Protocol.c_str(), statusCode.ToString().c_str())));
|
|
|
|
statusCode = SetUriStyle(s3UriStyle);
|
|
if (STATUS_CODE_SUCCESS != statusCode)
|
|
ACTIV_BOOST_THROW_EXCEPTION(StringException(Fmt::Format("Invalid S3 uri style {}, {}.", s3UriStyle.c_str(), statusCode.ToString().c_str())));
|
|
|
|
Initialize();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline AmazonS3Helper::~AmazonS3Helper()
|
|
{
|
|
CleanUp();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline void AmazonS3Helper::Initialize()
|
|
{
|
|
S3Status status = S3_initialize(m_agentContext.c_str(), S3_INIT_ALL, m_amazonEndpoint.c_str());
|
|
if (status != S3StatusOK)
|
|
ACTIV_BOOST_THROW_EXCEPTION(StringException((Fmt::Format("Failed to initialize libs3: {}", S3_get_status_name(status)))));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline void AmazonS3Helper::CleanUp()
|
|
{
|
|
S3_deinitialize();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline StatusCodeHelper AmazonS3Helper::PutObject(const String &localFilename, const String &bucketName, const String &objectKey, const String &accessKey, const String &secretKey)
|
|
{
|
|
S3PutObjectHandler putObjectHandler =
|
|
{
|
|
{
|
|
&ResponsePropertiesCallbackFunction,
|
|
&ResponseCompleteCallbackFunction
|
|
},
|
|
&PutObjectDataCallbackFunction
|
|
};
|
|
|
|
S3BucketContext bucketContext =
|
|
{
|
|
(m_amazonEndpoint.empty()) ? NULL : m_amazonEndpoint.c_str(),
|
|
bucketName.c_str(),
|
|
m_s3Protocol,
|
|
m_s3UriStyle,
|
|
accessKey.c_str(),
|
|
secretKey.c_str(),
|
|
0
|
|
};
|
|
|
|
PutObjectCallbackData putObjectCallbackData(m_maxTransferRate);
|
|
|
|
uint64_t contentLength = 0;
|
|
StatusCode statusCode = putObjectCallbackData.Initialize(localFilename, contentLength);
|
|
if (STATUS_CODE_SUCCESS != statusCode)
|
|
return StatusCodeHelper(statusCode, StatusCodeToString(statusCode));
|
|
|
|
S3_put_object(&bucketContext, objectKey.c_str(), contentLength, /*Optional Put Properties*/ NULL, /*Request Context for delaying*/ 0, &putObjectHandler, &putObjectCallbackData);
|
|
|
|
if (S3StatusOK != putObjectCallbackData.m_status)
|
|
return StatusCodeHelper(STATUS_CODE_FAILURE, S3_get_status_name(putObjectCallbackData.m_status));
|
|
|
|
return StatusCodeHelper(STATUS_CODE_SUCCESS, S3_get_status_name(putObjectCallbackData.m_status));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline S3Status AmazonS3Helper::ResponsePropertiesCallbackFunction(const S3ResponseProperties *properties, void *callbackData)
|
|
{
|
|
return S3StatusOK;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline void AmazonS3Helper::ResponseCompleteCallbackFunction(S3Status status, const S3ErrorDetails *error, void *callbackData)
|
|
{
|
|
CallbackData *pCallBackData = static_cast<CallbackData *>(callbackData);
|
|
|
|
pCallBackData->m_status = status;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline int AmazonS3Helper::PutObjectDataCallbackFunction(int bufferSize, char *buffer, void *callbackData)
|
|
{
|
|
PutObjectCallbackData *pCallBackData = static_cast<PutObjectCallbackData *>(callbackData);
|
|
|
|
return pCallBackData->AppendData(buffer, bufferSize);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline AmazonS3Helper::CallbackData::CallbackData(const size_t rateLimit) :
|
|
m_status(S3StatusOK),
|
|
m_messageRateThrottler(rateLimit, 100, false),
|
|
m_totalBytes(0),
|
|
m_lastSuspendByteCount(0)
|
|
{
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline AmazonS3Helper::CallbackData::~CallbackData()
|
|
{
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline int AmazonS3Helper::CallbackData::AppendData(void *ptr, int size)
|
|
{
|
|
return size;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline void AmazonS3Helper::CallbackData::Throttle(const size_t size)
|
|
{
|
|
m_totalBytes += size;
|
|
|
|
// Throttle on kB rather than bytes.
|
|
while (((m_totalBytes - m_lastSuspendByteCount) / 1024) > 0)
|
|
{
|
|
m_lastSuspendByteCount += 1024;
|
|
m_messageRateThrottler.Suspend();
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline AmazonS3Helper::PutObjectCallbackData::PutObjectCallbackData(const size_t rateLimit) :
|
|
CallbackData(rateLimit)
|
|
{
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline StatusCode AmazonS3Helper::PutObjectCallbackData::Initialize(const String &filename, uint64_t &size)
|
|
{
|
|
ACTIV_RETURN_RESULT_IF(STATUS_CODE_SUCCESS, !=, m_file.Open(filename, File::AccessFlags::Read(), File::ShareFlags::Read()));
|
|
|
|
return m_file.GetSize(size);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline int AmazonS3Helper::PutObjectCallbackData::AppendData(void *ptr, int size)
|
|
{
|
|
size_t sizeRead = 0;
|
|
|
|
try
|
|
{
|
|
ACTIV_THROW_RESULT_IF(STATUS_CODE_SUCCESS, !=, m_file.Read(static_cast<byte_t *>(ptr), size, sizeRead));
|
|
Throttle(sizeRead);
|
|
}
|
|
catch (StatusCodeException &statusCodeException)
|
|
{
|
|
LogMessage(LOG_TYPE_ERROR, "Failed to read data from file " + statusCodeException.ToString());
|
|
return -1;
|
|
}
|
|
|
|
return static_cast<int>(sizeRead);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline void AmazonS3Helper::SetAmazonEndpoint(const String &amazonEndpoint)
|
|
{
|
|
m_amazonEndpoint = amazonEndpoint;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline void AmazonS3Helper::SetAgentContext(const String &agentContext)
|
|
{
|
|
m_agentContext = agentContext;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline StatusCodeHelper AmazonS3Helper::SetProtocol(const String &s3Protocol)
|
|
{
|
|
if ("https" == s3Protocol)
|
|
m_s3Protocol = S3ProtocolHTTPS;
|
|
else if ("http" == s3Protocol)
|
|
m_s3Protocol = S3ProtocolHTTP;
|
|
else
|
|
return(StatusCodeHelper(STATUS_CODE_INVALID_PARAMETER, "Supported protocols are 'http' or 'https'"));
|
|
|
|
return StatusCodeHelper(STATUS_CODE_SUCCESS);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline StatusCodeHelper AmazonS3Helper::SetUriStyle(const String &s3UriStyle)
|
|
{
|
|
if ("virtual-host" == s3UriStyle)
|
|
m_s3UriStyle = S3UriStyleVirtualHost;
|
|
else if ("path" == s3UriStyle)
|
|
m_s3UriStyle = S3UriStylePath;
|
|
else
|
|
return(StatusCodeHelper(STATUS_CODE_INVALID_PARAMETER, "Suported uri styles are 'virtual-host' or 'path'"));
|
|
|
|
return StatusCodeHelper(STATUS_CODE_SUCCESS);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline void AmazonS3Helper::SetMaxTransferRate(const size_t &rate)
|
|
{
|
|
m_maxTransferRate = rate;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline String AmazonS3Helper::GetAmazonEndpoint() const
|
|
{
|
|
return m_amazonEndpoint;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline String AmazonS3Helper::GetAgentContext() const
|
|
{
|
|
return m_agentContext;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline String AmazonS3Helper::GetProtocol() const
|
|
{
|
|
if (S3ProtocolHTTPS == m_s3Protocol)
|
|
return "https";
|
|
else if (S3ProtocolHTTP == m_s3Protocol)
|
|
return "http";
|
|
|
|
return "uninitialized";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline String AmazonS3Helper::GetUriStyle() const
|
|
{
|
|
if (S3UriStyleVirtualHost == m_s3UriStyle)
|
|
return "virtual-host";
|
|
else if (S3UriStylePath == m_s3UriStyle)
|
|
return "path";
|
|
|
|
return "uninitialized";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------------------------------
|
|
|
|
inline size_t AmazonS3Helper::GetMaxTransferRate() const
|
|
{
|
|
return m_maxTransferRate;
|
|
}
|
|
|
|
} // namespace Activ
|
|
|
|
#endif // (!defined ACTIV_MISC_AMAZON_S3_HELPER_H)
|