Files

409 lines
14 KiB
C++

/**
* @file include/ActivMiddleware/Misc/AmazonS3Helper.h
*
* @brief Header file for the curl FTP helper class which uses cUrl and ftpparser.
*
* When using this file you will need to link to the curl library (CurlPackage_x86_win32_vc71_mdid.lib etc.).
*/
#if (!defined ACTIV_MISC_AMAZON_S3_HELPER_H)
#define ACTIV_MISC_AMAZON_S3_HELPER_H
#include "ActivMiddleware/ActivBase/FieldTypes/DateTime.h"
#include "ActivMiddleware/ActivBase/File.h"
#include "ActivMiddleware/ActivBase/MessageRateThrottler.h"
#include "ActivMiddleware/ActivBase/StatusCodeException.h"
#include "ActivMiddleware/ActivBase/StringException.h"
#include "ActivMiddleware/Misc/Fmt.h"
#include "ActivMiddleware/Misc/IFtpHelper.h"
#include "s3/libs3.h"
//! Activ namespace.
namespace Activ
{
/**
* @brief AmazonS3Helper class.
*/
class AmazonS3Helper
{
public:
/**
* @brief Constuctor
*
* @param amazonEndpoint the amazon enpoint for AWS S3.
* @param agentContext the agent context for reporting.
* @param maxTransferRate the maximum rate in kB at which to transfer files, 0 is unlimited.
* @param s3Protocol the protocol for S3 to use.
* @param s3UriStyle the uri style.
* @param logLevel the log level.
*/
AmazonS3Helper(const String &amazonEndpoint = "s3.amazonaws.com", const String &agentContext = "", const size_t maxTransferRate = 0, const String &s3Protocol = "S3ProtocolHTTP", const String &s3UriStyle = "S3UriStylePath", const bool logLevel = false);
virtual ~AmazonS3Helper();
/**
* @brief Put an object
*
* @param localFilename the local filename.
* @param bucketName the name of the bucket to push to.
* @param objectKey the remote object key.
* @param accessKey the access key (the amazon identity of the uploader, basically an AWS username).
* @param secretKey the secret key (the amazon identity confirmation, basically an AWS password).
*
* @return StatusCodeHelper containing the status and explaining text.
*/
virtual StatusCodeHelper PutObject(const String &localFilename, const String &bucketName, const String &objectKey, const String &accessKey, const String &secretKey);
/**
* @brief Set functions
*/
void SetAmazonEndpoint(const String &amazonEndpoint);
void SetAgentContext(const String &agentContext);
StatusCodeHelper SetProtocol(const String &s3Protocol);
StatusCodeHelper SetUriStyle(const String &s3UriStyle);
void SetMaxTransferRate(const size_t &rate);
/**
* @brief Get functions
*/
String GetAmazonEndpoint() const;
String GetAgentContext() const;
String GetProtocol() const;
String GetUriStyle() const;
size_t GetMaxTransferRate() const;
private:
/**
* @brief Initialize libS3 for use
*/
void Initialize();
/**
* @brief CleanUp libS3
*/
void CleanUp();
class CallbackData
{
public:
CallbackData(const size_t rate = 0);
virtual ~CallbackData();
virtual int AppendData(void *ptr, int size);
virtual void Throttle(const size_t size);
S3Status m_status;
//S3ResponseProperties m_responseProperties;
private:
MessageRateThrottler m_messageRateThrottler;
size_t m_totalBytes;
size_t m_lastSuspendByteCount;
};
class PutObjectCallbackData : public CallbackData
{
public:
PutObjectCallbackData(const size_t rateLimit = 0);
StatusCode Initialize(const String &filename, uint64_t &size);
virtual int AppendData(void *ptr, int size) ACTIV_OVERRIDE;
private:
File m_file;
};
static S3Status ResponsePropertiesCallbackFunction(const S3ResponseProperties *properties, void *callbackData);
static void ResponseCompleteCallbackFunction(S3Status status, const S3ErrorDetails *error, void *callbackData);
static int PutObjectDataCallbackFunction(int bufferSize, char *buffer, void *callbackData);
String m_amazonEndpoint; ///< the amazon enpoint for AWS S3.
String m_agentContext; ///< the agent context for reporting.
size_t m_maxTransferRate; ///< The max transfer rate in bytes.
S3Protocol m_s3Protocol; ///< the protocol for S3 to use.
S3UriStyle m_s3UriStyle; ///< the uri style.
bool m_logLevel; ///< the log level.
};
// ---------------------------------------------------------------------------------------------------------------------------------
// ---------------------------------------------------------------------------------------------------------------------------------
inline AmazonS3Helper::AmazonS3Helper(const String &amazonEndpoint, const String &agentContext, const size_t maxTransferRate, const String &s3Protocol, const String &s3UriStyle, const bool logLevel) :
m_amazonEndpoint(amazonEndpoint),
m_agentContext(agentContext),
m_maxTransferRate(maxTransferRate),
m_logLevel(logLevel)
{
StatusCodeHelper statusCode;
statusCode = SetProtocol(s3Protocol);
if (STATUS_CODE_SUCCESS != statusCode)
ACTIV_BOOST_THROW_EXCEPTION(StringException(Fmt::Format("Invalid S3 protocol {}, {}.", s3Protocol.c_str(), statusCode.ToString().c_str())));
statusCode = SetUriStyle(s3UriStyle);
if (STATUS_CODE_SUCCESS != statusCode)
ACTIV_BOOST_THROW_EXCEPTION(StringException(Fmt::Format("Invalid S3 uri style {}, {}.", s3UriStyle.c_str(), statusCode.ToString().c_str())));
Initialize();
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline AmazonS3Helper::~AmazonS3Helper()
{
CleanUp();
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline void AmazonS3Helper::Initialize()
{
S3Status status = S3_initialize(m_agentContext.c_str(), S3_INIT_ALL, m_amazonEndpoint.c_str());
if (status != S3StatusOK)
ACTIV_BOOST_THROW_EXCEPTION(StringException((Fmt::Format("Failed to initialize libs3: {}", S3_get_status_name(status)))));
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline void AmazonS3Helper::CleanUp()
{
S3_deinitialize();
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline StatusCodeHelper AmazonS3Helper::PutObject(const String &localFilename, const String &bucketName, const String &objectKey, const String &accessKey, const String &secretKey)
{
S3PutObjectHandler putObjectHandler =
{
{
&ResponsePropertiesCallbackFunction,
&ResponseCompleteCallbackFunction
},
&PutObjectDataCallbackFunction
};
S3BucketContext bucketContext =
{
(m_amazonEndpoint.empty()) ? NULL : m_amazonEndpoint.c_str(),
bucketName.c_str(),
m_s3Protocol,
m_s3UriStyle,
accessKey.c_str(),
secretKey.c_str(),
0
};
PutObjectCallbackData putObjectCallbackData(m_maxTransferRate);
uint64_t contentLength = 0;
StatusCode statusCode = putObjectCallbackData.Initialize(localFilename, contentLength);
if (STATUS_CODE_SUCCESS != statusCode)
return StatusCodeHelper(statusCode, StatusCodeToString(statusCode));
S3_put_object(&bucketContext, objectKey.c_str(), contentLength, /*Optional Put Properties*/ NULL, /*Request Context for delaying*/ 0, &putObjectHandler, &putObjectCallbackData);
if (S3StatusOK != putObjectCallbackData.m_status)
return StatusCodeHelper(STATUS_CODE_FAILURE, S3_get_status_name(putObjectCallbackData.m_status));
return StatusCodeHelper(STATUS_CODE_SUCCESS, S3_get_status_name(putObjectCallbackData.m_status));
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline S3Status AmazonS3Helper::ResponsePropertiesCallbackFunction(const S3ResponseProperties *properties, void *callbackData)
{
return S3StatusOK;
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline void AmazonS3Helper::ResponseCompleteCallbackFunction(S3Status status, const S3ErrorDetails *error, void *callbackData)
{
CallbackData *pCallBackData = static_cast<CallbackData *>(callbackData);
pCallBackData->m_status = status;
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline int AmazonS3Helper::PutObjectDataCallbackFunction(int bufferSize, char *buffer, void *callbackData)
{
PutObjectCallbackData *pCallBackData = static_cast<PutObjectCallbackData *>(callbackData);
return pCallBackData->AppendData(buffer, bufferSize);
}
// ---------------------------------------------------------------------------------------------------------------------------------
// ---------------------------------------------------------------------------------------------------------------------------------
inline AmazonS3Helper::CallbackData::CallbackData(const size_t rateLimit) :
m_status(S3StatusOK),
m_messageRateThrottler(rateLimit, 100, false),
m_totalBytes(0),
m_lastSuspendByteCount(0)
{
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline AmazonS3Helper::CallbackData::~CallbackData()
{
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline int AmazonS3Helper::CallbackData::AppendData(void *ptr, int size)
{
return size;
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline void AmazonS3Helper::CallbackData::Throttle(const size_t size)
{
m_totalBytes += size;
// Throttle on kB rather than bytes.
while (((m_totalBytes - m_lastSuspendByteCount) / 1024) > 0)
{
m_lastSuspendByteCount += 1024;
m_messageRateThrottler.Suspend();
}
}
// ---------------------------------------------------------------------------------------------------------------------------------
// ---------------------------------------------------------------------------------------------------------------------------------
inline AmazonS3Helper::PutObjectCallbackData::PutObjectCallbackData(const size_t rateLimit) :
CallbackData(rateLimit)
{
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline StatusCode AmazonS3Helper::PutObjectCallbackData::Initialize(const String &filename, uint64_t &size)
{
ACTIV_RETURN_RESULT_IF(STATUS_CODE_SUCCESS, !=, m_file.Open(filename, File::AccessFlags::Read(), File::ShareFlags::Read()));
return m_file.GetSize(size);
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline int AmazonS3Helper::PutObjectCallbackData::AppendData(void *ptr, int size)
{
size_t sizeRead = 0;
try
{
ACTIV_THROW_RESULT_IF(STATUS_CODE_SUCCESS, !=, m_file.Read(static_cast<byte_t *>(ptr), size, sizeRead));
Throttle(sizeRead);
}
catch (StatusCodeException &statusCodeException)
{
LogMessage(LOG_TYPE_ERROR, "Failed to read data from file " + statusCodeException.ToString());
return -1;
}
return static_cast<int>(sizeRead);
}
// ---------------------------------------------------------------------------------------------------------------------------------
// ---------------------------------------------------------------------------------------------------------------------------------
inline void AmazonS3Helper::SetAmazonEndpoint(const String &amazonEndpoint)
{
m_amazonEndpoint = amazonEndpoint;
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline void AmazonS3Helper::SetAgentContext(const String &agentContext)
{
m_agentContext = agentContext;
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline StatusCodeHelper AmazonS3Helper::SetProtocol(const String &s3Protocol)
{
if ("https" == s3Protocol)
m_s3Protocol = S3ProtocolHTTPS;
else if ("http" == s3Protocol)
m_s3Protocol = S3ProtocolHTTP;
else
return(StatusCodeHelper(STATUS_CODE_INVALID_PARAMETER, "Supported protocols are 'http' or 'https'"));
return StatusCodeHelper(STATUS_CODE_SUCCESS);
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline StatusCodeHelper AmazonS3Helper::SetUriStyle(const String &s3UriStyle)
{
if ("virtual-host" == s3UriStyle)
m_s3UriStyle = S3UriStyleVirtualHost;
else if ("path" == s3UriStyle)
m_s3UriStyle = S3UriStylePath;
else
return(StatusCodeHelper(STATUS_CODE_INVALID_PARAMETER, "Suported uri styles are 'virtual-host' or 'path'"));
return StatusCodeHelper(STATUS_CODE_SUCCESS);
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline void AmazonS3Helper::SetMaxTransferRate(const size_t &rate)
{
m_maxTransferRate = rate;
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline String AmazonS3Helper::GetAmazonEndpoint() const
{
return m_amazonEndpoint;
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline String AmazonS3Helper::GetAgentContext() const
{
return m_agentContext;
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline String AmazonS3Helper::GetProtocol() const
{
if (S3ProtocolHTTPS == m_s3Protocol)
return "https";
else if (S3ProtocolHTTP == m_s3Protocol)
return "http";
return "uninitialized";
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline String AmazonS3Helper::GetUriStyle() const
{
if (S3UriStyleVirtualHost == m_s3UriStyle)
return "virtual-host";
else if (S3UriStylePath == m_s3UriStyle)
return "path";
return "uninitialized";
}
// ---------------------------------------------------------------------------------------------------------------------------------
inline size_t AmazonS3Helper::GetMaxTransferRate() const
{
return m_maxTransferRate;
}
} // namespace Activ
#endif // (!defined ACTIV_MISC_AMAZON_S3_HELPER_H)