/** * @file include/ActivMiddleware/Misc/AmazonS3Helper.h * * @brief Header file for the curl FTP helper class which uses cUrl and ftpparser. * * When using this file you will need to link to the curl library (CurlPackage_x86_win32_vc71_mdid.lib etc.). */ #if (!defined ACTIV_MISC_AMAZON_S3_HELPER_H) #define ACTIV_MISC_AMAZON_S3_HELPER_H #include "ActivMiddleware/ActivBase/FieldTypes/DateTime.h" #include "ActivMiddleware/ActivBase/File.h" #include "ActivMiddleware/ActivBase/MessageRateThrottler.h" #include "ActivMiddleware/ActivBase/StatusCodeException.h" #include "ActivMiddleware/ActivBase/StringException.h" #include "ActivMiddleware/Misc/Fmt.h" #include "ActivMiddleware/Misc/IFtpHelper.h" #include "s3/libs3.h" //! Activ namespace. namespace Activ { /** * @brief AmazonS3Helper class. */ class AmazonS3Helper { public: /** * @brief Constuctor * * @param amazonEndpoint the amazon enpoint for AWS S3. * @param agentContext the agent context for reporting. * @param maxTransferRate the maximum rate in kB at which to transfer files, 0 is unlimited. * @param s3Protocol the protocol for S3 to use. * @param s3UriStyle the uri style. * @param logLevel the log level. */ AmazonS3Helper(const String &amazonEndpoint = "s3.amazonaws.com", const String &agentContext = "", const size_t maxTransferRate = 0, const String &s3Protocol = "S3ProtocolHTTP", const String &s3UriStyle = "S3UriStylePath", const bool logLevel = false); virtual ~AmazonS3Helper(); /** * @brief Put an object * * @param localFilename the local filename. * @param bucketName the name of the bucket to push to. * @param objectKey the remote object key. * @param accessKey the access key (the amazon identity of the uploader, basically an AWS username). * @param secretKey the secret key (the amazon identity confirmation, basically an AWS password). * * @return StatusCodeHelper containing the status and explaining text. */ virtual StatusCodeHelper PutObject(const String &localFilename, const String &bucketName, const String &objectKey, const String &accessKey, const String &secretKey); /** * @brief Set functions */ void SetAmazonEndpoint(const String &amazonEndpoint); void SetAgentContext(const String &agentContext); StatusCodeHelper SetProtocol(const String &s3Protocol); StatusCodeHelper SetUriStyle(const String &s3UriStyle); void SetMaxTransferRate(const size_t &rate); /** * @brief Get functions */ String GetAmazonEndpoint() const; String GetAgentContext() const; String GetProtocol() const; String GetUriStyle() const; size_t GetMaxTransferRate() const; private: /** * @brief Initialize libS3 for use */ void Initialize(); /** * @brief CleanUp libS3 */ void CleanUp(); class CallbackData { public: CallbackData(const size_t rate = 0); virtual ~CallbackData(); virtual int AppendData(void *ptr, int size); virtual void Throttle(const size_t size); S3Status m_status; //S3ResponseProperties m_responseProperties; private: MessageRateThrottler m_messageRateThrottler; size_t m_totalBytes; size_t m_lastSuspendByteCount; }; class PutObjectCallbackData : public CallbackData { public: PutObjectCallbackData(const size_t rateLimit = 0); StatusCode Initialize(const String &filename, uint64_t &size); virtual int AppendData(void *ptr, int size) ACTIV_OVERRIDE; private: File m_file; }; static S3Status ResponsePropertiesCallbackFunction(const S3ResponseProperties *properties, void *callbackData); static void ResponseCompleteCallbackFunction(S3Status status, const S3ErrorDetails *error, void *callbackData); static int PutObjectDataCallbackFunction(int bufferSize, char *buffer, void *callbackData); String m_amazonEndpoint; ///< the amazon enpoint for AWS S3. String m_agentContext; ///< the agent context for reporting. size_t m_maxTransferRate; ///< The max transfer rate in bytes. S3Protocol m_s3Protocol; ///< the protocol for S3 to use. S3UriStyle m_s3UriStyle; ///< the uri style. bool m_logLevel; ///< the log level. }; // --------------------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------------------- inline AmazonS3Helper::AmazonS3Helper(const String &amazonEndpoint, const String &agentContext, const size_t maxTransferRate, const String &s3Protocol, const String &s3UriStyle, const bool logLevel) : m_amazonEndpoint(amazonEndpoint), m_agentContext(agentContext), m_maxTransferRate(maxTransferRate), m_logLevel(logLevel) { StatusCodeHelper statusCode; statusCode = SetProtocol(s3Protocol); if (STATUS_CODE_SUCCESS != statusCode) ACTIV_BOOST_THROW_EXCEPTION(StringException(Fmt::Format("Invalid S3 protocol {}, {}.", s3Protocol.c_str(), statusCode.ToString().c_str()))); statusCode = SetUriStyle(s3UriStyle); if (STATUS_CODE_SUCCESS != statusCode) ACTIV_BOOST_THROW_EXCEPTION(StringException(Fmt::Format("Invalid S3 uri style {}, {}.", s3UriStyle.c_str(), statusCode.ToString().c_str()))); Initialize(); } // --------------------------------------------------------------------------------------------------------------------------------- inline AmazonS3Helper::~AmazonS3Helper() { CleanUp(); } // --------------------------------------------------------------------------------------------------------------------------------- inline void AmazonS3Helper::Initialize() { S3Status status = S3_initialize(m_agentContext.c_str(), S3_INIT_ALL, m_amazonEndpoint.c_str()); if (status != S3StatusOK) ACTIV_BOOST_THROW_EXCEPTION(StringException((Fmt::Format("Failed to initialize libs3: {}", S3_get_status_name(status))))); } // --------------------------------------------------------------------------------------------------------------------------------- inline void AmazonS3Helper::CleanUp() { S3_deinitialize(); } // --------------------------------------------------------------------------------------------------------------------------------- inline StatusCodeHelper AmazonS3Helper::PutObject(const String &localFilename, const String &bucketName, const String &objectKey, const String &accessKey, const String &secretKey) { S3PutObjectHandler putObjectHandler = { { &ResponsePropertiesCallbackFunction, &ResponseCompleteCallbackFunction }, &PutObjectDataCallbackFunction }; S3BucketContext bucketContext = { (m_amazonEndpoint.empty()) ? NULL : m_amazonEndpoint.c_str(), bucketName.c_str(), m_s3Protocol, m_s3UriStyle, accessKey.c_str(), secretKey.c_str(), 0 }; PutObjectCallbackData putObjectCallbackData(m_maxTransferRate); uint64_t contentLength = 0; StatusCode statusCode = putObjectCallbackData.Initialize(localFilename, contentLength); if (STATUS_CODE_SUCCESS != statusCode) return StatusCodeHelper(statusCode, StatusCodeToString(statusCode)); S3_put_object(&bucketContext, objectKey.c_str(), contentLength, /*Optional Put Properties*/ NULL, /*Request Context for delaying*/ 0, &putObjectHandler, &putObjectCallbackData); if (S3StatusOK != putObjectCallbackData.m_status) return StatusCodeHelper(STATUS_CODE_FAILURE, S3_get_status_name(putObjectCallbackData.m_status)); return StatusCodeHelper(STATUS_CODE_SUCCESS, S3_get_status_name(putObjectCallbackData.m_status)); } // --------------------------------------------------------------------------------------------------------------------------------- inline S3Status AmazonS3Helper::ResponsePropertiesCallbackFunction(const S3ResponseProperties *properties, void *callbackData) { return S3StatusOK; } // --------------------------------------------------------------------------------------------------------------------------------- inline void AmazonS3Helper::ResponseCompleteCallbackFunction(S3Status status, const S3ErrorDetails *error, void *callbackData) { CallbackData *pCallBackData = static_cast(callbackData); pCallBackData->m_status = status; } // --------------------------------------------------------------------------------------------------------------------------------- inline int AmazonS3Helper::PutObjectDataCallbackFunction(int bufferSize, char *buffer, void *callbackData) { PutObjectCallbackData *pCallBackData = static_cast(callbackData); return pCallBackData->AppendData(buffer, bufferSize); } // --------------------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------------------- inline AmazonS3Helper::CallbackData::CallbackData(const size_t rateLimit) : m_status(S3StatusOK), m_messageRateThrottler(rateLimit, 100, false), m_totalBytes(0), m_lastSuspendByteCount(0) { } // --------------------------------------------------------------------------------------------------------------------------------- inline AmazonS3Helper::CallbackData::~CallbackData() { } // --------------------------------------------------------------------------------------------------------------------------------- inline int AmazonS3Helper::CallbackData::AppendData(void *ptr, int size) { return size; } // --------------------------------------------------------------------------------------------------------------------------------- inline void AmazonS3Helper::CallbackData::Throttle(const size_t size) { m_totalBytes += size; // Throttle on kB rather than bytes. while (((m_totalBytes - m_lastSuspendByteCount) / 1024) > 0) { m_lastSuspendByteCount += 1024; m_messageRateThrottler.Suspend(); } } // --------------------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------------------- inline AmazonS3Helper::PutObjectCallbackData::PutObjectCallbackData(const size_t rateLimit) : CallbackData(rateLimit) { } // --------------------------------------------------------------------------------------------------------------------------------- inline StatusCode AmazonS3Helper::PutObjectCallbackData::Initialize(const String &filename, uint64_t &size) { ACTIV_RETURN_RESULT_IF(STATUS_CODE_SUCCESS, !=, m_file.Open(filename, File::AccessFlags::Read(), File::ShareFlags::Read())); return m_file.GetSize(size); } // --------------------------------------------------------------------------------------------------------------------------------- inline int AmazonS3Helper::PutObjectCallbackData::AppendData(void *ptr, int size) { size_t sizeRead = 0; try { ACTIV_THROW_RESULT_IF(STATUS_CODE_SUCCESS, !=, m_file.Read(static_cast(ptr), size, sizeRead)); Throttle(sizeRead); } catch (StatusCodeException &statusCodeException) { LogMessage(LOG_TYPE_ERROR, "Failed to read data from file " + statusCodeException.ToString()); return -1; } return static_cast(sizeRead); } // --------------------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------------------- inline void AmazonS3Helper::SetAmazonEndpoint(const String &amazonEndpoint) { m_amazonEndpoint = amazonEndpoint; } // --------------------------------------------------------------------------------------------------------------------------------- inline void AmazonS3Helper::SetAgentContext(const String &agentContext) { m_agentContext = agentContext; } // --------------------------------------------------------------------------------------------------------------------------------- inline StatusCodeHelper AmazonS3Helper::SetProtocol(const String &s3Protocol) { if ("https" == s3Protocol) m_s3Protocol = S3ProtocolHTTPS; else if ("http" == s3Protocol) m_s3Protocol = S3ProtocolHTTP; else return(StatusCodeHelper(STATUS_CODE_INVALID_PARAMETER, "Supported protocols are 'http' or 'https'")); return StatusCodeHelper(STATUS_CODE_SUCCESS); } // --------------------------------------------------------------------------------------------------------------------------------- inline StatusCodeHelper AmazonS3Helper::SetUriStyle(const String &s3UriStyle) { if ("virtual-host" == s3UriStyle) m_s3UriStyle = S3UriStyleVirtualHost; else if ("path" == s3UriStyle) m_s3UriStyle = S3UriStylePath; else return(StatusCodeHelper(STATUS_CODE_INVALID_PARAMETER, "Suported uri styles are 'virtual-host' or 'path'")); return StatusCodeHelper(STATUS_CODE_SUCCESS); } // --------------------------------------------------------------------------------------------------------------------------------- inline void AmazonS3Helper::SetMaxTransferRate(const size_t &rate) { m_maxTransferRate = rate; } // --------------------------------------------------------------------------------------------------------------------------------- inline String AmazonS3Helper::GetAmazonEndpoint() const { return m_amazonEndpoint; } // --------------------------------------------------------------------------------------------------------------------------------- inline String AmazonS3Helper::GetAgentContext() const { return m_agentContext; } // --------------------------------------------------------------------------------------------------------------------------------- inline String AmazonS3Helper::GetProtocol() const { if (S3ProtocolHTTPS == m_s3Protocol) return "https"; else if (S3ProtocolHTTP == m_s3Protocol) return "http"; return "uninitialized"; } // --------------------------------------------------------------------------------------------------------------------------------- inline String AmazonS3Helper::GetUriStyle() const { if (S3UriStyleVirtualHost == m_s3UriStyle) return "virtual-host"; else if (S3UriStylePath == m_s3UriStyle) return "path"; return "uninitialized"; } // --------------------------------------------------------------------------------------------------------------------------------- inline size_t AmazonS3Helper::GetMaxTransferRate() const { return m_maxTransferRate; } } // namespace Activ #endif // (!defined ACTIV_MISC_AMAZON_S3_HELPER_H)