Files
spark/src/lib/utils/HTTPStream.cpp
T

108 lines
3.2 KiB
C++
Executable File

#include "HTTPStream.h"
#include "lib/logging/logging.h"
#include "lib/utils/defer.h"
#include "lib/utils/bts_assert.h"
#include <boost/tokenizer.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <istream>
#include <fstream>
#include <iomanip>
#include <sys/types.h>
#include <sys/stat.h>
#include <boost/utility/value_init.hpp>
using namespace std;
using namespace boost;
using namespace boost::asio;
namespace bts {
namespace utils {
int read_headers(istream& input, map<string, string>& result) {
string line;
getline(input, line);
auto code_start = line.find_first_of(" ");
auto code_end = line.find_first_of(" ", code_start+1);
string http_code = line.substr(code_start+1, (code_end - code_start - 1));
boost::algorithm::trim(http_code);
if (http_code.empty()) return 418;
while(true) {
std::getline(input, line);
auto index = line.find_first_of(" ");
if (index == string::npos) break;
result.insert(std::make_pair(line.substr(0, index-1), boost::algorithm::trim_copy(line.substr(index+1))));
}
return boost::lexical_cast<int>(http_code);
}
HTTPStream::HTTPStream(const std::string& host, const std::string& path, const std::string& verb) :
ip::tcp::iostream(host, "http"),
_response(-1),
_headers()
{
// Send the request
*this << verb << " " << path << " HTTP/1.0\r\n";
*this << "HOST: " << host << "\r\n";
*this << "Accept: text/*" << "\r\n";
*this << "\r\n";
if (bad()) {
LOG(md,Error,"Failed to connect to host {}. Error: {}",
host,error().message() );
bts_assert_msg(!bad(), error().message());
}
// parse the headers
_response = read_headers(*this, _headers);
// Leave stream ready for reading body
}
HTTPStream::~HTTPStream() { }
// returns true if file was downloaded
bool download_file_if_newer(const string& host, const string& path, const string& destination) {
struct stat st = boost::initialized_value;
stat(destination.c_str(), &st);
// Begin download and check http headers
HTTPStream download(host, path, "GET");
tm modtm = boost::initialized_value;
const char* rfc_1123_fmt = "%a, %d %b %Y %H:%M:%S %Z";
time_t modified = 0;
const std::string last_modified_key = "Last-Modified";
if (download.good() && download.headers().find(last_modified_key) != download.headers().end()) {
stringstream ss(download.headers().at(last_modified_key));
ss >> std::get_time(&modtm, rfc_1123_fmt);
modified = mktime(&modtm);
}
// If our file mtime is newer do not download
if (difftime(modified, st.st_mtime) > 0) {
LOG(md,Info,"Downloading from: {}{}",host,path);
ofstream writer(destination.c_str(), ios::out | ios::binary);
if (writer) {
// This removes all '\n' characters
//copy(istream_iterator<char>(download), istream_iterator<char>(), os treambuf_iterator<char>(writer));
std::vector<char> buffer(1024);
while (!download.eof()) {
download.read(&buffer[0], buffer.size());
writer.write(&buffer[0], download.gcount());
}
return true;
}
}
else {
LOG(md,Info,"Skipping download as cache is newer");
}
return false;
}
}}//namespace