Working on adding seamless decompression.

This commit is contained in:
Pedro Pinto committed 2020-08-31 08:00:41 -04:00
1 parent 6b09e18948
commit f5fe552cd9
1 file changed
+52 -37
@@ -12,17 +12,21 @@
#include "../handler/ChannelHandler.h"
#include "../handler/IChannelObserver.h"
#include "../parsing/MDP3Parser.h"
#include "lib/utils/Uuid.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/overloaded.h"
#include <Pcap++/PcapFileDevice.h>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/date_time/posix_time/conversion.hpp>
#include <boost/date_time/posix_time/ptime.hpp>
#include <fmt/format.h>
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
#include <sys/stat.h>
#include <sys/types.h>
namespace bts::spartan::mdp3 {
@@ -348,48 +352,59 @@ int to_json(std::string const &pcap, size_t first_packet, size_t count) {
return 0;
}
void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
struct ObsWrapper : mdp3::IChannelObserver {
ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {}
IChannelObserver &real_obs;
size_t previous_packet = 0;
size_t previous_send_ts = 0;
TimeStamp last_print = Timer::now();
struct ObsWrapper : mdp3::IChannelObserver {
ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {}
IChannelObserver &real_obs;
size_t previous_packet = 0;
size_t previous_send_ts = 0;
TimeStamp last_print = Timer::now();
void on_recovery_complete(
std::vector<mdp3::InstrumentSnapshot const *> const &s) override {
real_obs.on_recovery_complete(s);
void on_recovery_complete(
std::vector<mdp3::InstrumentSnapshot const *> const &s) override {
real_obs.on_recovery_complete(s);
}
void on_real_time_event(mdp3::MDP3Event const &ev) override {
if (ms_diff(Timer::now(), last_print) >= 1000) {
last_print = Timer::now();
std::stringstream str;
mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str);
LOG(Spartan, Info, "Packet time: {} ({} pkt/sec {}x real time)",
str.str(), (ev.first_packet_sn - previous_packet),
(ev.first_packet_send_time.value() - previous_send_ts) /
1000000000);
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
void on_real_time_event(mdp3::MDP3Event const &ev) override {
if (ms_diff(Timer::now(), last_print) >= 1000) {
last_print = Timer::now();
std::stringstream str;
mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str);
LOG(Spartan, Info, "Packet time: {} ({} pkt/sec {}x real time)",
str.str(), (ev.first_packet_sn - previous_packet),
(ev.first_packet_send_time.value() - previous_send_ts) /
1000000000);
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
real_obs.on_real_time_event(ev);
}
void on_real_time_channel_loss() override {
real_obs.on_real_time_channel_loss();
}
} obs_wrapper(obs);
real_obs.on_real_time_event(ev);
}
void on_real_time_channel_loss() override {
real_obs.on_real_time_channel_loss();
}
};
struct Ctrl : mdp3::IFeedController {
void start_instrument_feed() override{};
void stop_instrument_feed() override{};
void start_snapshot_feed() override{};
void stop_snapshot_feed() override{};
void start_incremental_feed() override{};
void stop_incremental_feed() override{};
};
void observe_channel(std::string const &in_file, mdp3::IChannelObserver &obs) {
ObsWrapper obs_wrapper(obs);
Ctrl ctrl;
struct Ctrl : mdp3::IFeedController {
void start_instrument_feed() override{};
void stop_instrument_feed() override{};
void start_snapshot_feed() override{};
void stop_snapshot_feed() override{};
void start_incremental_feed() override{};
void stop_incremental_feed() override{};
} ctrl;
std::array<PacketBuffers, 2> buffers;
Guesser guesser;
std::string file = in_file;
// if (boost::algorithm::ends_with(file, "*.xz")) {
// file = fmt::format("/tmp/xz-decode-{}",
// utils::Uuid{}.to_text_string()); auto er = mkfifkkallo(file,0660);
// bts_assert(er == 0);
// }
auto reader = PCapReader{file};
reader.on_packet() = [&](auto const &info, auto const &payload) {
guesser.set_current_source(info);