utc / local errors in sparkcmd

This commit is contained in:
Kevin Darby committed 2020-02-18 16:53:07 -05:00
1 parent c1efbc00a5
commit 417fccb03c
16 files changed
+75 -54

No files matched your search

+8 -7
View File
@@ -105,13 +105,14 @@
"request": "launch",
"program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
"args": [
"historical", "--server"
// "--rstart", "11/25/2019",
// "--rend", "11/29/2019",
// "--rsymbol", "ZN",
// "--rsecuritytype", "Synthetic",
// "--post_process_file_type", "Trade",
// "--upload_to_aws",
"historical",
//"--server",
"--rstart", "11/22/2019",
"--rend", "11/25/2019",
"--rsymbol", "ES",
"--rsecuritytype", "Synthetic",
"--post_process_file_type", "Trade",
"--upload_to_aws",
],
"cwd": "${workspaceRoot}",
"stopAtEntry": false,
+1 -1
View File
@@ -79,7 +79,7 @@ namespace bts.excel
else
{
var dd = DateTime.FromOADate((double)dt);
var utc = dd.ToUniversalTime();
var utc = DateTime.SpecifyKind(dd, DateTimeKind.Utc);
var ts = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(utc);
s = JsonConvert.SerializeObject(ts, Formatting.None);
s = _formatter.Format(ts);
+3 -3
View File
@@ -22,8 +22,8 @@ namespace bts.history
async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
HistoricalDataOptions o)
{
start = start.ToUniversalTime();
end = end.ToUniversalTime();
start = DateTime.SpecifyKind(start, DateTimeKind.Utc);
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
var query = new MDQuery
{
Exchange = Venue.Glbx,
@@ -33,7 +33,7 @@ namespace bts.history
End = Timestamp.FromDateTime(end),
};
if(DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc));
List<IAnalyzer> anal = new List<IAnalyzer>();
string algo_security = o.algo_security ?? string.Empty;
+2 -2
View File
@@ -39,7 +39,7 @@ namespace bts.history
where P : IParser, new()
{
DateTime d = request.Begin.ToDateTime();
while (d < request.End.ToDateTime())
while (d <= request.End.ToDateTime())
{
try
{
@@ -107,7 +107,7 @@ namespace bts.history
Action<DateTime, Symbol, SecurityType, string, string> action, int peek_field = 0)
{
DateTime d = request.Begin.ToDateTime();
while (d < request.End.ToDateTime())
while (d <= request.End.ToDateTime())
{
try
{
@@ -37,7 +37,7 @@ namespace bts.history
protected override void send(DateTime dt, Symbol symbol, SecurityType sectype, MarketDataIncrementalRefresh md)
{
if( (md.MdEntries.Count > 0 && md.MdEntries[0].MdEntryType == _eventtype) || _eventtype == MDEntryType.Unset)
if( md.MdEntries[0].MdEntryType == _eventtype || _eventtype == MDEntryType.Unset )
{
for (int i = 0; i < md.MdEntries.Count; i++)
{
@@ -46,8 +46,8 @@ namespace bts.history
async virtual public Task generate(DateTime start, DateTime end, string symbol, SecurityType st, HistoricalDataOptions o)
{
start = start.ToUniversalTime();
end = end.ToUniversalTime();
start = DateTime.SpecifyKind(start, DateTimeKind.Utc);
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
var query = new MDQuery
{
Exchange = Venue.Glbx,
@@ -57,7 +57,7 @@ namespace bts.history
End = Timestamp.FromDateTime(end),
};
if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc));
int count = 0; Stopwatch watch = new Stopwatch();
watch.Start();
@@ -69,7 +69,7 @@ namespace bts.history
if(!line_validator(line))
{
if (++count % 10_000 == 1)
Console.WriteLine($"preprocessed {count.ToString("N0") } lines");
Console.WriteLine($"preprocessed {dt} {count.ToString("N0") } lines");
return;
}
var md = ProtoFix.parse_fix<MarketDataIncrementalRefresh>(line);
@@ -82,7 +82,7 @@ namespace bts.history
if (++count % 10_000 == 1)
{
var qlen = get_qlen();
Console.WriteLine($"preprocessed {count.ToString("N0") } lines, { qlen.ToString("N0") } write queue length");
Console.WriteLine($"preprocessed {dt} {count.ToString("N0") } lines, { qlen.ToString("N0") } write queue length");
if (qlen > 1_000_000)
{
var reps = 0;
@@ -97,8 +97,8 @@ namespace bts.history
async public Task generate(DateTime start, DateTime end, string symbol, SecurityType st,
HistoricalDataOptions o)
{
start = start.ToUniversalTime();
end = end.ToUniversalTime();
start = DateTime.SpecifyKind(start, DateTimeKind.Utc);
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
var query = new MDQuery
{
Exchange = Venue.Glbx,
@@ -108,7 +108,7 @@ namespace bts.history
End = Timestamp.FromDateTime(end),
};
if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc));
int count = 0;
var comparer = Comparer<MarketDataIncrementalRefresh>.Create((x, y)
+4 -4
View File
@@ -47,8 +47,8 @@ namespace bts.history
{
IMongoQueryable<H> res = queryable(collection);
Expression<Func<H, bool>> timeexpr = null;
begin = begin.ToUniversalTime();
end = end.ToUniversalTime();
begin = DateTime.SpecifyKind(begin, DateTimeKind.Utc);
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
if(use_sending_time)
timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end;
else
@@ -61,8 +61,8 @@ namespace bts.history
where H : IHasDateTimeStamp
{
Expression<Func<H, bool>> timeexpr = null;
begin = begin.ToUniversalTime();
end = end.ToUniversalTime();
begin = DateTime.SpecifyKind(begin, DateTimeKind.Utc);
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
if (use_sending_time)
timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end;
else
@@ -32,7 +32,7 @@ namespace bts.history
!string.IsNullOrEmpty(o.replaysymbol) )
{
var start = DateTime.Parse(o.replaystart);
var end = DateTime.Parse(o.replayend).AddDays(1.0).Date; //incl
var end = DateTime.Parse(o.replayend); //incl
SecurityType st = SecurityType.Future;
if(!string.IsNullOrEmpty(o.replaysecuritytype))
st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecuritytype, true);
@@ -54,17 +54,14 @@ OrderOpResult ArrivalPriceAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv
auto sectype = admin::AdminServiceUtils::security_type(sec);
const std::string& desc = bts::securities::descriptor(sec);
google::protobuf::Timestamp end = google::protobuf::util::TimeUtil::GetCurrentTime();
end.set_seconds(end.seconds() - 86400);
auto days = algo.distribution_params().time_horizon_days();
if(days <= 0)
return OrderOpResult{ ORDER_ERROR_INVALID_ORDER_DEFINITON, "time_horizon_days > 0" };
google::protobuf::Timestamp begin;
begin.set_seconds(end.seconds() - 86400*days);
begin.set_seconds(end.seconds() - 86400*(days+1));
//TODO
bool b = google::protobuf::util::TimeUtil::FromString("2019-11-25T00:00:00.00Z", &begin);
b = google::protobuf::util::TimeUtil::FromString("2019-11-29T00:00:00.00Z", &end);
b = b && b;
query_trading_distribution(serv, symbol, sectype, desc, begin, end, algo.duration().interval_in_seconds());
if (!serv.is_attaching())
@@ -74,7 +74,7 @@ void Slicer::query_trading_distribution(IBeforeLaunchAlgoServices& serv, const s
{
if(res.success())
{
auto* copy = new TradingDistribution(*res.value());
auto copy = new TradingDistribution(*res.value());
this->trading_distribution.reset(copy);
me->trading_distribution_status = TradingDistributionStatus::Received;
}
@@ -91,13 +91,28 @@ void Slicer::on_timer()
}
else if(me->trading_distribution_status == TradingDistributionStatus::Received)
{
volume_estimator().update_distribution(*this->trading_distribution);
price_estimator().update_distribution(*this->trading_distribution);
update_params();
me->trading_distribution_status = TradingDistributionStatus::Initialized;
const auto& td = *this->trading_distribution;
if(td.price_variance() == 0.0 ||
os::is_nan(td.price_variance()))
{
algo().terminate_error("Zero or Nan Trading Distribution");
}
else
{
volume_estimator().update_distribution(td);
price_estimator().update_distribution(td);
update_params();
me->trading_distribution_status = TradingDistributionStatus::Initialized;
}
}
}
bool Slicer::setup_schedule()
{
return me->trading_distribution_status == TradingDistributionStatus::NotRequired ||
me->trading_distribution_status == TradingDistributionStatus::Initialized;
}
OrderAggression& Slicer::aggression()
{
return me->aggression;
@@ -35,7 +35,7 @@ protected:
enum class Channel { Tactical, Aggressive, Market };
enum class TradingDistributionStatus { NotRequired, Waiting, Error, Received, Initialized};
IAlgoServices& algo() const;
IAlgoServices& algo() const;
void report_status();
bool all_children_complete();
void get_done();
@@ -52,6 +52,7 @@ protected:
protected:
virtual void update_params() = 0;
virtual bool setup_schedule();
virtual void compute_first_generation(Price& price, Quantity& size) = 0;
virtual void compute_next_generation(Price& price, Quantity& size) = 0;
@@ -83,7 +84,6 @@ protected:
TradeVector& last_trades();
VolumeEstimator& volume_estimator();
PriceEstimator& price_estimator();
bool trading_distribution_error();
void query_trading_distribution(IBeforeLaunchAlgoServices&, const std::string& symbol, api::SecurityType sectype, const std::string& secdesc,
const google::protobuf::Timestamp& begin, const google::protobuf::Timestamp& end,
@@ -57,7 +57,7 @@ void TimeSlicer::on_timer()
return;
}
setup_schedule();
return;
break;
}
case Slicer::Status::WORKING:
{
@@ -116,17 +116,20 @@ google::protobuf::Timestamp TimeSlicer::compute_end_time()
}
}
void TimeSlicer::setup_schedule()
bool TimeSlicer::setup_schedule()
{
bts_assert(status() == Slicer::Status::WAITING_FOR_START);
status() = Status::WORKING;
me->start_time = utils::pb::pb_now();
me->end_time = compute_end_time();
bool ready = Slicer::setup_schedule();
if(ready)
{
bts_assert(status() == Slicer::Status::WAITING_FOR_START);
status() = Status::WORKING;
me->start_time = utils::pb::pb_now();
me->end_time = compute_end_time();
expected_generations() = ( me->end_time.seconds() - me->start_time.seconds() ) / me->duration.interval_in_seconds();
if(me->start_time.seconds() > me->end_time.seconds() || expected_generations() <= 0 || expected_generations() > 86400)
algo().terminate_error("Invalid scheduling data");
}
return ready;
}
@@ -14,7 +14,7 @@ public:
void on_timer() override;
protected:
google::protobuf::Timestamp compute_end_time();
void setup_schedule();
bool setup_schedule() override;
virtual bool check_generation_advance();
protected:
@@ -59,7 +59,7 @@ void VolumeSlicer::on_ticker(SecurityKey k, const TradeVector& vec)
case Slicer::Status::WAITING_FOR_START:
{
setup_schedule();
return;
break;
}
case Slicer::Status::WORKING:
{
@@ -97,14 +97,19 @@ bool VolumeSlicer::check_generation_advance()
return false;
}
void VolumeSlicer::setup_schedule()
bool VolumeSlicer::setup_schedule()
{
bts_assert(status() == Slicer::Status::WAITING_FOR_START);
status() = Status::WORKING;
bool ready = Slicer::setup_schedule();
if(ready)
{
bts_assert(status() == Slicer::Status::WAITING_FOR_START);
status() = Status::WORKING;
expected_generations() = (uint32_t) duration().order_duration_in_lots() / duration().interval_in_lots();
if(expected_generations() <= 0 || expected_generations() > 2000000)
algo().terminate_error("Invalid scheduling data");
expected_generations() = (uint32_t) duration().order_duration_in_lots() / duration().interval_in_lots();
if(expected_generations() <= 0 || expected_generations() > 2000000)
algo().terminate_error("Invalid scheduling data");
}
return ready;
}
@@ -16,7 +16,7 @@ public:
void on_ticker(SecurityKey, const TradeVector&) override;
protected:
virtual bool check_generation_advance();
virtual void setup_schedule();
bool setup_schedule() override;
protected:
struct PImpl;