utc / local errors in sparkcmd
This commit is contained in:
1 parent
c1efbc00a5
commit
417fccb03c
16 files changed
+75
-54
No files matched your search
Vendored
+8
-7
@@ -105,13 +105,14 @@
|
||||
"request": "launch",
|
||||
"program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
|
||||
"args": [
|
||||
"historical", "--server"
|
||||
// "--rstart", "11/25/2019",
|
||||
// "--rend", "11/29/2019",
|
||||
// "--rsymbol", "ZN",
|
||||
// "--rsecuritytype", "Synthetic",
|
||||
// "--post_process_file_type", "Trade",
|
||||
// "--upload_to_aws",
|
||||
"historical",
|
||||
//"--server",
|
||||
"--rstart", "11/22/2019",
|
||||
"--rend", "11/25/2019",
|
||||
"--rsymbol", "ES",
|
||||
"--rsecuritytype", "Synthetic",
|
||||
"--post_process_file_type", "Trade",
|
||||
"--upload_to_aws",
|
||||
],
|
||||
"cwd": "${workspaceRoot}",
|
||||
"stopAtEntry": false,
|
||||
|
||||
@@ -79,7 +79,7 @@ namespace bts.excel
|
||||
else
|
||||
{
|
||||
var dd = DateTime.FromOADate((double)dt);
|
||||
var utc = dd.ToUniversalTime();
|
||||
var utc = DateTime.SpecifyKind(dd, DateTimeKind.Utc);
|
||||
var ts = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(utc);
|
||||
s = JsonConvert.SerializeObject(ts, Formatting.None);
|
||||
s = _formatter.Format(ts);
|
||||
|
||||
@@ -22,8 +22,8 @@ namespace bts.history
|
||||
async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
|
||||
HistoricalDataOptions o)
|
||||
{
|
||||
start = start.ToUniversalTime();
|
||||
end = end.ToUniversalTime();
|
||||
start = DateTime.SpecifyKind(start, DateTimeKind.Utc);
|
||||
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
|
||||
var query = new MDQuery
|
||||
{
|
||||
Exchange = Venue.Glbx,
|
||||
@@ -33,7 +33,7 @@ namespace bts.history
|
||||
End = Timestamp.FromDateTime(end),
|
||||
};
|
||||
if(DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
|
||||
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
|
||||
query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc));
|
||||
|
||||
List<IAnalyzer> anal = new List<IAnalyzer>();
|
||||
string algo_security = o.algo_security ?? string.Empty;
|
||||
|
||||
@@ -39,7 +39,7 @@ namespace bts.history
|
||||
where P : IParser, new()
|
||||
{
|
||||
DateTime d = request.Begin.ToDateTime();
|
||||
while (d < request.End.ToDateTime())
|
||||
while (d <= request.End.ToDateTime())
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -107,7 +107,7 @@ namespace bts.history
|
||||
Action<DateTime, Symbol, SecurityType, string, string> action, int peek_field = 0)
|
||||
{
|
||||
DateTime d = request.Begin.ToDateTime();
|
||||
while (d < request.End.ToDateTime())
|
||||
while (d <= request.End.ToDateTime())
|
||||
{
|
||||
try
|
||||
{
|
||||
|
||||
@@ -37,7 +37,7 @@ namespace bts.history
|
||||
|
||||
protected override void send(DateTime dt, Symbol symbol, SecurityType sectype, MarketDataIncrementalRefresh md)
|
||||
{
|
||||
if( (md.MdEntries.Count > 0 && md.MdEntries[0].MdEntryType == _eventtype) || _eventtype == MDEntryType.Unset)
|
||||
if( md.MdEntries[0].MdEntryType == _eventtype || _eventtype == MDEntryType.Unset )
|
||||
{
|
||||
for (int i = 0; i < md.MdEntries.Count; i++)
|
||||
{
|
||||
|
||||
@@ -46,8 +46,8 @@ namespace bts.history
|
||||
|
||||
async virtual public Task generate(DateTime start, DateTime end, string symbol, SecurityType st, HistoricalDataOptions o)
|
||||
{
|
||||
start = start.ToUniversalTime();
|
||||
end = end.ToUniversalTime();
|
||||
start = DateTime.SpecifyKind(start, DateTimeKind.Utc);
|
||||
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
|
||||
var query = new MDQuery
|
||||
{
|
||||
Exchange = Venue.Glbx,
|
||||
@@ -57,7 +57,7 @@ namespace bts.history
|
||||
End = Timestamp.FromDateTime(end),
|
||||
};
|
||||
if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
|
||||
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
|
||||
query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc));
|
||||
|
||||
int count = 0; Stopwatch watch = new Stopwatch();
|
||||
watch.Start();
|
||||
@@ -69,7 +69,7 @@ namespace bts.history
|
||||
if(!line_validator(line))
|
||||
{
|
||||
if (++count % 10_000 == 1)
|
||||
Console.WriteLine($"preprocessed {count.ToString("N0") } lines");
|
||||
Console.WriteLine($"preprocessed {dt} {count.ToString("N0") } lines");
|
||||
return;
|
||||
}
|
||||
var md = ProtoFix.parse_fix<MarketDataIncrementalRefresh>(line);
|
||||
@@ -82,7 +82,7 @@ namespace bts.history
|
||||
if (++count % 10_000 == 1)
|
||||
{
|
||||
var qlen = get_qlen();
|
||||
Console.WriteLine($"preprocessed {count.ToString("N0") } lines, { qlen.ToString("N0") } write queue length");
|
||||
Console.WriteLine($"preprocessed {dt} {count.ToString("N0") } lines, { qlen.ToString("N0") } write queue length");
|
||||
if (qlen > 1_000_000)
|
||||
{
|
||||
var reps = 0;
|
||||
|
||||
@@ -97,8 +97,8 @@ namespace bts.history
|
||||
async public Task generate(DateTime start, DateTime end, string symbol, SecurityType st,
|
||||
HistoricalDataOptions o)
|
||||
{
|
||||
start = start.ToUniversalTime();
|
||||
end = end.ToUniversalTime();
|
||||
start = DateTime.SpecifyKind(start, DateTimeKind.Utc);
|
||||
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
|
||||
var query = new MDQuery
|
||||
{
|
||||
Exchange = Venue.Glbx,
|
||||
@@ -108,7 +108,7 @@ namespace bts.history
|
||||
End = Timestamp.FromDateTime(end),
|
||||
};
|
||||
if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
|
||||
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
|
||||
query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc));
|
||||
|
||||
int count = 0;
|
||||
var comparer = Comparer<MarketDataIncrementalRefresh>.Create((x, y)
|
||||
|
||||
@@ -47,8 +47,8 @@ namespace bts.history
|
||||
{
|
||||
IMongoQueryable<H> res = queryable(collection);
|
||||
Expression<Func<H, bool>> timeexpr = null;
|
||||
begin = begin.ToUniversalTime();
|
||||
end = end.ToUniversalTime();
|
||||
begin = DateTime.SpecifyKind(begin, DateTimeKind.Utc);
|
||||
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
|
||||
if(use_sending_time)
|
||||
timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end;
|
||||
else
|
||||
@@ -61,8 +61,8 @@ namespace bts.history
|
||||
where H : IHasDateTimeStamp
|
||||
{
|
||||
Expression<Func<H, bool>> timeexpr = null;
|
||||
begin = begin.ToUniversalTime();
|
||||
end = end.ToUniversalTime();
|
||||
begin = DateTime.SpecifyKind(begin, DateTimeKind.Utc);
|
||||
end = DateTime.SpecifyKind(end, DateTimeKind.Utc);
|
||||
if (use_sending_time)
|
||||
timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end;
|
||||
else
|
||||
|
||||
@@ -32,7 +32,7 @@ namespace bts.history
|
||||
!string.IsNullOrEmpty(o.replaysymbol) )
|
||||
{
|
||||
var start = DateTime.Parse(o.replaystart);
|
||||
var end = DateTime.Parse(o.replayend).AddDays(1.0).Date; //incl
|
||||
var end = DateTime.Parse(o.replayend); //incl
|
||||
SecurityType st = SecurityType.Future;
|
||||
if(!string.IsNullOrEmpty(o.replaysecuritytype))
|
||||
st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecuritytype, true);
|
||||
|
||||
@@ -54,17 +54,14 @@ OrderOpResult ArrivalPriceAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv
|
||||
auto sectype = admin::AdminServiceUtils::security_type(sec);
|
||||
const std::string& desc = bts::securities::descriptor(sec);
|
||||
google::protobuf::Timestamp end = google::protobuf::util::TimeUtil::GetCurrentTime();
|
||||
end.set_seconds(end.seconds() - 86400);
|
||||
auto days = algo.distribution_params().time_horizon_days();
|
||||
if(days <= 0)
|
||||
return OrderOpResult{ ORDER_ERROR_INVALID_ORDER_DEFINITON, "time_horizon_days > 0" };
|
||||
|
||||
google::protobuf::Timestamp begin;
|
||||
begin.set_seconds(end.seconds() - 86400*days);
|
||||
begin.set_seconds(end.seconds() - 86400*(days+1));
|
||||
|
||||
//TODO
|
||||
bool b = google::protobuf::util::TimeUtil::FromString("2019-11-25T00:00:00.00Z", &begin);
|
||||
b = google::protobuf::util::TimeUtil::FromString("2019-11-29T00:00:00.00Z", &end);
|
||||
b = b && b;
|
||||
query_trading_distribution(serv, symbol, sectype, desc, begin, end, algo.duration().interval_in_seconds());
|
||||
|
||||
if (!serv.is_attaching())
|
||||
|
||||
@@ -74,7 +74,7 @@ void Slicer::query_trading_distribution(IBeforeLaunchAlgoServices& serv, const s
|
||||
{
|
||||
if(res.success())
|
||||
{
|
||||
auto* copy = new TradingDistribution(*res.value());
|
||||
auto copy = new TradingDistribution(*res.value());
|
||||
this->trading_distribution.reset(copy);
|
||||
me->trading_distribution_status = TradingDistributionStatus::Received;
|
||||
}
|
||||
@@ -91,13 +91,28 @@ void Slicer::on_timer()
|
||||
}
|
||||
else if(me->trading_distribution_status == TradingDistributionStatus::Received)
|
||||
{
|
||||
volume_estimator().update_distribution(*this->trading_distribution);
|
||||
price_estimator().update_distribution(*this->trading_distribution);
|
||||
update_params();
|
||||
me->trading_distribution_status = TradingDistributionStatus::Initialized;
|
||||
const auto& td = *this->trading_distribution;
|
||||
if(td.price_variance() == 0.0 ||
|
||||
os::is_nan(td.price_variance()))
|
||||
{
|
||||
algo().terminate_error("Zero or Nan Trading Distribution");
|
||||
}
|
||||
else
|
||||
{
|
||||
volume_estimator().update_distribution(td);
|
||||
price_estimator().update_distribution(td);
|
||||
update_params();
|
||||
me->trading_distribution_status = TradingDistributionStatus::Initialized;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Slicer::setup_schedule()
|
||||
{
|
||||
return me->trading_distribution_status == TradingDistributionStatus::NotRequired ||
|
||||
me->trading_distribution_status == TradingDistributionStatus::Initialized;
|
||||
}
|
||||
|
||||
OrderAggression& Slicer::aggression()
|
||||
{
|
||||
return me->aggression;
|
||||
|
||||
@@ -35,7 +35,7 @@ protected:
|
||||
enum class Channel { Tactical, Aggressive, Market };
|
||||
enum class TradingDistributionStatus { NotRequired, Waiting, Error, Received, Initialized};
|
||||
|
||||
IAlgoServices& algo() const;
|
||||
IAlgoServices& algo() const;
|
||||
void report_status();
|
||||
bool all_children_complete();
|
||||
void get_done();
|
||||
@@ -52,6 +52,7 @@ protected:
|
||||
|
||||
protected:
|
||||
virtual void update_params() = 0;
|
||||
virtual bool setup_schedule();
|
||||
virtual void compute_first_generation(Price& price, Quantity& size) = 0;
|
||||
virtual void compute_next_generation(Price& price, Quantity& size) = 0;
|
||||
|
||||
@@ -83,7 +84,6 @@ protected:
|
||||
TradeVector& last_trades();
|
||||
VolumeEstimator& volume_estimator();
|
||||
PriceEstimator& price_estimator();
|
||||
bool trading_distribution_error();
|
||||
|
||||
void query_trading_distribution(IBeforeLaunchAlgoServices&, const std::string& symbol, api::SecurityType sectype, const std::string& secdesc,
|
||||
const google::protobuf::Timestamp& begin, const google::protobuf::Timestamp& end,
|
||||
|
||||
@@ -57,7 +57,7 @@ void TimeSlicer::on_timer()
|
||||
return;
|
||||
}
|
||||
setup_schedule();
|
||||
return;
|
||||
break;
|
||||
}
|
||||
case Slicer::Status::WORKING:
|
||||
{
|
||||
@@ -116,17 +116,20 @@ google::protobuf::Timestamp TimeSlicer::compute_end_time()
|
||||
}
|
||||
}
|
||||
|
||||
void TimeSlicer::setup_schedule()
|
||||
bool TimeSlicer::setup_schedule()
|
||||
{
|
||||
bts_assert(status() == Slicer::Status::WAITING_FOR_START);
|
||||
status() = Status::WORKING;
|
||||
me->start_time = utils::pb::pb_now();
|
||||
me->end_time = compute_end_time();
|
||||
bool ready = Slicer::setup_schedule();
|
||||
if(ready)
|
||||
{
|
||||
bts_assert(status() == Slicer::Status::WAITING_FOR_START);
|
||||
status() = Status::WORKING;
|
||||
me->start_time = utils::pb::pb_now();
|
||||
me->end_time = compute_end_time();
|
||||
expected_generations() = ( me->end_time.seconds() - me->start_time.seconds() ) / me->duration.interval_in_seconds();
|
||||
if(me->start_time.seconds() > me->end_time.seconds() || expected_generations() <= 0 || expected_generations() > 86400)
|
||||
algo().terminate_error("Invalid scheduling data");
|
||||
}
|
||||
return ready;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ public:
|
||||
void on_timer() override;
|
||||
protected:
|
||||
google::protobuf::Timestamp compute_end_time();
|
||||
void setup_schedule();
|
||||
bool setup_schedule() override;
|
||||
virtual bool check_generation_advance();
|
||||
|
||||
protected:
|
||||
|
||||
@@ -59,7 +59,7 @@ void VolumeSlicer::on_ticker(SecurityKey k, const TradeVector& vec)
|
||||
case Slicer::Status::WAITING_FOR_START:
|
||||
{
|
||||
setup_schedule();
|
||||
return;
|
||||
break;
|
||||
}
|
||||
case Slicer::Status::WORKING:
|
||||
{
|
||||
@@ -97,14 +97,19 @@ bool VolumeSlicer::check_generation_advance()
|
||||
return false;
|
||||
}
|
||||
|
||||
void VolumeSlicer::setup_schedule()
|
||||
bool VolumeSlicer::setup_schedule()
|
||||
{
|
||||
bts_assert(status() == Slicer::Status::WAITING_FOR_START);
|
||||
status() = Status::WORKING;
|
||||
bool ready = Slicer::setup_schedule();
|
||||
if(ready)
|
||||
{
|
||||
bts_assert(status() == Slicer::Status::WAITING_FOR_START);
|
||||
status() = Status::WORKING;
|
||||
|
||||
expected_generations() = (uint32_t) duration().order_duration_in_lots() / duration().interval_in_lots();
|
||||
if(expected_generations() <= 0 || expected_generations() > 2000000)
|
||||
algo().terminate_error("Invalid scheduling data");
|
||||
expected_generations() = (uint32_t) duration().order_duration_in_lots() / duration().interval_in_lots();
|
||||
if(expected_generations() <= 0 || expected_generations() > 2000000)
|
||||
algo().terminate_error("Invalid scheduling data");
|
||||
}
|
||||
return ready;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ public:
|
||||
void on_ticker(SecurityKey, const TradeVector&) override;
|
||||
protected:
|
||||
virtual bool check_generation_advance();
|
||||
virtual void setup_schedule();
|
||||
bool setup_schedule() override;
|
||||
|
||||
protected:
|
||||
struct PImpl;
|
||||
|
||||
Reference in new issue
Block a user