Merge branch 'cleanup_mdclient' of bitbucket.org:bluetradesys/spark into fix

This commit is contained in:
Kevin Darby committed 2018-12-11 23:05:35 -05:00
commit 941f4bf837
115 files changed
+1669 -1151

No files matched your search

+10 -48
View File
@@ -17,31 +17,22 @@ using bts.api.user;
namespace bts {
[Export(typeof(IInitializable))]
class MDSource : IObserver<List<SecurityAndHandle>>, IInitializable, IDisposable {
class MDDaemon : bts.md.MDSource, IObserver<List<SecurityAndHandle>>, IInitializable, IDisposable {
readonly Lazy<IBus> _bus;
readonly Lazy<AuthToken> _token;
readonly Lazy<AuthTokenImmutable> _token;
readonly Lazy<SyntheticSecurityService> _ss;
readonly MDConsumer _consumer = new MDConsumer();
readonly Co2RelayControl _relay;
readonly Dictionary<SecurityKey, SecurityMD> _deleted_synthetics = new Dictionary<SecurityKey, SecurityMD>();
long _version = 0;
readonly ConcurrentDictionary<SecurityKey, SecurityMD> _exchange_sec_map = new ConcurrentDictionary<SecurityKey, SecurityMD>();
List<IDisposable> _dispose = new List<IDisposable>();
static uint get_naggle() => mbts.config.Configuration.get<bts.md.RelayOptions>().naggle_ms;
[ImportingConstructor]
public MDSource(
Lazy<IBus> bus,
Lazy<AuthToken> token,
Lazy<SyntheticSecurityService> ss) {
public MDDaemon(Lazy<IBus> bus, Lazy<AuthTokenImmutable> token, Lazy<SyntheticSecurityService> ss) : base(get_naggle(), true) {
_bus = bus;
_token = token;
_ss = ss;
_relay = new Co2RelayControl(_consumer);
_relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore();
_dispose.Add(_consumer.on_security.Subscribe(this));
_ss = ss;
}
public void initialize() {
override public void initialize() {
_relay.set_token(_token.Value);
_dispose.Add(
_ss.Value.subscribei_metadata()
@@ -66,6 +57,9 @@ namespace bts {
// Post full dictionary of syntheics here
bus.post(SparkTags.SyntheticstrategiesTag, t);
}));
_dispose.Add( this.on_securities.Subscribe( (md) => {
bus.post(SparkTags.SecuritiesTag, md);
}));
Task.Run(run);
}
@@ -74,37 +68,5 @@ namespace bts {
public int priority => 55;
IBus bus => _bus.Value;
public void OnCompleted() { }
public void OnError(Exception error) => Log.Md.error(error.ToString());
public void OnNext(List<SecurityAndHandle> values) {
if (values.Count > 0) {
foreach (var v in values) {
var smd = new SecurityMD(v.security, v.handle);
_exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd);
}
_version += 1;
}
}
public void Dispose() => Disposable.dispose(_dispose);
async Task run() {
long run_version = 0;
while (true) {
var exch = _exchange_sec_map;
long current_version = _version;
if (current_version > run_version) {
run_version = current_version;
try {
Log.Md.info($"MDSource publishing {exch.Count} securities");
bus.post(SparkTags.SecuritiesTag, exch);
} catch (Exception e) {
Log.Md.error($"Error in publish_securities: {e}");
}
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
}
}
}
+1 -4
View File
@@ -13,10 +13,7 @@ using System.ComponentModel.Composition;
namespace Spark {
public class ShellServices {
[Export]
public static ServiceContext context { get; } = new ServiceContext();
[Export]
public UserService user_service => context.locate<UserService>();
public ServiceContext context { get; } = new ServiceContext();
[Export("settings")]
public PersistenceService settings_service => context.locate<PersistenceService>("settings");
@@ -3,6 +3,7 @@ using bts.api.orderman;
using bts.shell;
using mbts.spark;
using mbts.ui;
using mbts;
using System;
using System.Collections.Generic;
@@ -218,9 +218,7 @@ namespace mbts
private Func<T, T, int> _comparer_func;
public DelegateComparer(Func<T, T, int> comparer_func) { _comparer_func = comparer_func; }
public int Compare(T x, T y) { return _comparer_func(x, y); }
}
public static void ignore(this Task t) { }
}
public static Google.Protobuf.WellKnownTypes.Timestamp ToGoogleTime(this DateTime self) {
return Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(self);
@@ -22,12 +22,12 @@ namespace mbts.ui.settings
readonly PersistenceService _persist;
[ImportingConstructor]
public UISettingsFactory(UserService login, [Import("settings")] PersistenceService persist) {
public UISettingsFactory(AuthService login, [Import("settings")] PersistenceService persist) {
_persist = persist;
_users = get_users(login);
}
async Task<User[]> get_users(UserService login) {
async Task<User[]> get_users(AuthService login) {
var users = await login.get_users_async(null);
if (users.success) return users.value.Users.ToArray();
else return new User[0];
@@ -120,7 +120,7 @@ namespace mbts.persist
public IObservable<IEnumerable<PersistResult<T>>> subscribe<T>(string ns, string pattern) {
var re = new Regex(pattern);
return _persist
.subscribe_ns(new NamespaceSubArgs() { Ns = uns(ns) })
.subscribe_ns(uns(ns))
.Select(update =>
update.added
.Where(item => re.IsMatch(item.Key))
+45 -69
View File
@@ -1,39 +1,29 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Threading;
using mbts.config;
using mbts.ui.native;
using mbts.utils;
using bts.co2;
using bts.api.user;
using bts;
using bts.api;
using bts.api.core;
using bts.api.persistence;
using bts;
using bts.api.user;
using mbts.config;
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Linq;
using System.Threading.Tasks;
using System.Windows.Threading;
namespace mbts.ui.shell
{
namespace mbts.ui.shell {
// The LoginContainer participates in the startup Import/Export dependency system
// Since the login form should not depend on *anything* it will be shown before this is created
// Once this is created it access the global LoginModel and retrieves the username
internal class LoginContainer
{
[ImportingConstructor]
public LoginContainer(UserService user, [Import("settings")] PersistenceService persist) {
public LoginContainer(ServiceContext context) {
var model = LoginModel.instance;
model.provide_services(user, persist);
model.provide_services(context);
// await_login pumps the dispatcher but doesn't allow control to return to MEF until the username and workspace are available
model.await_login();
var u = model.get_user();
UserName = u.name;
UserId = u.id;
Token = u.token;
(UserName,UserId,Token) = model.get_user();
Workspace = model.get_workspace();
model.Dispose();
}
@@ -44,21 +34,19 @@ namespace mbts.ui.shell
[Export("userid")]
public uint UserId { get; private set; }
[Export]
public AuthToken Token { get; private set; }
public AuthTokenImmutable Token { get; private set; }
[Export("workspace")]
public string Workspace { get; private set; }
}
public struct User {
public User(string name, uint id, AuthToken token) {
public User(string name, uint id) {
this.name = name;
this.id = id;
this.token = token;
}
public readonly string name;
public readonly UInt32 id;
public readonly AuthToken token;
}
public class LoginModel : IDisposable
@@ -66,11 +54,10 @@ namespace mbts.ui.shell
// Singleton to help bootstrap the system
private static LoginModel _instance = new LoginModel();
public static LoginModel instance { get { return _instance; } }
private User _user;
private string _workspace;
private PreloginWorkspaceEnumerator _enum_workspace;
UserService _logon;
ServiceContext _ctx;
DispatcherFrame _frame = new DispatcherFrame();
private LoginModel() {
@@ -82,13 +69,14 @@ namespace mbts.ui.shell
public string ErrorMessage { get; private set; }
public bool HasFatalError { get; set; }
public User[] Users { get; private set; }
// The final username
public User get_user() {
Log.assert_d(!string.IsNullOrEmpty(_user.name));
return _user;
public (string, uint, AuthTokenImmutable) get_user() {
Log.assert_d(!string.IsNullOrEmpty(_ctx.user));
return (_ctx.user, _ctx.user_id, _ctx.token.Value);
}
public string get_workspace() {
return _workspace ?? "";
}
@@ -112,13 +100,13 @@ namespace mbts.ui.shell
}
// Wait for the hub to create the login doodad
(var users, var persist) = await _hub_src.Task;
_logon = users;
_ctx = await _hub_src.Task;
// Populate the UI
_enum_workspace = new PreloginWorkspaceEnumerator(persist);
var response = await _logon.get_users_async(new GetUsersRequest());
_enum_workspace = new PreloginWorkspaceEnumerator(_ctx.locate<PersistenceService>("settings"));
var auth = _ctx.locate<AuthService>();
var response = await auth.get_users_async(new GetUsersRequest());
if (response.success) {
Users = response.value.Users.Select(x => new User(x.Login,x.Id, null)).OrderBy(u => u.name).ToArray();
Users = response.value.Users.Select(x => new User(x.Login,x.Id)).OrderBy(u => u.name).ToArray();
}
else {
ErrorMessage = "Logon: " + response.error_message;
@@ -129,49 +117,37 @@ namespace mbts.ui.shell
#endregion
public async Task<(bool success,string info)> do_login(string username, string workspace, string password) {
public async Task<LoginResult> do_login(string username, string workspace, string password) {
if (Users.Select(u => u.name).Contains(username)) {
var response = await _logon.authenticate_login_async(new AuthenticateLoginRequest() {
Login = username,
Password = password ??""
});
if (response.success) {
if (response.value.IsValid) {
_workspace = workspace;
set_authenticated_user(new User(username, response.value.UserId, response.value.Token));
return (true, "");
}
else {
return (false, response.value.Reason);
}
}
else {
return (false, response.error_message);
var result = await _ctx.login(username, password);
if (result.success) {
_workspace = workspace;
on_user_authenticated(_ctx.user, _ctx.token);
}
return result;
}
return (false, "Invalid username");
return new LoginResult("Invalid username");
}
private void set_authenticated_user(User user) {
Log.assert(!string.IsNullOrEmpty(user.name));
Log.assert(Users.Select(u => u.name).Contains(user.name));
_user = user;
Environment.SetEnvironmentVariable("BTS_USER", user.name);
private void on_user_authenticated(string username, AuthTokenImmutable? token) {
Log.assert(!string.IsNullOrEmpty(username));
Log.assert(Users.Select(u => u.name).Contains(username));
Environment.SetEnvironmentVariable("BTS_USER", username);
_frame.Continue = false;
}
public void await_login() {
if (string.IsNullOrEmpty(_user.name)) {
if (!_ctx.token.HasValue) {
Dispatcher.PushFrame(_frame);
}
}
#region provide hub
public void provide_services(UserService u, PersistenceService p) {
_hub_src.TrySetResult((u,p));
internal void provide_services(ServiceContext ctx) {
_hub_src.TrySetResult(ctx);
}
private TaskCompletionSource<(UserService, PersistenceService)> _hub_src = new TaskCompletionSource<(UserService, PersistenceService)>();
private TaskCompletionSource<ServiceContext> _hub_src = new TaskCompletionSource<ServiceContext>();
#endregion
#region Dispose
@@ -138,14 +138,15 @@ namespace mbts.ui.shell
if (selected_workspace == "New workspace...") {
selected_workspace = null;
}
(bool success, string message) = await _model.do_login(Name, selected_workspace, Password);
if (success) { VisualState = "Loading"; }
if (!success && !string.IsNullOrEmpty(message)) {
WarningMessage = message;
var result = await _model.do_login(Name, selected_workspace, Password);
if (result.success) { VisualState = "Loading"; }
if (!result.success && !string.IsNullOrEmpty(result.reason)) {
WarningMessage = result.reason;
VisualState = "BadPassword";
}
else if (!string.IsNullOrEmpty(message)) {
BTSMessageBox.Show(message, "Login Message", BTSMessageBoxButtons.OK, MessageBoxImage.Information);
else if (!string.IsNullOrEmpty(result.reason)) {
// Login was successful but we have a message for the user
BTSMessageBox.Show(result.reason, "Login Message", BTSMessageBoxButtons.OK, MessageBoxImage.Information);
}
}
private void OnCancel(object unused) {
@@ -11,7 +11,7 @@ namespace bts.settings {
public class SettingsManagerTool {
public static void run() {
var sc = new ServiceContext();
var users = sc.locate<UserService>();
var users = sc.locate<AuthService>();
var persist = sc.locate<PersistenceService>("settings");
var user_list = users.get_users(new GetUsersRequest());
var user_names = user_list.Users.Select(u => u.Login);
@@ -299,6 +299,7 @@ namespace mbts.spark {
_change.set(mid - md.settlement_price);
}
_volume.set(volume);
_misc_time = _model.md.misc_timestamp;
}
}
@@ -13,14 +13,9 @@ namespace mbts.co2
{
internal static unsafe T deserialize<T>(byte* buffer, ulong length) where T : new() {
var stream = new System.IO.UnmanagedMemoryStream(buffer, (long)length);
if (typeof(IMessage).IsAssignableFrom(typeof(T))) {
IMessage result = (IMessage)new T();
result.MergeFrom(stream);
return (T)result;
}
else {
return ProtoBuf.Serializer.Deserialize<T>(stream);
}
IMessage result = (IMessage)new T();
result.MergeFrom(stream);
return (T)result;
}
[DllImport("libspark")]
@@ -81,7 +76,7 @@ namespace mbts.co2
[DllImport("libspark")]
internal unsafe static extern IntPtr add_relay_symbol(IntPtr relay, UserSymbol symbol, bts.co2.MDRequest.Types.Flags flags);
[DllImport("libspark")]
internal unsafe static extern IntPtr set_relay_token(IntPtr relay, string user_name);
internal unsafe static extern IntPtr set_relay_token(IntPtr relay, byte[] token, long token_size);
[DllImport("libspark")]
internal unsafe static extern void post_on_scan_security_thread(Action callback);
@@ -14,17 +14,16 @@ namespace mbts.co2
{
MDConsumer _consumer;
IntPtr _relay;
public Co2RelayControl(MDConsumer consumer) {
public Co2RelayControl(MDConsumer consumer, uint naggle_ms) {
_consumer = consumer;
Log.assert(consumer.native_pointer() != IntPtr.Zero);
_relay =
Co2NativeMethods.start_private_relay(consumer.native_pointer(),
config.Configuration.get<bts.md.RelayOptions>().naggle_ms);
_relay = Co2NativeMethods.start_private_relay(consumer.native_pointer(), naggle_ms);
Log.assert(_relay != IntPtr.Zero);
}
public void set_token(AuthToken token) {
Co2NativeMethods.set_relay_token(_relay, token.Token);
public void set_token(AuthTokenImmutable token) {
var buffer = token.Token.ToByteArray();
Co2NativeMethods.set_relay_token(_relay, buffer, buffer.LongLength);
}
public async Task preload_securities(MDRequest.Types.Flags flags=MDRequest.Types.Flags.Security) {
File renamed without changes.
@@ -9,6 +9,7 @@ using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using security_key = System.UInt32;
using bts;
namespace mbts.co2
{
File renamed without changes.
+2 -2
View File
@@ -64,7 +64,7 @@ namespace bts.api.core {
internal static extern void api_client_send(IntPtr nchannel, byte[] buffer, int length, raw_pb_callback p, IntPtr extra);
[DllImport("libspark")]
internal static extern IntPtr api_client_subscribe(IntPtr client, string method_and_qualifier, raw_subscription_callback cb, IntPtr extra);
internal static extern IntPtr api_client_subscribe(IntPtr client, string method, string qualifier, raw_subscription_callback cb, IntPtr extra);
[DllImport("libspark")]
internal static extern void api_client_unsubscribe(IntPtr client, IntPtr subscription);
@@ -96,7 +96,7 @@ namespace bts.api.core {
}
[DllImport("libspark")]
internal static extern IntPtr api_server_create_table(IntPtr server, string method_and_qualifier);
internal static extern IntPtr api_server_create_table(IntPtr server, string method, string qualifier);
[DllImport("libspark")]
internal static extern void api_server_update_table(IntPtr table, [In] StreamUpdate[] changes, long change_count);
[DllImport("libspark")]
-9
View File
@@ -17,15 +17,6 @@ namespace bts.api.core
internal abstract class ApiServer : IApiServer
{
static JsonFormatter _formatter = new JsonFormatter(new JsonFormatter.Settings(true));
static Regex _fmt_cleanup = new Regex(@"[\s_+]", RegexOptions.Compiled | RegexOptions.Multiline);
public static string format_qualifier<T>(string method, T arg) where T : IMessage {
if (arg != null) {
string arg_fmt = _fmt_cleanup.Replace(_formatter.Format(arg), "");
return method + arg_fmt;
}
return method;
}
public abstract void dispatch(RequestHeader header, Stream request, Action<byte[]> on_response);
protected void respond<T>(int request_id, Task<T> task, Action<byte[]> on_response) where T : IMessage, IImmutableMessage {
+61 -61
View File
@@ -1,68 +1,68 @@
using Google.Protobuf;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace bts.api {
internal static class ApiUtils {
public static byte[] serialize<T>(this RequestHeader header, T body) where T : IImmutableMessage {
int body_size = body?.CalculateSize() ?? 0;
int header_size = header.CalculateSize() + 1; // 1 byte header length
var buffer = new byte[header_size+body_size];
using bts.api.core;
using Google.Protobuf;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace bts.api {
internal static class ApiUtils {
public static byte[] serialize<T>(this RequestHeader header, T body) where T : IImmutableMessage {
int body_size = body?.CalculateSize() ?? 0;
int header_size = header.CalculateSize() + 1; // 1 byte header length
var buffer = new byte[header_size + body_size];
using (var stream = new MemoryStream(buffer)) {
checked { stream.WriteByte((byte)header_size); }
header.WriteTo(stream);
if (body != null) body.WriteTo(stream);
}
return buffer;
}
public struct DeserializeResult<THeader,TBody> {
public THeader header;
public TBody body;
}
private static DeserializeResult<THeader, TBody> mkresult<THeader,TBody>(THeader header, TBody body) {
return new DeserializeResult<THeader, TBody>() { header = header, body = body }; }
public static unsafe DeserializeResult<THeader, TBody> deserialize<THeader,TBody>(byte* buffer, long buffer_length) where THeader : IMessage,new() where TBody : class,IMessage,new() {
var header_size = buffer[0];
Debug.Assert(header_size <= buffer_length);
THeader header = new THeader();
header.MergeFrom(new UnmanagedMemoryStream(buffer+1, header_size - 1));
TBody body = new TBody();
if (buffer_length > header_size) {
body.MergeFrom(new UnmanagedMemoryStream(buffer + header_size, buffer_length - header_size));
}
return mkresult(header, body);
}
public static byte[] serialize<T>(this ResponseHeader header, T body) where T : IImmutableMessage {
int header_size = 1 + header.CalculateSize();
int body_size = body == null ? 0 : body.CalculateSize();
}
return buffer;
}
public struct DeserializeResult<THeader, TBody> {
public THeader header;
public TBody body;
}
private static DeserializeResult<THeader, TBody> mkresult<THeader, TBody>(THeader header, TBody body) {
return new DeserializeResult<THeader, TBody>() { header = header, body = body }; }
public static unsafe DeserializeResult<THeader, TBody> deserialize<THeader, TBody>(byte* buffer, long buffer_length) where THeader : IMessage, new() where TBody : class, IMessage, new() {
var header_size = buffer[0];
Debug.Assert(header_size <= buffer_length);
THeader header = new THeader();
header.MergeFrom(new UnmanagedMemoryStream(buffer + 1, header_size - 1));
TBody body = new TBody();
if (buffer_length > header_size) {
body.MergeFrom(new UnmanagedMemoryStream(buffer + header_size, buffer_length - header_size));
}
return mkresult(header, body);
}
public static byte[] serialize<T>(this ResponseHeader header, T body) where T : IImmutableMessage {
int header_size = 1 + header.CalculateSize();
int body_size = body == null ? 0 : body.CalculateSize();
var buffer = new byte[header_size + body_size];
using (var stream = new MemoryStream(buffer)) {
checked { stream.WriteByte((byte)header_size); }
header.WriteTo(stream);
if (body != null) body.WriteTo(stream);
using (var stream = new MemoryStream(buffer)) {
checked { stream.WriteByte((byte)header_size); }
header.WriteTo(stream);
if (body != null) body.WriteTo(stream);
}
return buffer;
}
public static byte[] serialize(this ResponseHeader header) {
int header_size = 1 + header.CalculateSize();
var buffer = new byte[header_size];
using (var stream = new MemoryStream(buffer)) {
checked { stream.WriteByte((byte)header_size); }
header.WriteTo(stream);
}
return buffer;
}
}
}
}
public static byte[] serialize(this ResponseHeader header) {
int header_size = 1 + header.CalculateSize();
var buffer = new byte[header_size];
using (var stream = new MemoryStream(buffer)) {
checked { stream.WriteByte((byte)header_size); }
header.WriteTo(stream);
}
return buffer;
}
}
}
@@ -35,6 +35,7 @@ namespace bts.api.core {
var header = new RequestHeader() {
Method = method_id,
RequestId = id,
Token = _context.token.HasValue ? new AuthToken() { Token = _context.token?.Token } : null
};
var buffer = header.serialize(request);
var tcs = new TaskCompletionSource<Result<Response>>();
@@ -56,60 +57,99 @@ namespace bts.api.core {
return tcs.Task;
}
public unsafe override IObservable<RawUpdate<TOut>> subscribe<TParse,TOut>(string method_and_qualifier, MessageParser<TParse> parser, Func<TParse,TOut> converter) {
var result = Observable.Create<RawUpdate<TOut>>(observer => {
bool first_update = true;
// Memory optimization. There is only one update per subscription.
RawUpdate<TOut> the_update = new RawUpdate<TOut>();
ApiNativeMethods.raw_subscription_callback cb = (event_type, changes, change_count, unused) => {
the_update.Clear();
try {
if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) {
foreach (var change in changes) {
switch (change.type) {
case ApiNativeMethods.ChangeType.REMOVE:
the_update.removed_keys.Add(change.key);
break;
case ApiNativeMethods.ChangeType.REPLACE:
the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len))));
the_update.changed_keys.Add(change.key);
break;
case ApiNativeMethods.ChangeType.CREATE:
the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len))));
the_update.added_keys.Add(change.key);
break;
}
}
observer.OnNext(the_update);
public override IObservable<RawUpdate<TOut>> subscribe<TParse,TOut>(int method_id, string method, string qualifier, MessageParser<TParse> parser, Func<TParse,TOut> converter) {
var result = Observable.Create<RawUpdate<TOut>>(async observer => {
IntPtr temp_subscription = IntPtr.Zero;
void unsubscribe_temp_sub() {
// Unsubscribe late to keep qb refcount > 0 continuously
if (temp_subscription != IntPtr.Zero) ApiNativeMethods.api_client_unsubscribe(_client, temp_subscription);
}
while (true) {
var response = await check_stream_access(method_id, qualifier);
if (response.success) {
if (response.value.Authorized) {
// success. Break loop and subscribe
break;
}
else if (event_type == ApiNativeMethods.EventType.TABLE_CREATED) {
if (!first_update) {
observer.OnNext(the_update.MarkAsReset());
}
first_update = false;
else {
// Not authorized. return empty stream
unsubscribe_temp_sub();
observer.OnCompleted();
return System.Reactive.Disposables.Disposable.Empty;
}
}
catch (Exception e) {
ApiNativeMethods.api_client_log_error($"Exception in subscription callback for sub: '{method_and_qualifier}' \n--\n '{e}'");
// Sleep to let the log message get asyncronously written to disk
Thread.Sleep(100);
Environment.Exit(-1);
else {
// Wait for stream to fire once before trying again
unsubscribe_temp_sub();
var tcs = new TaskCompletionSource<bool>();
ApiNativeMethods.raw_subscription_callback cb = (event_type, changes, change_count, unused) => {
tcs.TrySetResult(true);
};
temp_subscription = ApiNativeMethods.api_client_subscribe(_client, method, qualifier, cb, IntPtr.Zero);
await tcs.Task;
GC.KeepAlive(cb);
}
};
var subscription = ApiNativeMethods.api_client_subscribe(_client, method_and_qualifier, cb, IntPtr.Zero);
return FinalizeableAnonymousDisposable.Create((disposing) => {
ApiNativeMethods.api_client_unsubscribe(_client, subscription);
subscription = IntPtr.Zero;
GC.KeepAlive(cb);
cb = null;
// Cannot (ever) call logging from the finalizer thread
// if (!disposing) {
// ApiNativeMethods.api_client_log_error($"Subscription has been terminated because the returned IDisposable has been finalized. Subscription: {method_and_qualifier}");
// }
});
}
// Authorized. Stream events
unsubscribe_temp_sub();
return SubscribeNative(method, qualifier, parser, converter, observer);
});
return result;
}
private Task<Result<StreamAccessResponse>> check_stream_access(int method_id, string qualifier) {
var request = new StreamAccessRequest() { Qualifier = qualifier ?? "" };
return send<StreamAccessRequest, StreamAccessResponse>(method_id, request);
}
private unsafe IDisposable SubscribeNative<TParse, TOut>(string method, string qualifier, MessageParser<TParse> parser, Func<TParse, TOut> converter, IObserver<RawUpdate<TOut>> observer) where TParse : IMessage<TParse> {
bool first_update = true;
// Memory optimization. There is only one update per subscription.
RawUpdate<TOut> the_update = new RawUpdate<TOut>();
ApiNativeMethods.raw_subscription_callback cb = (event_type, changes, change_count, unused) => {
the_update.Clear();
try {
if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) {
foreach (var change in changes) {
switch (change.type) {
case ApiNativeMethods.ChangeType.REMOVE:
the_update.removed_keys.Add(change.key);
break;
case ApiNativeMethods.ChangeType.REPLACE:
the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len))));
the_update.changed_keys.Add(change.key);
break;
case ApiNativeMethods.ChangeType.CREATE:
the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len))));
the_update.added_keys.Add(change.key);
break;
}
}
observer.OnNext(the_update);
}
else if (event_type == ApiNativeMethods.EventType.TABLE_CREATED) {
if (!first_update) {
observer.OnNext(the_update.MarkAsReset());
}
first_update = false;
}
}
catch (Exception e) {
ApiNativeMethods.api_client_log_error($"Exception in subscription callback for sub: '{method}/{qualifier}' \n--\n '{e}'");
// Sleep to let the log message get asyncronously written to disk
Thread.Sleep(100);
Environment.Exit(-1);
}
};
var subscription = ApiNativeMethods.api_client_subscribe(_client, method, qualifier, cb, IntPtr.Zero);
return FinalizeableAnonymousDisposable.Create((disposing) => {
ApiNativeMethods.api_client_unsubscribe(_client, subscription);
subscription = IntPtr.Zero;
GC.KeepAlive(cb);
cb = null;
});
}
private class FinalizeableAnonymousDisposable : IDisposable {
private Action<bool> _action;
public FinalizeableAnonymousDisposable(Action<bool> dispose) {
+2 -2
View File
@@ -5,7 +5,7 @@ using System.Threading.Tasks;
namespace bts.api.core {
internal abstract class ClientChannel : IDisposable
{
readonly ServiceContext _context;
protected readonly ServiceContext _context;
public ClientChannel(ServiceContext context) {
_context = context;
@@ -13,7 +13,7 @@ namespace bts.api.core {
public abstract void Dispose();
public abstract Task<Result<Response>> send<Request, Response>(int method_id, Request request)
where Request: IImmutableMessage where Response : class,IMessage,new();
public abstract IObservable<RawUpdate<TOut>> subscribe<TParse,TOut>(string method_and_qualifier, MessageParser<TParse> parser, Func<TParse,TOut> converter=null)
public abstract IObservable<RawUpdate<TOut>> subscribe<TParse,TOut>(int method_id, string method, string qualifier, MessageParser<TParse> parser, Func<TParse,TOut> converter=null)
where TParse : IMessage<TParse>;
public abstract bool is_sync_call_ok();
+6 -1
View File
@@ -37,7 +37,12 @@ namespace bts.api.core
public T value {
get {
if (success) return _result;
else throw new ApiException(error_message);
else {
if (typeof(T).IsValueType) { throw new ApiException(error_message); }
else {
return default(T);// always null
}
}
}
}
public bool success { get { return _code == ApiError.Success; } }
+2 -2
View File
@@ -34,8 +34,8 @@ namespace bts.api.core {
}
public Action<RequestHeader, Stream, Action<byte[]>> on_request;
public IntPtr create_table(string method_and_qualifier) {
return ApiNativeMethods.api_server_create_table(_server, method_and_qualifier);
public IntPtr create_table(string method, string qualifier) {
return ApiNativeMethods.api_server_create_table(_server, method, qualifier);
}
public void Dispose() {
+43 -3
View File
@@ -4,12 +4,14 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Reflection;
using bts.api.user;
namespace bts.api.core
{
namespace bts.api.core {
public class ServiceContext : IDisposable
{
IntPtr _qbus;
AuthToken _token;
public ServiceContext() {
_qbus = ApiNativeMethods.api_create_quickbus(null, 0);
}
@@ -22,7 +24,6 @@ namespace bts.api.core
Type[] arg_types = new[] { typeof(ClientChannel) };
ConstructorInfo ctor = typeof(T).GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance).First(ci => ci.GetParameters().SingleOrDefault()?.ParameterType == typeof(ClientChannel));
return (T)ctor.Invoke(new object[] { channel });
//return (T)Activator.CreateInstance(typeof(T), channel);
}
internal ServerChannel publish<T>(T service, string service_type, string qualifier) where T : IApiServer {
@@ -31,6 +32,38 @@ namespace bts.api.core
return ch;
}
public AuthTokenImmutable? token => _token ?? (AuthTokenImmutable?)null;
public string user { get; private set; }
public uint user_id { get; private set; }
public bool is_authenticated => token.HasValue;
public async Task<LoginResult> login(string username, string password) {
var auth_service = locate<AuthService>();
var result = await auth_service.authenticate_login_async(new AuthenticateLoginRequest() {
Login = username ?? "",
Password = password ?? ""
});
if (result.success) {
if (result.value.IsValid) {
_token = result.value.Token;
user = username;
user_id = result.value.UserId;
return new LoginResult();
}
else {
return new LoginResult(result.value.Reason);
}
}
else {
return new LoginResult(result.error_message);
}
}
public void login(AuthTokenImmutable token) {
this.user = "";
this.user_id = 0;
this._token = token.CloneToMutable();
}
#region IDisposable Support
protected virtual void Dispose(bool disposing) {
@@ -52,4 +85,11 @@ namespace bts.api.core
}
#endregion
}
public class LoginResult {
public LoginResult() { success = true; }
public LoginResult(string reason) { success = false; this.reason = reason; }
public bool success { get; }
public string reason { get; }
}
}
-23
View File
@@ -1,23 +0,0 @@
using System;
using System.Threading.Tasks;
using bts.api;
using bts.api.core;
namespace bts.api
{
public class Authenticator
{
readonly static Lazy<user.UserService> _auth = new Lazy<user.UserService>(() => Locator.locate<user.UserService>(), true);
user.AuthToken _token = null;
public Authenticator() { }
public user.AuthToken last_token => _token;
public async Task<user.AuthToken> login(string login, string password)
{
var res = await _auth.Value.authenticate_login_async(new user.AuthenticateLoginRequest { Login = login, Password = password });
if (res.success)
_token = res.value.Token;
return _token;
}
}
}
+59
View File
@@ -0,0 +1,59 @@
using System;
using System.Collections.Generic;
using System.Linq;
using bts.api;
using mbts;
using mbts.co2;
namespace bts.md
{
interface IMDUpdate
{
bool update(SecurityMD smd, ref ulong last_update_time);
}
public interface IMDData
{
SecurityImmutable security { get; }
}
public interface IMDBook : IMDData
{
List<PriceSize> bids { get; }
List<PriceSize> asks { get; }
}
class MDBook : IMDBook, IMDUpdate
{
public MDBook()
{
}
public SecurityImmutable security { get; private set; }
public List<PriceSize> bids { get; private set; }
public List<PriceSize> asks { get; private set; }
bool IMDUpdate.update(SecurityMD smd, ref ulong last_update_time)
{
this.security = smd.security;
NativePriceSize[] bids = null, asks = null;
if (smd.md.get_depth(ref last_update_time, ref bids, ref asks, NativePriceHelpers.COCONUT_MAX_DEPTH))
{
this.bids = bids.Select(x => new PriceSize(x.size, x.direct_size, x.implied_size, x.price)).ToList();
this.asks = asks.Select(x => new PriceSize(x.size, x.direct_size, x.implied_size, x.price)).ToList();
return true;
}
return false;
}
public override string ToString()
{
return security.ToString() + Environment.NewLine +
"Bid BidSz BidISz" + Environment.NewLine +
string.Join("\n", bids.Select(x => x.ToString())) + Environment.NewLine +
"Ask AskSz AskISz" + Environment.NewLine +
string.Join("\n", asks.Select(x => x.ToString()));
}
}
}
+61
View File
@@ -0,0 +1,61 @@
using System;
using System.Collections.Generic;
using System.Linq;
using bts.api;
using mbts;
namespace bts.md
{
public interface IMDCombined : IMDData, IMDMisc, IMDLast, IMDBook
{
}
class MDCombined : IMDCombined, IMDUpdate
{
MDBook _book = new MDBook();
MDLast _last = new MDLast();
MDMisc _misc = new MDMisc();
public MDCombined()
{
}
public SecurityImmutable security { get; private set; }
public ulong total_volume => _misc.total_volume;
public double settlement_price => _misc.settlement_price;
public DateTime settlement_date => _misc.settlement_date;
public double previous_settlement_price => _misc.previous_settlement_price;
public DateTime previous_settlement_date => _misc.previous_settlement_date;
public double open_price => _misc.open_price;
public DateTime open_date => _misc.open_date;
public double close_price => _misc.close_price;
public DateTime close_date => _misc.close_date;
public double session_high => _misc.session_high;
public double session_low => _misc.session_low;
public double fixing_price => _misc.fixing_price;
public PriceSize last => _last.last;
public ulong total_volume2 => _last.total_volume;
public DateTime last_time => _last.last_time;
public List<PriceSize> bids => _book.bids;
public List<PriceSize> asks => _book.asks;
bool IMDUpdate.update(SecurityMD smd, ref ulong last_update_time)
{
this.security = smd.security;
var bl = last_update_time;
bool b = ((IMDUpdate)_book).update(smd, ref bl);
var ll = last_update_time;
bool l = ((IMDUpdate)_last).update(smd, ref ll);
var ml = last_update_time;
bool m = ((IMDUpdate)_misc).update(smd, ref ml);
last_update_time = new[] { bl, ll, ml }.Max();
return b || l || m;
}
}
}
+40
View File
@@ -0,0 +1,40 @@
using System;
using bts.api;
using mbts;
using mbts.co2;
namespace bts.md
{
public interface IMDLast : IMDData
{
PriceSize last{ get; }
ulong total_volume { get; }
DateTime last_time { get; }
}
class MDLast : IMDLast, IMDUpdate
{
public MDLast()
{
}
public SecurityImmutable security { get; private set; }
public PriceSize last { get; private set; }
public ulong total_volume { get; private set; }
public DateTime last_time { get; private set; }
bool IMDUpdate.update(SecurityMD smd, ref ulong last_update_time)
{
this.security = smd.security;
NativePriceSize x; DateTime ts; ulong vol;
if (smd.md.get_last_trade(ref last_update_time, out x, out ts, out vol))
{
last = new PriceSize(x.size, x.direct_size, x.implied_size, x.price);
total_volume = vol;
last_time = ts;
return true;
}
return false;
}
}
}
+67
View File
@@ -0,0 +1,67 @@
using System;
using bts.api;
using mbts;
namespace bts.md
{
public interface IMDMisc : IMDData
{
ulong total_volume { get; }
double settlement_price { get; }
DateTime settlement_date { get; }
double previous_settlement_price { get; }
DateTime previous_settlement_date { get; }
double open_price { get; }
DateTime open_date { get; }
double close_price { get; }
DateTime close_date { get; }
double session_high { get; }
double session_low { get; }
double fixing_price { get; }
}
class MDMisc : IMDMisc, IMDUpdate
{
public MDMisc()
{
}
public SecurityImmutable security { get; private set; }
public ulong total_volume { get; private set; }
public double settlement_price { get; private set; }
public DateTime settlement_date { get; private set; }
public double previous_settlement_price { get; private set; }
public DateTime previous_settlement_date { get; private set; }
public double open_price { get; private set; }
public DateTime open_date { get; private set; }
public double close_price { get; private set; }
public DateTime close_date { get; private set; }
public double session_high { get; private set; }
public double session_low { get; private set; }
public double fixing_price { get; private set; }
bool IMDUpdate.update(SecurityMD smd, ref ulong last_update_time)
{
this.security = smd.security;
if (smd.md.misc_timestamp > last_update_time)
{
total_volume = smd.md.total_volume;
settlement_price = smd.md.settlement_price;
settlement_date = smd.md.settlement_date;
previous_settlement_price = smd.md.previous_settlement_price;
previous_settlement_date = smd.md.previous_settlement_date;
open_price = smd.md.open_price;
open_date = smd.md.open_date;
close_price = smd.md.close_price;
close_date = smd.md.close_date;
session_high = smd.md.session_high;
session_low = smd.md.session_low;
fixing_price = smd.md.fixing_price;
}
return false;
}
}
}
+90
View File
@@ -0,0 +1,90 @@
using bts.api;
using bts.api.synthetic;
using mbts;
using mbts.co2;
using System;
using System.Collections.Generic;
using System.Reactive.Linq;
using System.Threading.Tasks;
using System.Linq;
using SecurityKey = System.UInt32;
using System.Collections.Concurrent;
using System.Threading;
using bts.api.user;
using System.Reactive.Subjects;
namespace bts.md
{
class MDSource : IObserver<List<SecurityAndHandle>>, IDisposable
{
protected readonly MDConsumer _consumer = new MDConsumer();
protected readonly Co2RelayControl _relay;
protected long _version = 0;
protected readonly ConcurrentDictionary<SecurityKey, SecurityMD> _exchange_sec_map = new ConcurrentDictionary<SecurityKey, SecurityMD>();
protected List<IDisposable> _dispose = new List<IDisposable>();
public MDSource(uint naggle_ms, bool defer_init = false)
{
_relay = new Co2RelayControl(_consumer, naggle_ms);
_relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore();
_dispose.Add(_consumer.on_security.Subscribe(this));
if(!defer_init)
initialize();
}
readonly BehaviorSubject<ConcurrentDictionary<SecurityKey, SecurityMD>> _on_securities =
new BehaviorSubject<ConcurrentDictionary<SecurityKey, SecurityMD>>( new ConcurrentDictionary<SecurityKey, SecurityMD>() );
public IObservable<ConcurrentDictionary<SecurityKey, SecurityMD>> on_securities => _on_securities;
virtual public void initialize()
{
Task.Run(run);
}
public void set_token(AuthTokenImmutable token)
{
_relay.set_token(token);
}
public void OnCompleted() { }
public void OnError(Exception error) => Log.Md.error(error.ToString());
public void OnNext(List<SecurityAndHandle> values)
{
if (values.Count > 0)
{
foreach (var v in values)
{
var smd = new SecurityMD(v.security, v.handle);
_exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd);
}
_version += 1;
}
}
public void Dispose() => Disposable.dispose(_dispose);
async protected Task run()
{
long run_version = 0;
while (true)
{
var exch = _exchange_sec_map;
long current_version = _version;
if (current_version > run_version)
{
run_version = current_version;
try
{
Log.Md.info($"MDSource publishing {exch.Count} securities");
_on_securities.OnNext(exch);
}
catch (Exception e)
{
Log.Md.error($"Error in publish_securities: {e}");
}
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
}
}
}
+189 -276
View File
@@ -11,108 +11,224 @@ using System.Reactive.Subjects;
using bts.api;
using bts.api.core;
using bts.co2;
using mbts;
using SecurityKey = System.UInt32;
namespace bts.api.md
namespace bts.md
{
using security_key = System.UInt32;
public class MarketDataClient : IDisposable
public class MarketDataClient : IDisposable
{
class Data
{
public long last_update_time;
public ChannelUpdate data;
}
static uint naggle_ms = 100;
readonly static bts.md.MDSource _subscription = new MDSource(naggle_ms);
const UInt32 _all_flags = (uint)(
MDRequest.Types.Flags.Security |
MDRequest.Types.Flags.Bbo |
MDRequest.Types.Flags.Direct |
MDRequest.Types.Flags.Implied |
MDRequest.Types.Flags.LastTrade);
readonly ConcurrentDictionary<security_key, Data> _data = new ConcurrentDictionary<security_key, Data>();
readonly ConcurrentDictionary<security_key, string> _security_key_to_desc= new ConcurrentDictionary<security_key, string>();
readonly ConcurrentDictionary<string, security_key> _security_desc_to_key = new ConcurrentDictionary<string, security_key>();
readonly MDRequest _subscription;
readonly Venue _venue;
readonly static Lazy<MarketDataRelay> _md = new Lazy<MarketDataRelay>(() => Locator.locate<MarketDataRelay>(), true);
readonly static ConcurrentDictionary<api.Venue, MarketDataClient> _venue_cache = new ConcurrentDictionary<api.Venue, api.md.MarketDataClient>();
static readonly List<Product> _all_products;
static MarketDataClient()
readonly ConcurrentDictionary<SecurityKey, SecurityMD> _data = new ConcurrentDictionary<SecurityKey, SecurityMD>();
readonly List<IDisposable> _disp = new List<IDisposable>();
readonly uint _default_pump_ms;
public MarketDataClient(uint default_pump_ms = 250, AuthTokenImmutable? token = null)
{
_all_products = _md.Value.request_products(new VoidMessage()).Products.ToList();
}
_default_pump_ms = default_pump_ms;
if (token != null)
set_auth_token(token);
public static IEnumerable<api.Venue> cached_exchanges => _venue_cache.Keys;
public static bool create_or_get(api.Venue exchange, out MarketDataClient md, user.AuthToken token = null)
{
if (!_venue_cache.TryGetValue(exchange, out md))
_disp.Add(_subscription.on_securities.Subscribe( md =>
{
_venue_cache[exchange] = md = new MarketDataClient(exchange, token);
return true;
}
return false;
foreach (var entry in md)
_data.AddOrUpdate(entry.Key, entry.Value, (_, __) => entry.Value);
}));
}
public IEnumerable<SecurityImmutable> securities => _data.Values.Select(x => x.security);
public IObservable<IEnumerable<SecurityImmutable>> on_securities
{
get { return _subscription.on_securities.Select(x => x.Values.Select(xx => xx.security)); }
}
public void set_auth_token(AuthTokenImmutable? toke)
{
if(toke.HasValue)
_subscription.set_token(toke.Value);
}
public IObservable< IEnumerable<IMDBook> > subscribe_book(Func<SecurityImmutable, bool> filter,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
return _subscribe<MDBook>(filter, update_interval, scheduler);
}
public IObservable< IEnumerable<IMDLast> > subscribe_last(Func<SecurityImmutable, bool> filter,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
return _subscribe<MDLast>(filter, update_interval, scheduler);
}
public IObservable< IEnumerable<IMDMisc> > subscribe_misc(Func<SecurityImmutable, bool> filter,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
return _subscribe<MDMisc>(filter, update_interval, scheduler);
}
public IObservable< IEnumerable<IMDCombined> > subscribe_combined(Func<SecurityImmutable, bool> filter,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
return _subscribe<MDCombined>(filter, update_interval, scheduler);
}
IObservable<IEnumerable<T>> _subscribe<T>(Func<SecurityImmutable, bool> filter,
TimeSpan? update_interval = null, IScheduler scheduler = null) where T : IMDData, IMDUpdate, new()
{
List<Slot<T>> slots = new List<Slot<T>>(); int data_size = 0;
return timer_pump<IEnumerable<T>>((last_timer_tick, observer) =>
{
//if the number of securities has changed, or this is the first call
if (data_size != _data.Count)
{
slots = _data.Values.Where(x => filter(x.security)).Select(xx => new Slot<T>(xx)).ToList();
data_size = _data.Count;
}
List<T> updates = new List<T>();
foreach (var slot in slots)
{
bool did_update = slot.data.update(slot.secmd, ref slot.last_update_time);
//if (slot.last_update_time > last_timer_tick)
if(did_update)
updates.Add(slot.data);
}
if(updates.Count > 0)
observer.OnNext(updates);
},
update_interval, scheduler);
}
public IObservable<IEnumerable<IMDBook>> subscribe_book(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
IScheduler scheduler = null)
{
return _subscribe<MDBook>(ids, update_interval, scheduler);
}
public IObservable<IEnumerable<IMDLast>> subscribe_last(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
IScheduler scheduler = null)
{
return _subscribe<MDLast>(ids, update_interval, scheduler);
}
public IObservable<IEnumerable<IMDMisc>> subscribe_misc(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
IScheduler scheduler = null)
{
return _subscribe<MDMisc>(ids, update_interval, scheduler);
}
public IObservable<IEnumerable<IMDCombined>> subscribe_combined(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
IScheduler scheduler = null)
{
return _subscribe<MDCombined>(ids, update_interval, scheduler);
}
IObservable<IEnumerable<T>> _subscribe<T>(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
IScheduler scheduler = null) where T : IMDData, IMDUpdate, new()
{
List<Slot<T>> slots = ids.Where(ii => _data.TryGetValue(ii, out SecurityMD _)).Select(x =>
{
_data.TryGetValue(x, out SecurityMD smd);
return new Slot<T>(smd);
}).ToList();
return timer_pump<IEnumerable<T>>((last_timer_tick, observer) =>
{
List<T> updates = new List<T>();
foreach (var slot in slots)
{
bool did_update = slot.data.update(slot.secmd, ref slot.last_update_time);
//if (slot.last_update_time > last_timer_tick)
if(did_update)
updates.Add(slot.data);
}
if(updates.Count > 0)
observer.OnNext(updates);
},
update_interval, scheduler);
}
public MarketDataClient(bts.api.Venue exchange = Venue.Unset, user.AuthToken token = null)
class Slot<T> where T : IMDData, IMDUpdate, new()
{
_venue = exchange;
_subscription = load_request(exchange, token);
public Slot(SecurityMD md)
{
secmd = md;
last_update_time = 0;
data = new T();
}
public SecurityMD secmd;
public ulong last_update_time;
public T data;
}
IObservable<T> timer_pump<T>(Action<ulong, IObserver<T>> tick, TimeSpan? interval, IScheduler scheduler = null)
{
if(!interval.HasValue)
interval = TimeSpan.FromMilliseconds(_default_pump_ms);
ulong last_update_time = 0;
return Observable.Create<T>(observer =>
{
return Observable.Timer(interval.Value, interval.Value, scheduler ?? Scheduler.Default).Subscribe(_ =>
{
tick(last_update_time, observer);
last_update_time = (ulong)DateTime.UtcNow.Ticks;
});
});
}
public void Dispose()
{
_disp.ForEach(x => x.Dispose());
}
public void set_auth_token(user.AuthToken toke)
{
_subscription.Token = toke?.Token ?? string.Empty;
}
#region security search
MDRequest load_request(bts.api.Venue exchange, user.AuthToken token)
public SecurityImmutable? lookup(SecurityKey key)
{
var req = new MDRequest() { Token = token?.Token ?? string.Empty } ;
var prods = _all_products.Where(x=>x.Exchange == exchange || exchange == bts.api.Venue.Unset);
foreach(var prod in prods)
req.Channels.Add(new SecurityChannel() { Exchange = prod.Exchange, Type = prod.Type, Symbol = prod.Symbol, Flags = 0 });
return req;
}
public IEnumerable<SecurityImmutable> get_securities()
{
return _data.Values.Select(x => new SecurityImmutable(x.data.Security));
}
public SecurityImmutable? lookup(security_key key)
{
Data value;
if(_data.TryGetValue(key, out value))
return value.data.Security;
if (_data.TryGetValue(key, out SecurityMD value))
return value.security;
else
return null;
}
public bool lookup_security_desc(security_key key, out string security_desc)
public bool lookup_security_desc(SecurityKey key, out string security_desc)
{
return _security_key_to_desc.TryGetValue(key, out security_desc);
security_desc = null;
var sec = lookup(key);
if (sec.HasValue)
{
security_desc = sec.Value.glbx_desc();
return true;
}
return false;
}
public bool lookup_security_key_from_desc(string security_desc, out security_key key)
public bool lookup_security_key_from_desc(string security_desc, out SecurityKey key)
{
return _security_desc_to_key.TryGetValue(security_desc, out key);
key = 0;
var fnd = find(x => x.glbx_desc() == security_desc).TryFirst();
if (fnd.HasValue)
{
key = fnd.Value.security.Id;
return true;
}
return false;
}
public Security find_security(Func<Security, bool> filter)
IEnumerable<SecurityMD> find(Func<SecurityImmutable, bool> filter)
{
return _data.Values.Select(x => x.data.Security).Where(xx => filter(xx)).FirstOrDefault();
Func<SecurityMD, bool> filt = (SecurityMD secmd) => filter(secmd.security);
return _data.Values.Where(filt);
}
public async Task<Security> find_security_async(Func<Security, bool> filter, TimeSpan? timeout = null)
public SecurityImmutable? find_security(Func<SecurityImmutable, bool> filter)
{
if(!timeout.HasValue) {
timeout = TimeSpan.FromMilliseconds(int.MaxValue-1);
var fnd = find(filter).TryFirst();
return fnd?.security;
}
public async Task<SecurityImmutable?> find_security_async(Func<SecurityImmutable, bool> filter, TimeSpan? timeout = null)
{
if (!timeout.HasValue)
{
timeout = TimeSpan.FromMilliseconds(10000);
}
CancellationTokenSource src = new CancellationTokenSource(timeout.Value);
Security security = await Task.Run<Security>(async () =>
SecurityImmutable? security = await Task.Run<SecurityImmutable?>(async () =>
{
Security sec = null;
SecurityImmutable? sec = null;
do
{
sec = find_security(filter);
@@ -122,214 +238,11 @@ namespace bts.api.md
while (sec == null);
return sec;
}, src.Token);
return security ?? throw new KeyNotFoundException("Can't find security for filter"); ;
}
public IObservable<ChannelUpdate> subscribe(Product product,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
return subscribe(product, _all_flags, update_interval, scheduler);
}
public IObservable<ChannelUpdate> subscribe(Product product, uint flags,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
check_subscriptions(flags, x =>
(x.Exchange == product.Exchange && x.Type == product.Type && x.Symbol == product.Symbol));
return _subscribe(
x => (x.Exchange == product.Exchange && ((SecurityImmutable)x).security_type() == product.Type && x.Symbol == product.Symbol),
update_interval,
scheduler);
}
IObservable<ChannelUpdate> _subscribe(Func<Security, bool> filter,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
Data[] slots = new Data[0]; long local_security_count = 0;
return timer_pump<Data>((last_update, observer) =>
{
//if the number of securities has changed, or this is the first call
if (slots.Length == 0 || local_security_count != _data.Count)
slots = _data.Values.Where(xx => filter(xx.data.Security)).ToArray();
foreach (var slot in slots)
{
Data data;
if (_data.TryGetValue(slot.data.Security.Id, out data))
{
if (data.last_update_time > last_update)
observer.OnNext(data);
}
}
},
update_interval, scheduler).Select(x => x.data);
}
/// <summary>
/// Poll for updates for a security on a specified update_interval, on the specified scheduler with default flags
/// </summary>
/// <param name="security"></param>
/// <param name="update_interval"></param>
/// <param name="scheduler"></param>
/// <returns></returns>
public IObservable<ChannelUpdate> subscribe(Security security,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
return subscribe(security, _all_flags, update_interval, scheduler);
}
/// <summary>
/// Poll for updates for a security on a specified update_interval, on the specified scheduler
/// </summary>
/// <param name="security"></param>
/// <param name="flags"></param>
/// <param name="update_interval"></param>
/// <param name="scheduler"></param>
/// <returns></returns>
public IObservable<ChannelUpdate> subscribe(Security security, uint flags,
TimeSpan? update_interval = null, IScheduler scheduler = null)
{
check_subscriptions(flags, x =>
(x.Exchange == security.Exchange && x.Type == ((SecurityImmutable)security).security_type() && security.Symbol == security.Symbol));
return timer_pump<Data>( (last_update, observer) =>
{
Data data;
if (_data.TryGetValue(security.Id, out data))
{
if (data.last_update_time > last_update)
observer.OnNext(data);
}
},
update_interval, scheduler).Select(x => x.data);
}
IObservable<T> timer_pump<T>(Action<long, IObserver<T>> tick, TimeSpan? interval, IScheduler scheduler = null)
{
if(!interval.HasValue)
interval = TimeSpan.FromMilliseconds(25);
long last_update_time = 0;
return Observable.Create<T>(observer =>
{
return Observable.Timer(interval.Value, interval.Value, scheduler ?? Scheduler.Default).Subscribe(_ =>
{
tick(last_update_time, observer);
last_update_time = DateTime.UtcNow.Ticks;
});
});
}
bool check_subscriptions(uint flags, Func<SecurityChannel, bool> selector)
{
foreach (var sub in _subscription.Channels)
{
if (selector(sub))
{
sub.Flags |= flags;
return true;
}
}
return false;
}
/// <summary>
/// Start pumping market data for all subscribed channels and securities. Call this without await
/// </summary>
/// <param name="interval"></param>
/// <returns></returns>
public async Task start_pump(TimeSpan? interval = null)
{
await _start_pump(interval);
}
async Task _start_pump(TimeSpan? interval = null, int num_pumps = -1)
{
var rep = await _md.Value.request_names_async(new VoidMessage());
int count = 0;
do
{
foreach (var s in rep.value.Securities)
{
foreach (var ss in s.Names)
{
_security_desc_to_key[ss.Value] = ss.Key;
_security_key_to_desc[ss.Key] = ss.Value;
}
}
}
while (!rep.success && count++ < 5);
count = 0;
while (num_pumps < 0 || count++ < num_pumps)
{
var reply = await _md.Value.requestmd_async(_subscription);
if (reply.success)
{
foreach (var channel in reply.value.Channels)
{
long now = DateTime.UtcNow.Ticks;
foreach (var update in channel.Updates)
{
Data data;
if (_data.TryGetValue(update.Security.Id, out data))
{
//mergefrom concats repeated fields
if (update.Direct != null)
data.data.Direct = null;
//this can tear, but the book won't tear, so maybe it's ok
data.data.MergeFrom(update);
data.last_update_time = now;
}
else
{
_data[update.Security.Id] = new Data() { last_update_time = now, data = update };
}
}
}
await Task.Delay(interval ?? TimeSpan.FromMilliseconds(100), _cts.Token);
}
}
}
Task _pump_task;
public void run_pump(TimeSpan? interval = null)
{
if (_pump_task == null)
{
_pump_task = this.start_pump(interval)
.ContinueWith(ex =>
{
System.Diagnostics.Debug.Assert(ex == null, ex.ToString());
}
, TaskContinuationOptions.OnlyOnFaulted);
}
}
public async void run_pump_async(TimeSpan? interval = null)
{
await this.start_pump(interval);
}
public void pump_once()
{
if (_pump_task == null)
{
_pump_task = this._start_pump(null, 1)
.ContinueWith(ex =>
{
System.Diagnostics.Debug.Assert(ex == null, ex.ToString());
}
, TaskContinuationOptions.OnlyOnFaulted);
}
}
readonly CancellationTokenSource _cts = new CancellationTokenSource();
public void Dispose()
{
_cts.Cancel();
_cts.Dispose();
if (_md.IsValueCreated)
_md.Value.Dispose();
}
#endregion
}
}
+24
View File
@@ -0,0 +1,24 @@
using bts.api;
using mbts.co2;
namespace bts.md
{
public class PriceSize
{
public uint size { get; }
public uint direct_size { get; }
public uint implied_size { get; }
public double price { get; }
public PriceSize(uint sz, uint direct_sz, uint implied_sz, double prx)
{
size = sz;
direct_size = direct_sz;
implied_size = implied_sz;
price = prx;
}
public override string ToString()
{
return $"{price} {size} {implied_size} ";
}
}
}
@@ -6,7 +6,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace mbts.spark {
namespace mbts {
struct SecurityMD : IEquatable<SecurityMD> {
public SecurityImmutable security { get; }
public MDConsumerHandle md { get; }
+22 -3
View File
@@ -1,15 +1,34 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace bts
namespace mbts
{
public static class APIExtensions
static class APIExtensions
{
public static T? TryFirst<T>(this IEnumerable<T> src) where T : struct
{
foreach (var v in src)
return v;
return null;
}
}
public static DateTime lldatetime(this ulong ll)
{
// Warning: Do not remove these consts, or try to simplify this method.
// System.DateTime() requires bits 63-64 to be empty in the "DateTime(long ticks)" constructor.
// The only way to set the DateTimeKind is through the "DateTime(long ticks, DateTimeKind kind)" constructor.
// (Yes, it is annoying. Internally, System.DateTime stores the DateTimeKind in bits 63-64 - in the exact format we already have!!!)
const ulong ticks_mask = 0x3FFFFFFFFFFFFFFF;
const int kind_shift = 62;
return new DateTime((long)(ll & ticks_mask), (DateTimeKind)(ll >> kind_shift));
}
public static string fmt(this string input, params object[] vals)
{
return string.Format(input, vals);
}
public static void ignore(this Task t) { }
}
}
@@ -2,7 +2,7 @@
using System;
using System.Threading.Tasks;
namespace bts
namespace bts.utils
{
public class AsyncLazy<T> : Lazy<Task<T>>
{
@@ -11,6 +11,11 @@ namespace bts.api
public static T locate<T>(string qualifier="") where T : class, IApiClient
{
return _context.Value.locate<T>(qualifier);
}
}
static public async Task<LoginResult> login(string username, string password)
{
return await _context.Value.login(username, password);
}
static public AuthTokenImmutable? token => _context.Value.token;
}
}
@@ -6,9 +6,6 @@ using System.Text;
namespace mbts {
public static class GlobalExtensions {
public static string fmt(this string input, params object[] vals) {
return string.Format(input, vals);
}
public static string Repeat(this string s, uint count) {
if (count == 0) return "";
if (count == 1) return s;
@@ -27,15 +24,6 @@ namespace mbts {
return range.Skip(index).First();
}
public static DateTime lldatetime(this ulong ll) {
// Warning: Do not remove these consts, or try to simplify this method.
// System.DateTime() requires bits 63-64 to be empty in the "DateTime(long ticks)" constructor.
// The only way to set the DateTimeKind is through the "DateTime(long ticks, DateTimeKind kind)" constructor.
// (Yes, it is annoying. Internally, System.DateTime stores the DateTimeKind in bits 63-64 - in the exact format we already have!!!)
const ulong ticks_mask = 0x3FFFFFFFFFFFFFFF;
const int kind_shift = 62;
return new DateTime((long)(ll & ticks_mask), (DateTimeKind)(ll >> kind_shift));
}
//this overloaded declaration, (as opposed to one function with comp = null) allows observable_collection.sync_with to be passed into functions as a parameterless Action
public static void sync_with<T>(this IList<T> dest, IEnumerable<T> src) { dest.sync_with(src, EqualityComparer<T>.Default); }
@@ -17,7 +17,7 @@ namespace bts.api.orderman
{
public partial class OrderService_Excel
{
readonly static Lazy<user.UserService> _users = new Lazy<user.UserService>(() => Locator.locate<user.UserService>(), true);
readonly static Lazy<user.AuthService> _users = new Lazy<user.AuthService>(() => Locator.locate<user.AuthService>(), true);
readonly static Lazy<admin.AdminService> _admin = new Lazy<admin.AdminService>(() => Locator.locate<admin.AdminService>(), true);
[ExcelFunction(Category = "BTS", Description = "Get All Accessible Accounts")]
@@ -26,7 +26,7 @@ namespace bts.api.orderman
try
{
List<string> accts = new List<string>();
var admin = await _admin.Value.get_filtered_admin_data_async(new FilteredAdminDataRequest { Token = login_token });
var admin = await _admin.Value.get_filtered_admin_data_async(new FilteredAdminDataRequest());
if (admin.success)
{
foreach (var acc in admin.value.Accounts)
@@ -1,21 +1,22 @@
using System;
using bts.md;
namespace bts.excel.rtd
{
class BookMDFormatter : IExcelFormatter
{
readonly co2.ChannelUpdate _update;
public BookMDFormatter(co2.ChannelUpdate upd)
readonly IMDBook _update;
public BookMDFormatter(IMDBook upd)
{
_update = upd;
}
public static double format_md(long mantissa, int exponent)
public object[,] format(bool include_headers = false)
{
return mantissa * Math.Pow(10, exponent);
return format(_update, include_headers);
}
public object[,] format(bool include_headers = false)
static public object[,] format(IMDBook update, bool include_headers = false)
{
const int cols = 5;
const int rows = 30;
@@ -24,21 +25,21 @@ namespace bts.excel.rtd
for (int j = 0; j < cols; j++)
ret[i, j] = string.Empty;
if (_update.Direct != null)
if (update != null)
{
int idx = 0;
for (int j = _update.Direct.Asks.Count - 1; j >= 0 && idx < rows; j--)
for (int j = update.asks.Count - 1; j >= 0 && idx < rows; j--)
{
var ask = _update.Direct.Asks[j];
ret[idx, 1] = format_md(ask.Mantissa, ask.Exponent);
ret[idx, 2] = ask.Size;
var ask = update.asks[j];
ret[idx, 1] = ask.price;
ret[idx, 2] = ask.size;
idx++;
}
for (int j = 0; j < _update.Direct.Bids.Count && idx < rows; j++)
for (int j = 0; j < update.bids.Count && idx < rows; j++)
{
var bid = _update.Direct.Bids[j];
ret[idx, 1] = format_md(bid.Mantissa, bid.Exponent);
ret[idx, 0] = bid.Size;
var bid = update.bids[j];
ret[idx, 1] = bid.price;
ret[idx, 0] = bid.size;
idx++;
}
}
@@ -1,4 +1,5 @@
using System;
using bts.md;
namespace bts.excel.rtd
{
@@ -6,7 +7,7 @@ namespace bts.excel.rtd
{
public static IExcelFormatter make_formatter(object data)
{
if(data is co2.ChannelUpdate upd)
if(data is IMDBook upd)
return new BookMDFormatter(upd);
else
{
@@ -0,0 +1,35 @@
using System;
using bts.md;
namespace bts.excel.rtd
{
class LastMDFormatter : IExcelFormatter
{
readonly IMDLast _update;
public LastMDFormatter(IMDLast upd)
{
_update = upd;
}
public object[,] format(bool include_headers = false)
{
return format(_update, include_headers);
}
static public object[,] format(IMDLast update, bool include_headers = false)
{
const int cols = 3;
const int rows = 1;
var ret = new object[rows, cols];
if (update != null)
{
ret[0, 0] = update.last?.price;
ret[0, 1] = update.last?.size;
ret[0, 1] = update.last_time;
}
return ret;
}
}
}
@@ -1,16 +1,22 @@
using System;
using bts.md;
namespace bts.excel.rtd
{
class MiscMDFormatter : IExcelFormatter
{
readonly co2.ChannelUpdate _update;
public MiscMDFormatter(co2.ChannelUpdate upd)
readonly IMDMisc _update;
public MiscMDFormatter(IMDMisc upd)
{
_update = upd;
}
public object[,] format(bool include_headers = false)
{
return format(_update, include_headers);
}
static public object[,] format(IMDMisc update, bool include_headers = false)
{
return new object[0, 0];
}
@@ -1,17 +1,22 @@
using System;
using bts.md;
namespace bts.excel.rtd
{
class TopOfBookMDFormatter : IExcelFormatter
{
readonly co2.ChannelUpdate _update;
public TopOfBookMDFormatter(co2.ChannelUpdate upd)
readonly IMDBook _update;
public TopOfBookMDFormatter(IMDBook upd)
{
_update = upd;
}
public object[,] format(bool include_headers = false)
{
return format(_update, include_headers);
}
static public object[,] format(IMDBook update, bool include_headers = false)
{
const int cols = 4;
const int rows = 1;
@@ -20,22 +25,22 @@ namespace bts.excel.rtd
for (int j = 0; j < cols; j++)
ret[i, j] = string.Empty;
if (_update.Direct != null)
if (update != null)
{
int idx = 0;
for (int j = 0; j < _update.Direct.Asks.Count && idx < rows; j++)
for (int j = 0; j < update.asks.Count && idx < rows; j++)
{
var ask = _update.Direct.Asks[j];
ret[idx, 2] = BookMDFormatter.format_md(ask.Mantissa, ask.Exponent);
ret[idx, 3] = ask.Size;
var ask = update.asks[j];
ret[idx, 2] = ask.price;
ret[idx, 3] = ask.size;
idx++;
}
idx = 0;
for (int j = 0; j < _update.Direct.Bids.Count && idx < rows; j++)
for (int j = 0; j < update.bids.Count && idx < rows; j++)
{
var bid = _update.Direct.Bids[j];
ret[idx, 1] = BookMDFormatter.format_md(bid.Mantissa, bid.Exponent);
ret[idx, 0] = bid.Size;
var bid = update.bids[j];
ret[idx, 1] = bid.price;
ret[idx, 0] = bid.size;
idx++;
}
}
@@ -1,86 +1,95 @@
using System;
using bts.md;
namespace bts.excel.rtd
{
class TopicMDFormatter : IExcelFormatter
{
readonly co2.ChannelUpdate _update;
readonly IMDCombined _update;
readonly string _topic;
public TopicMDFormatter(co2.ChannelUpdate upd, string topic)
public TopicMDFormatter(IMDCombined upd, string topic)
{
_update = upd;
_topic = topic;
}
public object[,] format(bool include_headers = false)
{
return format(_update, _topic, include_headers);
}
static public object[,] format(IMDCombined update, string topic, bool include_headers = false)
{
var retval = new object[1, 1];
switch (_topic.ToLower())
if(update == null || string.IsNullOrEmpty(topic))
return retval;
switch (topic.ToLower())
{
case "bid":
case "bidprice":
retval[0, 0] = BookMDFormatter.format_md(_update?.Direct?.Bids?[0]?.Mantissa ?? 0, _update?.Direct?.Bids?[0]?.Exponent ?? 0);
retval[0, 0] = update.bids?[0]?.price;
break;
case "ask":
case "askprice":
retval[0, 0] = BookMDFormatter.format_md(_update?.Direct?.Asks?[0]?.Mantissa ?? 0, _update?.Direct?.Asks?[0]?.Exponent ?? 0);
retval[0, 0] = update.asks?[0]?.price;
break;
case "bidsize":
case "bidqty":
retval[0, 0] = _update?.Direct?.Bids?[0]?.Size;
retval[0, 0] = update.bids?[0]?.size;
break;
case "asksize":
case "askqty":
retval[0, 0] = _update?.Direct?.Asks?[0]?.Size;
retval[0, 0] = update.asks?[0]?.size;
break;
case "impliedbid":
case "impliedbidprice":
retval[0, 0] = BookMDFormatter.format_md(_update?.Implied?.Bids?[0]?.Mantissa ?? 0, _update?.Implied?.Bids?[0]?.Exponent ?? 0);
retval[0, 0] = update.bids?[0]?.price; //TODO is this the first price with impsz > 0?
break;
case "impliedask":
case "impliedaskprice":
retval[0, 0] = BookMDFormatter.format_md(_update?.Implied?.Asks?[0]?.Mantissa ?? 0, _update?.Implied?.Asks?[0]?.Exponent ?? 0);
retval[0, 0] = update.asks?[0]?.price;
break;
case "impliedbidsize":
case "impliedbidqty":
retval[0, 0] = _update?.Implied?.Bids?[0]?.Size;
retval[0, 0] = update.bids?[0]?.implied_size;
break;
case "impliedasksize":
case "impliedaskqty":
retval[0, 0] = _update?.Implied?.Asks?[0]?.Size;
retval[0, 0] = update.asks?[0]?.implied_size;
break;
case "settlement":
case "settlementprice":
retval[0, 0] = BookMDFormatter.format_md(_update?.Misc?.Settlement?.Mantissa ?? 0, _update?.Misc?.Settlement?.Exponent ?? 0);
retval[0, 0] = update.settlement_price;
break;
case "previoussettlement":
case "previoussettlementprice":
retval[0, 0] = BookMDFormatter.format_md(_update?.Misc?.PreviousSettlementPrice?.Mantissa ?? 0, _update?.Misc?.PreviousSettlementPrice?.Exponent ?? 0);
retval[0, 0] = update.previous_settlement_price;
break;
case "open":
case "openprice":
retval[0, 0] = _update?.Misc?.OpenPrice;
retval[0, 0] = update.open_price;
break;
case "close":
case "closeprice":
retval[0, 0] = _update?.Misc?.ClosePrice;
retval[0, 0] = update.close_price;
break;
case "high":
case "highprice":
retval[0, 0] = _update?.Misc?.SessionHigh;
retval[0, 0] = update.session_high;
break;
case "low":
case "lowprice":
retval[0, 0] = _update?.Misc?.SessionLow;
retval[0, 0] = update.session_low;
break;
case "last":
case "lastprice":
retval[0, 0] = BookMDFormatter.format_md(_update?.LastTrade?.Mantissa ?? 0, _update?.LastTrade?.Exponent ?? 0);
retval[0, 0] = update?.last?.price;
break;
case "lastsize":
case "lastqty":
retval[0, 0] = _update?.LastTrade?.Size;
retval[0, 0] = update?.last?.size;
break;
}
return retval;
@@ -7,26 +7,26 @@ using ExcelDna.Integration.Rtd;
using ExcelDna.Registration.Utils;
using System.Reactive.Linq;
using bts.api;
using bts.api.md;
using bts.md;
using bts.api.orderman;
namespace bts.excel.rtd
{
public class MDSubscribe
{
static Authenticator _auth = new Authenticator();
[ExcelFunction(Category = "BTS", Description = "Log in to BTS Spark")]
async public static void spark_login(string login, string password, ExcelAsyncHandle handle)
{
try
{
var toke = await _auth.login(login, password);
foreach (var ven in MarketDataClient.cached_exchanges)
var res = await Locator.login(login, password);
if (res.success && Locator.token.HasValue)
{
MarketDataClient.create_or_get(ven, out MarketDataClient md);
md?.set_auth_token(toke);
_md.set_auth_token(Locator.token);
handle.SetResult("Login Successful");
}
handle.SetResult(toke?.Token ?? "Could not log in check user/pass");
else
handle.SetResult("Login Error: Could not log in check user/pass " + res.reason);
}
catch(Exception ex)
{
@@ -34,12 +34,10 @@ namespace bts.excel.rtd
}
}
static MarketDataClient _md = new MarketDataClient();
public static MarketDataClient market_data(api.Venue exchange)
{
var newc = MarketDataClient.create_or_get(exchange, out MarketDataClient md);
if(newc)
md.run_pump_async(TimeSpan.FromMilliseconds(333));
return md;
return _md;
}
static public object spark_lookup_security_desc(string exchange, long security_key)
@@ -51,7 +49,7 @@ namespace bts.excel.rtd
return desc ?? string.Empty;
}
static IObservable<object> md_observable(string contract, string symbol, string security_type, string exchange)
static IObservable<IEnumerable<object>> md_observable(string contract, string symbol, string security_type, string exchange)
{
security_type = string.IsNullOrEmpty(security_type) ? SecurityType.Future.ToString() : security_type;
exchange = string.IsNullOrEmpty(exchange) ? Security.ExchangeDataOneofCase.Glbx.ToString() : exchange;
@@ -60,18 +58,18 @@ namespace bts.excel.rtd
Venue ven;
SecurityType sectype;
if (!Utilities.parse_venue(exchange, out ven))
return Observable.Return("Can't Parse Exchange: " + exchange ?? string.Empty);
return Observable.Return(new [] {"Can't Parse Exchange: " + exchange ?? string.Empty});
if (!Utilities.parse_security_type(security_type, out sectype))
return Observable.Return("Can't Parse Security Type: " + security_type ?? string.Empty);
return Observable.Return(new [] {"Can't Parse Security Type: " + security_type ?? string.Empty});
if (string.IsNullOrEmpty(symbol))
return Observable.Return("Can't Parse Symbol: " + symbol ?? string.Empty);
return Observable.Return(new []{"Can't Parse Symbol: " + symbol ?? string.Empty});
if (string.IsNullOrEmpty(contract))
return Observable.Return("Can't Parse Contract: " + contract ?? string.Empty);
return Observable.Return(new []{"Can't Parse Contract: " + contract ?? string.Empty});
Product prod = new Product() { Exchange = ven, Type = sectype, Symbol = symbol };
Func<Security, bool> filter = (x => x.Glbx.SecurityDesc == contract); //TODO GLBX
Func<SecurityImmutable, bool> where = (x => x.Exchange == ven && x.Symbol == symbol && x.security_type() == sectype &&
x.Glbx.SecurityDesc == contract);
var stream = market_data(ven).subscribe(prod).Where(x => filter(x.Security));
var stream = market_data(ven).subscribe_combined(where);
return stream;
}
@@ -85,10 +83,11 @@ namespace bts.excel.rtd
var param = new object[] { contract, symbol, security_type, exchange };
return ObservableRtdUtil.Observe(nameof(spark_md_book), param,
() => md_observable(contract, symbol, security_type, exchange).Select(x =>
() => md_observable(contract, symbol, security_type, exchange).Select(data =>
{
if (x is co2.ChannelUpdate upd)
return new BookMDFormatter(upd).format();
var x = data.FirstOrDefault();
if (x != null && x is IMDBook upd)
return BookMDFormatter.format(upd);
return x;
}));
// security_type = string.IsNullOrEmpty(security_type) ? SecurityType.Future.ToString() : security_type;
@@ -111,13 +110,33 @@ namespace bts.excel.rtd
{
var param = new object[] { contract, symbol, security_type, exchange };
return ObservableRtdUtil.Observe(nameof(spark_md_top_of_book), param,
() => md_observable(contract, symbol, security_type, exchange).Select(x =>
() => md_observable(contract, symbol, security_type, exchange).Select(data =>
{
if (x is co2.ChannelUpdate upd)
return new TopOfBookMDFormatter(upd).format();
var x = data.FirstOrDefault();
if (x != null && x is IMDBook upd)
return TopOfBookMDFormatter.format(upd);
return x;
}));
}
}
[ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark last trade market data")]
public static object spark_md_last(
[ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
[ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,
[ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
[ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
{
var param = new object[] { contract, symbol, security_type, exchange };
return ObservableRtdUtil.Observe(nameof(spark_md_last), param,
() => md_observable(contract, symbol, security_type, exchange).Select(data =>
{
var x = data.FirstOrDefault();
if (x != null && x is IMDLast upd)
return LastMDFormatter.format(upd);
return x;
}));
}
[ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark miscellaneous market data")]
public static object spark_md_misc(
@@ -128,16 +147,17 @@ namespace bts.excel.rtd
{
var param = new object[] { contract, symbol, security_type, exchange };
return ObservableRtdUtil.Observe(nameof(spark_md_misc), param,
() => md_observable(contract, symbol, security_type, exchange).Select(x =>
() => md_observable(contract, symbol, security_type, exchange).Select(data =>
{
if (x is co2.ChannelUpdate upd)
return new MiscMDFormatter(upd).format();
var x = data.FirstOrDefault();
if (x != null && x is IMDMisc upd)
return MiscMDFormatter.format(upd);
return x;
}));
}
[ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark top of book market data")]
[ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark topic based market data")]
public static object spark_md(
[ExcelArgument("Required. The 'Topic' string, 'bid', 'ask', etc")] string topic,
[ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
@@ -147,12 +167,14 @@ namespace bts.excel.rtd
{
var param = new object[] { topic, contract, symbol, security_type, exchange };
return ObservableRtdUtil.Observe(nameof(spark_md), param,
() => md_observable(contract, symbol, security_type, exchange).Select(x =>
() => md_observable(contract, symbol, security_type, exchange).Select(data =>
{
if (x is co2.ChannelUpdate upd)
return new TopicMDFormatter(upd, topic).format();
var x = data.FirstOrDefault();
if (x != null && x is IMDCombined upd)
return TopicMDFormatter.format(upd, topic);
return x;
}));
}
}
}
Binary file not shown.
+1 -1
View File
@@ -13,7 +13,7 @@ using bts.api;
using bts.api.core;
using bts.api.orderman;
using bts.api.user;
using bts.api.md;
using bts.md;
using bts.api.admin;
using security_key = System.UInt32;
@@ -9,8 +9,7 @@ using System.Reactive.Concurrency;
using bts.api;
using bts.api.core;
using bts.co2;
using bts.api.md;
using bts.md;
namespace bts.sdk
{
@@ -24,10 +23,13 @@ namespace bts.sdk
_md = client;
}
void print_update(ChannelUpdate update)
void print_update(IEnumerable<IMDData> updates)
{
Console.WriteLine(update.ToString());
Console.WriteLine();
foreach (var update in updates)
{
Console.WriteLine(update.ToString());
Console.WriteLine();
}
}
public static Venue parse_venue(string exch)
@@ -46,12 +48,14 @@ namespace bts.sdk
return st;
}
public void print_channel(string exch, string sectype, string symbol)
public void print_product(string exch, string sectype, string symbol)
{
Venue ven = parse_venue(exch);
SecurityType st = parse_security_type(sectype);
Product prod = new Product() { Exchange = ven, Type = st, Symbol = symbol };
_disps.Add(_md.subscribe(prod).Subscribe(x =>
Func<SecurityImmutable, bool> where = (x => x.Exchange == ven && x.Symbol == symbol && x.security_type() == st);
_disps.Add(_md.subscribe_book(where).Subscribe(x =>
{
print_update(x);
}));
@@ -61,11 +65,11 @@ namespace bts.sdk
{
Venue ven = parse_venue(exch);
SecurityType st = parse_security_type(sectype);
Product prod = new Product() { Exchange = ven, Type = st, Symbol = symbol };
Func<Security, bool> filter = (x => x.Glbx.SecurityDesc == security_desc);
//if you have a security_id, we could make a more specifics subscription, but this is a command line driver....
_disps.Add(_md.subscribe(prod).Where(x => filter(x.Security)).Subscribe(x =>
Func<SecurityImmutable, bool> where = (x => x.Exchange == ven && x.Symbol == symbol && x.security_type() == st &&
x.Glbx.SecurityDesc == security_desc);
_disps.Add(_md.subscribe_book(where).Subscribe(x =>
{
print_update(x);
}));
+5 -5
View File
@@ -7,7 +7,7 @@ using System.Threading.Tasks;
using System.Reactive.Linq;
using bts.api;
using bts.api.md;
using bts.md;
using bts.api.core;
using bts.api.orderman;
using bts.api.user;
@@ -27,7 +27,7 @@ namespace bts.sdk
}
readonly static Lazy<OrderService> _orders = new Lazy<OrderService>(() => Locator.locate<OrderService>(), true);
readonly static Lazy<UserService> _users = new Lazy<UserService>(() => Locator.locate<UserService>(), true);
readonly static Lazy<AuthService> _users = new Lazy<AuthService>(() => Locator.locate<AuthService>(), true);
readonly static Lazy<AdminService> _admin = new Lazy<AdminService>(() => Locator.locate<AdminService>(), true);
void print_update<T>(T update)
@@ -36,8 +36,8 @@ namespace bts.sdk
Console.WriteLine();
}
public async Task<Result<Response>> send_order(string exch, string sectype, string symbol, string security_desc, string qty,
string price = null, string user = null, string account = null)
public async Task<Result<Response>> send_order(string user, string account ,
string exch, string sectype, string symbol, string security_desc, string qty, string price = null)
{
Venue ven = MarketDataSample.parse_venue(exch);
SecurityType st = MarketDataSample.parse_security_type(sectype);
@@ -58,7 +58,7 @@ namespace bts.sdk
else
cr.NewOrder.CustomDef = new CustomOrderDefinition() { Market = new MarketOrderDefinition() };
var sec = await _md.find_security_async(x => x.Glbx.SecurityDesc == security_desc.ToUpper());
cr.NewOrder.SecurityKey = sec.Id;
cr.NewOrder.SecurityKey = sec.Value.Id;
Result<Response> resp = await _orders.Value.create_order_async(cr);
print_update(resp);
if(resp.value != null)
+45 -54
View File
@@ -6,46 +6,54 @@ using System.Reactive.Disposables;
using bts.api;
using bts.api.core;
using bts.api.md;
using bts.md;
namespace bts.sdk
{
public class SDKMain
{
private static void user_test()
static async Task<LoginResult> login(string user, string pass)
{
var ctx = new ServiceContext();
var s = ctx.locate<api.user.UserService>();
return await Locator.login(user, pass);
}
private static void user_test()
{
var s = Locator.locate<api.user.AuthService>();
var r = new api.user.GetUsersRequest();
var resp = s.get_users(r);
Console.WriteLine("{0}", resp);
}
public static void Main(string[] args)
{
public async static Task Main(string[] args)
{
Log.ClientSdk.info("Welcome to the BTS Client Side SDK example");
string u = args.FirstOrDefault(x => x.Contains("--user="));
string p = args.FirstOrDefault(x => x.Contains("--pass="));
u = u?.Replace("--user=", string.Empty) ?? string.Empty;
p = p?.Replace("--pass=", string.Empty) ?? string.Empty;
var lr = await login(u, p);
if (!lr.success)
{
Console.WriteLine("Login failure: " + lr.reason);
help();
}
if (args.FirstOrDefault() == "users")
{
user_test();
}
else
{
MarketDataClient client = new MarketDataClient();
uint ms = 100;
string pu = args.FirstOrDefault(x => x.Contains("--pump="));
if (pu != null)
ms = uint.Parse(p.Replace("--pump=", string.Empty));
MarketDataClient client = new MarketDataClient(ms, Locator.token);
CompositeDisposable disp = new CompositeDisposable(client);
if (args.FirstOrDefault() == "ui")
{
if (args.Length >= 6)
{
using (var ui = new ui.UISample(client))
{
ui.simple_price_ui(args[1], args[2], args[3], args[4], args[5]);
}
}
else
ui_help();
}
else if (args.Length > 0 && args.FirstOrDefault().Contains("md"))
if (args.Length > 0 && args.FirstOrDefault().Contains("md"))
{
var s = new MarketDataSample(client);
if (args.FirstOrDefault() == "md")
@@ -56,7 +64,7 @@ namespace bts.sdk
}
else if (args.Length >= 4)
{
s.print_channel(args[1], args[2], args[3]);
s.print_product(args[1], args[2], args[3]);
}
else
md_help();
@@ -68,33 +76,24 @@ namespace bts.sdk
{
OrderSample os = new OrderSample(client);
disp.Add(os);
order_sample(os, args);
order_sample(os, u, args);
}
else
help();
int ms = 100;
string p = args.FirstOrDefault(x => x.Contains("--pump="));
if (p != null)
ms = int.Parse(p.Replace("--pump=", string.Empty));
client.run_pump(TimeSpan.FromMilliseconds(ms));
wait_for_stop(disp);
}
}
static async void order_sample(OrderSample os, string[] args)
static async void order_sample(OrderSample os, string user, string[] args)
{
if (args.Length >= 9)
if (args.Length >= 10) //--user and --pass are in args[]
{
await os.send_order(args[1], args[2], args[3], args[4], args[5], args[6], args[7], args[8]);
await os.send_order(user, args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
}
else if (args.Length >= 7)
else if (args.Length >= 9)
{
await os.send_order(args[1], args[2], args[3], args[4], args[5], args[6]);
}
else if (args.Length >= 6)
{
await os.send_order(args[1], args[2], args[3], args[4], args[5]);
await os.send_order(user, args[1], args[2], args[3], args[4], args[5], args[6]);
}
else
order_help();
@@ -119,9 +118,10 @@ namespace bts.sdk
Console.WriteLine();
Console.WriteLine("Commands:");
Console.WriteLine(" md <exchange> <security_type> <symbol> [<security_desc>]");
Console.WriteLine(" ui <exchange> <security_type> <symbol> <security_desc> <poll_interval_in_ms>");
Console.WriteLine(" order <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>]");
Console.WriteLine(" order <account> <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>]");
Console.WriteLine("Arguments:");
Console.WriteLine(" --user=<bts user name>");
Console.WriteLine(" --pass=<password>");
Console.WriteLine(" --pump=<pump_interval_in_ms=100 default>");
Environment.Exit(1);
}
@@ -130,10 +130,9 @@ namespace bts.sdk
{
Console.WriteLine("order <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>] [<user>] [<account>]");
Console.WriteLine("This command sends a limit or market DAY order for the specified security_desc. Leave off the price for a market order");
Console.WriteLine(" example: order GLBX Future ES ESM8 -2 2650.25");
Console.WriteLine(" order GLBX Future ES ESM8 -2");
Console.WriteLine(" order GLBX Future ES ESM8 10 2650.0 bts \"ACC A1\"");
Console.WriteLine(" order GLBX Future ES ESM8 10 2650.0");
Console.WriteLine(" example: order \"ACC A1\" GLBX Future ES ESM8 10 2650.0");
Console.WriteLine(" order \"ACC A1\" SIM Future ES SIM:ESM8 -10 2650.0");
Console.WriteLine(" order \"ACC A1\" GLBX Future ES ESM8 10 (this is a market order)");
Environment.Exit(1);
}
@@ -142,19 +141,11 @@ namespace bts.sdk
Console.WriteLine("md <exchange> <security_type> <symbol>");
Console.WriteLine("This command subscribes to market data for an exchange-type-symbol channel and writes to console.");
Console.WriteLine("Marketdata is read off the wire on the same thread as is used to read and print");
Console.WriteLine(" example: md GLBX Future ES");
Console.WriteLine(" md GLBX Future ES ESZ8");
Console.WriteLine(" md GLBX Spread ES");
Console.WriteLine(" example: md GLBX Future ES --user=bts --pass=password");
Console.WriteLine(" md GLBX Future ES ESZ8 --user=bts --pass=password");
Console.WriteLine(" md GLBX Spread ES --user=bts --pass=password");
Environment.Exit(1);
}
static void ui_help()
{
Console.WriteLine("ui <exchange> <security_type> <symbol> <security_desc> <poll_interval_in_ms>");
Console.WriteLine("This command subscribes to market data for a specific security and writes to a UI on a specified refresh interval (in ms).");
Console.WriteLine("Marketdata is read off the wire on one thread and posted to the GUI on another through Observable.Timer()");
Console.WriteLine(" example: ui GLBX Future ES ESZ8 25");
Environment.Exit(1);
}
}
}
+1 -1
View File
@@ -2,7 +2,7 @@ using System;
using System.Linq;
using bts.api;
using bts.api.md;
using bts.md;
namespace bts.sdk.ui
{
+1
View File
@@ -18,6 +18,7 @@
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<WarningsAsErrors />
<Version>1.0.0</Version>
<LangVersion>7.2</LangVersion>
</PropertyGroup>
<ItemGroup>
+1 -1
View File
@@ -11,7 +11,7 @@ using System.Threading.Tasks;
using System.Windows.Forms;
using bts.api;
using bts.api.md;
using bts.md;
namespace bts.sdk.ui
{
-32
View File
@@ -34,25 +34,6 @@ namespace mbts.api.Test {
Disposable.dispose(ref _context);
}
[Test,Ignore("Timeout")]
public void test_format_qualifier() {
var simple = new QualifierArgument() { Simple = new SimpleQualifier() { Symbol = "ES" } };
var s1 = _mtest.format_qualifier(simple);
var s2 = _ntest.format_qualifier(simple);
Assert.AreEqual(s1.Str, s2.Str);
var complex = new ComplexQualifier() {
Name = "name",
};
complex.Probability.AddRange(new[] { 1.0, 2.0 });
complex.Symbols.Add(simple.Simple);
complex.Symbols.Add(new SimpleQualifier() { Symbol = "$50\"+" });
var carg = new QualifierArgument() { Complex = complex };
var c1 = _mtest.format_qualifier(carg);
var c2 = _ntest.format_qualifier(carg);
Assert.AreEqual(c1.Str, c2.Str);
}
[Test, Ignore("Times out")]
public void test_empty_reply_is_not_null() {
foreach (var svc in test_services()) {
@@ -127,19 +108,6 @@ namespace mbts.api.Test {
public ApiTestServiceServer(ServiceContext context) : base(context, "c#") { _context = context; }
public override Task<UnitTestResult> format_qualifier(QualifierArgument request) {
UnitTestResult result = new UnitTestResult();
switch (request.QualifierCase) {
case QualifierArgument.QualifierOneofCase.Simple:
result.Str = format_qualifier("format_qualifier", request.Simple);
break;
case QualifierArgument.QualifierOneofCase.Complex:
result.Str = format_qualifier("format_qualifier", request.Complex);
break;
}
return Task.FromResult(result);
}
public override Task<UnitTestResult> run_hello_service(EmptyRequest request) {
stop_hello_service(null);
_hello = new HelloServiceServer(_context);
@@ -4,7 +4,7 @@ using bts.api.synthetic;
using bts.test;
using ManyConsole.CommandLineUtils;
using System;
using System.Linq;
using System.Threading;
namespace bts {
public class ServiceTest : ConsoleCommand {
@@ -86,7 +86,7 @@ namespace bts {
private static void user_test() {
var ctx = new ServiceContext();
var s = ctx.locate<api.user.UserService>();
var s = ctx.locate<api.user.AuthService>();
var r = new api.user.GetUsersRequest();
var resp = s.get_users(r);
Console.WriteLine("{0}", resp);
@@ -96,8 +96,13 @@ namespace bts {
var ctx = new ServiceContext();
var s = ctx.locate<api.admin.AdminService>();
var resp = s.get_admin_data(new VoidMessage());
Console.WriteLine("{0}", resp);
var d = s.subscribei_admin_data().Subscribe(Console.WriteLine);
//var resp = s.get_admin_data(new VoidMessage());
//Console.WriteLine("{0}", resp);
while (true) {
Thread.Sleep(100);
}
}
}
}
+1 -1
View File
@@ -1 +1 @@
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/thread_manager.cpp)
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp)
+11
View File
@@ -12,9 +12,14 @@ message PingPong {
// RequestHeader
// T Request
message AuthToken {
bytes token = 1;
}
message RequestHeader {
int32 request_id = 1;
int32 method = 2;
AuthToken token = 3;
}
enum ApiError {
@@ -32,6 +37,12 @@ message ResponseHeader {
ApiError code = 2;
}
message StreamAccessRequest {
string qualifier = 1;
}
message StreamAccessResponse {
bool authorized = 1;
}
// Some Useful buffers for use as arguments/returns
+3 -2
View File
@@ -1,6 +1,7 @@
syntax="proto3";
package bts.api.gateway;
option csharp_namespace="bts.api.gateway";
import "lib/api/apicore.proto";
message GatewayLogin {
string username = 1;
@@ -9,13 +10,13 @@ message GatewayLogin {
message GatewayLoginResult {
bool is_valid = 1;
string reason = 2;
string token = 3;
api.AuthToken token = 3;
bool can_create = 4;
}
message GatewayCallHeader {
int32 request_id = 1;
int32 method = 2;
string token = 3;
api.AuthToken token = 3;
string service_type = 4;
string service_qualifier = 5;
}
+9 -4
View File
@@ -181,11 +181,16 @@ void safe_call_subscription_callback(raw_subscription_callback cb, EventType typ
}
bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method_and_qualifier) {
return qbus::Topic(base_topic + "-" + method_and_qualifier);
bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method, const char* qualifier) {
if (qualifier == nullptr || strlen(qualifier) == 0) {
return qbus::Topic(base_topic + "-" + method);
}
else {
return qbus::Topic(base_topic + "-" + method + "/" + qualifier);
}
}
CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* method_and_qualifier, raw_subscription_callback cb, void* extra) {
CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* method, const char* qualifier, raw_subscription_callback cb, void* extra) {
bts_assert(pclient);
static_assert((int)qbus::TableEvent::EventType::TABLE_CREATED == EventType::TABLE_CREATED, "Keep these Enums in sync");
static_assert((int)qbus::TableEvent::EventType::TABLE_DESTROYED == EventType::TABLE_DESTROYED, "");
@@ -194,7 +199,7 @@ CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* meth
static_assert((int)qbus::Change::Type::REPLACE == StreamUpdate::REPLACE, "");
static_assert((int)qbus::Change::Type::CREATE == StreamUpdate::CREATE, "");
auto topic = stream_topic(pclient->service_name, method_and_qualifier);
auto topic = stream_topic(pclient->service_name, method, qualifier);
LOG(Api,Diag,"api subscribe: {}" ,topic.value());
auto result = new qbus::TableSubscriber(pclient->qbus, equal_topic(topic), [cb, extra](const qbus::TableEvent& ev) {
+1 -1
View File
@@ -21,7 +21,7 @@ enum EventType {
typedef void* TableSubscription;
typedef void(*raw_subscription_callback)(EventType event_type, const StreamUpdate* changes, int count, void* extra);
CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* method_and_qualifier, raw_subscription_callback cb, void* extra);
CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* method, const char* qualifier, raw_subscription_callback cb, void* extra);
CAPI void api_client_unsubscribe(ApiClient* pclient, TableSubscription ptable);
}}//namespace
+3 -3
View File
@@ -9,7 +9,7 @@ namespace api {
// defined in ApiClientInterop.h
std::string base_topic(const char* type_name, const char* service_qualifier);
bts::qbus::RPCServiceName rpc_topic(const std::string& base_topic);
bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method_and_qualifier);
bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method, const char* qualifier);
struct ApiServerCore {
ApiServerCore(Quickbus& qbus, const std::string topic, const qbus::OnRPCRequest& handler) :
@@ -51,9 +51,9 @@ CAPI void api_server_call_callback(ReplyHandler** ppvhandler, const uint8_t* buf
*pphandler = nullptr;
}
CAPI TableHandle api_server_create_table(ApiServerCore* pserver, const char* method_and_qualifier) {
CAPI TableHandle api_server_create_table(ApiServerCore* pserver, const char* method, const char* qualifier) {
bts_assert(pserver);
auto topic = stream_topic(pserver->topic, method_and_qualifier);
auto topic = stream_topic(pserver->topic, method, qualifier);
LOG(Api,Info,"Create table: {}",topic.value());
return new qbus::TablePublisher(pserver->qbus, topic);
}
+1 -1
View File
@@ -12,7 +12,7 @@ CAPI ApiServerCore* api_server_create(Quickbus* pbus, const char* type_name, con
CAPI void api_server_call_callback(ReplyHandler** pphandler, const uint8_t* buffer, int length);
typedef void* TableHandle;
CAPI TableHandle api_server_create_table(ApiServerCore* pserver, const char* method_and_qualifier);
CAPI TableHandle api_server_create_table(ApiServerCore* pserver, const char* method, const char* qualifier);
CAPI void api_server_update_table(TableHandle htable, StreamUpdate* raw_changes, int change_count);
}}//namespace
-17
View File
@@ -18,22 +18,5 @@ namespace api {
else return "";
}
template <class T>
const std::string format_qualifier(const char* type_name, const T& t) {
google::protobuf::util::JsonPrintOptions options;
options.add_whitespace = false;
options.always_print_primitive_fields = true;
std::string result;
auto status = google::protobuf::util::MessageToJsonString(t, &result, options);
assert(status.ok());
result.erase( std::remove_if(result.begin(), result.end(), [](char c) {
return
c == '+' ||
c == '_';
}), result.end());
return type_name + result;
}
}}//namespace
+1 -1
View File
@@ -3,7 +3,7 @@
namespace bts {
namespace api {
ClientChannel::ClientChannel() : _id(1)
ClientChannel::ClientChannel(const ServiceContext& ctx) : _id(1), _context(ctx)
{
}
+53 -36
View File
@@ -4,7 +4,7 @@
#include "Result.h"
#include "ApiException.h"
#include "RawUpdate.h"
//#include "Update.h"
#include "ServiceContext.h"
#include <rxcpp/rx.hpp>
#include <functional>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
@@ -12,19 +12,21 @@
namespace bts {
namespace api {
class ClientChannel
{
int _id;
ServiceContext _context;
public:
ClientChannel();
ClientChannel(const ServiceContext&);
virtual ~ClientChannel();
template <typename Request, typename Response>
void send(int method_id, const Request& request, std::function<void(Result<Response>)> on_response) {
void send(int method_id, const Request& request, std::function<void(const Result<Response>)> on_response) {
RequestHeader header;
header.set_method(method_id);
header.set_request_id(++_id);
auto ptoken = _context.token();
if (ptoken) header.mutable_token()->CopyFrom(*ptoken);
int header_size = header.ByteSize()+1;
assert(header_size < 128);
@@ -62,42 +64,57 @@ public:
template <typename T>
rxcpp::observable<RawUpdate<std::shared_ptr<const T>>> subscribe(const std::string& method_and_qualifier) {
rxcpp::observable<RawUpdate<std::shared_ptr<const T>>> subscribe(int method_id, const char* method, const std::string& qualifier) {
return rxcpp::observable<>::create<RawUpdate<std::shared_ptr<const T>>>([=](rxcpp::subscriber<RawUpdate<std::shared_ptr<const T>>> observer) {
bool first_update = true;
auto cb = [observer, first_update](EventType event_type, const StreamUpdate* changes, int change_count) mutable {
if (event_type == EventType::TABLE_UPDATED) {
std::vector<std::shared_ptr<const T>> added_items;
std::vector<int> added_keys,
removed_keys;
for (size_t x = 0; x < change_count; ++x) {
const StreamUpdate& change = changes[x];
if (change.type != StreamUpdate::REMOVE) {
added_keys.reserve(change_count);
added_items.reserve(change_count);
added_keys.push_back(change.key);
google::protobuf::io::ArrayInputStream stream(change.buffer, change.buffer_len);
auto item = std::make_shared<T>();
if (item->ParseFromZeroCopyStream(&stream)) {
added_items.push_back(move(item));
StreamAccessRequest request;
request.set_qualifier(qualifier);
send<StreamAccessRequest,StreamAccessResponse>(method_id, request, [=](const Result<StreamAccessResponse>& response) {
// There is a race on server startup. If the server is not yet we can't return a dead stream
// if (!response) {
// observer.on_error(
// std::make_exception_ptr(
// ApiException(response.error_message())));
// }
if (response && !response.value()->authorized()) {
observer.on_completed();
}
else {
bool first_update = true;
auto cb = [observer, first_update](EventType event_type, const StreamUpdate* changes, int change_count) mutable {
if (event_type == EventType::TABLE_UPDATED) {
std::vector<std::shared_ptr<const T>> added_items;
std::vector<int> added_keys,
removed_keys;
for (size_t x = 0; x < change_count; ++x) {
const StreamUpdate& change = changes[x];
if (change.type != StreamUpdate::REMOVE) {
added_keys.reserve(change_count);
added_items.reserve(change_count);
added_keys.push_back(change.key);
google::protobuf::io::ArrayInputStream stream(change.buffer, change.buffer_len);
auto item = std::make_shared<T>();
if (item->ParseFromZeroCopyStream(&stream)) {
added_items.push_back(move(item));
}
}
if (change.type != StreamUpdate::CREATE) {
removed_keys.reserve(change_count);
removed_keys.push_back(change.key);
}
}
observer.on_next(RawUpdate<std::shared_ptr<const T>>(added_items, added_keys, removed_keys));
}
if (change.type != StreamUpdate::CREATE) {
removed_keys.reserve(change_count);
removed_keys.push_back(change.key);
else if (event_type == EventType::TABLE_CREATED) {
if (!first_update) {
observer.on_next(RawUpdate<std::shared_ptr<const T>>::Reset());
}
first_update = false;
}
}
observer.on_next(RawUpdate<std::shared_ptr<const T>>(added_items, added_keys, removed_keys));
};
observer.add(
subscribe_core(method, qualifier.c_str(), cb));
}
else if (event_type == EventType::TABLE_CREATED) {
if (!first_update) {
observer.on_next(RawUpdate<std::shared_ptr<const T>>::Reset());
}
first_update = false;
}
};
observer.add(
subscribe_core(method_and_qualifier, cb));
});
});
}
@@ -108,7 +125,7 @@ protected:
typedef std::function<void()> Unsubscribe;
typedef std::function<void(EventType,const StreamUpdate*,int)> OnSubscriptionCB;
virtual void send_core(const std::string& request_buffer, OnReply on_response) = 0;
virtual Unsubscribe subscribe_core(const std::string& method_and_qualifier, OnSubscriptionCB cb) = 0;
virtual Unsubscribe subscribe_core(const char* method, const char* qualifier, OnSubscriptionCB cb) = 0;
private:
ClientChannel(const ClientChannel& other);// = delete;
+4 -3
View File
@@ -4,7 +4,8 @@
namespace bts {
namespace api {
QuickbusClientChannel::QuickbusClientChannel(Quickbus& bus, const char* type_name, const char* service_qualifier) :
QuickbusClientChannel::QuickbusClientChannel(Quickbus& bus, const ServiceContext& context, const char* type_name, const char* service_qualifier) :
ClientChannel(context),
_pclient(api_client_create(&bus, type_name, service_qualifier))
{
}
@@ -27,14 +28,14 @@ void QuickbusClientChannel::send_core(const std::string& request_envalope, OnRep
api_client_send(_pclient, request_envalope.c_str(), request_envalope.size(), trampoline, heap_on_response);
}
ClientChannel::Unsubscribe QuickbusClientChannel::subscribe_core(const std::string & method_and_qualifier, OnSubscriptionCB cb) {
ClientChannel::Unsubscribe QuickbusClientChannel::subscribe_core(const char* method, const char* qualifier, OnSubscriptionCB cb) {
raw_subscription_callback trampoline = [](EventType event_type, const StreamUpdate* changes, int count, void* extra) {
OnSubscriptionCB* heap_cb = static_cast<OnSubscriptionCB*>(extra);
assert(heap_cb != nullptr);
(*heap_cb)(event_type, changes, count);
};
OnSubscriptionCB* heap_cb = new OnSubscriptionCB(cb);
auto subscription = api_client_subscribe(_pclient, method_and_qualifier.c_str(), trampoline, heap_cb);
auto subscription = api_client_subscribe(_pclient, method, qualifier, trampoline, heap_cb);
return [this,subscription,heap_cb]() {
api_client_unsubscribe(_pclient, subscription);
delete heap_cb;
+3 -2
View File
@@ -6,19 +6,20 @@
namespace bts {
namespace api {
class ServiceContext;
class QuickbusClientChannel : public ClientChannel
{
ApiClient* _pclient;
public:
QuickbusClientChannel(Quickbus&, const char* type_name, const char* service_qualifier);
QuickbusClientChannel(Quickbus&, const ServiceContext&, const char* type_name, const char* service_qualifier);
~QuickbusClientChannel();
private:
BTS_NO_COPY(QuickbusClientChannel);
virtual void send_core(const std::string& request_envalope, OnReply on_response) override;
virtual Unsubscribe subscribe_core(const std::string& method_and_qualifier, OnSubscriptionCB cb) override;
virtual Unsubscribe subscribe_core(const char* method, const char* qualifier, OnSubscriptionCB cb) override;
virtual bool is_sync_call_ok() override;
};
+2 -7
View File
@@ -55,9 +55,9 @@ public:
{}
#endif
std::shared_ptr<T> result() const {
std::shared_ptr<T> value() const {
if (success()) return _t;
else throw ApiException(std::string(error_message()));
else return nullptr;
}
bool success() const { return _code == ApiError::SUCCESS; }
@@ -66,13 +66,8 @@ public:
const char* error_message() const { return error_string(_code); }
~Result() {}
// Is not a type??
// typedef std::function<void(Result<T>)> OnResponse;
};
//template <typename T>
//using OnResponse = std::function<void(Result<T>)>;
}}//namespace
+2 -2
View File
@@ -51,8 +51,8 @@ ServerChannel::ServerChannel(ServerChannel&& other) : _me(std::move(other._me))
ServerChannel::~ServerChannel() { }
TableHandle ServerChannel::create_table(const std::string& method_and_qualifier) const {
return api_server_create_table(_me->_core, method_and_qualifier.c_str());
TableHandle ServerChannel::create_table(const char* method, const char* qualifier) const {
return api_server_create_table(_me->_core, method, qualifier);
}
void ServerChannel::publish(TableHandle table, StreamUpdate* changes, int change_count) const {
api_server_update_table(table, changes, change_count);
+1 -1
View File
@@ -15,7 +15,7 @@ public:
ServerChannel(ServerChannel&& other);// : _me(std::move(other._me)) {}
~ServerChannel();
TableHandle create_table(const std::string& method_and_qualifier) const;
TableHandle create_table(const char* method, const char* qualifier) const;
void publish(TableHandle table, StreamUpdate* changes, int change_count) const;
private:
+71 -10
View File
@@ -3,33 +3,94 @@
#include "QuickbusClientChannel.h"
#include "ApiServer.h"
#include "lib/logging/logging.h"
#include "lib/api/services/AuthService.client.pb.h"
#include <memory>
namespace bts {
namespace api {
LoginResult::LoginResult():
success(true),
reason()
{}
LoginResult::LoginResult(const std::string& why):
success(true),
reason(why)
{}
struct ServiceContextImpl {
ServiceContextImpl(Quickbus* maybe_bus):
_qbus(maybe_bus ? *maybe_bus : *api_create_quickbus(nullptr, 0)),
_owns_qb(maybe_bus == nullptr),
_token(),
_user()
{}
~ServiceContextImpl() {
if (_owns_qb) {
api_destroy_quickbus(&_qbus);
}
}
void set_token(const AuthToken& t) {
auto st = std::make_shared<AuthToken>();
st->CopyFrom(t);
_token = st;
}
Quickbus& _qbus;
bool _owns_qb;
std::shared_ptr<AuthToken> _token;
std::string _user;
};
ServiceContext::ServiceContext(Quickbus* maybe_bus) :
_qbus(maybe_bus ? *maybe_bus : *api_create_quickbus(nullptr, 0)),
_owns_qb(maybe_bus == nullptr)
_self(std::make_shared<ServiceContextImpl>(maybe_bus))
{
}
ServiceContext::~ServiceContext() {
if (_owns_qb) {
api_destroy_quickbus(&_qbus);
}
}
ServiceContext::~ServiceContext() {}
std::unique_ptr<ClientChannel> ServiceContext::create_channel(const char * type_name, const char * service_qualifier) const {
LOG(Api,Info,"Locate Service: {}{}",std::string(type_name) ,std::string(service_qualifier));
return std::unique_ptr<ClientChannel>(new QuickbusClientChannel(_qbus, type_name, service_qualifier));
return std::unique_ptr<ClientChannel>(new QuickbusClientChannel(_self->_qbus, *this, type_name, service_qualifier));
}
ServerChannel ServiceContext::publish(ApiServer& service, const std::string& service_type, const std::string& qualifier) const {
return ServerChannel(_qbus, service_type, qualifier, service);
return ServerChannel(_self->_qbus, service_type, qualifier, service);
}
const LoginResult ServiceContext::login(const std::string& username, const std::string& password) {
auto auth = locate<user::AuthService>();
user::AuthenticateLoginRequest request;
request.set_login(username);
request.set_password(password);
auto result = auth.authenticate_login(request);
if (result.success()) {
if (result.value()->is_valid()) {
_self->set_token(result.value()->token());
_self->_user = username;
return {};
}
else {
return {result.value()->reason()};
}
}
else {
return {result.error_message()};
}
}
void ServiceContext::login(const AuthToken& token) {
_self->set_token(token);
_self->_user.clear();
}
void ServiceContext::post(const Action& action, uint64_t delay) {
api_post(_qbus, action, delay);
api_post(_self->_qbus, action, delay);
}
Quickbus& ServiceContext::bus() const { return _self->_qbus; }
bool ServiceContext::is_authenticated() const { return _self->_token != nullptr; }
const std::shared_ptr<AuthToken> ServiceContext::token() const {
return _self->_token;
}
}}//namespace
+18 -5
View File
@@ -5,7 +5,6 @@
// client
#include "../core/ApiClientInterop.h"
#include "../core/ApiServerInterop.h"
#include "ClientChannel.h"
// server
#include "ServerChannel.h"
@@ -16,13 +15,23 @@
namespace bts {
namespace api {
class ClientChannel;
struct ServiceContextImpl;
struct LoginResult {
LoginResult();
LoginResult(const std::string&);
bool success;
std::string reason;
};
class BTS_EXPORT ServiceContext
{
Quickbus& _qbus;
bool _owns_qb;
std::shared_ptr<ServiceContextImpl> _self;
public:
ServiceContext(Quickbus* maybe_bus=nullptr);
// ServiceContext(ServiceContext& other) = default;
~ServiceContext();
template <class T>
@@ -30,11 +39,15 @@ public:
return T(create_channel(T::name(), qualifier));
}
Quickbus& bus() const;
bool is_authenticated() const;
const std::shared_ptr<AuthToken> token() const;
const LoginResult login(const std::string& username, const std::string& password);
void login(const AuthToken& token);
typedef std::function<void()> Action;
void post(const Action&, uint64_t delayms=0);
Quickbus& bus() const { return _qbus; }
private:
friend class ApiServer;
std::unique_ptr<ClientChannel> create_channel(const char*, const char*) const;
@@ -18,24 +18,24 @@ public sealed class {{name}} : IApiClient, IDisposable {
{{#each method}}
{{#if server_streaming}}
// {{comment}}
public IObservable<Update<{{output_type_full}},{{key_type.cstype}}>> subscribe_{{name}}({{input_type_full}} qualifier=null) {
return _channel.subscribe<{{output_type_full}}, {{output_type_full}}>(ApiServer.format_qualifier("{{name}}",qualifier), {{output_type_full}}.Parser, a=>a).reconstitute(key);
}
public IObservable<Update<{{output_type_full}}Immutable,{{key_type.cstype}}>> subscribei_{{name}}({{input_type_full}} qualifier=null) {
return _channel.subscribe<{{output_type_full}}, {{output_type_full}}Immutable>(ApiServer.format_qualifier("{{name}}",qualifier), {{output_type_full}}.Parser, a=>({{output_type_full}}Immutable)a).reconstitute(key);
}
private static {{key_type.cstype}} key({{output_type_full}} row) {
return row.{{camel key_field}};
}
{{#if key_type.ismessage}}
private static {{key_type.cstype}} key({{output_type_full}}Immutable row) {
return row.{{camel key_field}}.CloneToMutable();
}
{{else}}
private static {{key_type.cstype}} key({{output_type_full}}Immutable row) {
return row.{{camel key_field}};
// {{comment}}
public IObservable<Update<{{output_type_full}},{{key_type.cstype}}>> subscribe_{{name}}(string qualifier=null) {
return _channel.subscribe<{{output_type_full}}, {{output_type_full}}>({{@index}}, "{{name}}", qualifier, {{output_type_full}}.Parser, a=>a).reconstitute(key);
}
public IObservable<Update<{{output_type_full}}Immutable,{{key_type.cstype}}>> subscribei_{{name}}(string qualifier=null) {
return _channel.subscribe<{{output_type_full}}, {{output_type_full}}Immutable>({{@index}}, "{{name}}", qualifier, {{output_type_full}}.Parser, a=>({{output_type_full}}Immutable)a).reconstitute(key);
}
private static {{key_type.cstype}} key({{output_type_full}} row) {
return row.{{camel key_field}};
}
{{#if key_type.ismessage}}
private static {{key_type.cstype}} key({{output_type_full}}Immutable row) {
return row.{{camel key_field}}.CloneToMutable();
}
{{else}}
private static {{key_type.cstype}} key({{output_type_full}}Immutable row) {
return row.{{camel key_field}};
}
{{/if}}
{{else}}
// {{comment}}
@@ -14,9 +14,9 @@ namespace {{package}}
{{#each service}}
// {{comment}}
internal abstract class {{name}}Base : ApiServer, IDisposable {
internal abstract class Authenticated{{name}}Base : ApiServer, IDisposable {
readonly ServerChannel _channel;
public {{name}}Base(ServiceContext context, string qualifier="") {
public Authenticated{{name}}Base(ServiceContext context, string qualifier="") {
_channel = context.publish(this, "{{name}}", qualifier);
}
@@ -122,16 +122,22 @@ internal abstract class {{name}}Base : ApiServer, IDisposable {
}
//{{comment}}
protected {{cap name}}Publisher make_{{name}}_publisher({{input_type_full}} qualifier=null) {
IntPtr table = _channel.create_table(format_qualifier("{{name}}", qualifier));
protected {{cap name}}Publisher make_{{name}}_publisher(string qualifier=null) {
IntPtr table = _channel.create_table("{{name}}",qualifier);
return new {{cap name}}Publisher(table);
}
protected static {{key_type.cstype}} {{name}}_key({{output_type_full}} row) {
return row.{{camel key_field}};
}
public virtual Task<StreamAccessResponse> allow_{{name}}(bts.api.AuthTokenImmutable? token, string qualifier) {
var r = new StreamAccessResponse() { Authorized = true };
return Task.FromResult(r);
}
{{else}}
//{{comment}}
public abstract Task<{{output_type_full}}> {{name}}({{input_type_full}} request);
public abstract Task<{{output_type_full}}> {{name}}(bts.api.AuthTokenImmutable? token, {{input_type_full}} request);
{{/if}}
{{/each}}
@@ -150,15 +156,18 @@ internal abstract class {{name}}Base : ApiServer, IDisposable {
break;
}
{{#each method}}
{{#unless server_streaming}}
case ServiceMethods.{{upper name}}: {
{{#if server_streaming}}
var msg = bts.api.StreamAccessRequest.Parser.ParseFrom(request_stream);
var result = allow_{{name}}(header.Token, msg.Qualifier);
{{else}}
var msg = {{input_type_full}}.Parser.ParseFrom(request_stream);
var result = {{name}}(msg);
var result = {{name}}(header.Token, msg);
{{/if}}
var id = header.RequestId;
result.ContinueWith(t => respond(id, t, on_response));
break;
}
{{/unless}}
}
{{/each}}
default:
var buffer = new ResponseHeader(){
@@ -170,6 +179,20 @@ internal abstract class {{name}}Base : ApiServer, IDisposable {
}
}
// {{comment}}
internal abstract class {{name}}Base : Authenticated{{name}}Base , IDisposable {
public {{name}}Base(ServiceContext context, string qualifier="") : base(context, qualifier) { }
{{#each method}}
{{#unless server_streaming}}
public override Task<{{output_type_full}}> {{name}}(bts.api.AuthTokenImmutable? token, {{input_type_full}} request) {
return {{name}}(request);
}
// {{comment}}
public abstract Task<{{output_type_full}}> {{name}}({{input_type_full}} request);
{{/unless}}
{{/each}}
}
{{/each}}
}
#endregion
@@ -16,19 +16,14 @@ using namespace rxcpp::operators;
{{#each method}}
{{#if server_streaming}}
rxcpp::observable<{{cap name}}Update> {{../name}}::subscribe_{{name}}(const {{input_type_cpp}}& qualifier) {
return
_pchannel->subscribe<{{output_type_cpp}}>(format_qualifier("{{name}}", qualifier))
| (bts::api::reconstitute<std::shared_ptr<const {{output_type_cpp}}>,const {{key_type.cpptype}}, {{../name}}>());
}
rxcpp::observable<{{cap name}}Update> {{../name}}::subscribe_{{name}}(std::string const&qualifier) {
return
_pchannel->subscribe<{{output_type_cpp}}>(std::string("{{name}}/") + qualifier)
_pchannel->subscribe<{{output_type_cpp}}>({{@index}}, "{{name}}/", qualifier)
| (bts::api::reconstitute<std::shared_ptr<const {{output_type_cpp}}>,const {{key_type.cpptype}}, {{../name}}>());
}
rxcpp::observable<{{cap name}}Update> {{../name}}::subscribe_{{name}}() {
return
_pchannel->subscribe<{{output_type_cpp}}>("{{name}}")
_pchannel->subscribe<{{output_type_cpp}}>({{@index}}, "{{name}}", "")
| (bts::api::reconstitute<std::shared_ptr<const {{output_type_cpp}}>,const {{key_type.cpptype}}, {{../name}}>());
}
@@ -44,7 +39,7 @@ using namespace rxcpp::operators;
{{else}}
void {{../name}}::{{name}}(const {{input_type_cpp}}& input, std::function<void(bts::api::Result<{{output_type_cpp}}>)> callback, uint32_t timeout_ms) {
void {{../name}}::{{name}}(const {{input_type_cpp}}& input, std::function<void(const bts::api::Result<{{output_type_cpp}}>)> callback, uint32_t timeout_ms) {
_pchannel->send({{@index}}, input, callback);
}
@@ -47,9 +47,7 @@ class BTS_EXPORT {{name}} {
{{#each method}}
{{#if server_streaming}}
// {{comment}}
rxcpp::observable<{{cap name}}Update> subscribe_{{name}}(const {{input_type_cpp}}& qualifier);
// {{comment}}
rxcpp::observable<{{cap name}}Update> subscribe_{{name}}(std::string const&);
rxcpp::observable<{{cap name}}Update> subscribe_{{name}}(std::string const& qualifier);
// {{comment}}
rxcpp::observable<{{cap name}}Update> subscribe_{{name}}();
{{else}}
@@ -4,6 +4,7 @@
#include "lib/api/public/ApiUtils.h"
#include "lib/api/core/ApiServerInterop.h"
#include <unordered_map>
#include <mutex>
{{ns-open package}}
{{#each service}}
@@ -19,13 +20,13 @@ const {{key_type.cpptype}}& {{name}}_key(const {{output_type_cpp}}& row) {
{{/if}}
{{/if}}{{/each}}
{{name}}Base::{{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier) :
Authenticated{{name}}Base::Authenticated{{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier) :
bts::api::ApiServer(context, "{{name}}", qualifier)
{}
const char* {{name}}Base::name() const { return "{{name}}"; };
const char* Authenticated{{name}}Base::name() const { return "{{name}}"; };
void {{name}}Base::dispatch(const bts::api::RequestHeader& header, google::protobuf::io::ZeroCopyInputStream& stream, OnResponse on_response) {
void Authenticated{{name}}Base::dispatch(const bts::api::RequestHeader& header, google::protobuf::io::ZeroCopyInputStream& stream, OnResponse on_response) {
switch((ServiceMethods)header.method()) {
case SYS_PING: {
bts::api::PingPong pp;
@@ -33,17 +34,30 @@ void {{name}}Base::dispatch(const bts::api::RequestHeader& header, google::proto
respond(header.request_id(), pp, on_response);
}
} break;
{{#each method}} {{#unless server_streaming}}
{{#each method}}
case {{upper name}}: {
{{#if server_streaming}}
auto id = header.request_id();
bts::api::StreamAccessRequest msg;
if (msg.ParsePartialFromZeroCopyStream(&stream)) {
allow_{{name}}(
header.token(),
msg.qualifier(),
[this,id,on_response](const bts::api::StreamAccessResponse& result) {
respond(id,result,on_response);
});
}
{{else}}
{{input_type_cpp}} msg;
if (msg.ParsePartialFromZeroCopyStream(&stream)) {
auto id = header.request_id();
{{name}}(msg, [this,id,on_response](const {{output_type_cpp}}& result) {
{{name}}(header.token(), msg, [this,id,on_response](const {{output_type_cpp}}& result) {
respond(id, result, on_response);
});
}
{{/if}}
} break;
{{/unless}}{{/each}}
{{/each}}
default: {
bts::api::ResponseHeader response;
response.set_request_id(header.request_id());
@@ -58,19 +72,11 @@ void {{name}}Base::dispatch(const bts::api::RequestHeader& header, google::proto
{{#each method}}{{#if server_streaming}}
{{../name}}Base::{{cap name}}Publisher {{../name}}Base::make_{{name}}_publisher(std::string const&qualifier) const {
bts::api::TableHandle handle = _channel.create_table(std::string("{{name}}/") + qualifier);
{{../name}}Base::{{cap name}}Publisher Authenticated{{../name}}Base::make_{{name}}_publisher(const char* qualifier) const {
bts::api::TableHandle handle = _channel.create_table("{{name}}", qualifier);
return make_{{name}}_publisher(handle);
}
{{../name}}Base::{{cap name}}Publisher {{../name}}Base::make_{{name}}_publisher(const {{input_type_cpp}}& qualifier) const {
bts::api::TableHandle handle = _channel.create_table(format_qualifier("{{name}}", qualifier));
return make_{{name}}_publisher(handle);
}
{{../name}}Base::{{cap name}}Publisher {{../name}}Base::make_{{name}}_publisher() const {
bts::api::TableHandle handle = _channel.create_table("{{name}}");
return make_{{name}}_publisher(handle);
}
{{../name}}Base::{{cap name}}Publisher {{../name}}Base::make_{{name}}_publisher(bts::api::TableHandle handle) const {
{{../name}}Base::{{cap name}}Publisher Authenticated{{../name}}Base::make_{{name}}_publisher(bts::api::TableHandle handle) const {
{{#unless key_type.natural_key}}
std::unordered_map<{{key_type.cpptype}}, int> _map;
int _next_id;
@@ -165,17 +171,15 @@ void {{name}}Base::dispatch(const bts::api::RequestHeader& header, google::proto
};
}
{{../name}}Base::{{cap name}}StatefulPublisher {{../name}}Base::make_{{name}}_stateful_publisher() const {
return bts::api::StatefulPublisher<{{output_type_cpp}},{{key_type.cpptype}}>(make_{{name}}_publisher(), {{name}}_key);
}
{{../name}}Base::{{cap name}}StatefulPublisher {{../name}}Base::make_{{name}}_stateful_publisher(const std::string& qualifier) const {
return bts::api::StatefulPublisher<{{output_type_cpp}},{{key_type.cpptype}}>(make_{{name}}_publisher(qualifier), {{name}}_key);
}
{{../name}}Base::{{cap name}}StatefulPublisher {{../name}}Base::make_{{name}}_stateful_publisher(const {{input_type_cpp}}& qualifier) const {
{{../name}}Base::{{cap name}}StatefulPublisher Authenticated{{../name}}Base::make_{{name}}_stateful_publisher(const char* qualifier) const {
return bts::api::StatefulPublisher<{{output_type_cpp}},{{key_type.cpptype}}>(make_{{name}}_publisher(qualifier), {{name}}_key);
}
void Authenticated{{../name}}Base::allow_{{name}}(const bts::api::AuthToken&, const std::string&, const api::Response<bts::api::StreamAccessResponse>& reply) {
bts::api::StreamAccessResponse r;
r.set_authorized(true);
reply(r);
}
{{/if}} {{/each}}
@@ -10,6 +10,7 @@
namespace bts { namespace api {
class ServiceContext;
class AuthToken;
}}
{{ns-open package}}
@@ -20,7 +21,7 @@ class {{name}};{{/each}}
{{#each service}}
// {{comment}}
class {{name}}Base : public bts::api::ApiServer {
class Authenticated{{name}}Base : public bts::api::ApiServer {
public:
{{#each method}}
{{#if server_streaming}}
@@ -30,22 +31,23 @@ public:
{{/each}}
protected:
{{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier="");
Authenticated{{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier="");
const char* name() const override;
{{#each method}}
{{#if server_streaming}}
{{cap name}}Publisher make_{{name}}_publisher() const;
{{cap name}}Publisher make_{{name}}_publisher(const std::string &) const;
{{cap name}}Publisher make_{{name}}_publisher(const {{input_type_cpp}}& qualifier) const;
{{cap name}}Publisher make_{{name}}_publisher(const std::string& qualifier) const { return make_{{name}}_publisher(qualifier.c_str()); }
{{cap name}}Publisher make_{{name}}_publisher(const char* qualifier=nullptr) const;
{{cap name}}StatefulPublisher make_{{name}}_stateful_publisher(const char* qualifier=nullptr) const;
virtual void allow_{{name}}(const bts::api::AuthToken& token, const std::string& qualifier, const api::Response<bts::api::StreamAccessResponse>& reply);
{{cap name}}StatefulPublisher make_{{name}}_stateful_publisher() const;
{{cap name}}StatefulPublisher make_{{name}}_stateful_publisher(const std::string& qualifier) const;
{{cap name}}StatefulPublisher make_{{name}}_stateful_publisher(const {{input_type_cpp}}& qualifier) const;
{{else}}
// {{comment}}
virtual void {{name}}(
const api::AuthToken& token,
const {{input_type_cpp}}& request,
const api::Response<{{output_type_cpp}}>& reply) = 0;
{{/if}}
@@ -58,11 +60,33 @@ protected:
{{/each}}
};
private:
void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override;
{{#each method}}{{#if server_streaming}}
{{cap name}}Publisher make_{{name}}_publisher(bts::api::TableHandle) const;
{{/if}}{{/each}}
private:
void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override;
};
// {{comment}}
class {{name}}Base : public Authenticated{{name}}Base {
protected:
{{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier="") : Authenticated{{name}}Base(context, qualifier) {}
public:
{{#each method}}
{{#unless server_streaming}}
void {{name}}(
const api::AuthToken&,
const {{input_type_cpp}}& request,
const api::Response<{{output_type_cpp}}>& reply) override {
{{name}}(request, reply);
}
// {{comment}}
virtual void {{name}}(
const {{input_type_cpp}}& request,
const api::Response<{{output_type_cpp}}>& reply) = 0;
{{/unless}}
{{/each}}
};
{{/each}}
+1 -3
View File
@@ -459,9 +459,7 @@ message AdminServiceCreateAccountResult {
uint32 new_id = 30;
}
message FilteredAdminDataRequest {
string token = 1;
}
message FilteredAdminDataRequest { }
service AdminService {
option (bts.api.service_description) = "Provides a management interface to the user model";
@@ -31,7 +31,6 @@ message QualifierArgument {
}
service ApiTestService {
rpc format_qualifier(QualifierArgument) returns (UnitTestResult);
rpc test_empty_reply(EmptyRequest) returns (UnitTestResult);
@@ -1,6 +1,7 @@
syntax="proto3";
package bts.api.user;
option csharp_namespace="bts.api.user";
import "lib/api/apicore.proto";
import "lib/api/apioptions.proto";
message OperatorID { string tag50 = 10; }
@@ -25,10 +26,6 @@ message User {
repeated Entitlement entitlements = 200;
}
message AuthToken {
string token = 1;
}
message TokenPayload {
uint32 date = 1;
uint32 id = 2;
@@ -37,22 +34,22 @@ message TokenPayload {
message GetUsersRequest {}
message GetUsersResponse { repeated User users = 1; string error = 2; }
message AddOrUpdateUserRequest {AuthToken token = 1; User user = 2;}
message RemoveUserRequest {AuthToken token = 1; string login = 2;}
message AddOrUpdateUserRequest { api.AuthToken token = 1; User user = 2;}
message RemoveUserRequest {api.AuthToken token = 1; string login = 2;}
message UserOperationResponse { bool is_valid = 1; string reason = 2;}
message AuthenticateLoginRequest {string login = 1; string password = 2;}
message AuthenticateLoginResponse {
bool is_valid = 1;
string reason = 2;
AuthToken token = 3;
api.AuthToken token = 3;
uint32 user_id = 4;
bool can_create = 5;
}
message AuthenticateTokenRequest {AuthToken token = 1;}
message AuthenticateTokenRequest {api.AuthToken token = 1;}
message AuthenticateTokenResponse {bool is_valid = 1; string reason = 2; User user = 3;}
service UserService {
service AuthService {
option (bts.api.service_description) = "Provides a management interface to the system user list";
rpc authenticate_login(AuthenticateLoginRequest) returns (AuthenticateLoginResponse)
-19
View File
@@ -25,25 +25,6 @@ namespace test {
_hello(),
_shout()
{}
// Inherited via ApiTestServiceBase
virtual void format_qualifier(const QualifierArgument & request, const std::function<void(const UnitTestResult&)>& reply) override {
UnitTestResult result;
switch (request.qualifier_case()) {
case QualifierArgument::kSimple:
result.set_str(
bts::api::format_qualifier("format_qualifier", request.simple()));
break;
case QualifierArgument::kComplex:
result.set_str(
bts::api::format_qualifier("format_qualifier", request.complex()));
break;
default:
result.set_str("Unknown qualifier test case");
break;
}
reply(result);
}
// Inherited via ApiTestServiceBase
virtual void test_empty_reply(const EmptyRequest&, const std::function<void(const UnitTestResult&)>& reply) override {
UnitTestResult result;
+1 -1
View File
@@ -44,7 +44,7 @@ void HelloServiceServer::yell_hello(const HelloRequest & request, const std::fun
_shouter.upper(sr, [=](Result<ShoutResponse> response) {
if (response.success()) {
HelloResponse hr;
hr.set_message(response.result()->msg());
hr.set_message(response.value()->msg());
int rmid = _id;
publish(hr.message(), &rmid);
reply(hr);
+2 -2
View File
@@ -162,8 +162,8 @@ CAPI void add_relay_symbol(md::RelayControl* prelay, BlitUserSymbol& symbol, uin
}
}
CAPI void set_relay_token(md::RelayControl* prelay, const char* token) {
md::set_relay_token(prelay, std::string(token));
CAPI void set_relay_token(md::RelayControl* prelay, const char* token, size_t token_size) {
md::set_relay_token(prelay, std::string(token,token_size));
}
+2 -1
View File
@@ -62,7 +62,8 @@ message MDRequest {
IMPLIED = 0x40;
}
string host = 2;
string token = 3;
// bytes token = 3;
reserved 3;
}
message ChannelUpdate {
+7 -7
View File
@@ -387,7 +387,7 @@ public:
};
struct MDRelayServer : bts::co2::MarketDataRelayBase {
struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase {
const QueryServerImpl& _db;
utils::Dispatcher& _threadpool;
std::shared_ptr<std::unordered_map<uint32_t,Entitlements>> _entitlements;
@@ -396,7 +396,7 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
#endif
MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx, metrics::MetricsContainer& stats) :
bts::co2::MarketDataRelayBase(ctx),
bts::co2::AuthenticatedMarketDataRelayBase(ctx),
_db(db), _threadpool(threadpool), _entitlements()
#ifndef _WIN32
, _time(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_response_time_ms", std::array<double,5>({ 5.0, 10.0, 20.0, 50.0, 100.0 }))),
@@ -412,6 +412,7 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
void update_entitlements(std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>& entitlements) {
_entitlements = entitlements;
}
static const Entitlements* get_entitlements(uint32_t user_id, std::shared_ptr<std::unordered_map<uint32_t,Entitlements>>& entitlements) {
if (entitlements) {
auto iter = entitlements->find(user_id);
@@ -422,7 +423,7 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
return nullptr;
}
virtual void requestmd(const MDRequest& request, const std::function<void(const MDReply&)>& reply) override {
virtual void requestmd(const api::AuthToken& token, const MDRequest& request, const std::function<void(const MDReply&)>& reply) override {
TimeStamp start = TimeStamp::now();
_threadpool.post([=]() {
google::protobuf::Arena arena;
@@ -432,13 +433,12 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
auto ents = _entitlements;
const Entitlements* user_entitlement = nullptr;
bool bypass_entitlement = false;
if (api::auth::decode_token(request.token(), &payload)) {
if (api::auth::decode_token(token, &payload)) {
bypass_entitlement = payload.wheel();
if (!bypass_entitlement) {
user_entitlement = get_entitlements(payload.id(), ents);
}
}
// Temporarily bypass all entitlements
_db.query(request, preply, user_entitlement, bypass_entitlement);
reply(*preply);
#ifndef _WIN32
@@ -449,7 +449,7 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
});
}
virtual void request_products(const api::VoidMessage&, const std::function<void(const ProductListReply&)>& reply) override {
virtual void request_products(const api::AuthToken&, const api::VoidMessage&, const std::function<void(const ProductListReply&)>& reply) override {
ProductListReply list;
std::set<UserSymbol> syms;
get_system_md_symbols(syms);
@@ -463,7 +463,7 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
reply(list);
}
virtual void request_names(const api::VoidMessage&, const std::function<void(const ExpirationNameListReply&)>& reply) override {
virtual void request_names(const api::AuthToken&, const api::VoidMessage&, const std::function<void(const ExpirationNameListReply&)>& reply) override {
_threadpool.post([=]() {
google::protobuf::Arena arena;
ExpirationNameListReply* preply = google::protobuf::Arena::CreateMessage<ExpirationNameListReply>(&arena);
+14 -13
View File
@@ -219,14 +219,9 @@ public:
}
}
void set_token(const std::string& token) {
_request.set_token(token);
}
std::string compact_log_format() const {
ostringstream log;
log << "{";
log << std::endl << "token: " << _request.token();
for (const auto& ch : _request.channels()) {
log << std::endl;
log << ch.symbol() << " ";
@@ -259,7 +254,7 @@ public:
if (result.success()) {
// Write data to shared mem
bool has_new_security = false;
_writer.write(*result.result(), &has_new_security);
_writer.write(*result.value(), &has_new_security);
// If the reply included and we have a consumer have it scan them
if (has_new_security && _pconsumer) {
scan_securities_thread().post([=]() {
@@ -269,7 +264,7 @@ public:
}
// Update reply timetamps
for (auto& req : *_request.mutable_channels()) {
for (auto& reply : result.result()->channels()) {
for (auto& reply : result.value()->channels()) {
if (req.type() == reply.channel().type() &&
req.exchange() == reply.channel().exchange() &&
req.symbol() == reply.channel().symbol()) {
@@ -315,11 +310,15 @@ struct RelayScheduler {
}
};
api::ServiceContext elevated_relay_context() {
api::ServiceContext ctx;
ctx.login(api::auth::generate_wheel_token());
return ctx;
}
void relay_selected(api::ServiceContext& ctx, const std::set<UserSymbol>& symbols, IRelayWriter& writer) {
auto relay = ctx.locate<MarketDataRelay>();
RelayClient client(relay, nullptr, symbols, writer);
client.set_token(api::auth::generate_wheel_token().token());
RelayScheduler scheduler(ctx, config::get<RelayOptions>().naggle_ms(), client);
ctx.post([&]() { scheduler.run(); });
@@ -328,7 +327,7 @@ void relay_selected(api::ServiceContext& ctx, const std::set<UserSymbol>& symbol
}
void relay_everything() {
api::ServiceContext ctx;
api::ServiceContext ctx = elevated_relay_context();
std::set<UserSymbol> system_symbols;
get_system_md_symbols(system_symbols);
RelayProducer producer(true);
@@ -336,7 +335,8 @@ void relay_everything() {
}
void local_relay(IRelayWriter& writer) {
api::ServiceContext ctx;
api::ServiceContext ctx = elevated_relay_context();
ctx.login(api::auth::generate_wheel_token());
auto relay = ctx.locate<MarketDataRelay>();
std::set<UserSymbol> symbols;
{
@@ -378,14 +378,13 @@ void record_everything(std::string path, std::string filename) {
auto naggle_ms = config::get<RelayOptions>().naggle_ms();
RelayRecorder recorder(path, filename, naggle_ms);
api::ServiceContext ctx;
api::ServiceContext ctx = elevated_relay_context();
auto relay = ctx.locate<MarketDataRelay>();
std::set<UserSymbol> system_symbols;
get_system_md_symbols(system_symbols);
RelayClient client(relay, nullptr, system_symbols, recorder);
client.set_token(api::auth::generate_wheel_token().token());
RelayScheduler scheduler(ctx, naggle_ms, client);
scheduler.run();
@@ -435,7 +434,9 @@ void add_relay_symbol(RelayControl* relay, const SecurityChannel& chan) {
void set_relay_token(RelayControl* relay, const std::string& token) {
bts_assert(relay);
relay->context.post([=]() {
relay->client.set_token(token);
api::AuthToken auth_token;
auth_token.set_token(token);
relay->context.login(auth_token);
});
}
@@ -19,6 +19,7 @@
#include "lib/api/services/OrderService.pb.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "lib/orderman/orders/Order.h"
#include <future>
namespace bts {
namespace orderman {
@@ -95,7 +95,7 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
req.set_ns("my_ns");
auto resp = cli.retrieve_all(req,5000);
REQUIRE(resp.success());
REQUIRE(resp.result()->result() == RetrieveResponse::NO_SUCH_NS);
REQUIRE(resp.value()->result() == RetrieveResponse::NO_SUCH_NS);
}
SECTION("store_works")
@@ -113,16 +113,16 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
auto resp = cli.store(req,5000);
REQUIRE(resp.success());
REQUIRE(resp.result()->result() == StoreResponse::OK);
REQUIRE(resp.value()->result() == StoreResponse::OK);
RetrieveAllRequest req2;
req2.set_ns("my_ns1");
auto resp2 = cli.retrieve_all(req2,5000);
REQUIRE(resp2.success());
REQUIRE(resp2.result()->result() == RetrieveResponse::OK);
REQUIRE(resp2.result()->record_size() == 2);
REQUIRE(resp2.result()->record(0).key() == "1");
REQUIRE(resp2.result()->record(0).value() == "a");
REQUIRE(resp2.value()->result() == RetrieveResponse::OK);
REQUIRE(resp2.value()->record_size() == 2);
REQUIRE(resp2.value()->record(0).key() == "1");
REQUIRE(resp2.value()->record(0).value() == "a");
}
SECTION("retrieve_works")
@@ -144,7 +144,7 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
auto resp = cli.store(req,5000);
REQUIRE(resp.success());
REQUIRE(resp.result()->result() == StoreResponse::OK);
REQUIRE(resp.value()->result() == StoreResponse::OK);
RetrieveRequest req2;
req2.set_ns("my_ns1");
@@ -152,10 +152,10 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
auto resp2 = cli.retrieve(req2,5000);
REQUIRE(resp2.success());
REQUIRE(resp2.result()->result() == RetrieveResponse::OK);
REQUIRE(resp2.result()->record_size() == 1);
REQUIRE(resp2.result()->record(0).key() == "2");
REQUIRE(resp2.result()->record(0).value() == "b");
REQUIRE(resp2.value()->result() == RetrieveResponse::OK);
REQUIRE(resp2.value()->record_size() == 1);
REQUIRE(resp2.value()->record(0).key() == "2");
REQUIRE(resp2.value()->record(0).value() == "b");
}
SECTION("remove_works")
@@ -173,7 +173,7 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
auto resp = cli.store(req,5000);
REQUIRE(resp.success());
REQUIRE(resp.result()->result() == StoreResponse::OK);
REQUIRE(resp.value()->result() == StoreResponse::OK);
RemoveRequest req2;
req2.set_ns("my_ns1");
@@ -181,16 +181,16 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
auto resp2 = cli.remove(req2,5000);
REQUIRE(resp2.success());
REQUIRE(resp2.result()->result() == RemoveResponse::OK);
REQUIRE(resp2.value()->result() == RemoveResponse::OK);
RetrieveAllRequest req3;
req3.set_ns("my_ns1");
auto resp3 = cli.retrieve_all(req3,5000);
REQUIRE(resp3.success());
REQUIRE(resp3.result()->result() == RetrieveResponse::OK);
REQUIRE(resp3.result()->record_size() == 1);
REQUIRE(resp3.result()->record(0).key() == "1");
REQUIRE(resp3.result()->record(0).value() == "a");
REQUIRE(resp3.value()->result() == RetrieveResponse::OK);
REQUIRE(resp3.value()->record_size() == 1);
REQUIRE(resp3.value()->record(0).key() == "1");
REQUIRE(resp3.value()->record(0).value() == "a");
}
SECTION("remove_all_works")
@@ -203,7 +203,7 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
auto resp = cli.store(req,5000);
REQUIRE(resp.success());
REQUIRE(resp.result()->result() == StoreResponse::OK);
REQUIRE(resp.value()->result() == StoreResponse::OK);
req.set_ns("my_ns2");
rec = req.add_record();
@@ -212,21 +212,21 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
resp = cli.store(req,5000);
REQUIRE(resp.success());
REQUIRE(resp.result()->result() == StoreResponse::OK);
REQUIRE(resp.value()->result() == StoreResponse::OK);
RemoveAllRequest req2;
req2.set_ns("my_ns1");
auto resp2 = cli.remove_all(req2,5000);
REQUIRE(resp2.success());
REQUIRE(resp2.result()->result() == RemoveResponse::OK);
REQUIRE(resp2.value()->result() == RemoveResponse::OK);
RetrieveAllRequest req3;
req3.set_ns("my_ns1");
auto resp3 = cli.retrieve_all(req3,5000);
REQUIRE(resp3.success());
REQUIRE(resp3.result()->result() == RetrieveResponse::OK);
REQUIRE(resp3.result()->record_size() == 0);
REQUIRE(resp3.value()->result() == RetrieveResponse::OK);
REQUIRE(resp3.value()->record_size() == 0);
}
SECTION("streaming_works")
@@ -239,7 +239,7 @@ TEST_CASE ("PersistenceService Service API", "[persistence]")
auto resp = cli.store(req,5000);
REQUIRE(resp.success());
REQUIRE(resp.result()->result() == StoreResponse::OK);
REQUIRE(resp.value()->result() == StoreResponse::OK);
std::promise<void> p;
auto f = p.get_future();
+67 -18
View File
@@ -1,50 +1,99 @@
#include "AuthToken.h"
#ifndef _WIN32
#include "lib/utils/base64.hpp"
#include "lib/utils/bts_assert.h"
#include "datetime.h"
#include "base64.hpp"
// cryptopp
#include <cryptopp/osrng.h>//temp
#include <cryptopp/cryptlib.h>
#include <cryptopp/hex.h>
#include <cryptopp/filters.h>
#include <cryptopp/twofish.h>
#include <cryptopp/eax.h>
#include <cryptopp/secblock.h>
namespace bts {
namespace api {
namespace auth {
#ifndef _WIN32
user::AuthToken generate_token(const admin::BTSUser& user) {
using namespace CryptoPP;
// chosen by fair dice roll.
// guaranteed to be random.
constexpr byte raw_key[AES::DEFAULT_KEYLENGTH] = { 75, 22, 52, 77, 93, 194, 234, 176, 155, 181, 215, 7, 185, 211, 20, 43 };
const SecByteBlock bts_key(raw_key, AES::DEFAULT_KEYLENGTH);
constexpr byte raw_ivector[16] = { 182, 169, 236, 110, 146, 158, 187, 4, 5, 160, 205, 242, 174, 92, 127, 175 };
const SecByteBlock bts_iv(raw_ivector, sizeof(raw_ivector));
std::string encrypt(const std::string& buffer) {
CFB_Mode<AES>::Encryption enc(bts_key, bts_key.size(), bts_iv);
std::string result;
result.resize(buffer.size());
enc.ProcessData((byte*)result.c_str(), (byte*)buffer.c_str(), buffer.size());
return result;
}
std::string decrypt(const std::string& buffer) {
CFB_Mode<AES>::Decryption dec(bts_key, bts_key.size(), bts_iv);
std::string result;
result.resize(buffer.size());
dec.ProcessData((byte*)result.c_str(), (byte*)buffer.c_str(), buffer.size());
return result;
}
AuthToken generate_token(const admin::BTSUser& user) {
api::user::TokenPayload payload;
payload.set_date(datetime::ll_now());
payload.set_id(user.id());
if(user.name() == "bts")
if(user.name() == "bts") {
payload.set_wheel(true);
auto token = websocketpp::base64_encode(payload.SerializeAsString());
user::AuthToken result;
}
auto token = encrypt(payload.SerializeAsString());
bts_assert_d(token.size() < 255-8);
AuthToken result;
result.set_token(token);
return result;
}
user::AuthToken generate_wheel_token() {
AuthToken generate_wheel_token() {
api::user::TokenPayload payload;
payload.set_date(datetime::ll_now());
payload.set_id(99999);
payload.set_wheel(true);
auto token = websocketpp::base64_encode(payload.SerializeAsString());
user::AuthToken result;
auto token = encrypt(payload.SerializeAsString());
AuthToken result;
result.set_token(token);
return result;
}
#else
user::AuthToken generate_wheel_token() {
return user::AuthToken();
}
#endif
bool decode_token(const std::string& token, user::TokenPayload* payload) {
std::string temp = websocketpp::base64_decode(token);
bool decode_token(const api::AuthToken& token, user::TokenPayload* payload) {
if (token.token().empty()) return false;
std::string temp = decrypt(token.token());
if (payload) {
return payload->ParseFromString(temp);
return payload->ParseFromString(temp) && !is_expired(*payload);
}
else {
// validate the token but don't return it
user::TokenPayload payload;
return payload.ParseFromString(temp);
return payload.ParseFromString(temp) && !is_expired(payload);
}
}
bool is_expired(const user::TokenPayload&) {
return false;
}
#else
namespace bts {
namespace api {
namespace auth {
AuthToken generate_wheel_token() {
return AuthToken();
}
bool decode_token(const api::AuthToken& token, user::TokenPayload* payload) {
return false;
}
#endif
}}}
+5 -4
View File
@@ -1,5 +1,5 @@
#pragma once
#include "lib/api/services/UserService.pb.h"
#include "lib/api/services/AuthService.pb.h"
#include "lib/api/services/AdminService.client.pb.h"
#include <string>
@@ -7,9 +7,10 @@ namespace bts {
namespace api {
namespace auth {
user::AuthToken generate_token(const admin::BTSUser&);
user::AuthToken generate_wheel_token();
bool decode_token(const std::string& token, user::TokenPayload* payload);
AuthToken generate_token(const admin::BTSUser&);
AuthToken generate_wheel_token();
bool decode_token(const api::AuthToken& token, user::TokenPayload* payload);
bool is_expired(const user::TokenPayload&);
}}}
+41
View File
@@ -0,0 +1,41 @@
#ifndef _WIN32
#include "../AuthToken.h"
#include "catch/catch.hpp"
namespace bts {
namespace test {
TEST_CASE("AuthToken", "[security]") {
api::admin::BTSUser user;
user.set_id(42);
user.set_name("user1");
SECTION("round trip") {
auto token = api::auth::generate_token(user);
// We use 1 byte for api header size
REQUIRE(token.ByteSize() <= 240);
api::user::TokenPayload payload;
REQUIRE(api::auth::decode_token(token, &payload));
REQUIRE(payload.id() == user.id());
REQUIRE(payload.wheel() == false);
}
SECTION("empty token") {
api::user::TokenPayload payload;
api::AuthToken empty;
REQUIRE(!api::auth::decode_token(empty, &payload));
REQUIRE(payload.id() == 0);
REQUIRE(!payload.wheel());
}
SECTION("bts is the big wheel") {
user.set_name("bts");
auto token = api::auth::generate_token(user);
api::user::TokenPayload payload;
REQUIRE(api::auth::decode_token(token, &payload));
REQUIRE(payload.id() == user.id());
REQUIRE(payload.wheel());
}
}
}}//ns
#endif
+1 -1
View File
@@ -52,7 +52,7 @@ int AdminReport::md_report()
auto res = admin.get_admin_data(api::VoidMessage());
if(res.success())
{
const api::admin::AdminData& data = *res.result();
const api::admin::AdminData& data = *res.value();
std::cout << "User,Fullname,Company,Exchange,MIC,Display,Pro/NonPro" << std::endl;
for(const auto& u : data.users())
{
Loaded 100 of 115 files, more files were not shown because too many files have changed in this diff. Show more