diff --git a/.vscode/launch.json b/.vscode/launch.json
index 552dad725..92f6ca8af 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -140,12 +140,9 @@
"program": "",
"windows": {
"program": "DataStreamQueryAnalyzer.exe",
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
- ],
"cwd": "build/Debug/bin/net8.0-windows",
"env": {
+ "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
},
},
@@ -345,6 +342,15 @@
],
"default": "PubSubServer",
},
+ {
+ "id": "dataStreamQueryAnalyzerEnvironment",
+ "type": "pickString",
+ "description": "DataStreamQueryAnalyzer Configuration",
+ "options": [
+ "Development",
+ ],
+ "default": "Development",
+ },
{
"id": "sparkConfigServer",
"type": "promptString",
diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj b/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj
index 44579304e..3ec8b893b 100644
--- a/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj
+++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj
@@ -14,6 +14,7 @@
+
@@ -25,4 +26,9 @@
+
+
+
+
+
diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs
index bf71a9960..bb96f3958 100644
--- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs
+++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs
@@ -17,12 +17,19 @@ using System.Diagnostics;
using BCG.Utilities;
// using BCG.Utilities.BlueConfiguration;
using BCG.Utilities.General;
+using Spark.Core.Disposables;
using Spark.DataStream;
+using Bts.DataStream;
using Bts.DataStreamExpressions;
// using BCG.Evaluator.DataStreamExpression;
// using BCG.Utilities.Win32;
using System.Threading.Tasks;
+using mbts.utils;
using Bts.DataStreamCore;
+using System.Linq;
+using System.Text.Json.Nodes;
+using System.Threading;
+using System.Reactive.Linq;
namespace BCG
{
@@ -40,12 +47,13 @@ namespace BCG
private bool myChangeStyle = false;
private PersistWindowState myWindowState;
private string? myLastFileName;
- private ViewStream? myCurrentStream;
+ IDisposable? currStreamSub;
private List mySiteItems = new List();
// private string mySite = SitesCore.getMySiteName ();
private string mySite = "TODO";
private string myLastClassName = "MyView";
- private ViewStream? myAllTagsStream;
+ IBlueRowCollection? allTagsRows = null;
+ IDisposable? allTagsSub;
private string myKQ;
private TextWriter? myConsoleOutput;
readonly PrimitiveManager primitiveManager;
@@ -162,7 +170,7 @@ namespace BCG
private void myHandleTableListChange(object o)
{
- IBlueRowCollection rows = (o as DataPacket).newRows;
+ allTagsRows = (o as DataPacket)?.newRows;
myUpdateTableList();
}
@@ -176,6 +184,11 @@ namespace BCG
#endregion
private Stopwatch _stopwatch = new Stopwatch();
+ object locker = new();
+ (IDataPacket DataPacket, long Version)? pendingData = null;
+ long pendingVersion = 0;
+ bool isBusy = false;
+
private void myTryExpression(string expStr)
{
_stopwatch.Restart();
@@ -193,14 +206,41 @@ namespace BCG
myErrorTextBox.Text = "";
// myDataGrid.CaptionText = "";
- if (myCurrentStream != null)
- myCurrentStream.onChange -= myCallback;
- myCurrentStream = null;
+ Disposable.dispose(ref currStreamSub);
myExp = exp;
myCrono.Restart();
- (myCurrentStream = myExp.getView(new Dictionary(0), streamFactory)).onChange += myCallback;
+ // object locker = new();
+ // IDataPacket? pendingDataPacket = null;
+ // bool isBusy = false;
+ var version = Interlocked.Increment(ref pendingVersion);
+ currStreamSub = myExp.getView(new Dictionary(0), streamFactory)
+ ?.AsObservable()
+ .Sample(TimeSpan.FromMilliseconds(333))
+ .Subscribe(dp => {
+ lock (locker) {
+ pendingData = (dp, version);
+ if (isBusy) {
+ return;
+ }
+ isBusy = true;
+ }
+ // BeginInvoke(() => {
+ // while (true) {
+ // IDataPacket? currDp = null;
+ // lock (locker) {
+ // currDp = pendingDataPacket;
+ // pendingDataPacket = null;
+ // if (currDp is null) {
+ // isBusy = false;
+ // break;
+ // }
+ // }
+ // myCallback(currDp);
+ // }
+ // });
+ });
myChangeStyle = true;
//new SimpleBindableStream(myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback);
/*
@@ -350,20 +390,12 @@ namespace BCG
}
// this is the stream callback
- private void myCallback(object o)
- {
+ private void myCallback(IDataPacket dp, bool resetGrid) {
_stopwatch.Stop();
var timer = Stopwatch.StartNew();
- DataPacket dp = o as DataPacket;
myCrono.Stop();
- Log.Ui.diag($"Received callback for {dp.schema.tableName}");
- if (o == null)
- {
- System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
- "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
- return;
- }
- Log.Ui.diag(dp.toIncrementalIDString());
+ // Log.Ui.diag($"[dsqa_upd.json]{{ \"table\": \"{dp.Schema.tableName}\", \"added\": {string.Join(", ", dp.addedRows.Cast().Select(r => r.Id().ToString()))} {dp.GetIncrementalIdString()} }}");
+ Log.Ui.diag($"[dsqa_upd.json]{dp.GetIncrementalIdString()}");
myToolStripStatusLabelTime.BackColor = Color.Black;
myToolStripStatusLabelRows.BackColor = Color.Black;
myToolStripStatusLabelTime.ForeColor = Color.White;
@@ -374,13 +406,17 @@ namespace BCG
Log.Ui.diag(Text);
int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
+ // if (myDataGrid.DataSource is not SimpleBindableCollection currDataSource || resetGrid) {
+ // currDataSource = new SimpleBindableCollection(dp.newRows.schema);
+ // myDataGrid.DataSource = currDataSource;
+ // }
+ // currDataSource.Clear();
myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
var force = false;
- if (myChangeStyle && force)
- {
+ if (myChangeStyle && force) {
myChangeStyle = false;
myDataGrid.Columns.Clear();
- myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
+ myDataGrid.Columns.AddRange(Array.ConvertAll(dp.Schema.columnInfo, _column_factory));
}
myCrono.Restart();
// myDataGrid.CaptionText = myCrono.time.TotalMilliseconds + " ms ";
@@ -390,26 +426,38 @@ namespace BCG
myDataGrid.HorizScrollBarValue = hscrollval;
var varNames = new List(0);
- var code = myExp?.print(9, varNames);
- myPrintCode(dp.schema, varNames, code);
+ var code = myExp?.print(9, varNames) ?? string.Empty;
+ myPrintCode(dp.Schema, varNames, code);
myConsoleRichTextBox.Clear();
// this was was very very slow
//myConsoleRichTextBox.Text = BlueDataUtility.get_printed_rows(dp.newRows, (col) => 30);
timer.Stop();
Log.Gen.info(
- $"DSQA MainForm.myCallback table: {dp.schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms");
+ $"DSQA MainForm.myCallback table: {dp.Schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms");
}
private Stopwatch myCrono = new();
private DataStreamExpression? myExp;
- private void myTimer_Tick(object sender, System.EventArgs e)
- {
+ long lastProcessedVersion = -2;
+ private void myTimer_Tick(object sender, System.EventArgs e) {
+ IDataPacket? dp = null;
+ bool reset = false;
+ lock (locker) {
+ if (!pendingData.HasValue) {
+ return;
+ }
+ dp = pendingData.Value.DataPacket;
+ reset = pendingData.Value.Version != lastProcessedVersion;
+ lastProcessedVersion = pendingData.Value.Version;
+ pendingData = null;
+ }
+ myCallback(dp, reset);
// try
// {
- ViewStreamDatabase.instance.doWork();
- MasterWorker.getInstance(false).doWork();
+ // ViewStreamDatabase.instance.doWork();
+ // MasterWorker.getInstance(false).doWork();
//}
// catch (Exception exc) {
// Log.Ui.warn(exc.ToString());
@@ -437,8 +485,8 @@ namespace BCG
private void MainForm_Closed(object sender, System.EventArgs e)
{
- if (myExp != null)
- myExp.getView(new Dictionary(0), primitiveManager).onChange -= myCallback;
+ Disposable.dispose(ref currStreamSub);
+ Disposable.dispose(ref allTagsSub);
}
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
@@ -595,8 +643,9 @@ namespace BCG
myLoadAssemblies();
- myAllTagsStream = myGetAllTagsStream();
- myAllTagsStream.onChange += myHandleTableListChange;
+ Disposable.dispose(ref allTagsSub);
+ allTagsSub = myGetAllTagsStream().AsObservable()
+ .Subscribe(obj => BeginInvoke(() => myHandleTableListChange(obj)));
myTabControl.SelectedIndex = 3;
myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
//do this lazily so we don't have to have bps, etc up to run dsqa
@@ -614,13 +663,13 @@ namespace BCG
m.Checked = false;
mi.Checked = true;
- myAllTagsStream.onChange -= myHandleTableListChange;
+ Disposable.dispose(ref allTagsSub);
myTableListBox.Items.Clear();
// SitesCore.setMySiteName (mySite);
- myAllTagsStream = myGetAllTagsStream();
- myAllTagsStream.onChange += myHandleTableListChange;
+ allTagsSub = myGetAllTagsStream().AsObservable()
+ .Subscribe(obj => BeginInvoke(() => myHandleTableListChange(obj)));
myDataGrid.DataSource = null;
myTabControl.SelectedIndex = 3;
@@ -680,11 +729,11 @@ namespace BCG
private void myToolStripMenuItemTest_Click(object sender, EventArgs e)
{
- if (myCurrentStream != null)
- {
- myCurrentStream.onChange -= myCallback;
- myCurrentStream.onChange += myCallback;
- }
+ // if (myCurrentStream != null)
+ // {
+ // myCurrentStream.onChange -= myCallback;
+ // myCurrentStream.onChange += myCallback;
+ // }
}
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e)
@@ -700,7 +749,7 @@ namespace BCG
private void myUpdateTableList()
{
- IBlueRowCollection rows = myAllTagsStream.rows;
+ IBlueRowCollection? rows = allTagsRows;
if (rows == null)
return;
diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs
index b9e205ffc..cdd7748d1 100644
--- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs
+++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs
@@ -1,54 +1,66 @@
using BCG;
-using bts;
-using bts.api.core;
-
+using Bts.DataStream.Services;
using Bts.DataStreamCore;
-using Bts.DataStreamQueryAnalyzer;
-using Bts.PubSub;
+using Bts.DataStreamCore.Services;
+using Bts.Extensions.Hosting;
+using Bts.PubSub.Services;
+using Bts.SharedStreams.Services;
-using CommandLine;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
-using mbts.config;
+using Spark.Api.Services;
+using Spark.Core.Services;
using System;
+using System.Linq;
using System.Threading.Tasks;
using System.Windows.Forms;
static class Program {
- [STAThread]
- static async Task Main(string[] args) {
- await Parser.Default.ParseArguments(args)
- .MapResult(RunAsync, errs => Task.FromResult(1));
-
- async Task RunAsync(DataStreamQueryAnalyzerConfig cfg) {
- try {
- cfg.MergeWithConfigFile();
- cfg.MergeWith(DataStreamQueryAnalyzerConfig.Default());
- PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
- await Configuration.InitializeAsync(cfg.Sparkd);
- Log.Gen.info("Starting DataStreamQueryAnalyzer");
- var primitiveManager = new PrimitiveManager(
- new PrimitiveManagerConfig() {
- Assemblies = new() {
- "DataStream.dll",
- "SharedStreams.dll"
- },
- });
- var mainForm = new MainForm(primitiveManager);
- if (cfg.StreamFile is not null) {
- mainForm.loadFile(cfg.StreamFile);
- }
- Application.Run(mainForm);
- } catch (Exception ex) {
- // Write the error to console first, in case the logging system is compromised.
- var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
- Console.Error.WriteLine(error_msg);
- Console.Error.Flush();
- bts.Log.Gen.error(error_msg);
- return 1;
- }
- return 0;
+ static (IServiceProvider Services, Task RunAsync) init(string[] args) {
+ var tcs = new TaskCompletionSource();
+ var host = HostBuilderExtensions.CreateBtsBuilder(args)
+ .ConfigureServices(ConfigureServices)
+ .Build();
+ RunQuiet();
+ return (host.Services, tcs.Task);
+ async void RunQuiet() {
+ await Task.Delay(100);
+ await host.RunAsync();
+ tcs.SetResult();
}
+ }
+
+ [STAThread]
+ static async Task Main(string[] args) {
+ Console.Write("Starting DataStreamQueryAnalyzer");
+ try {
+ var (services, runAsync) = init(args);
+ var streamFactoryList = services.GetServices().ToList();
+ var streamFactory = streamFactoryList.Count == 1
+ ? streamFactoryList[0]
+ : new CompositeDataStreamFactory(streamFactoryList);
+ var primitiveManager = streamFactoryList.OfType().Single();
+ var win = new MainForm(primitiveManager, streamFactory);
+ Application.Run(win);
+ await runAsync;
+ } catch (Exception ex) {
+ Console.Error.Write($"DataStreamQueryAnalyzer crashed: {ex}");
+ return 1;
+ }
+ return 0;
}
+
+ static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
+ services.AddSpark(context.Configuration.GetSection("Spark"))
+ .AddSparkApi(context.Configuration.GetSection("SparkApi"))
+ .AddMasterWorker(context.Configuration.GetSection("MasterWorker"))
+ .AddViewStreamDatabase()
+ .AddPubSubClient(context.Configuration.GetSection("PubSub"))
+ .AddPubTableBroker()
+ .AddRemoteQueryManager()
+ .AddPrimitiveStreams(context.Configuration.GetSection("PrimitiveStream"))
+ .AddMarketDataStreams();
}
diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.Development.json b/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.Development.json
new file mode 100644
index 000000000..19899ec21
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.Development.json
@@ -0,0 +1,27 @@
+{
+ "SparkApi": {
+ "IsEnabled": true,
+ "Credentials": {
+ "User": "bts",
+ "Password": ""
+ }
+ },
+ "PubSub": {
+ "Routing": {
+ "PublisherRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://127.0.0.1:22019",
+ "TagRegExp": ".*"
+ }
+ ],
+ "SubscriberRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://127.0.0.1:22019",
+ "TagRegExp": ".*"
+ }
+ ]
+ }
+ }
+}
diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json b/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json
new file mode 100644
index 000000000..065abc5d9
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json
@@ -0,0 +1,86 @@
+{
+ "Logging": {
+ "LogLevel": {
+ "Default": "Information",
+ "Microsoft.AspNetCore": "Warning"
+ }
+ },
+ "Spark": {},
+ "SparkApi": {
+ "IsEnabled": false
+ },
+ "MasterWorker": {
+ "Interval": "00:00:00.1"
+ },
+ "ViewStreamDatabase": {
+ "Interval": "00:00:00.1"
+ },
+ "PrimitiveStream": {
+ "Assemblies": [
+ "DataStream.dll",
+ "SharedStreams.dll"
+ ],
+ "ExcludedTypes": [
+ "SettlementPriceStream",
+ "OHLCVStream"
+ ]
+ },
+ "PubSub": {
+ "HeartbeatInterval": 15000,
+ "MissingHeartbeatsBeforeGc": 50,
+ "DisposedConnectionCheckPeriod": 1000,
+ "ServerPort": 22019,
+ "Routing": {
+ "PublisherRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://pubsub:22019",
+ "TagRegExp": ".*"
+ }
+ ],
+ "SubscriberRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://pubsub:22019",
+ "TagRegExp": ".*"
+ }
+ ]
+ },
+ "TagTypeToAssemblyMap": {
+ "Entries": [
+ {
+ "HandlerClass": "BCG.PubSubServerTagHandler",
+ "Assembly": "DataStreamCore.dll"
+ }
+ ]
+ },
+ "TagHandlerFactory": {
+ "Handlers": [
+ {
+ "TagType": "DataStream",
+ "Assembly": "DataStreamCore",
+ "Class": "bts.PubsubServerStreamTagHandler"
+ },
+ {
+ "TagType": "System_Sub",
+ "Assembly": "PubSub",
+ "Class": "BCG.SystemSubscriptionMapPublisher"
+ },
+ {
+ "TagType": "System_Stats",
+ "Assembly": "PubSub",
+ "Class": "BCG.SystemStatsPublisher"
+ },
+ {
+ "TagType": ".*",
+ "Assembly": "DataStreamCore",
+ "Class": "bts.PubsubServerGenericDataHandler"
+ }
+ ]
+ }
+ },
+ "Database": {
+ "Path": "",
+ "Query": ""
+ }
+}