refactor: Use dependency injection in DSQA

This commit is contained in:
John Colvin committed 2025-11-14 14:52:07 -05:00
1 parent ada7d4bed1
commit 4877355fee
6 files changed
+271 -85

No files matched your search

+10 -4
View File
@@ -140,12 +140,9 @@
"program": "",
"windows": {
"program": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
],
"cwd": "build/Debug/bin/net8.0-windows",
"env": {
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
},
},
@@ -345,6 +342,15 @@
],
"default": "PubSubServer",
},
{
"id": "dataStreamQueryAnalyzerEnvironment",
"type": "pickString",
"description": "DataStreamQueryAnalyzer Configuration",
"options": [
"Development",
],
"default": "Development",
},
{
"id": "sparkConfigServer",
"type": "promptString",
@@ -14,6 +14,7 @@
<ItemGroup>
<ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
<ProjectReference Include="..\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj" />
<ProjectReference Include="..\PubSub\PubSub.csproj" />
<ProjectReference Include="..\DataStream\DataStream.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
@@ -25,4 +26,9 @@
<ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
</ItemGroup>
<ItemGroup>
<ConfigurationProfile Include="appsettings.Development.json" />
<ConfigurationProfile Include="appsettings.json" />
</ItemGroup>
</Project>
@@ -17,12 +17,19 @@ using System.Diagnostics;
using BCG.Utilities;
// using BCG.Utilities.BlueConfiguration;
using BCG.Utilities.General;
using Spark.Core.Disposables;
using Spark.DataStream;
using Bts.DataStream;
using Bts.DataStreamExpressions;
// using BCG.Evaluator.DataStreamExpression;
// using BCG.Utilities.Win32;
using System.Threading.Tasks;
using mbts.utils;
using Bts.DataStreamCore;
using System.Linq;
using System.Text.Json.Nodes;
using System.Threading;
using System.Reactive.Linq;
namespace BCG
{
@@ -40,12 +47,13 @@ namespace BCG
private bool myChangeStyle = false;
private PersistWindowState myWindowState;
private string? myLastFileName;
private ViewStream? myCurrentStream;
IDisposable? currStreamSub;
private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
// private string mySite = SitesCore.getMySiteName ();
private string mySite = "TODO";
private string myLastClassName = "MyView";
private ViewStream? myAllTagsStream;
IBlueRowCollection? allTagsRows = null;
IDisposable? allTagsSub;
private string myKQ;
private TextWriter? myConsoleOutput;
readonly PrimitiveManager primitiveManager;
@@ -162,7 +170,7 @@ namespace BCG
private void myHandleTableListChange(object o)
{
IBlueRowCollection rows = (o as DataPacket).newRows;
allTagsRows = (o as DataPacket)?.newRows;
myUpdateTableList();
}
@@ -176,6 +184,11 @@ namespace BCG
#endregion
private Stopwatch _stopwatch = new Stopwatch();
object locker = new();
(IDataPacket DataPacket, long Version)? pendingData = null;
long pendingVersion = 0;
bool isBusy = false;
private void myTryExpression(string expStr)
{
_stopwatch.Restart();
@@ -193,14 +206,41 @@ namespace BCG
myErrorTextBox.Text = "";
// myDataGrid.CaptionText = "";
if (myCurrentStream != null)
myCurrentStream.onChange -= myCallback;
myCurrentStream = null;
Disposable.dispose(ref currStreamSub);
myExp = exp;
myCrono.Restart();
(myCurrentStream = myExp.getView(new Dictionary<string, StatementExpression>(0), streamFactory)).onChange += myCallback;
// object locker = new();
// IDataPacket? pendingDataPacket = null;
// bool isBusy = false;
var version = Interlocked.Increment(ref pendingVersion);
currStreamSub = myExp.getView(new Dictionary<string, StatementExpression>(0), streamFactory)
?.AsObservable()
.Sample(TimeSpan.FromMilliseconds(333))
.Subscribe(dp => {
lock (locker) {
pendingData = (dp, version);
if (isBusy) {
return;
}
isBusy = true;
}
// BeginInvoke(() => {
// while (true) {
// IDataPacket? currDp = null;
// lock (locker) {
// currDp = pendingDataPacket;
// pendingDataPacket = null;
// if (currDp is null) {
// isBusy = false;
// break;
// }
// }
// myCallback(currDp);
// }
// });
});
myChangeStyle = true;
//new SimpleBindableStream(myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback);
/*
@@ -350,20 +390,12 @@ namespace BCG
}
// this is the stream callback
private void myCallback(object o)
{
private void myCallback(IDataPacket dp, bool resetGrid) {
_stopwatch.Stop();
var timer = Stopwatch.StartNew();
DataPacket dp = o as DataPacket;
myCrono.Stop();
Log.Ui.diag($"Received callback for {dp.schema.tableName}");
if (o == null)
{
System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
"Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
Log.Ui.diag(dp.toIncrementalIDString());
// Log.Ui.diag($"[dsqa_upd.json]{{ \"table\": \"{dp.Schema.tableName}\", \"added\": {string.Join(", ", dp.addedRows.Cast<IBlueRow>().Select(r => r.Id().ToString()))} {dp.GetIncrementalIdString()} }}");
Log.Ui.diag($"[dsqa_upd.json]{dp.GetIncrementalIdString()}");
myToolStripStatusLabelTime.BackColor = Color.Black;
myToolStripStatusLabelRows.BackColor = Color.Black;
myToolStripStatusLabelTime.ForeColor = Color.White;
@@ -374,13 +406,17 @@ namespace BCG
Log.Ui.diag(Text);
int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
// if (myDataGrid.DataSource is not SimpleBindableCollection currDataSource || resetGrid) {
// currDataSource = new SimpleBindableCollection(dp.newRows.schema);
// myDataGrid.DataSource = currDataSource;
// }
// currDataSource.Clear();
myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
var force = false;
if (myChangeStyle && force)
{
if (myChangeStyle && force) {
myChangeStyle = false;
myDataGrid.Columns.Clear();
myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
myDataGrid.Columns.AddRange(Array.ConvertAll(dp.Schema.columnInfo, _column_factory));
}
myCrono.Restart();
// myDataGrid.CaptionText = myCrono.time.TotalMilliseconds + " ms ";
@@ -390,26 +426,38 @@ namespace BCG
myDataGrid.HorizScrollBarValue = hscrollval;
var varNames = new List<string>(0);
var code = myExp?.print(9, varNames);
myPrintCode(dp.schema, varNames, code);
var code = myExp?.print(9, varNames) ?? string.Empty;
myPrintCode(dp.Schema, varNames, code);
myConsoleRichTextBox.Clear();
// this was was very very slow
//myConsoleRichTextBox.Text = BlueDataUtility.get_printed_rows(dp.newRows, (col) => 30);
timer.Stop();
Log.Gen.info(
$"DSQA MainForm.myCallback table: {dp.schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms");
$"DSQA MainForm.myCallback table: {dp.Schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms");
}
private Stopwatch myCrono = new();
private DataStreamExpression? myExp;
private void myTimer_Tick(object sender, System.EventArgs e)
{
long lastProcessedVersion = -2;
private void myTimer_Tick(object sender, System.EventArgs e) {
IDataPacket? dp = null;
bool reset = false;
lock (locker) {
if (!pendingData.HasValue) {
return;
}
dp = pendingData.Value.DataPacket;
reset = pendingData.Value.Version != lastProcessedVersion;
lastProcessedVersion = pendingData.Value.Version;
pendingData = null;
}
myCallback(dp, reset);
// try
// {
ViewStreamDatabase.instance.doWork();
MasterWorker.getInstance(false).doWork();
// ViewStreamDatabase.instance.doWork();
// MasterWorker.getInstance(false).doWork();
//}
// catch (Exception exc) {
// Log.Ui.warn(exc.ToString());
@@ -437,8 +485,8 @@ namespace BCG
private void MainForm_Closed(object sender, System.EventArgs e)
{
if (myExp != null)
myExp.getView(new Dictionary<string, StatementExpression>(0), primitiveManager).onChange -= myCallback;
Disposable.dispose(ref currStreamSub);
Disposable.dispose(ref allTagsSub);
}
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
@@ -595,8 +643,9 @@ namespace BCG
myLoadAssemblies();
myAllTagsStream = myGetAllTagsStream();
myAllTagsStream.onChange += myHandleTableListChange;
Disposable.dispose(ref allTagsSub);
allTagsSub = myGetAllTagsStream().AsObservable()
.Subscribe(obj => BeginInvoke(() => myHandleTableListChange(obj)));
myTabControl.SelectedIndex = 3;
myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
//do this lazily so we don't have to have bps, etc up to run dsqa
@@ -614,13 +663,13 @@ namespace BCG
m.Checked = false;
mi.Checked = true;
myAllTagsStream.onChange -= myHandleTableListChange;
Disposable.dispose(ref allTagsSub);
myTableListBox.Items.Clear();
// SitesCore.setMySiteName (mySite);
myAllTagsStream = myGetAllTagsStream();
myAllTagsStream.onChange += myHandleTableListChange;
allTagsSub = myGetAllTagsStream().AsObservable()
.Subscribe(obj => BeginInvoke(() => myHandleTableListChange(obj)));
myDataGrid.DataSource = null;
myTabControl.SelectedIndex = 3;
@@ -680,11 +729,11 @@ namespace BCG
private void myToolStripMenuItemTest_Click(object sender, EventArgs e)
{
if (myCurrentStream != null)
{
myCurrentStream.onChange -= myCallback;
myCurrentStream.onChange += myCallback;
}
// if (myCurrentStream != null)
// {
// myCurrentStream.onChange -= myCallback;
// myCurrentStream.onChange += myCallback;
// }
}
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e)
@@ -700,7 +749,7 @@ namespace BCG
private void myUpdateTableList()
{
IBlueRowCollection rows = myAllTagsStream.rows;
IBlueRowCollection? rows = allTagsRows;
if (rows == null)
return;
@@ -1,54 +1,66 @@
using BCG;
using bts;
using bts.api.core;
using Bts.DataStream.Services;
using Bts.DataStreamCore;
using Bts.DataStreamQueryAnalyzer;
using Bts.PubSub;
using Bts.DataStreamCore.Services;
using Bts.Extensions.Hosting;
using Bts.PubSub.Services;
using Bts.SharedStreams.Services;
using CommandLine;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using mbts.config;
using Spark.Api.Services;
using Spark.Core.Services;
using System;
using System.Linq;
using System.Threading.Tasks;
using System.Windows.Forms;
static class Program {
[STAThread]
static async Task Main(string[] args) {
await Parser.Default.ParseArguments<DataStreamQueryAnalyzerConfig>(args)
.MapResult(RunAsync, errs => Task.FromResult(1));
async Task<int> RunAsync(DataStreamQueryAnalyzerConfig cfg) {
try {
cfg.MergeWithConfigFile();
cfg.MergeWith(DataStreamQueryAnalyzerConfig.Default());
PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
await Configuration.InitializeAsync(cfg.Sparkd);
Log.Gen.info("Starting DataStreamQueryAnalyzer");
var primitiveManager = new PrimitiveManager(
new PrimitiveManagerConfig() {
Assemblies = new() {
"DataStream.dll",
"SharedStreams.dll"
},
});
var mainForm = new MainForm(primitiveManager);
if (cfg.StreamFile is not null) {
mainForm.loadFile(cfg.StreamFile);
}
Application.Run(mainForm);
} catch (Exception ex) {
// Write the error to console first, in case the logging system is compromised.
var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
Console.Error.WriteLine(error_msg);
Console.Error.Flush();
bts.Log.Gen.error(error_msg);
return 1;
}
return 0;
static (IServiceProvider Services, Task RunAsync) init(string[] args) {
var tcs = new TaskCompletionSource();
var host = HostBuilderExtensions.CreateBtsBuilder(args)
.ConfigureServices(ConfigureServices)
.Build();
RunQuiet();
return (host.Services, tcs.Task);
async void RunQuiet() {
await Task.Delay(100);
await host.RunAsync();
tcs.SetResult();
}
}
[STAThread]
static async Task<int> Main(string[] args) {
Console.Write("Starting DataStreamQueryAnalyzer");
try {
var (services, runAsync) = init(args);
var streamFactoryList = services.GetServices<IDataStreamFactory>().ToList();
var streamFactory = streamFactoryList.Count == 1
? streamFactoryList[0]
: new CompositeDataStreamFactory(streamFactoryList);
var primitiveManager = streamFactoryList.OfType<PrimitiveManager>().Single();
var win = new MainForm(primitiveManager, streamFactory);
Application.Run(win);
await runAsync;
} catch (Exception ex) {
Console.Error.Write($"DataStreamQueryAnalyzer crashed: {ex}");
return 1;
}
return 0;
}
static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
services.AddSpark(context.Configuration.GetSection("Spark"))
.AddSparkApi(context.Configuration.GetSection("SparkApi"))
.AddMasterWorker(context.Configuration.GetSection("MasterWorker"))
.AddViewStreamDatabase()
.AddPubSubClient(context.Configuration.GetSection("PubSub"))
.AddPubTableBroker()
.AddRemoteQueryManager()
.AddPrimitiveStreams(context.Configuration.GetSection("PrimitiveStream"))
.AddMarketDataStreams();
}
@@ -0,0 +1,27 @@
{
"SparkApi": {
"IsEnabled": true,
"Credentials": {
"User": "bts",
"Password": ""
}
},
"PubSub": {
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:22019",
"TagRegExp": ".*"
}
]
}
}
}
@@ -0,0 +1,86 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"Spark": {},
"SparkApi": {
"IsEnabled": false
},
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"PrimitiveStream": {
"Assemblies": [
"DataStream.dll",
"SharedStreams.dll"
],
"ExcludedTypes": [
"SettlementPriceStream",
"OHLCVStream"
]
},
"PubSub": {
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"DisposedConnectionCheckPeriod": 1000,
"ServerPort": 22019,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
]
},
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
},
"Database": {
"Path": "",
"Query": ""
}
}