TimeSeries and TimeBucketStreams, Stream offloading etc

This commit is contained in:
Taha committed 2023-10-23 14:18:31 -04:00
1 parent 33030b1212
commit 88bf04e5f1
31 files changed
+1823 -836

No files matched your search

@@ -25,17 +25,19 @@ using System.Threading.Tasks;
namespace BCG {
public partial class MainForm : System.Windows.Forms.Form {
static async Task _run() {
var login = "bts";
var passwd = String.Empty;
Console.WriteLine("Starting DSQA... ");
// Log.notnull(login);
// Log.notnull(passwd);
var service_ctx = bts.api.core.ServiceContext.create();
await service_ctx.login(login, passwd);
}
namespace BCG
{
public partial class MainForm : System.Windows.Forms.Form
{
// static async Task _run() {
// var login = "bts";
// var passwd = String.Empty;
// Console.WriteLine("Starting DSQA... ");
// // Log.notnull(login);
// // Log.notnull(passwd);
// var service_ctx = bts.api.core.ServiceContext.create();
// await service_ctx.login(login, passwd);
// }
private bool myChangeStyle = false;
private PubSubRoute[] myRoutes = PubSub3Config.instance.getSubRouting();
private PersistWindowState myWindowState;
@@ -50,8 +52,9 @@ namespace BCG {
private string myKQ;
private TextWriter myConsoleOutput;
public MainForm() {
Task.Run(async () => await _run());
public MainForm()
{
//Task.Run(async () => await _run());
InitializeComponent();
// myKQ = ConfigurationRequester.getApplicationQualifier();
myKQ = "TODO";
@@ -65,17 +68,20 @@ namespace BCG {
this.Closing += new CancelEventHandler(myClosing);
}
public string query_text {
public string query_text
{
get { return myQueryTextBox.Text; }
set { myQueryTextBox.Text = value; }
}
private ViewStream myGetAllTagsStream() {
private ViewStream myGetAllTagsStream()
{
// TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
return new FilterStream(new PubSubServerStateStream(), "tag~='DS_'");
//return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
}
private void mySetText() {
private void mySetText()
{
this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
myToolStripStatusLabelTime.Text = "Ready";
myToolStripStatusLabelRows.Text = string.Empty;
@@ -85,10 +91,12 @@ namespace BCG {
/// Gets the data in the form of a multilevel IDictionary
/// </summary>
/// <returns></returns>
private IDictionary myGetLibraryData() {
private IDictionary myGetLibraryData()
{
SortedList retVal = new SortedList();
string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
foreach (string pName in primitiveNames) {
foreach (string pName in primitiveNames)
{
string aName = PrimitiveManager.instance.getAssemblyName(pName);
ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
if (!retVal.Contains(aName))
@@ -103,12 +111,15 @@ namespace BCG {
}
private void myLoadAssemblies() {
private void myLoadAssemblies()
{
myTreeView.Nodes.Clear();
IDictionary libraryData = myGetLibraryData();
foreach (DictionaryEntry de in libraryData) {
foreach (DictionaryEntry de in libraryData)
{
TreeNode assemblyNode = new TreeNode(de.Key as String);
foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) {
foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
{
TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
foreach (string con in (innerDe.Value as ICollection))
primitiveNode.Nodes.Add(con);
@@ -128,23 +139,27 @@ namespace BCG {
}*/
}
private void myHandleTableListChange(object o) {
private void myHandleTableListChange(object o)
{
IBlueRowCollection rows = (o as DataPacket).newRows;
myUpdateTableList();
}
#region Persistence Keys
public int textBoxHeight {
public int textBoxHeight
{
get { return myQueryTextBox.Height; }
set { myQueryTextBox.Height = value; }
}
#endregion
private void myTryExpression(string expStr) {
private void myTryExpression(string expStr)
{
myFreezeView(false); // force unfreeze the view
DataStreamExpression oldExp = myExp;
try {
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(expStr);
if (exp == null)
@@ -171,7 +186,9 @@ namespace BCG {
oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback);
*/
//myQueryTextBox.Text = expStr;
} catch (Exception exc) {
}
catch (Exception exc)
{
Log.Ui.warn(exc.ToString());
myDataGrid.DataSource = null;
@@ -185,7 +202,8 @@ namespace BCG {
}
}
private string myExtractFromClass(TextReader r) {
private string myExtractFromClass(TextReader r)
{
string line = r.ReadLine();
while (line != null && line.IndexOf("*Expression*") == -1)
line = r.ReadLine();
@@ -198,7 +216,8 @@ namespace BCG {
string result = "";
line = r.ReadLine();
while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) {
while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1)
{
if (line != "")
result += line.Substring(" ".Length) + "\n";
line = r.ReadLine();
@@ -209,17 +228,24 @@ namespace BCG {
return result;
}
public void loadFile(string fileName) {
try {
public void loadFile(string fileName)
{
try
{
TextReader r = new StreamReader(fileName);
try {
try
{
myQueryTextBox.Text = myExtractFromClass(r);
} finally {
}
finally
{
r.Close();
}
myLastFileName = fileName;
this.Text = fileName;
} catch (Exception exc) {
}
catch (Exception exc)
{
Log.Ui.warn(exc.ToString());
myDataGrid.DataSource = null;
@@ -230,7 +256,8 @@ namespace BCG {
}
private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) {
private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code)
{
myCSharpText.Text = "using System;\n\nnamespace BCG\n{";
@@ -250,7 +277,8 @@ namespace BCG {
string lastVar = null;
string variableDeclarations = "";
foreach (string var in vars) {
foreach (string var in vars)
{
variableDeclarations += " ViewStream " + var + ";\n";
lastVar = var;
}
@@ -280,13 +308,16 @@ namespace BCG {
myCSharpText.Text += "}";
}
static DataGridViewColumn _column_factory(ColumnInfo ci) {
return new DataGridViewTextBoxColumn {
static DataGridViewColumn _column_factory(ColumnInfo ci)
{
return new DataGridViewTextBoxColumn
{
HeaderText = ci.colName,
Width = get_col_width(ci)
};
static int get_col_width(ColumnInfo ci) {
static int get_col_width(ColumnInfo ci)
{
const int CHAR_WIDTH = 10;
const int MIN_WIDTH = 50;
const int MIN_DATE_WIDTH = 200;
@@ -297,12 +328,14 @@ namespace BCG {
}
private HighPerformanceTimer _timer = new HighPerformanceTimer();
// this is the stream callback
private void myCallback(object o) {
private void myCallback(object o)
{
_timer.start();
DataPacket dp = o as DataPacket;
myCrono.stop();
Log.Ui.diag($"Received callback for {dp.schema.tableName}");
if (o == null) {
if (o == null)
{
System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
"Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
@@ -319,7 +352,8 @@ namespace BCG {
myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
var force = false;
if (myChangeStyle && force) {
if (myChangeStyle && force)
{
myChangeStyle = false;
myDataGrid.Columns.Clear();
myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
@@ -346,11 +380,15 @@ namespace BCG {
private DataStreamExpression myExp;
private void myTimer_Tick(object sender, System.EventArgs e) {
try {
private void myTimer_Tick(object sender, System.EventArgs e)
{
try
{
ViewStreamDatabase.instance.doWork();
MasterWorker.getInstance(false).doWork();
} catch (Exception exc) {
}
catch (Exception exc)
{
Log.Ui.warn(exc.ToString());
myDataGrid.DataSource = null;
@@ -361,22 +399,27 @@ namespace BCG {
}
}
class StringWrapper {
public StringWrapper(string s) {
class StringWrapper
{
public StringWrapper(string s)
{
myTableName = s;
}
public string tableName {
public string tableName
{
get { return myTableName; }
}
private string myTableName;
}
private void MainForm_Closed(object sender, System.EventArgs e) {
private void MainForm_Closed(object sender, System.EventArgs e)
{
if (myExp != null)
myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= new DataStream.ChangeDelegate(myCallback);
}
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
{
myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionColor = Color.Black;
@@ -384,22 +427,24 @@ namespace BCG {
myTryExpression(myQueryTextBox.Text);
}
private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) {
int selstart = myQueryTextBox.SelectionStart;
int sellength = myQueryTextBox.SelectionLength;
myQueryTextBox.SelectionStart = 0;
myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionColor = Color.Black;
myQueryTextBox.SelectionStart = selstart;
myQueryTextBox.SelectionLength = sellength;
myQueryTextBox.SelectionColor = Color.Crimson;
myTryExpression(myQueryTextBox.SelectedText);
myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionLength = 0;
myQueryTextBox.SelectionColor = Color.Black;
private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e)
{
// int selstart = myQueryTextBox.SelectionStart;
// int sellength = myQueryTextBox.SelectionLength;
// myQueryTextBox.SelectionStart = 0;
// myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
// myQueryTextBox.SelectionColor = Color.White;
// myQueryTextBox.SelectionStart = selstart;
// myQueryTextBox.SelectionLength = sellength;
// myQueryTextBox.SelectionColor = Color.Crimson;
// myTryExpression(myQueryTextBox.SelectedText);
// myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
// myQueryTextBox.SelectionLength = 0;
// myQueryTextBox.SelectionColor = Color.White;
}
private void myMenuItemOpenClick(object sender, System.EventArgs e) {
private void myMenuItemOpenClick(object sender, System.EventArgs e)
{
OpenFileDialog od = new OpenFileDialog();
od.Filter = "C# class (*.cs)|*.cs";
@@ -407,21 +452,24 @@ namespace BCG {
loadFile(od.FileName);
}
private void mySave(string fileName) {
private void mySave(string fileName)
{
using (TextWriter w = new StreamWriter(fileName))
w.Write(myCSharpText.Text);
myLastFileName = fileName;
this.Text = fileName;
}
private void myMenuItemSaveClick(object sender, System.EventArgs e) {
private void myMenuItemSaveClick(object sender, System.EventArgs e)
{
if (myLastFileName == null)
myMenuItemSaveAsClick(sender, e);
else
mySave(myLastFileName);
}
private void myMenuItemSaveAsClick(object sender, System.EventArgs e) {
private void myMenuItemSaveAsClick(object sender, System.EventArgs e)
{
SaveFileDialog sd = new SaveFileDialog();
if (myLastFileName != null)
sd.FileName = myLastFileName;
@@ -432,47 +480,56 @@ namespace BCG {
mySave(sd.FileName);
}
private void myTableListBox_DoubleClick(object sender, System.EventArgs e) {
if (myTableListBox.SelectedItem != null) {
private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
{
if (myTableListBox.SelectedItem != null)
{
string tableName = myTableListBox.SelectedItem.ToString();
myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName;
myTryExpression(myQueryTextBox.Text);
}
}
private void myDataGrid_MouseEnter(object sender, System.EventArgs e) {
private void myDataGrid_MouseEnter(object sender, System.EventArgs e)
{
//myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue;
// if (myDataGrid.TableStyles.Count > 0)
// myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue;
}
private void myDataGrid_MouseLeave(object sender, System.EventArgs e) {
private void myDataGrid_MouseLeave(object sender, System.EventArgs e)
{
//myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro;
// if (myDataGrid.TableStyles.Count > 0)
// myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro;
}
private void myCSharpText_MouseEnter(object sender, System.EventArgs e) {
private void myCSharpText_MouseEnter(object sender, System.EventArgs e)
{
//myCSharpText.BackColor = Color.Gainsboro;
myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
}
private void myCSharpText_MouseLeave(object sender, System.EventArgs e) {
private void myCSharpText_MouseLeave(object sender, System.EventArgs e)
{
//myCSharpText.BackColor = Color.White;
myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
}
private void myTableListBox_MouseEnter(object sender, System.EventArgs e) {
private void myTableListBox_MouseEnter(object sender, System.EventArgs e)
{
//myTableListBox.BackColor = Color.Gainsboro;
myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
}
private void myTableListBox_MouseLeave(object sender, System.EventArgs e) {
private void myTableListBox_MouseLeave(object sender, System.EventArgs e)
{
//myTableListBox.BackColor = Color.White;
myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
}
private void myLoad(object sender, EventArgs e) {
private void myLoad(object sender, EventArgs e)
{
myFreezeView(false);
myToolStripMenuItemSite.DropDownItems.Clear();
@@ -496,7 +553,9 @@ namespace BCG {
{
var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error())
Console.WriteLine(ex.Message);
} else {
}
else
{
Console.WriteLine("Set stdout worked.");
}
@@ -504,7 +563,9 @@ namespace BCG {
{
var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error())
Console.WriteLine(ex.Message);
} else {
}
else
{
Console.WriteLine("Set stderr worked.");
}
Console.WriteLine("Redirecting console ouput to this window...");
@@ -520,10 +581,12 @@ namespace BCG {
//_library = new LibraryForm(this);
}
private void myMenuItemSiteClick(object sender, System.EventArgs e) {
private void myMenuItemSiteClick(object sender, System.EventArgs e)
{
ToolStripMenuItem mi = (ToolStripMenuItem)sender;
if (mi.Text != mySite) {
if (mi.Text != mySite)
{
mySite = mi.Text;
foreach (var m in mySiteItems)
m.Checked = false;
@@ -544,63 +607,78 @@ namespace BCG {
}
}
private void myClosing(object sender, CancelEventArgs e) {
private void myClosing(object sender, CancelEventArgs e)
{
}
private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) {
if (myTreeView.SelectedNode != null) {
if (myTreeView.SelectedNode.LastNode == null) {
private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e)
{
if (myTreeView.SelectedNode != null)
{
if (myTreeView.SelectedNode.LastNode == null)
{
string sn = myTreeView.SelectedNode.Parent.Text + "()";
myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
}
}
}
private void myToolStripMenuItemExit_Click(object sender, EventArgs e) {
private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
{
Close();
}
private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) {
private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e)
{
myFreezeView(myTimer.Enabled);
}
private void myFreezeView(bool freeze) {
private void myFreezeView(bool freeze)
{
myTimer.Enabled = !freeze;
myToolStripMenuItemFreeze.Checked = freeze;
myToolStripMenuItemStep.Enabled = freeze;
}
private void myToolStripMenuItemStep_Click(object sender, EventArgs e) {
private void myToolStripMenuItemStep_Click(object sender, EventArgs e)
{
ViewStreamDatabase.instance.doWork();
}
private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) {
private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog();
ofd.ShowReadOnly = true;
DialogResult dr = ofd.ShowDialog(this);
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) {
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
{
PrimitiveManager.instance.loadAssembly(ofd.FileName);
myLoadAssemblies();
}
}
private void myToolStripMenuItemTest_Click(object sender, EventArgs e) {
if (myCurrentStream != null) {
private void myToolStripMenuItemTest_Click(object sender, EventArgs e)
{
if (myCurrentStream != null)
{
myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback);
myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback);
}
}
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) {
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e)
{
myConsoleRichTextBox.Clear();
Console.WriteLine("Redirecting console ouput to this window...");
}
private void myTableFilterBox_TextChanged(object sender, EventArgs e) {
private void myTableFilterBox_TextChanged(object sender, EventArgs e)
{
myUpdateTableList();
}
private void myUpdateTableList() {
private void myUpdateTableList()
{
IBlueRowCollection rows = myAllTagsStream.rows;
if (rows == null)
return;
@@ -615,12 +693,15 @@ namespace BCG {
myTableListBox.Items.Clear();
SortedList<string, bool> sl = new SortedList<string, bool>();
if (string.IsNullOrEmpty(myTableFilterBox.Text))
foreach (IBlueRow r in rows) {
foreach (IBlueRow r in rows)
{
sl[r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length)] = true;
}
else {
else
{
Regex tableRegex = new Regex(myTableFilterBox.Text);
foreach (IBlueRow r in rows) {
foreach (IBlueRow r in rows)
{
string thisTable = r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length);
if (tableRegex.IsMatch(thisTable))
sl[thisTable] = true;
@@ -638,7 +719,8 @@ namespace BCG {
myTableListBox.EndUpdate();
}
private void testGridToolStripMenuItem_Click(object sender, EventArgs e) {
private void testGridToolStripMenuItem_Click(object sender, EventArgs e)
{
HighPerformanceTimer timer = new HighPerformanceTimer();
//while (true)
//{
@@ -660,9 +742,11 @@ namespace BCG {
foreach (IBlueRow r in s.rows)
rows.Add(r);
for (int i = 0; i < 100; i++) {
for (int i = 0; i < 100; i++)
{
timer.start();
rows.Sort(delegate (IBlueRow r1, IBlueRow r2) {
rows.Sort(delegate (IBlueRow r1, IBlueRow r2)
{
return _sort_dicts_index(r1, r2, 5, true);
});
timer.stop();
@@ -670,19 +754,22 @@ namespace BCG {
}
}
private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) {
private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high)
{
if (r1.Contains(column) && r2.Contains(column))
return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high);
else
return 0;
}
private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) {
private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high)
{
//return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high);
return ((IComparable)r1[col_no]).CompareTo(r2[col_no]);
}
internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high)
{
if (myIsDontCare(o1))
return myIsDontCare(o2) ? 0 : 1;
else
@@ -690,14 +777,16 @@ namespace BCG {
: ((IComparable)o2).CompareTo(o1));
}
private static bool myIsDontCare(object o) {
private static bool myIsDontCare(object o)
{
return o == null || o == DBNull.Value ||
o is string && String.IsNullOrEmpty((string)o) ||
o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o));
}
private void _new_grid_form(object args) {
private void _new_grid_form(object args)
{
//System.Timers.Timer t = new System.Timers.Timer();
////t.SynchronizingObject
//BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId),
@@ -709,11 +798,13 @@ namespace BCG {
private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { }
private void myTableFilterLabel_Click(object sender, EventArgs e) {
private void myTableFilterLabel_Click(object sender, EventArgs e)
{
}
private void MainForm_Load(object sender, EventArgs e) {
private void MainForm_Load(object sender, EventArgs e)
{
}
}