diff --git a/docker/linux b/docker/linux new file mode 160000 index 000000000..653781226 --- /dev/null +++ b/docker/linux @@ -0,0 +1 @@ +Subproject commit 65378122620c7a26561641cb49b477c822b315f2 diff --git a/docker/windows b/docker/windows new file mode 160000 index 000000000..de7d40c09 --- /dev/null +++ b/docker/windows @@ -0,0 +1 @@ +Subproject commit de7d40c09de473e1b2802c4b27e03dc7f0c360d6 diff --git a/src/DataStreamSuite/DataStreamCore/DataPacket.cs b/src/DataStreamSuite/DataStreamCore/DataPacket.cs index 50c88a253..c51b62a50 100644 --- a/src/DataStreamSuite/DataStreamCore/DataPacket.cs +++ b/src/DataStreamSuite/DataStreamCore/DataPacket.cs @@ -1,185 +1,163 @@ using System; using System.Collections; +using System.Collections.Generic; using System.Data; + using BCG.Utilities.General; using Bts.DataStreamCore; -namespace BCG -{ - [Serializable] - public class DataPacket : ICloneable, IDataPacket - { - public class BlueRowIdComparator - { - private string[] myIdColumns = new string[]{"id"}; +namespace BCG { + [Serializable] + public class DataPacket : ICloneable, IDataPacket { + public class BlueRowIdComparator { + private string[] myIdColumns = new string[] { "id" }; - public BlueRowIdComparator(string[] idCols) - { - if ( idCols.Length != 0 ) - { + public BlueRowIdComparator(string[] idCols) { + if (idCols.Length != 0) { myIdColumns = idCols; } } - - public bool blueRowIdEqualEvaluator(object a, object b) - { - if ( a == null || b == null ) - { + + public bool blueRowIdEqualEvaluator(object a, object b) { + if (a == null || b == null) { return false; } - if ( !(a is IBlueRow) || !(b is IBlueRow) ) - { + if (!(a is IBlueRow) || !(b is IBlueRow)) { return false; } IBlueRow lhs = (IBlueRow)a; IBlueRow rhs = (IBlueRow)b; - + bool result = true; - foreach(string idColumn in myIdColumns) - { + foreach (string idColumn in myIdColumns) { result = result && lhs[idColumn].Equals(rhs[idColumn]); - if ( !result ) + if (!result) break; } - return result; + return result; } } public IBlueRowCollection newRows, addedRows, deletedRows; - public DataPacket () {} - public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) - { + public IDictionary meta_data = null; + public DataPacket() { } + public DataPacket(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) { this.newRows = newRows; this.addedRows = addedRows; this.deletedRows = deletedRows; } - public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) - { + public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { new_rows = newRows; added_rows = addedRows; deleted_rows = deletedRows; } - public IBlueRowCollection getTrulyAddedRows(params string[] idColumns) - { + public IBlueRowCollection getTrulyAddedRows(params string[] idColumns) { BlueTable truly = new BlueTable(addedRows.schema); BlueRowIdComparator c = new BlueRowIdComparator(idColumns); - CollectionHelpers.EqualDelegate ed = + CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); - - foreach(IBlueRow r in ta) - { + + foreach (IBlueRow r in ta) { truly.addRow(r); } - + return truly; - + } - public virtual IBlueRowCollection getTrulyAddedRows () - { - BlueTable retVal = new BlueTable (schema); + public virtual IBlueRowCollection getTrulyAddedRows() { + BlueTable retVal = new BlueTable(schema); foreach (IBlueRow r in addedRows) - if (!deletedRows.containsID ((int)r["id"])) - retVal.addRow (r); + if (!deletedRows.containsID((int)r["id"])) + retVal.addRow(r); return retVal; } - public virtual IBlueRowCollection getTrulyDeletedRows () - { - BlueTable retVal = new BlueTable (schema); + public virtual IBlueRowCollection getTrulyDeletedRows() { + BlueTable retVal = new BlueTable(schema); foreach (IBlueRow r in deletedRows) - if (!addedRows.containsID ((int)r["id"])) - retVal.addRow (r); + if (!addedRows.containsID((int)r["id"])) + retVal.addRow(r); return retVal; } - public virtual IBlueRowCollection getTrulyChangedRows () - { - BlueTable retVal = new BlueTable (schema); + public virtual IBlueRowCollection getTrulyChangedRows() { + BlueTable retVal = new BlueTable(schema); foreach (IBlueRow r in addedRows) - if (deletedRows.containsID ((int)r["id"])) - retVal.addRow (r); + if (deletedRows.containsID((int)r["id"])) + retVal.addRow(r); return retVal; } - - public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns) - { + + public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns) { BlueTable truly = new BlueTable(deletedRows.schema); BlueRowIdComparator c = new BlueRowIdComparator(idColumns); - CollectionHelpers.EqualDelegate ed = + CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); - - foreach(IBlueRow r in td) - { + + foreach (IBlueRow r in td) { truly.addRow(r); } - return truly; + return truly; } - - public IBlueRowCollection getTrulyChangedRows(params string[] idColumns) - { - + + public IBlueRowCollection getTrulyChangedRows(params string[] idColumns) { + BlueTable truly = new BlueTable(addedRows.schema); BlueRowIdComparator c = new BlueRowIdComparator(idColumns); - CollectionHelpers.EqualDelegate ed = + CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); ICollection tc = CollectionHelpers.difference(addedRows, ta, ed); - - foreach(IBlueRow r in tc) - { + + foreach (IBlueRow r in tc) { truly.addRow(r); - } - - return truly; + } + + return truly; } - - public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns) - { + + public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns) { BlueTable truly = new BlueTable(deletedRows.schema); BlueRowIdComparator c = new BlueRowIdComparator(idColumns); - CollectionHelpers.EqualDelegate ed = + CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); ICollection tco = CollectionHelpers.difference(deletedRows, td, ed); - - foreach(IBlueRow r in tco) - { + + foreach (IBlueRow r in tco) { truly.addRow(r); - } + } - return truly; - } - - public TablePacket toTablePacket () - { - return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows), - BlueDataUtility.toDataTable (deletedRows)); - + return truly; } - public IBlueTableSchema schema - { - get - { + public TablePacket toTablePacket() { + return new TablePacket(BlueDataUtility.toDataTable(newRows), BlueDataUtility.toDataTable(addedRows), + BlueDataUtility.toDataTable(deletedRows)); + + } + + public IBlueTableSchema schema { + get { IBlueTableSchema retVal = newRows.schema != null ? newRows.schema : addedRows.schema != null ? addedRows.schema : deletedRows.schema; return retVal; @@ -209,19 +187,17 @@ namespace BCG d' = dthis - (dthis intersect a) + d // keep the old delete a' = a - (dthis intersect a) + athis // keep the new add * */ - public static DataPacket combineWithIds(ICollection packets, string[] idColumns) - { - if ( packets.Count == 0 ) - { + public static DataPacket combineWithIds(ICollection packets, string[] idColumns) { + if (packets.Count == 0) { return new DataPacket( new BlueTable(new DataTable()), new BlueTable(new DataTable()), - new BlueTable(new DataTable())); + new BlueTable(new DataTable())); } DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns); - - CollectionHelpers.EqualDelegate ed = + + CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); ArrayList added = new ArrayList(); @@ -230,80 +206,70 @@ namespace BCG BlueTable at = null; IBlueRowCollection nt = null; - foreach(DataPacket dp in packets) - { + foreach (DataPacket dp in packets) { ArrayList oldDeleted = deleted; ArrayList oldAdded = added; added = new ArrayList(); deleted = new ArrayList(); - if ( dt == null ) - { + if (dt == null) { dt = new BlueTable(dp.schema); at = new BlueTable(dp.schema); } - + deleted.AddRange(oldDeleted); - deleted.AddRange( CollectionHelpers.difference( + deleted.AddRange(CollectionHelpers.difference( CollectionHelpers.difference(dp.deletedRows, oldAdded, ed), - oldDeleted, ed) ); + oldDeleted, ed)); added.AddRange(dp.addedRows); - added.AddRange( CollectionHelpers.difference( + added.AddRange(CollectionHelpers.difference( CollectionHelpers.difference(oldAdded, dp.deletedRows, ed), - added, ed) ); + added, ed)); nt = dp.newRows; - } - - foreach ( IBlueRow dr in added ) - { + } + + foreach (IBlueRow dr in added) { at.addRow(dr); } - foreach ( IBlueRow dr in deleted ) - { + foreach (IBlueRow dr in deleted) { dt.addRow(dr); } - return new DataPacket(nt,at,dt); + return new DataPacket(nt, at, dt); } - public DataPacket deepCopy () - { - return new DataPacket (new OrderedBlueTable (newRows), - new OrderedBlueTable (newRows), - new BlueTable (deletedRows)); + public DataPacket deepCopy() { + return new DataPacket(new OrderedBlueTable(newRows), + new OrderedBlueTable(newRows), + new BlueTable(deletedRows)); } - public virtual DataPacket createImmutable () - { - return + public virtual DataPacket createImmutable() { + return new DataPacket( newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows), - addedRows, + addedRows, deletedRows); } #region ICloneable Members - public object Clone() - { + public object Clone() { return MemberwiseClone(); } #endregion - public string toIncrementalIDString() - { + public string toIncrementalIDString() { System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName); - sb.Append ("\ndeleted : "); - foreach (IBlueRow r in deletedRows) - { + sb.Append("\ndeleted : "); + foreach (IBlueRow r in deletedRows) { sb.Append(r["id"]); sb.Append(','); } sb.Append("\nadded : "); - foreach (IBlueRow r in addedRows) - { + foreach (IBlueRow r in addedRows) { sb.Append(r["id"]); sb.Append(','); } @@ -311,8 +277,7 @@ namespace BCG return sb.ToString(); } - public override string ToString() - { + public override string ToString() { return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count); } } diff --git a/src/DataStreamSuite/DataStreamSuite.sln b/src/DataStreamSuite/DataStreamSuite.sln index 5a3ed169e..5a6420d6b 100644 --- a/src/DataStreamSuite/DataStreamSuite.sln +++ b/src/DataStreamSuite/DataStreamSuite.sln @@ -36,8 +36,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}" -EndProject +/*Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}" +EndProject*/ Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU diff --git a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs index ff4d1ee91..330dfc309 100644 --- a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs @@ -11,6 +11,7 @@ public class BlueOrbNLStream : ViewStream { private readonly string _nl_prompt; private ViewStream _nl_to_dsl_stream, _dsl_stream; + private IDictionary _dp_metadata = new Dictionary(); public BlueOrbNLStream(string nl_prompt) { _nl_prompt = nl_prompt; } @@ -24,6 +25,7 @@ DataPacket dp = dp_obj as DataPacket; var r = dp.newRows.to_list().ElementAt(0); string dsl = r.get("response"); + _dp_metadata["dsl"] = dsl; Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}"); _dsl_stream = new DSLStream(dsl); _dsl_stream.onChange += _handle_data_cb; @@ -31,6 +33,7 @@ private void _handle_data_cb(object dp_obj) { var dp = dp_obj as DataPacket; + dp.meta_data = new Dictionary(_dp_metadata); DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket, true); myFireEvent(dp_out); } diff --git a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs index 8863e199a..46cfc81a5 100644 --- a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs @@ -28,7 +28,7 @@ mlp_file_path = Path.Combine(bldPath, "BlueOrbDSL_MLP_v1.9.md"); } - + string mlp_text = File.ReadAllText(mlp_file_path); string system_prompt = $"You are a deterministic BlueOrbDSL generator. " + $"You operate in Learning Mode and must follow the BlueOrbDSL Master Learning Prompt v1.9 strictly. " +