diff --git a/docker/linux b/docker/linux
new file mode 160000
index 000000000..653781226
--- /dev/null
+++ b/docker/linux
@@ -0,0 +1 @@
+Subproject commit 65378122620c7a26561641cb49b477c822b315f2
diff --git a/docker/windows b/docker/windows
new file mode 160000
index 000000000..de7d40c09
--- /dev/null
+++ b/docker/windows
@@ -0,0 +1 @@
+Subproject commit de7d40c09de473e1b2802c4b27e03dc7f0c360d6
diff --git a/src/DataStreamSuite/DataStream/Config/DataStreamLoadFileConfig.cs b/src/DataStreamSuite/DataStream/Config/DataStreamLoadFileConfig.cs
new file mode 100644
index 000000000..72dd6f81f
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Config/DataStreamLoadFileConfig.cs
@@ -0,0 +1,116 @@
+using System;
+
+//using BCG.Utilities.BlueConfiguration;
+
+namespace BCG
+{
+ public class DataStreamLoadFileConfig //: ConfigurationRequest
+ {
+ private static DataStreamLoadFileConfig myInstance;
+ private string[] myAssemblies;
+ private string myIsTestMode = "false";
+ private bool myProfileStreams = false;
+ private double myProfileThreshold = 500;
+ private bool myUseNewJoin, myUseIncrementsForMultipleOps, myUseNewAppend, myUseNewFilter, myUseNewMap,
+ myUseUncopiedNewRowsInFireEvent, myUseModularizedBridge;
+
+ private bool myLogBlobPackets = false;
+ private bool myUseStaticParser = true;
+
+ //[Description("DataStream: list of assemblies providing the streams")]
+ public string [] loadedAssemblies
+ {
+ get { return myAssemblies; }
+ set { myAssemblies = value; }
+ }
+
+ public bool useStaticParser
+ {
+ get { return myUseStaticParser; }
+ set { myUseStaticParser = value; }
+ }
+
+ public bool logBlobPackets
+ {
+ get { return myLogBlobPackets; }
+ set { myLogBlobPackets = value; }
+ }
+
+ public string isTestMode
+ {
+ get {return myIsTestMode;}
+ set {myIsTestMode = value;}
+ }
+
+ public bool profileStreams
+ {
+ get { return myProfileStreams; }
+ set { myProfileStreams = Convert.ToBoolean(value); }
+ }
+
+ public double profileThreshold
+ {
+ get { return myProfileThreshold; }
+ set { myProfileThreshold = Convert.ToDouble(value); }
+ }
+
+ public bool isValid()
+ {
+ return true;
+ //return myVerify (myAssemblies);
+ }
+
+ public bool useNewJoin
+ {
+ get { return myUseNewJoin; }
+ set { myUseNewJoin = value; }
+ }
+
+ public bool useNewAppend
+ {
+ get { return myUseNewAppend; }
+ set { myUseNewAppend = value; }
+ }
+
+ public bool useNewFilter
+ {
+ get { return myUseNewFilter; }
+ set { myUseNewFilter = value; }
+ }
+
+ public bool useNewMap
+ {
+ get { return myUseNewMap; }
+ set { myUseNewMap = value; }
+ }
+
+ public bool useIncrementsForMultipleOps
+ {
+ get { return myUseIncrementsForMultipleOps; }
+ set { myUseIncrementsForMultipleOps = value; }
+ }
+
+ public bool useUncopiedNewRowsInFireEvent
+ {
+ get { return myUseUncopiedNewRowsInFireEvent; }
+ set { myUseUncopiedNewRowsInFireEvent = value; }
+ }
+ public bool useModularizedDataStreamBridge
+ {
+ get { return myUseModularizedBridge; }
+ set { myUseModularizedBridge = value; }
+ }
+ public static DataStreamLoadFileConfig instance
+ {
+ get
+ {
+ return myInstance = new DataStreamLoadFileConfig();
+ return myInstance;
+ // myInstance == null ?
+ // ConfigurationRequester.request (typeof (DataStreamLoadFileConfig)) as DataStreamLoadFileConfig
+ // as DataStreamLoadFileConfig :
+ // myInstance;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Core/DataPacket.cs b/src/DataStreamSuite/DataStream/Core/DataPacket.cs
new file mode 100644
index 000000000..441d2a06e
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/DataPacket.cs
@@ -0,0 +1,316 @@
+using System;
+using System.Collections;
+using System.Data;
+using BCG.Utilities.General;
+
+namespace BCG
+{
+ [Serializable]
+ public class DataPacket : ICloneable, IDataPacket
+ {
+ public class BlueRowIdComparator
+ {
+ private string[] myIdColumns = new string[]{"id"};
+
+ public BlueRowIdComparator(string[] idCols)
+ {
+ if ( idCols.Length != 0 )
+ {
+ myIdColumns = idCols;
+ }
+ }
+
+ public bool blueRowIdEqualEvaluator(object a, object b)
+ {
+ if ( a == null || b == null )
+ {
+ return false;
+ }
+
+ if ( !(a is IBlueRow) || !(b is IBlueRow) )
+ {
+ return false;
+ }
+
+ IBlueRow lhs = (IBlueRow)a;
+ IBlueRow rhs = (IBlueRow)b;
+
+ bool result = true;
+ foreach(string idColumn in myIdColumns)
+ {
+ result = result && lhs[idColumn].Equals(rhs[idColumn]);
+ if ( !result )
+ break;
+ }
+
+ return result;
+ }
+ }
+
+ public IBlueRowCollection newRows, addedRows, deletedRows;
+ public DataPacket () {}
+ public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
+ {
+ this.newRows = newRows;
+ this.addedRows = addedRows;
+ this.deletedRows = deletedRows;
+ }
+
+ public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
+ {
+ new_rows = newRows;
+ added_rows = addedRows;
+ deleted_rows = deletedRows;
+ }
+
+ public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
+ {
+ BlueTable truly = new BlueTable(addedRows.schema);
+ BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+
+ CollectionHelpers.EqualDelegate ed =
+ new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+
+ ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
+
+ foreach(IBlueRow r in ta)
+ {
+ truly.addRow(r);
+ }
+
+ return truly;
+
+ }
+
+ public virtual IBlueRowCollection getTrulyAddedRows ()
+ {
+ BlueTable retVal = new BlueTable (schema);
+ foreach (IBlueRow r in addedRows)
+ if (!deletedRows.containsID ((int)r["id"]))
+ retVal.addRow (r);
+ return retVal;
+ }
+
+ public virtual IBlueRowCollection getTrulyDeletedRows ()
+ {
+ BlueTable retVal = new BlueTable (schema);
+ foreach (IBlueRow r in deletedRows)
+ if (!addedRows.containsID ((int)r["id"]))
+ retVal.addRow (r);
+ return retVal;
+ }
+
+ public virtual IBlueRowCollection getTrulyChangedRows ()
+ {
+ BlueTable retVal = new BlueTable (schema);
+ foreach (IBlueRow r in addedRows)
+ if (deletedRows.containsID ((int)r["id"]))
+ retVal.addRow (r);
+ return retVal;
+ }
+
+
+ public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
+ {
+ BlueTable truly = new BlueTable(deletedRows.schema);
+ BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+
+ CollectionHelpers.EqualDelegate ed =
+ new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+
+ ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
+
+ foreach(IBlueRow r in td)
+ {
+ truly.addRow(r);
+ }
+
+ return truly;
+ }
+
+ public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
+ {
+
+ BlueTable truly = new BlueTable(addedRows.schema);
+ BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+
+ CollectionHelpers.EqualDelegate ed =
+ new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+
+ ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
+ ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
+
+ foreach(IBlueRow r in tc)
+ {
+ truly.addRow(r);
+ }
+
+ return truly;
+ }
+
+ public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
+ {
+ BlueTable truly = new BlueTable(deletedRows.schema);
+ BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+
+ CollectionHelpers.EqualDelegate ed =
+ new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+
+ ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
+ ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
+
+ foreach(IBlueRow r in tco)
+ {
+ truly.addRow(r);
+ }
+
+ return truly;
+ }
+
+ public TablePacket toTablePacket ()
+ {
+ return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
+ BlueDataUtility.toDataTable (deletedRows));
+
+ }
+
+ public IBlueTableSchema schema
+ {
+ get
+ {
+ IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
+ addedRows.schema != null ? addedRows.schema : deletedRows.schema;
+ return retVal;
+ }
+ }
+
+ /*
+ status D total A total D this A This D' total A' Total
+ x 0 0 0 0 0 0
+ 0 0 0 1 1 this+
+ 0 0 1 0 1this
+ 0 0 1 1 1 this 1 this+
+ x 0 1 0 0
+ error 0 1 0 1
+ 0 1 1 0 1 total-
+ 0 1 1 1 1 total-, 1 this +
+ x 1 0 0 0
+ 1 0 0 1 1 this+
+ error 1 0 1 0
+ error 1 0 1 1
+ x 1 1 0 0
+ error 1 1 0 1
+ 1 1 1 0 1 total -1 keep the old delete and remove the old add
+ 1 1 1 1 1 total-, 1 this +
+
+ d' = dthis - (dthis intersect a) + d // keep the old delete
+ a' = a - (dthis intersect a) + athis // keep the new add
+ * */
+ public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
+ {
+ if ( packets.Count == 0 )
+ {
+ return new DataPacket(
+ new BlueTable(new DataTable()),
+ new BlueTable(new DataTable()),
+ new BlueTable(new DataTable()));
+ }
+
+ DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
+
+ CollectionHelpers.EqualDelegate ed =
+ new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+
+ ArrayList added = new ArrayList();
+ ArrayList deleted = new ArrayList();
+ BlueTable dt = null;
+ BlueTable at = null;
+ IBlueRowCollection nt = null;
+
+ foreach(DataPacket dp in packets)
+ {
+ ArrayList oldDeleted = deleted;
+ ArrayList oldAdded = added;
+
+ added = new ArrayList();
+ deleted = new ArrayList();
+ if ( dt == null )
+ {
+ dt = new BlueTable(dp.schema);
+ at = new BlueTable(dp.schema);
+ }
+
+ deleted.AddRange(oldDeleted);
+ deleted.AddRange( CollectionHelpers.difference(
+ CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
+ oldDeleted, ed) );
+
+ added.AddRange(dp.addedRows);
+ added.AddRange( CollectionHelpers.difference(
+ CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
+ added, ed) );
+ nt = dp.newRows;
+ }
+
+ foreach ( IBlueRow dr in added )
+ {
+ at.addRow(dr);
+ }
+
+ foreach ( IBlueRow dr in deleted )
+ {
+ dt.addRow(dr);
+ }
+
+ return new DataPacket(nt,at,dt);
+ }
+
+ public DataPacket deepCopy ()
+ {
+ return new DataPacket (new OrderedBlueTable (newRows),
+ new OrderedBlueTable (newRows),
+ new BlueTable (deletedRows));
+ }
+
+ public virtual DataPacket createImmutable ()
+ {
+ return
+ new DataPacket(
+ newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
+ addedRows,
+ deletedRows);
+ }
+ #region ICloneable Members
+
+ public object Clone()
+ {
+ return MemberwiseClone();
+ }
+
+ #endregion
+
+ public string toIncrementalIDString()
+ {
+ System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
+ sb.Append ("\ndeleted : ");
+ foreach (IBlueRow r in deletedRows)
+ {
+ sb.Append(r["id"]);
+ sb.Append(',');
+ }
+ sb.Append("\nadded : ");
+ foreach (IBlueRow r in addedRows)
+ {
+ sb.Append(r["id"]);
+ sb.Append(',');
+ }
+ sb.Append("\n");
+ return sb.ToString();
+ }
+
+ public override string ToString()
+ {
+ return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
+ }
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/Core/DataStream.cs b/src/DataStreamSuite/DataStream/Core/DataStream.cs
new file mode 100644
index 000000000..1f14a3027
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/DataStream.cs
@@ -0,0 +1,180 @@
+using System;
+using System.Collections;
+
+//using BCG.Utilities;
+//using BCG.Utilities.General;
+using mbts;
+
+namespace BCG
+{
+ ///
+ /// Base class for all the sreams
+ ///
+ public class DataStream
+ {
+ internal static event EventHandler onFireStart;
+ protected bool myTestMode = Convert.ToBoolean (DataStreamLoadFileConfig.instance.isTestMode);
+ protected bool myFirstSubImplemented = true,
+ myLastSubImplemented = true;
+ protected IDictionary myDelMap = new Hashtable ();
+ private Guid myID = Guid.Empty;
+ public DateTime last_fired_time = DateTime.MinValue;
+ public int n_updates = 0;
+
+ public DataStream ()
+ :
+ this (null)
+ {}
+
+ public DataStream (DataStream realSource)
+ {
+ myRealSource = realSource;
+ myOnChangeDelegate = null;
+ myPendingAdds = new Stack();
+ }
+
+ public int numberOfSubscribers
+ {
+ get
+ {
+ return
+ myOnChangeDelegate != null?
+ myOnChangeDelegate.GetInvocationList().Length
+ :
+ 0;
+ }
+ }
+
+ public delegate void ChangeDelegate(object newValue);
+ public event ChangeDelegate onChange
+ {
+ add
+ {
+ ChangeDelegate currValue = value;
+ if (myTestMode)
+ {
+ ChangeDelegateWrapper w = new ChangeDelegateWrapper (value);
+ currValue = new ChangeDelegate (w.wrapperFunction);
+ myDelMap.Add (value, currValue);
+ }
+ if (myRealSource != null)
+ myRealSource.onChange += currValue;
+ else
+ {
+ myOnChangeDelegate =
+ (ChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);
+
+
+ if (myOnChangeDelegate.GetInvocationList().Length == 1) // first subscriber
+ {
+ myPendingAdds.Push(new ArrayList());
+ myOnFirstSubscriber();
+ ArrayList calledDelegates = (ArrayList) myPendingAdds.Pop();
+
+ object v = myGetValueForInitialSubscription();
+ //BlueTrace.WriteViewStreamSubscribe(this.ID, v != null);
+ if (v != null && calledDelegates.IndexOf(currValue) == -1)
+ currValue (v);
+ }
+ else
+ {
+ object v = myGetValueForInitialSubscription();
+ // BlueTrace.WriteViewStreamSubscribe(this.ID, v != null);
+ if (v != null)
+ currValue (v);
+ }
+ }
+ }
+ remove
+ {
+ ChangeDelegate currValue = value;
+ // BlueTrace.WriteViewStreamUnsubscribe(this.ID);
+ if (myTestMode)
+ {
+ currValue = myDelMap[value] as ChangeDelegate;
+ myDelMap.Remove (value);
+ }
+ if (myRealSource != null)
+ myRealSource.onChange -= currValue;
+ else
+ {
+ myOnChangeDelegate =
+ (ChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);
+
+ // this is need to avoid a callback in a reentrant add.
+ if (myPendingAdds.Count > 0)
+ ((ArrayList) (myPendingAdds.Peek())).Add(currValue);
+
+ if (myOnChangeDelegate == null) // last subscriber
+ {
+ myOnLastSubscriber();
+ myLastValue = null;
+ }
+ }
+ }
+ }
+
+ protected virtual object myGetValueForInitialSubscription()
+ {return myLastValue;}
+
+ protected void mySetRealSource (DataStream src)
+ {myRealSource = src;}
+
+ protected virtual void myOnFirstSubscriber() {myFirstSubImplemented = false;}
+ protected virtual void myOnLastSubscriber () {myLastSubImplemented = false;}
+ protected virtual void myFireEvent(object o)
+ {
+ last_fired_time = DateTime.Now;
+ n_updates++;
+ // BlueEventLog.logAssert (myRealSource == null);
+ myLastValue = o;
+ if (myOnChangeDelegate != null)
+ {
+ // this is need to avoid a callback in a reentrant add.
+ if (myPendingAdds.Count > 0)
+ foreach (Delegate d in myOnChangeDelegate.GetInvocationList())
+ ((ArrayList) (myPendingAdds.Peek())).Add(d);
+ if (onFireStart != null)
+ onFireStart(this, EventArgs.Empty);
+ myOnChangeDelegate (o);
+ }
+ }
+
+ public Guid ID
+ {
+ get
+ {
+ if (this.myID == Guid.Empty) // && BlueTraceHelper.IsEnabled(BlueTraceLevel.Verbose, BlueTraceKeywords.Streams))
+ {
+ // BlueTraceHelper.FastInitGuid(ref this.myID);
+ // BlueTrace.WriteViewStreamCreate(this.GetType().ToString(), this.myID);
+ }
+ return this.myID;
+ }
+ }
+
+ // this was made internal for testing purposes only.
+ // don't mess with it.
+ internal ChangeDelegate myOnChangeDelegate;
+ protected object myLastValue;
+ protected DataStream myRealSource;
+ private Stack myPendingAdds;
+ }
+
+ internal class ChangeDelegateWrapper
+ {
+ private HighPerformanceTimer myTimer = new HighPerformanceTimer ();
+ private DataStream.ChangeDelegate myDel;
+
+ internal ChangeDelegateWrapper (DataStream.ChangeDelegate del) {myDel = del;}
+
+ internal void wrapperFunction (object o)
+ {
+ myTimer.start ();
+ myDel (o);
+ myTimer.stop ();
+ string minfo = "Type = " + myDel.Method.DeclaringType.FullName + " , method = " + myDel.Method.Name;
+ // BlueEventLog.logEvent (minfo + " , time = " + myTimer.lastIntervalTime.ToString (), BlueEventLevel.Performance);
+ }
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/Core/IExportableDataStream.cs b/src/DataStreamSuite/DataStream/Core/IExportableDataStream.cs
new file mode 100644
index 000000000..1116fe3eb
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/IExportableDataStream.cs
@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BCG
+{
+ public delegate void ExportableChangeDelegate(object data_packet);
+ public interface IExportableDataStream
+ {
+ void begin_subscription();
+ event ExportableChangeDelegate on_change;
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/Core/LightTableStream.cs b/src/DataStreamSuite/DataStream/Core/LightTableStream.cs
new file mode 100644
index 000000000..561dbd769
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/LightTableStream.cs
@@ -0,0 +1,51 @@
+using System;
+using System.Collections;
+using System.Data;
+
+using BCG.Utilities;
+
+
+/* this class reprsents a table stream for use with CppDataStreams. It does not hold on to the last value fired.
+ * This way, the last value is purged from managed memory and only the slimmer, sexier Cpp implementation of the row remains.
+ *
+ */
+namespace BCG
+{
+ public class LightTableStream : TableStream
+ {
+ public LightTableStream(string tableName)
+ :
+ base(tableName, null, null)
+ {
+ }
+
+ public LightTableStream(string tableName, ViewStreamDatabase db)
+ :
+ base(tableName, db, null)
+ {}
+
+ public LightTableStream(string tableName, string colExp)
+ :
+ base(tableName,
+ null,
+ colExp)
+ {}
+
+
+ public LightTableStream(string ttableName, ViewStreamDatabase db, string colExp) :
+ base(ttableName,db,colExp)
+ {
+ }
+
+
+ protected override void myHandleUpdates(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
+ {
+
+ base.myHandleUpdates(newRows, addedRows, deletedRows);
+ //once you've fired the rows, delete them becuase we don't want to hold onto them in memory
+ myLastValue = null;
+
+ }
+
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/Core/ProxyStream.cs b/src/DataStreamSuite/DataStream/Core/ProxyStream.cs
new file mode 100644
index 000000000..9ec179502
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/ProxyStream.cs
@@ -0,0 +1,526 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Diagnostics;
+using System.Reflection;
+
+using BCG.Utilities;
+using BCG.Utilities.General;
+
+namespace BCG
+{
+ /// Derive from this class to create a new proxy stream for a stream that
+ /// should be computed out of process. In the proxy class add constructors
+ /// that exactly mirror the ones you want to expose from the real stream.
+ /// Each of the proxy constructors should just call the base constructor
+ /// passing it a, possibly null VSDB, and the arguments it received in
+ /// exactly the same order. Only non nullparameters of types string, int, bool or
+ /// DateTime are supported. DateTime arguments will loose their time portion,
+ /// string arguments can only have digits or letters and ints have to be positive.
+ ///
+ /// For example, if you real stream is:
+ ///
+ /// class TradeLog : ViewStream
+ /// {
+ /// public TradeLog(string forAccount, string forTrader) {....}
+ /// }
+ ///
+ /// Then your proxy stream would like:
+ ///
+ /// class TradeLogProxy : ProxyStream
+ /// {
+ /// public TradeLogProxy(string forAccount, string forTrader) : base(null,forTrader, forAccount) {}
+ /// }
+ ///
+ /// Unfortunatly, PubSub tags have a very limited alphabet, one that does not even support upper and lower
+ /// case dinstinctions, so some special care needs to be taken by the proxy or the real stream when dealing
+ /// with case sensitive arguments or with string arguments that can contain _ (the only non alpha numeric
+ /// character allowed in a string). The proxy tries to help by supporting the following conventions:
+ ///
+ /// - When passing a symbol, mark the parameter with the kind of symbol you mean, option or
+ /// underlyier. Example:
+ /// public TheosProxy([Option] string symbol) : base(symbol) {} or
+ /// public FuturePositionProxy([Underlyer] string future) : base(symbol) {}
+ ///
+ /// - When passing an exchange mark it with the Exchange attribute. Example:
+ /// public QuoteWidthsProxy([Symbol] string symbol,
+ /// [Exchange] string e) : base(e) {}
+ ///
+ /// - When passing a group name mark it with the group. Since group names can have _ and presumably numbers
+ /// there can be only one group argument which has to be last one.
+ /// attribute. Ex:
+ /// public RiskTheosProxy([Group] string g) : base(g) {}
+ ///
+ /// - When passing an instrument type mark it as InstrumentType. Example:
+ /// public SymbolsOfTypeProxy([InstrumentType] string it) : base(it) {}
+ ///
+ /// - When passing a trader mark it as trader. Example:
+ /// public TraderTrades([trader] string joes) : base(joes) {}
+ ///
+ /// - When passing a string that can contain _, the string argument must
+ /// be the last one in the constructor and should be marked as [LongString]
+ /// public WorkerHealth([LongString] string machineName) : base(machineName) {}
+ ///
+ /// - When passing a symbol, instrument type combo combine both in one string
+ /// with the symbol:InstrumentType format and market it with BCGSym:
+ /// public PositionProxy([BCGSym] string bcgsym) : base(bcgsym) {}
+ /// More complex cases should be handled by the real stream.
+
+
+ public abstract class ProxyStream : ViewStream
+ {
+ public enum ProxyParameterType
+ {
+ clrType,
+ optionSymbol,
+ trader,
+ underlyerSymbol,
+ group,
+ exchange,
+ instrumentType,
+ bcgSym
+ }
+
+ public class ProxyParameter
+ {
+ public ProxyParameterType type;
+ public object value;
+ }
+
+ public class MarshallingException : SystemException
+ {
+ public MarshallingException (string format, params object [] args)
+ : base(String.Format(format,args) )
+ {}
+ }
+
+ [AttributeUsage(AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)]
+ protected class ProxyParameterAttribute : Attribute
+ {}
+
+ protected class OptionAttribute : ProxyParameterAttribute
+ { }
+
+ protected class LongStringAttribute : ProxyParameterAttribute
+ { }
+
+ protected class UnderlyierAttribute : ProxyParameterAttribute
+ { }
+
+ protected class BCGSymAttribute : ProxyParameterAttribute
+ { }
+
+ protected class GroupAttribute : ProxyParameterAttribute
+ { }
+
+ protected class ExchangeAttribute : ProxyParameterAttribute
+ { }
+
+ protected class TraderAttribute: ProxyParameterAttribute
+ { }
+
+ protected class InstrumentTypeAttribute : ProxyParameterAttribute
+ { }
+
+ public string getTag() { return myTag; }
+ private string myTag;
+
+ protected ProxyStream(params object[] args)
+ {myInit(null, args);}
+
+ protected ProxyStream(ViewStreamDatabase db, params object[] args)
+ {myInit(db, args);}
+
+ class MyException : System.Exception
+ {
+ public MyException (string msg) : base (msg)
+ {
+ }
+ }
+
+ private static void myThrowIfFalse(bool condition, string msg)
+ {
+ if (!condition)
+ throw new MyException(msg);
+ }
+
+ private void myInit(ViewStreamDatabase db, params object[] args)
+ {
+ try
+ {
+ ConstructorInfo ci = GetType().GetConstructor(args.Select(t => t.GetType()).ToArray());
+ foreach (object o in args)
+ myThrowIfFalse(o != null,
+ "Null arguments are not supported.");
+
+ myTag = myGetTag(ci, args, db != null);
+ mySetRealSource(new TableStream("prepub_" + myTag, db));
+ }
+ catch (MyException e)
+ {
+ // BlueEventLog2.instance().logEvent(e.Message, BlueEventLog2Level.Fatal);
+ mySetRealSource(new TableStream("error_tag"));
+ }
+ }
+
+ public static string getProxyName(string tag)
+ { return tag.Split('_')[0]; }
+
+ public static List unmarshal(Type proxyType, string tag)
+ {
+ myThrowIfFalse (proxyType.IsSubclassOf(typeof(ProxyStream)),
+ String.Format("{0} is not a subclass of ProxyStream", proxyType.Name));
+
+ myThrowIfFalse (tag.ToLower().StartsWith(proxyType.Name.ToLower()),
+ String.Format("Tag {0} does not match proxy type {1}.", tag, proxyType.Name));
+
+ string encodedParameters =
+ tag.Length == proxyType.Name.Length ? "" :
+ tag.Substring((proxyType.Name + "_").Length);
+
+ ConstructorInfo ci;
+ if (myConstructorsNeedToBeEnconded(proxyType))
+ {
+ List signature = myUnmarshalConstructorSignature(ref encodedParameters);
+ ci = proxyType.GetConstructor(signature.ToArray());
+ if (ci == null)
+ {
+ signature.Insert(0, typeof(ViewStreamDatabase));
+ ci = proxyType.GetConstructor(signature.ToArray());
+ if (ci == null)
+ throw new MarshallingException("Cannot find constructor for {0} matching signature {1}",
+ proxyType.Name, encodedParameters);
+ }
+ }
+ else
+ ci = proxyType.GetConstructors()[0];
+ ParameterInfo[] parameters = ci.GetParameters();
+
+ List result = new List();
+
+ ParameterInfo [] pars = ci.GetParameters();
+ // skip VSDB if first parameter
+ int i = pars.Length > 0 && pars[0].ParameterType.Equals(typeof(ViewStreamDatabase)) ? 1 : 0;
+ for (;i < pars.Length; i++)
+ result.Add(new ProxyParameter
+ {
+ type = myGetParameterType(pars[i]),
+ value = myGetParameterValue(pars[i], ref encodedParameters)
+ });
+ return result;
+ }
+
+ private static List myUnmarshalConstructorSignature(ref string encodedParameters)
+ {
+ List result = new List ();
+ for (int i = 0; i < encodedParameters.Length; i++)
+ if (encodedParameters[i] == 's')
+ result.Add(typeof(string));
+ else if (encodedParameters[i] == 'd')
+ result.Add(typeof(DateTime));
+ else if (encodedParameters[i] == 'i')
+ result.Add(typeof(Int32));
+ else if (encodedParameters[i] == 'b')
+ result.Add(typeof(bool));
+ else if (encodedParameters[i] == '_')
+ break;
+ else
+ throw new MarshallingException("'{0}' does not start with a valid constructor signature",
+ encodedParameters);
+ if (result.Count > 0)
+ encodedParameters = encodedParameters.Substring(result.Count + 1);
+ return result;
+ }
+
+ private static ProxyParameterType myGetParameterType(ParameterInfo pi)
+ {
+ if (pi.ParameterType == typeof(string))
+ {
+ ProxyParameterType result = ProxyParameterType.clrType;
+ object[] attr = pi.GetCustomAttributes(false);
+ if (attr.Length > 0)
+ {
+ foreach (Attribute a in attr)
+ if (a is OptionAttribute)
+ result = ProxyParameterType.optionSymbol;
+ else if (a is ExchangeAttribute)
+ result = ProxyParameterType.exchange;
+ else if (a is TraderAttribute)
+ result = ProxyParameterType.trader;
+ else if (a is GroupAttribute)
+ result = ProxyParameterType.group;
+ else if (a is InstrumentTypeAttribute)
+ result = ProxyParameterType.instrumentType;
+ else if (a is UnderlyierAttribute)
+ result = ProxyParameterType.underlyerSymbol;
+ else if (a is BCGSymAttribute)
+ result = ProxyParameterType.bcgSym;
+ else if (a is LongStringAttribute)
+ result = ProxyParameterType.clrType;
+ }
+ return result;
+ }
+ else if (pi.ParameterType == typeof(Int32) ||
+ pi.ParameterType == typeof(DateTime) ||
+ pi.ParameterType == typeof(bool))
+ return ProxyParameterType.clrType;
+ else
+ {
+ myThrowIfFalse (false,String.Format(
+ "{0} is not a supported proxy parameter type", pi.ParameterType.Name));
+ return ProxyParameterType.clrType;
+ }
+ }
+
+ private static object myGetParameterValue(ParameterInfo pi, ref string encodedParameters)
+ {
+ if (pi.ParameterType == typeof(DateTime))
+ {
+ string[] parts = encodedParameters.Split('_');
+ if (parts.Length < 3)
+ throw new MarshallingException("{0} does not start with a valid date", encodedParameters);
+
+ int year;
+ if (!Int32.TryParse(parts[0], out year))
+ throw new MarshallingException("{0} is not a valid year string", parts[0]);
+ int month;
+ if (!Int32.TryParse(parts[1], out month))
+ throw new MarshallingException("{0} is not a valid month string", parts[1]);
+ int day;
+ if (!Int32.TryParse(parts[2], out day))
+ throw new MarshallingException("{0} is not a valid day string", parts[2]);
+
+ DateTime result;
+ try {
+ result = new DateTime(year, month, day);
+ }
+ catch (ArgumentException)
+ {
+ throw new MarshallingException("{0}/{1}/{2} is not a valid date", year, month, day );
+ }
+
+ if (parts.Length > 3)
+ encodedParameters = encodedParameters.Substring(parts[0].Length + parts[1].Length + parts[2].Length + 3);
+ else
+ encodedParameters = "";
+ return
+ result;
+ }
+ else
+ {
+ if (pi.ParameterType == typeof(string) && myIsBCGSym(pi))
+ {
+ string[] parts = encodedParameters.Split('_');
+ if (parts.Length < 2)
+ throw new MarshallingException("{0} does not start with a valid BCGSym string", encodedParameters);
+ else
+ {
+ if (parts.Length > 2)
+ encodedParameters = encodedParameters.Substring(parts[0].Length + parts[1].Length + 2);
+ else
+ encodedParameters = "";
+ return parts[0] + ":" + parts[1];
+ }
+ }
+ else if (pi.ParameterType == typeof(string) && myIsLongString(pi))
+ {
+ string result = encodedParameters;
+ encodedParameters = "";
+ return result;
+ }
+ else
+ {
+ string[] parts = encodedParameters.Split('_');
+ object result = null;
+ if (pi.ParameterType == typeof(string))
+ result = parts[0];
+ else if (pi.ParameterType == typeof(Int32))
+ {
+ int n;
+ if (!Int32.TryParse(parts[0], out n))
+ throw new MarshallingException("{0} is not a valid int", parts[0]);
+ result = n;
+ }
+ else if (pi.ParameterType == typeof(Boolean))
+ {
+ bool b;
+ if (!bool.TryParse(parts[0], out b))
+ throw new MarshallingException("{0} is not a valid bool", parts[0]);
+ result = b;
+ }
+ else
+ myThrowIfFalse (false,
+ String.Format("{0} is not a supported proxy parameter type", pi.ParameterType.Name));
+ if (parts.Length > 1)
+ encodedParameters = encodedParameters.Substring(parts[0].Length + 1);
+ else
+ encodedParameters = "";
+ return result;
+ }
+ }
+ }
+
+ private bool myValidateString(string s, bool allowUnderscore)
+ {
+ foreach (char c in s)
+ if (!Char.IsDigit(c) && !Char.IsLetter(c) && !(allowUnderscore && c.Equals('_')))
+ return false;
+ return true;
+ }
+
+ private static bool myHasAttribute(ParameterInfo pi, Type attrType)
+ {
+ object[] attr = pi.GetCustomAttributes(false);
+ if (attr.Length > 0)
+ foreach (Attribute at in attr)
+ if (at.GetType().Equals(attrType))
+ return true;
+ return false;
+ }
+
+ private static bool myIsBCGSym(ParameterInfo pi)
+ {return myHasAttribute(pi, typeof(BCGSymAttribute));}
+
+ private static bool myIsLongString(ParameterInfo pi)
+ { return myHasAttribute(pi, typeof(LongStringAttribute)) || myHasAttribute(pi, typeof(GroupAttribute)); }
+
+ private static bool myFirstParameterIsVSDB(ConstructorInfo ci)
+ {
+ return
+ ci.GetParameters().Length > 0 &&
+ ci.GetParameters()[0].ParameterType.Equals(typeof(ViewStreamDatabase));
+ }
+
+ private static bool myIsSameModuloVSDB(ConstructorInfo ci0, ConstructorInfo ci1)
+ {
+ if ((ci1.GetParameters().Length - ci0.GetParameters().Length) != 1)
+ return false;
+
+ if (!myFirstParameterIsVSDB(ci1))
+ return false;
+
+ for (int i = 0; i < ci0.GetParameters().Length; i++)
+ if (!ci1.GetParameters()[i + 1].ParameterType.Equals(
+ ci0.GetParameters()[i].ParameterType))
+ return false;
+ return true;
+ }
+
+ private static bool myConstructorsNeedToBeEnconded(Type t)
+ {
+ ConstructorInfo [] cis = t.GetConstructors();
+ if (cis.Length == 1)
+ return false;
+ else if (cis.Length > 2)
+ return true;
+ else
+ {
+ ConstructorInfo ci0 = cis[0];
+ ConstructorInfo ci1 = cis[1];
+
+ return !(myIsSameModuloVSDB(ci0, ci1) || myIsSameModuloVSDB(ci1, ci0));
+ }
+ }
+
+ private string myGetConstructorSignature(ConstructorInfo ci)
+ {
+ string result ="";
+ int i = myFirstParameterIsVSDB(ci) ? 1 : 0;
+ for (; i < ci.GetParameters().Length; i++)
+ {
+ Type t = ci.GetParameters()[i].ParameterType;
+ if (t.Equals(typeof(string)))
+ result += "s";
+ else if (t.Equals(typeof(DateTime)))
+ result += "d";
+ else if (t.Equals(typeof(bool)))
+ result += "b";
+ else if (t.Equals(typeof(Int32)))
+ result += "i";
+ else
+ myThrowIfFalse (false,
+ String.Format("{0} is not a supported proxy parameter type", t.Name));
+ }
+ return result;
+ }
+
+ private string myGetTag(ConstructorInfo ci, object[] args, bool firstArgIsVSDB)
+ {
+ Type t = ci.DeclaringType;
+ string tag = t.Name;
+ if (myConstructorsNeedToBeEnconded(t))
+ tag += "_" + myGetConstructorSignature(ci);
+
+ int i = myFirstParameterIsVSDB(ci) ? 1 : 0;
+ foreach (object a in args)
+ {
+ Debug.Assert(a != null);
+ if (a.GetType() == typeof(DateTime))
+ {
+ DateTime dt = (DateTime)a;
+ tag += String.Format("_{0}_{1}_{2}", dt.Year, dt.Month, dt.Day);
+ }
+ else if (a.GetType() == typeof(string))
+ {
+ if (myIsBCGSym(ci.GetParameters()[i]))
+ {
+ string[] parts = a.ToString().Split(':');
+ myThrowIfFalse (parts.Length == 2,
+ String.Format("Invalid BCGSym string argument {0}. Use sym:InstrumentType",
+ a));
+ if (!myValidateString(parts[0], false) || !myValidateString(parts[1], false))
+ {
+ // BlueEventLog2.instance().logEvent(
+ // String.Format(
+ // "Invalid string argument {0}. Currently, only letters and digits are supported in string arguments",
+ // a),BlueEventLog2Level.Error);
+ return "error_tag";
+ }
+ tag += String.Format("_{0}_{1}", parts[0], parts[1]);
+ }
+ else if (myIsLongString(ci.GetParameters()[i]))
+ {
+ // not a BCGSym
+ if (!myValidateString(a.ToString(), true))
+ {
+ // BlueEventLog2.instance().logEvent(
+ // String.Format("Invalid string argument {0}. Currently, only letters, digits and _'s are supported in long string arguments",
+ // a),BlueEventLog2Level.Error);
+ return "error_tag";
+ }
+ //can't do this-- ci.GetParameters() order is undefined
+ //myThrowIfFalse (i.Equals(ci.GetParameters().Length - 1),
+ // "Long string argument must be last");
+ tag += "_" + a.ToString();
+ }
+ else
+ {
+ // not a BCGSym
+ if (!myValidateString(a.ToString(), false))
+ {
+ // BlueEventLog2.instance().logEvent(String.Format(
+ // "Invalid string argument {0}. Currently, only letters and digits are supported in string arguments", a),
+ // BlueEventLog2Level.Error);
+ return "error_tag";
+ }
+ tag += "_" + a.ToString();
+ }
+ }
+ else if (a.GetType() == typeof(Int32))
+ {
+ myThrowIfFalse ((int) a >= 0,
+ String.Format("Invalid int argument {0}. Currently only positive integers are supported.",
+ a));
+ tag += "_" + a.ToString();
+ } else if (a.GetType() == typeof(bool))
+ tag += "_" + a.ToString();
+ else
+ myThrowIfFalse (false,
+ String.Format("{0} is not a supported proxy parameter type", a.GetType().Name));
+ i++;
+ }
+ return tag;
+ }
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/Core/TablePacket.cs b/src/DataStreamSuite/DataStream/Core/TablePacket.cs
new file mode 100644
index 000000000..89a0e75d3
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/TablePacket.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Data;
+using System.Collections;
+
+namespace BCG
+{
+ ///
+ /// Defines the type that will be passed as the callback from the TableStream class
+ ///
+ public class TablePacket
+ {
+ public DataTable newTable = new DataTable();
+ public DataTable addedRows = new DataTable();
+ public DataTable deletedRows = new DataTable();
+
+ public TablePacket () {}
+
+ public TablePacket (DataTable nt, DataTable ar, DataTable dr)
+ {
+ newTable = nt;
+ addedRows = ar;
+ deletedRows = dr;
+ }
+
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/Core/TableStream.cs b/src/DataStreamSuite/DataStream/Core/TableStream.cs
new file mode 100644
index 000000000..fef2b2524
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/TableStream.cs
@@ -0,0 +1,184 @@
+using System;
+using System.Collections;
+using System.Data;
+
+using BCG.Utilities;
+
+namespace BCG
+{
+ public class TableStream : ViewStream
+ {
+ public string tableName;
+ private ITableAccumulator myTA;
+ private bool myGotData = false;
+
+ public TableStream(string tableName)
+ :
+ this(tableName, null, null)
+ {
+ }
+
+ public TableStream(string tableName, ViewStreamDatabase db)
+ :
+ this(tableName, db, null)
+ {}
+
+ public TableStream(string tableName, string colExp)
+ :
+ this(tableName,
+ null,
+ colExp)
+ {}
+
+
+ public TableStream(string ttableName, ViewStreamDatabase db, string colExp)
+ {
+ tableName = PubStreamPrefixer.prefix (ttableName);
+ // if (colExp != null)
+ // mySetRealSource (new ProjectStream (new TableStream (tableName, db), colExp));
+ // else
+ {
+ if (db == null)
+ myTA = ViewStreamDatabase.instance.getITableAccumulator (tableName);
+ else
+ myTA = db.getITableAccumulator (tableName);
+ }
+ }
+
+ protected override void myOnFirstSubscriber()
+ {
+ myTA.onUpdate += new TableUpdateDelegate (myHandleUpdates);
+ }
+
+ protected override void myOnLastSubscriber()
+ {
+ myTA.onUpdate -= new TableUpdateDelegate (myHandleUpdates);
+ }
+
+ protected virtual void myHandleUpdates (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
+ {
+ if (!myGotData)
+ {
+ myGotData = true;
+ // BlueEventLog.logEvent("Received data for " + tableName + " for the first time", BlueEventLevel.Information);
+ }
+ myFireEvent (new DataPacket (newRows, addedRows, deletedRows));
+ }
+
+ public override string ToString()
+ {
+ return GetType() + " table = " + tableName;
+ }
+ }
+
+ public class ThreadedTableStream : TableStream, IExportableDataStream
+ {
+ static private Lazy _vsd = new Lazy(() => new ViewStreamDatabase(), true);
+ public ThreadedTableStream(string table) : base(table, _vsd.Value)
+ {
+ }
+
+ public void begin_subscription()
+ {
+ this.onChange += _stream_onChange;
+ }
+
+ public event ExportableChangeDelegate on_change;
+ void _stream_onChange(object newValue)
+ {
+ if(on_change != null)
+ on_change(newValue);
+ }
+ }
+
+ // public class ExportableTableStream : IExportableDataStream
+ // {
+ // TableStream _stream;
+ // public ExportableTableStream(string table, IDataPump pump)
+ // {
+ // _stream = new TableStream(table, ((ExportableDataPump)pump).vsd.Value);
+ // }
+
+ // public void begin_subscription()
+ // {
+ // _stream.onChange += _stream_onChange;
+ // }
+
+ // public event ExportableChangeDelegate on_change;
+ // void _stream_onChange(object newValue)
+ // {
+ // if(on_change != null)
+ // on_change(newValue);
+ // }
+ // }
+
+ // public class ExportableQuery : IExportableDataStream
+ // {
+ // ViewStream _stream;
+ // public ExportableQuery(string query, BCG.IDataPump _)
+ // {
+ // // BCG.Evaluator.DataStreamExpression.syntax parser = new BCG.Evaluator.DataStreamExpression.syntax();
+ // // BCG.Evaluator.DataStreamExpression.DataStreamExpression exp =
+ // // (BCG.Evaluator.DataStreamExpression.DataStreamExpression)parser.Parse(query);
+
+ // _stream = exp.getView(new System.Collections.Hashtable());
+ // }
+
+ // public void begin_subscription()
+ // {
+ // _stream.onChange += _stream_onChange;
+ // }
+
+ // public event ExportableChangeDelegate on_change;
+ // void _stream_onChange(object newValue)
+ // {
+ // if(on_change != null)
+ // on_change(newValue);
+ // }
+ // }
+
+ // public class ExportableDataPump : IDataPump
+ // {
+ // public Lazy vsd = new Lazy(() => new ViewStreamDatabase(), true);
+ // System.Threading.Thread _thread;
+ // bool _continue = true;
+ // public ExportableDataPump(string pumpname)
+ // {
+ // tag = pumpname;
+ // _thread = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart((_) =>
+ // {
+ // while(_continue)
+ // {
+ // vsd.Value.doWork();
+ // System.Threading.Thread.Sleep(50);
+ // }
+ // }));
+ // _thread.Name = "DataPump " + pumpname;
+ // _thread.IsBackground = true;
+ // }
+ // public string tag
+ // {
+ // get;
+ // private set;
+ // }
+ // public void start()
+ // {
+ // try
+ // {
+ // if(_thread.ThreadState != System.Threading.ThreadState.Running)
+ // _thread.Start();
+ // }
+ // catch(System.Threading.ThreadStateException) { }
+ // }
+ // public void stop()
+ // {
+ // try
+ // {
+ // _continue = false;
+ // if(_thread.ThreadState == System.Threading.ThreadState.Running)
+ // _thread.Join(500);
+ // }
+ // catch(System.Threading.ThreadStateException) { }
+ // }
+ // }
+}
diff --git a/src/DataStreamSuite/DataStream/Core/ViewStream.cs b/src/DataStreamSuite/DataStream/Core/ViewStream.cs
new file mode 100644
index 000000000..22084a5e4
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/ViewStream.cs
@@ -0,0 +1,97 @@
+using System;
+//using BCG.Utilities.BlueConfiguration;
+
+using BCG.Utilities;
+
+namespace BCG
+{
+ ///
+ /// Marker abstract class
+ ///
+ public abstract class ViewStream : DataStream
+ {
+ private IBlueRowCollection myLastValueForTestMode;
+ protected bool myUseUncopiedNewRowsInFireEvent = DataStreamLoadFileConfig.instance.useUncopiedNewRowsInFireEvent;
+
+ public ViewStream (ViewStream sourceStream)
+ :
+ base(sourceStream)
+ {}
+
+ public ViewStream()
+ {}
+
+ public IBlueRowCollection rows
+ {
+ get
+ {
+ if (myRealSource != null)
+ {
+ // BlueEventLog.logAssert (myRealSource is ViewStream);
+
+ return (myRealSource as ViewStream).rows;
+ }
+ else
+ {
+ // used to force a computation of a stream which
+ // happens to have no subscribers
+ onChange += new DataStream.ChangeDelegate(myDummy);
+ try
+ {
+
+ if (myLastValue != null)
+ return ((DataPacket) myLastValue).newRows;
+ else
+ return null;
+ }
+ finally
+ {
+ onChange -= new DataStream.ChangeDelegate(myDummy);
+ }
+ }
+ }
+ }
+
+ protected override object myGetValueForInitialSubscription()
+ {
+ if (myLastValue == null)
+ return null;
+ else
+ {
+ return new DataPacket(
+ (myLastValue as DataPacket).newRows,
+ (myLastValue as DataPacket).newRows,
+ new BlueTable((myLastValue as DataPacket).newRows.schema));
+ }
+
+ }
+
+ protected override void myFireEvent(object o)
+ {
+ DataPacket dp = (DataPacket) o;
+
+ // BlueTrace.WriteViewStreamFireEvent(this.ID,
+ // dp.newRows == null ? 0 : dp.newRows.Count,
+ // dp.addedRows == null ? 0 : dp.addedRows.Count,
+ // dp.deletedRows == null ? 0 : dp.deletedRows.Count);
+
+ if (myTestMode)
+ {
+ // BlueEventLog.logAssert ((myFirstSubImplemented && myLastSubImplemented) | myRealSource != null,
+ // @"either myOnFirstSub/myOnLastSub should be implemented or mySetRealSource
+ // called with a non-null arg in " + this.GetType().FullName);
+ if (myLastValueForTestMode != null)
+ BlueDataUtility.checkConsistency(myLastValueForTestMode,
+ dp.newRows, dp.deletedRows, dp.addedRows);
+ myLastValueForTestMode = new ImmutableBlueTable(dp.newRows);
+ }
+ // the view already garantees that added & deleted are immutable.
+ // this garantess that new is also immutable
+ base.myFireEvent(myUseUncopiedNewRowsInFireEvent ? dp : dp.createImmutable());
+
+ }
+
+ private void myDummy (object o)
+ {}
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/Core/ViewStreamDatabase.cs b/src/DataStreamSuite/DataStream/Core/ViewStreamDatabase.cs
new file mode 100644
index 000000000..1c5b3987b
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Core/ViewStreamDatabase.cs
@@ -0,0 +1,312 @@
+using System;
+using System.Collections;
+using System.IO;
+using BCG.Utilities;
+using BCG.Utilities.General;
+//using BCG.Utilities.BlueConfiguration;
+using System.Collections.Generic;
+using System.Threading;
+using mbts;
+
+namespace BCG
+{
+ public class ViewStreamDatabase : IWorker, IDisposable
+ {
+ internal delegate void TagFireDelegate(string tag);
+ public delegate void DoWorkDelegate ();
+ public event DoWorkDelegate onDoWorkStart,
+ onDoWorkEnd;
+ internal event TagFireDelegate onTagFireStart,
+ onTagFireEnd;
+ //currently onTagUpdate is usd by the DataStreamPerfRecorder to keep track of how often the updates for various streams come in
+ //and how big they are.
+ //public delegate void OnTagUpdateDelegate(PubMetaData metaData);
+ //public event OnTagUpdateDelegate onTagUpdate;
+ private IDictionary myITableAccumulatorMap = new Hashtable ();
+ private static ViewStreamDatabase mySelf;
+ private ISubscriberCore mySubscriberCore;
+ private ArrayList myWorkers = new ArrayList ();
+ private bool myIsDisposed = false;
+ private bool myIsInDoWork = false;
+ private static object myTimeInterval;
+ private HighPerformanceTimer myTimer = new HighPerformanceTimer();
+ private double myLogTimeThreshold = 10000;
+ private readonly double _dowork_threshold = 500;
+ private Thread myThreadOwner = null; // ViewStreamDatabase takes a thread affinity when created. If called from a different thread it will assert.
+
+ private static readonly object myInstanceLockObject = new object();
+ private readonly object myStateLock = new object();
+
+ public ViewStreamDatabase ()
+ : this (new SubscriberCore ())
+ {
+ }
+
+ public ViewStreamDatabase (ISubscriberCore subscriberCore)
+ {
+ mySubscriberCore = subscriberCore;
+ // if (DataStreamLoadFileConfig.instance.profileStreams)
+ // new DataStreamPerfRecorder(this);
+ }
+
+ private static int myGetTimeInterval()
+ {
+ if (myTimeInterval == null)
+ {
+ // TODO
+ string timeInterval = String.Empty; //ConfigurationRequester.request("viewStreamDatabaseTimeOut");
+ if (timeInterval == null || timeInterval == "")
+ myTimeInterval = 0;
+ else
+ myTimeInterval = Int32.Parse(timeInterval);
+ }
+ return (int) myTimeInterval;
+ }
+
+ public static int instance_hashcode = -1;
+ public static ViewStreamDatabase instance
+ {
+ get
+ {
+ if(mySelf == null || mySelf.myIsDisposed)
+ lock(myInstanceLockObject)
+ {
+ if (mySelf == null || mySelf.myIsDisposed)
+ {
+ mySelf = new ViewStreamDatabase();
+ instance_hashcode = mySelf.GetHashCode();
+ }
+ }
+ return mySelf;
+ }
+ set
+ {
+ lock(myInstanceLockObject)
+ {
+ if (mySelf != null)
+ mySelf.mySubscriberCore.Dispose ();
+ mySelf = value;
+ instance_hashcode = mySelf.GetHashCode();
+ }
+ }
+ }
+
+ public ICollection getCachedTables()
+ {
+ List result = new List();
+
+
+ foreach (string s in myITableAccumulatorMap.Keys)
+ if (((ITableAccumulator)myITableAccumulatorMap[s]).isActive())
+ result.Add(s);
+
+ return result;
+ }
+
+ public ISubscriberCore subscriber{get{return mySubscriberCore;}}
+
+
+ public ITableAccumulator getITableAccumulator (string tableName)
+ {
+ // If theres a ITableAccumulator for the tableName its returned (to share subscriptions), otherwise
+ // a new one is obtained from the Subscriber factory
+
+ myITableAccumulatorMap[tableName] = myITableAccumulatorMap[tableName] == null ?
+ AccumulatorFactory.getAccumulator(tableName, mySubscriberCore) : myITableAccumulatorMap[tableName];
+
+ return myITableAccumulatorMap [tableName] as ITableAccumulator;
+ }
+
+ public void doWork ()
+ {
+ verifyAccess();
+
+ if (myIsSuspended)
+ return;
+ if (myIsInDoWork)
+ return;
+ else
+ {
+ myIsInDoWork = true; // to allow reentrancy
+ try
+ {
+ if (onDoWorkStart != null)
+ onDoWorkStart();
+
+ ArrayList updateList = mySubscriberCore.getUpdates(myGetTimeInterval());
+ IDictionary updateMap = myGetUpdateMap(updateList);
+
+ foreach (DictionaryEntry de in updateMap)
+ {
+ Stream[] allUpdates = (de.Value as ArrayList).ToArray(typeof(Stream)) as Stream[];
+ string tag = de.Key.ToString();
+ if (myITableAccumulatorMap[tag] == null){}
+ // BlueEventLog.logEvent("Received a SubscriberCore's callback for a tag that was never subscribed", BlueEventLevel.Warning);
+ else
+ {
+ myTimer.start();
+
+
+ if (onTagFireStart != null)
+ onTagFireStart(tag);
+
+ (myITableAccumulatorMap[tag] as ITableAccumulator).update(allUpdates);
+ if (onTagFireEnd != null)
+ onTagFireEnd(tag);
+ myTimer.stop();
+ if (myTimer.lastIntervalTime > myLogTimeThreshold)
+ {
+ // BlueEventLog.logEvent("callback on tag " + tag.ToUpper() + " took " +
+ // myTimer.lastIntervalTime + " msecs", BlueEventLevel.Warning);
+ }
+ }
+ }
+
+ // to allow for registering of workers on the worker callback
+ myIsCallingWorkers = true;
+ myPendingOps = new ArrayList();
+ foreach (IWorker w in myWorkers)
+ {
+ myTimer.start();
+ w.doWork();
+ myTimer.stop();
+ if (myTimer.lastIntervalTime > _dowork_threshold){}
+ // mbts.Logger.log (bts.log.Level.Warn, bts.log.Subsystem.gen, String.Format ("VSD.dowork on worker {0} took {1}", w.GetType().ToString (), myTimer.lastIntervalTime));
+ }
+ myIsCallingWorkers = false;
+
+ foreach (PendingAddRemoveWorker op in myPendingOps)
+ if (op.isAdd)
+ addWorker(op.w);
+ else
+ removeWorker(op.w);
+ }
+ finally
+ {
+ myPendingOps = null;
+ myIsInDoWork = false;
+ myIsCallingWorkers = false;
+ if (onDoWorkEnd != null)
+ onDoWorkEnd ();
+ }
+ }//end if we're not in do work
+
+ }
+
+ private bool myIsSuspended = false;
+
+ public void suspend() { myIsSuspended = true; }
+
+ public void resume() { myIsSuspended = false; }
+
+ public bool checkAccess()
+ {
+ if (this.myThreadOwner == null) this.myThreadOwner = Thread.CurrentThread;
+ return this.myThreadOwner == Thread.CurrentThread;
+ }
+
+ public void verifyAccess()
+ {
+ if (!checkAccess())
+ {
+ string errorMessage = string.Format("Invalid cross thread access on ViewStreamDatabase. Owning thread: {0}. Accessing thread: {1}", this.myThreadOwner.ManagedThreadId, Thread.CurrentThread.ManagedThreadId);
+
+//#if DEBUG
+// BlueEventLog2.instance().assert(false, errorMessage);
+//#else
+// BlueEventLog2.instance().logEvent(errorMessage, BlueEventLog2Level.Fatal);
+//#endif
+
+ }
+ }
+
+ private IDictionary myGetUpdateMap (ArrayList updates)
+ {
+ IDictionary retVal = new Hashtable ();
+ foreach (PubSubUpdate update in updates)
+ {
+ if (retVal[update.tag] == null)
+ retVal[update.tag] = new ArrayList ();
+ (retVal[update.tag] as ArrayList).Add (update.body);
+ }
+ if (retVal.Count > 0)
+ {
+ string tagString = "";
+ foreach (string tag in retVal.Keys)
+ tagString += tag + " , ";
+ }
+ return retVal;
+ }
+
+ public bool isConsistent{get {return true;}}
+
+ public void addWorker(IWorker w)
+ {
+ verifyAccess();
+ if (myIsCallingWorkers)
+ myPendingOps.Add(new PendingAddRemoveWorker(true,w));
+ else
+ myWorkers.Add(w);
+ }
+
+
+ public void removeWorker(IWorker w)
+ {
+ verifyAccess();
+ if (myIsCallingWorkers)
+ myPendingOps.Add(new PendingAddRemoveWorker(false,w));
+ else
+ myWorkers.Remove(w);
+ }
+
+ #region IDisposable Members
+
+ public void Dispose()
+ {
+ if (!myIsDisposed)
+ {
+ myIsDisposed = true;
+ mySubscriberCore.Dispose ();
+ }
+ }
+
+ private ArrayList myPendingOps;
+ private bool myIsCallingWorkers = false;
+ class PendingAddRemoveWorker
+ {
+ public PendingAddRemoveWorker(bool isAdd, IWorker w)
+ {
+ this.isAdd = isAdd;
+ this.w = w;
+ }
+
+ public bool isAdd;
+ public IWorker w;
+ }
+
+ #endregion
+ }
+
+ /*
+ ///
+ /// Returns a subscriber based on the name of the tag. Hardcoded right now, will change to config
+ /// later
+ ///
+ public class SubscriberFactory
+ {
+ public static ITableAccumulator getITableAccumulator (string tag, SubscriberCore subCore)
+ {
+ string tableName = tag.Substring (3);
+ if (!DataStreamGatewaySettings.instance.isSerializableUnderNewScheme (tableName))
+ return new TableSubscriber (tag, subCore);
+ else
+ {
+ string subName = DataStreamGatewaySettings.instance.getSubscriber (tableName);
+ if (subName.Equals ("BCG.NonIncrementalTableSubscriber"))
+ return new NonIncrementalTableSubscriber (tag, subCore);
+ else
+ return new TableSubscriber (tag, subCore);
+ }
+ }
+ }*/
+}
diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj
index 9d6b6f5de..db079e6d4 100644
--- a/src/DataStreamSuite/DataStream/DataStream.csproj
+++ b/src/DataStreamSuite/DataStream/DataStream.csproj
@@ -1,5 +1,7 @@
-
+
+
+
net7.0
enable
diff --git a/src/DataStreamSuite/DataStream/DynamicLoading/DataStreamUserRespresentation.cs b/src/DataStreamSuite/DataStream/DynamicLoading/DataStreamUserRespresentation.cs
new file mode 100644
index 000000000..86dda7cc9
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/DynamicLoading/DataStreamUserRespresentation.cs
@@ -0,0 +1,24 @@
+using System;
+
+namespace BCG
+{
+ public class DataStreamUserRespresentation : Attribute
+ {
+ private bool myIsVisible = true;
+ private string myFriendlyName;
+
+ public DataStreamUserRespresentation (string friendlyName)
+ {
+ myFriendlyName = friendlyName;
+ }
+
+ public DataStreamUserRespresentation (bool isVisible)
+ {
+ myIsVisible = isVisible;
+ }
+
+ public bool isVisible {get{return myIsVisible;}}
+
+ public string friendlyName {get{return myFriendlyName;}}
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/DynamicLoading/PrimitiveManager.cs b/src/DataStreamSuite/DataStream/DynamicLoading/PrimitiveManager.cs
new file mode 100644
index 000000000..b39832da0
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/DynamicLoading/PrimitiveManager.cs
@@ -0,0 +1,192 @@
+using System;
+using System.Collections;
+using System.Reflection;
+using System.IO;
+
+//using BCG.Utilities.BlueConfiguration;
+//using BCG.Utilities;
+
+namespace BCG
+{
+ public class PrimitiveManager
+ {
+ private IDictionary myTypeLookupTable = new Hashtable ();
+ private static PrimitiveManager mySelf;
+ private ArrayList myAssemblyNames = new ArrayList ();
+
+ private PrimitiveManager()
+ {
+ myLoadAllTypes ();
+ }
+
+ private void myLoadAllTypes ()
+ {
+ DataStreamLoadFileConfig config = DataStreamLoadFileConfig.instance;
+
+ AppDomain currentDomain = AppDomain.CurrentDomain;
+ currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
+ loadAssembly (this.GetType ().Assembly.FullName);
+
+ foreach (string assemblyName in config.loadedAssemblies)
+ {
+ string resolvedAssemblyPath = assemblyName;
+ if (!System.IO.Path.IsPathRooted(assemblyName))
+ {
+ string appPath = Path.GetDirectoryName(
+ Assembly.GetAssembly(typeof(PrimitiveManager)).CodeBase.Substring("file:///".Length));
+ resolvedAssemblyPath = System.IO.Path.GetFullPath(System.IO.Path.Combine(appPath, assemblyName));
+ }
+ loadAssembly(resolvedAssemblyPath);
+ }
+ }
+
+ private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args)
+ {
+ foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
+ if (args.Name.Equals(a.FullName))
+ return a;
+ return null;
+ }
+
+ ///
+ /// Loads all the Stream types associated with the assembly
+ ///
+ ///
+ public void loadAssembly (string assemblyName)
+ {
+ if (myAssemblyNames.Contains (assemblyName))
+ return;
+ myAssemblyNames.Add (assemblyName);
+ Assembly loadedAssembly = GetType ().Assembly.FullName.Equals (assemblyName) ?
+ GetType ().Assembly :
+ Assembly.LoadFrom (assemblyName);
+
+ foreach (Type type in loadedAssembly.GetTypes())
+ if (!type.IsInterface
+ && !type.IsAbstract
+ && type.IsSubclassOf (typeof(ViewStream))
+ && type.IsPublic)
+ {
+ bool isVisible = true;
+ string userRep = createDefaultName (type.Name);
+ foreach (Attribute att in type.GetCustomAttributes (false))
+ if (att is DataStreamUserRespresentation)
+ {
+ DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
+ isVisible = dsatt.isVisible;
+ userRep = isVisible ?
+ dsatt.friendlyName == null ? userRep : dsatt.friendlyName
+ : null;
+ break;
+ }
+ if (isVisible)
+ {
+ // BlueEventLog.logAssert(myTypeLookupTable [userRep] == null);
+ myTypeLookupTable[userRep] = type;
+ }
+ }
+ }
+
+ public static string createDefaultName (string typeName)
+ {
+ if (!typeName.EndsWith ("Stream"))
+ return typeName;
+ string retVal = typeName.Remove (typeName.Length - "Stream".Length, "Stream".Length);
+ string startString = new String (new char[] {retVal[0]}).ToLower ();
+ retVal = retVal.Remove (0, 1);
+ retVal = retVal.Insert (0, startString);
+ return retVal;
+ }
+
+
+
+ public static PrimitiveManager instance
+ {
+ get {return mySelf = mySelf == null ? new PrimitiveManager () : mySelf;}
+ }
+
+ ///
+ /// Returns the Type name corresponding to the primitive name
+ ///
+ ///
+ ///
+ public string find (string friendlyName)
+ {
+ if (myTypeLookupTable[friendlyName] == null)
+ throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types");
+ return (myTypeLookupTable[friendlyName] as Type).Name;
+ }
+
+ ///
+ /// Constructs and returns a ViewStream of type corresponding to the friendly name
+ ///
+ /// friendly primitive name
+ /// args can be null or an empty ICollection if the paramterless constructor
+ /// is the desired one
+ ///
+ public ViewStream build (string friendlyName, ICollection args)
+ {
+ object[] constructorArgs = null;
+ if (args != null && args.Count > 0)
+ {
+ constructorArgs = new object [args.Count];
+ args.CopyTo (constructorArgs, 0);
+ }
+ if (myTypeLookupTable[friendlyName] == null)
+ throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types");
+ return
+ constructorArgs == null ?
+ Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type) as ViewStream :
+ Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
+ }
+
+ ///
+ /// Returns the name of all the primitives
+ ///
+ ///
+ public string[] getPrimitiveNames ()
+ {
+ string[] retVal = new string [myTypeLookupTable.Count];
+ myTypeLookupTable.Keys.CopyTo (retVal, 0);
+ return retVal;
+ }
+
+ ///
+ /// Return the types of all the streams
+ ///
+ ///
+ public Type[] getStreamTypes ()
+ {
+ Type[] retVal = new Type [myTypeLookupTable.Count];
+ myTypeLookupTable.Values.CopyTo (retVal, 0);
+ return retVal;
+ }
+
+ public ICollection getConstructorSigns (string primitiveName)
+ {
+ ArrayList retVal = new ArrayList ();
+ // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null);
+ Type t = myTypeLookupTable[primitiveName] as Type;
+ foreach (ConstructorInfo ci in t.GetConstructors())
+ {
+ string sign = "";
+ sign = t.Name + " (";
+ foreach (ParameterInfo pi in ci.GetParameters ())
+ sign += pi.ParameterType.Name +
+ " " +
+ pi.Name + ", ";
+ sign = sign.Length == (t.Name + " (").Length ? sign + ")": sign.Remove (sign.Length - 2, 2) + ")";
+ retVal.Add (sign);
+ }
+ return retVal;
+ }
+
+ public string getAssemblyName (string primitiveName)
+ {
+ // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null);
+ return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
+
+ }
+
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/PrimitiveManager.cs b/src/DataStreamSuite/DataStream/PrimitiveManager.cs
deleted file mode 100644
index 8c986d696..000000000
--- a/src/DataStreamSuite/DataStream/PrimitiveManager.cs
+++ /dev/null
@@ -1,14 +0,0 @@
-namespace BCG {
- using System;
- using System.Collections;
-
- public class PrimitiveManager {
- public static PrimitiveManager instance => throw new NotImplementedException();
-
- public ViewStream build(string friendlyName, ICollection args) =>
- throw new NotImplementedException();
-
- public string find(string friendlyName) =>
- throw new NotImplementedException();
- }
-}
diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs
new file mode 100644
index 000000000..73c3e72f2
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs
@@ -0,0 +1,105 @@
+using System;
+using System.Collections;
+
+namespace BCG
+{
+ internal class AddColumnsRow : IBlueRow
+ {
+ private IBlueTableSchema mySchema;
+ private IBlueRow myRow;
+ //private object[] myCachedValues;
+
+ #region Constructors
+ public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
+ {
+ mySchema = schema;
+ myRow = row;
+ //myCachedValues = new object[schema.columnNames.Length];
+ }
+ #endregion
+
+ #region IBlueRow members
+ public IBlueTableSchema schema{get{return mySchema;}}
+ #endregion
+
+ #region ICollection members
+ public int Count {get{return mySchema.columnNames.Length;}}
+
+ public bool IsSynchronized {get{return false;}}
+
+ public object SyncRoot {get{return this;}}
+
+ public void CopyTo (Array array, int index)
+ {
+ int i = index;
+ foreach (object o in this as IEnumerable)
+ array.SetValue (o, i++);
+ }
+ #endregion
+
+ #region IDictionary members
+ public bool IsFixedSize {get{return true;}}
+
+ public bool IsReadOnly {get{return true;}}
+
+ ///
+ /// The key can either be the string corrsponding to the column name or
+ /// the order of the column
+ ///
+ public object this [object key]
+ {
+ get
+ {
+ if (key is string)
+ return this[(string)key];
+ return this[(int)key];
+ }
+ set {throw new NotSupportedException ("The IBlueRow is readonly");}
+ }
+
+ public object this[int index]
+ {
+ get
+ {
+ //if (myCachedValues[index] == null)
+ // myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+ //return myCachedValues[index];
+ return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+ }
+ }
+
+ public object this[string colName] {get{return this[mySchema[colName].colNo];}}
+
+ public ICollection Keys{get{return mySchema.columnNames;}}
+
+ public ICollection Values
+ {
+ get
+ {
+ object[] retVal = new object[mySchema.columnNames.Length];
+ CopyTo (retVal, 0);
+ return retVal;
+ }
+ }
+
+ public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+ public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+ public bool Contains (object key){return mySchema[(string)key] != null;}
+
+ ///
+ /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+ /// iterate through the values, typecast this into an IEnumerable
+ ///
+ ///
+ public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
+
+ IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
+
+ public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+ #endregion
+
+ }
+
+}
diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs
new file mode 100644
index 000000000..bb2500791
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs
@@ -0,0 +1,13 @@
+// using System;
+// using System.Data;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Utilities.General;
+// namespace BCG
+// {
+
+// public class AddColumnsStream : MapStream
+// {
+// public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef)) {}
+// }
+// }
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddRegexColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddRegexColumnsStream.cs
new file mode 100644
index 000000000..4f6492719
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddRegexColumnsStream.cs
@@ -0,0 +1,13 @@
+
+// using System;
+
+// namespace BCG {
+
+// public class AddRegexColumnsStream : ViewStream {
+
+// public AddRegexColumnsStream (ViewStream vs, string srcCol, string regex, string newColDefs) {
+// IStreamMapper mapper = new RegexMatchAdder (srcCol, newColDefs, regex);
+// mySetRealSource (new MapStream (vs, mapper));
+// }
+// }
+// }
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs
new file mode 100644
index 000000000..ac1336fe6
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs
@@ -0,0 +1,73 @@
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.AddColumnsExpression;
+
+// namespace BCG
+// {
+// ///
+// /// Provides methods to add columns to the schema
+// ///
+// public class ColumnAdder : IStreamMapper
+// {
+// protected IDictionary myNewColumns;
+// protected IBlueTableSchema mySchema;
+
+// static syntax myParser =
+// new syntax();
+
+// ///
+// /// string represting the columns to be added to the output table.
+// ///
+// ///
+// public ColumnAdder (string columnDef)
+// {
+// AddColumnsExpression ace;
+
+// if (DataStreamLoadFileConfig.instance.useStaticParser)
+// {
+// lock (myParser)
+// {
+// ace =
+// (AddColumnsExpression)myParser.Parse(columnDef);
+// }
+// }
+// else
+// {
+// syntax parser = new syntax();
+// ace =
+// (AddColumnsExpression)parser.Parse(columnDef);
+// }
+
+// if (ace == null)
+// throw new Exception ("Invalid columnn expression: " + columnDef);
+// myNewColumns = ace.columns;
+// }
+
+// public virtual IBlueRow map (IBlueRow row)
+// {
+// if (mySchema == null)
+// myConstructSchema (row.schema);
+// return new AddColumnsRow (mySchema, row);
+// }
+
+// public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
+// {
+// myConstructSchema (inputSchema);
+// return mySchema;
+// }
+
+// private void myConstructSchema (IBlueTableSchema schema)
+// {
+// ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
+// int ciIndex = 0;
+// foreach (ColumnInfo ci in schema.columnInfo)
+// cis[ciIndex++] = ci;
+// foreach (DictionaryEntry de in myNewColumns)
+// cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++,
+// ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
+// mySchema = new BlueTableSchema (cis, schema.tableName);
+// }
+
+// }
+// }
diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/MathStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/MathStream.cs
new file mode 100644
index 000000000..38ace9708
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/MathStream.cs
@@ -0,0 +1,100 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+
+// namespace BCG
+// {
+
+// public class BooleanStringReplaceStream : ViewStream, IStreamMapper
+// {
+// private BooleanExpressionFilter _bool;
+// private string _column, _value;
+// private IBlueTableSchema _schema;
+// public BooleanStringReplaceStream(ViewStream vs, string boolean_expression, string column, string value)
+// {
+// _column = column;
+// _value = value;
+// if(!string.IsNullOrEmpty(boolean_expression))
+// _bool = new BooleanExpressionFilter(boolean_expression);
+// mySetRealSource(new MapStream(vs, this));
+// }
+// public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+// {
+// _schema = inputSchema;
+// return inputSchema;
+// }
+// public IBlueRow map(IBlueRow row)
+// {
+// MutableBlueRow result = new MutableBlueRow(_schema, row);
+// if(_bool == null || _bool.filter(row))
+// result.setValue(_column, _value);
+// return result;
+// }
+
+// }
+
+// public class MathStream : ViewStream, IStreamMapper
+// {
+// private ArithmeticExpressionEvaluator.ArithmeticExpression myExp;
+// private BooleanExpressionFilter _bool;
+// List myExtraColumns = new List();
+// BlueTableSchema mySchema;
+
+
+// public MathStream(ViewStream vs, string mathExpression)
+// : this(vs, mathExpression, string.Empty)
+// {
+// }
+
+// public MathStream(ViewStream vs, string mathExpression, string boolean_expression)
+// {
+// myExp = ArithmeticExpressionEvaluator.compile(mathExpression);
+// myExtraColumns = ArithmeticExpressionEvaluator.getOutputIdentifiers(myExp);
+// if(!string.IsNullOrEmpty(boolean_expression))
+// _bool = new BooleanExpressionFilter(boolean_expression);
+// mySetRealSource(new MapStream(vs, this));
+// }
+
+// #region IStreamMapper Members
+
+// public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+// {
+// if (mySchema == null)
+// {
+// Dictionary colIdx = new Dictionary();
+// foreach (ColumnInfo ci in inputSchema.columnInfo)
+// colIdx[ci.colName] = ci;
+
+// foreach (string ncn in myExtraColumns)
+// {
+// ColumnInfo newC;
+// if (colIdx.ContainsKey(ncn))
+// newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
+// else
+// newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
+// colIdx[ncn] = newC;
+// }
+// mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
+// }
+// return mySchema;
+// }
+
+// #endregion
+
+// #region IDynamicStreamMapper Members
+
+// public IBlueRow map(IBlueRow row)
+// {
+// MutableBlueRow result = new MutableBlueRow(mySchema, row);
+// if(_bool == null || _bool.filter(row))
+// {
+// foreach(var kvp in ArithmeticExpressionEvaluator.eval(myExp, (System.Collections.IDictionary)result))
+// result.setValue(kvp.Key, kvp.Value);
+// }
+// return result;
+// }
+
+// #endregion
+// }
+// }
diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/RegexMatchAdder.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/RegexMatchAdder.cs
new file mode 100644
index 000000000..75e72a870
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/RegexMatchAdder.cs
@@ -0,0 +1,48 @@
+
+// using System;
+// using System.Collections;
+// using System.Collections.Generic;
+// using System.Text.RegularExpressions;
+// using System.ComponentModel;
+
+// using BCG.Utilities;
+
+// namespace BCG {
+
+// public class RegexMatchAdder : ColumnAdder {
+
+// private int[] myAddedColNos;
+// private string mySrcCol;
+// private Regex myRegex;
+
+// public RegexMatchAdder (string srcCol, string columnDef, string regex) : base (columnDef) {
+// mySrcCol = srcCol;
+// myRegex = new Regex (regex);
+// }
+
+// public override IBlueRow map (IBlueRow row) {
+// MutableBlueRow result = new MutableBlueRow (mySchema, row);
+
+// if (myAddedColNos == null) {
+// myAddedColNos = new int[myNewColumns.Count];
+// for (int i = 0; i < myAddedColNos.Length; ++i)
+// myAddedColNos[i] = i + row.Count;
+// }
+
+// object[] values = new object[myNewColumns.Count];
+// string src = (string)row[mySrcCol];
+// // extract the requested regex substrings
+// Match m = myRegex.Match (src);
+// for (int icol = 0; icol < myAddedColNos.Length; ++icol) {
+// int i = icol + row.Count;
+// Group g = m.Groups[mySchema.columnNames[i]];
+// object val = TypeDescriptor.GetConverter (mySchema[i].colType)
+// .ConvertFromString (g.Success ? g.Value: string.Empty);
+// result.setValue (i, val);
+// }
+
+// return result;
+// }
+
+// }
+// }
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/XYTranspositionStream.cs b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/XYTranspositionStream.cs
new file mode 100644
index 000000000..9b31ce7c1
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/XYTranspositionStream.cs
@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BCG
+{
+ public class XYTranspositionStream : ViewStream
+ {
+ private ViewStream _stream;
+ private IBlueTableSchema _schema;
+ private string[] _transposed_cols;
+
+ public XYTranspositionStream(ViewStream stream, string transposed_cols)
+ {
+ _transposed_cols = transposed_cols.Split(' ');
+ _stream = stream;
+ }
+
+ protected override void myOnFirstSubscriber()
+ {
+ _stream.onChange += _stream_onChange;
+ }
+
+ protected override void myOnLastSubscriber()
+ {
+ if (_stream != null)
+ _stream.onChange -= _stream_onChange;
+ }
+
+ private void _stream_onChange(object newValue)
+ {
+ DataPacket dp = newValue as DataPacket;
+ if (dp != null)
+ {
+ if (_schema == null)
+ _make_schema(dp.schema);
+ BlueTable result = new BlueTable(_schema);
+ int id = 1;
+ foreach (IBlueRow row in dp.newRows)
+ {
+ foreach (string transposed_col in _transposed_cols)
+ {
+ MutableBlueRow mbr = new MutableBlueRow(_schema);
+ for (int i = 0; i < _schema.columnNames.Length; i++)
+ {
+ if (_schema.columnNames[i] == "id")
+ mbr.setValue(i, id++);
+ else if (_schema.columnNames[i] == "param_name")
+ mbr.setValue(i, transposed_col);
+ else if (_schema.columnNames[i] == "param_value")
+ mbr.setValue(i, row[transposed_col]);
+ else
+ mbr.setValue(i, row[_schema.columnNames[i]]);
+ }
+ result.addRow(mbr);
+ }
+ }
+ myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(_schema) : (myLastValue as DataPacket).newRows));
+ }
+ }
+
+ private void _make_schema(IBlueTableSchema input_schema)
+ {
+ List info = new List();
+ int col_counter = 0;
+ info.Add(new ColumnInfo("id", col_counter++, typeof(int)));
+ info.Add(new ColumnInfo("param_name", col_counter++, typeof(string)));
+ info.Add(new ColumnInfo("param_value", col_counter++, typeof(object)));
+ foreach (ColumnInfo ci in input_schema.columnInfo)
+ if (!_is_transposing_column(ci.colName) && ci.colName != "id")
+ info.Add(new ColumnInfo(ci.colName, col_counter++, ci.colType));
+ _schema = new BlueTableSchema(info.ToArray(), String.Format("xy_transposed_{0}", input_schema.tableName));
+ }
+
+ private bool _is_transposing_column(string col_name)
+ {
+ foreach (string tc in _transposed_cols)
+ if (col_name == tc)
+ return true;
+ return false;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs
new file mode 100644
index 000000000..9ef52fdfe
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs
@@ -0,0 +1,244 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
+
+namespace BCG
+{
+ public class YXTranspositionStream : ViewStream, INonIncGroupByEvaluator
+ {
+ private class OnlyFireOnceStream : ViewStream
+ {
+ private ViewStream _stream;
+ private bool _hooked = false;
+
+ public OnlyFireOnceStream(ViewStream stream)
+ {
+ _stream = stream;
+ }
+
+ protected override void myOnFirstSubscriber()
+ {
+ _stream.onChange += _stream_onChange;
+ _hooked = true;
+ }
+
+ protected override void myOnLastSubscriber()
+ {
+ if (_hooked && _stream != null)
+ _stream.onChange -= _stream_onChange;
+ }
+
+ private void _stream_onChange(object newValue)
+ {
+ DataPacket dp = newValue as DataPacket;
+ if (dp != null && dp.newRows.Count > 0)
+ {
+ _stream.onChange -= _stream_onChange;
+ myFireEvent(new DataPacket(dp.newRows, dp.newRows, new BlueTable(dp.newRows.schema)));
+ }
+ }
+ }
+
+ public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
+ public delegate string GetColumnPrefix(object[] transposing_col_values);
+
+ public delegate string GetColumnSuffix(string col_name);
+ private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
+
+ private IncrementValues _increment_values;
+ private GetColumnPrefix _get_column_prefix;
+
+ private IBlueTableSchema _schema;
+
+ private string[] _group_by_cols;
+ private Pair[] _cols_per_transposed;
+ private string[] _transposing_cols;
+
+ private int[] _transposing_col_indices;
+
+ private Comparison> _transpsed_col_value_comparer_wrapper;
+
+ public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
+ IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison