diff --git a/docker/linux b/docker/linux new file mode 160000 index 000000000..653781226 --- /dev/null +++ b/docker/linux @@ -0,0 +1 @@ +Subproject commit 65378122620c7a26561641cb49b477c822b315f2 diff --git a/docker/windows b/docker/windows new file mode 160000 index 000000000..de7d40c09 --- /dev/null +++ b/docker/windows @@ -0,0 +1 @@ +Subproject commit de7d40c09de473e1b2802c4b27e03dc7f0c360d6 diff --git a/src/DataStreamSuite/DataStream/Config/DataStreamLoadFileConfig.cs b/src/DataStreamSuite/DataStream/Config/DataStreamLoadFileConfig.cs new file mode 100644 index 000000000..72dd6f81f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Config/DataStreamLoadFileConfig.cs @@ -0,0 +1,116 @@ +using System; + +//using BCG.Utilities.BlueConfiguration; + +namespace BCG +{ + public class DataStreamLoadFileConfig //: ConfigurationRequest + { + private static DataStreamLoadFileConfig myInstance; + private string[] myAssemblies; + private string myIsTestMode = "false"; + private bool myProfileStreams = false; + private double myProfileThreshold = 500; + private bool myUseNewJoin, myUseIncrementsForMultipleOps, myUseNewAppend, myUseNewFilter, myUseNewMap, + myUseUncopiedNewRowsInFireEvent, myUseModularizedBridge; + + private bool myLogBlobPackets = false; + private bool myUseStaticParser = true; + + //[Description("DataStream: list of assemblies providing the streams")] + public string [] loadedAssemblies + { + get { return myAssemblies; } + set { myAssemblies = value; } + } + + public bool useStaticParser + { + get { return myUseStaticParser; } + set { myUseStaticParser = value; } + } + + public bool logBlobPackets + { + get { return myLogBlobPackets; } + set { myLogBlobPackets = value; } + } + + public string isTestMode + { + get {return myIsTestMode;} + set {myIsTestMode = value;} + } + + public bool profileStreams + { + get { return myProfileStreams; } + set { myProfileStreams = Convert.ToBoolean(value); } + } + + public double profileThreshold + { + get { return myProfileThreshold; } + set { myProfileThreshold = Convert.ToDouble(value); } + } + + public bool isValid() + { + return true; + //return myVerify (myAssemblies); + } + + public bool useNewJoin + { + get { return myUseNewJoin; } + set { myUseNewJoin = value; } + } + + public bool useNewAppend + { + get { return myUseNewAppend; } + set { myUseNewAppend = value; } + } + + public bool useNewFilter + { + get { return myUseNewFilter; } + set { myUseNewFilter = value; } + } + + public bool useNewMap + { + get { return myUseNewMap; } + set { myUseNewMap = value; } + } + + public bool useIncrementsForMultipleOps + { + get { return myUseIncrementsForMultipleOps; } + set { myUseIncrementsForMultipleOps = value; } + } + + public bool useUncopiedNewRowsInFireEvent + { + get { return myUseUncopiedNewRowsInFireEvent; } + set { myUseUncopiedNewRowsInFireEvent = value; } + } + public bool useModularizedDataStreamBridge + { + get { return myUseModularizedBridge; } + set { myUseModularizedBridge = value; } + } + public static DataStreamLoadFileConfig instance + { + get + { + return myInstance = new DataStreamLoadFileConfig(); + return myInstance; + // myInstance == null ? + // ConfigurationRequester.request (typeof (DataStreamLoadFileConfig)) as DataStreamLoadFileConfig + // as DataStreamLoadFileConfig : + // myInstance; + } + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Core/DataPacket.cs b/src/DataStreamSuite/DataStream/Core/DataPacket.cs new file mode 100644 index 000000000..441d2a06e --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/DataPacket.cs @@ -0,0 +1,316 @@ +using System; +using System.Collections; +using System.Data; +using BCG.Utilities.General; + +namespace BCG +{ + [Serializable] + public class DataPacket : ICloneable, IDataPacket + { + public class BlueRowIdComparator + { + private string[] myIdColumns = new string[]{"id"}; + + public BlueRowIdComparator(string[] idCols) + { + if ( idCols.Length != 0 ) + { + myIdColumns = idCols; + } + } + + public bool blueRowIdEqualEvaluator(object a, object b) + { + if ( a == null || b == null ) + { + return false; + } + + if ( !(a is IBlueRow) || !(b is IBlueRow) ) + { + return false; + } + + IBlueRow lhs = (IBlueRow)a; + IBlueRow rhs = (IBlueRow)b; + + bool result = true; + foreach(string idColumn in myIdColumns) + { + result = result && lhs[idColumn].Equals(rhs[idColumn]); + if ( !result ) + break; + } + + return result; + } + } + + public IBlueRowCollection newRows, addedRows, deletedRows; + public DataPacket () {} + public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) + { + this.newRows = newRows; + this.addedRows = addedRows; + this.deletedRows = deletedRows; + } + + public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) + { + new_rows = newRows; + added_rows = addedRows; + deleted_rows = deletedRows; + } + + public IBlueRowCollection getTrulyAddedRows(params string[] idColumns) + { + BlueTable truly = new BlueTable(addedRows.schema); + BlueRowIdComparator c = new BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); + + foreach(IBlueRow r in ta) + { + truly.addRow(r); + } + + return truly; + + } + + public virtual IBlueRowCollection getTrulyAddedRows () + { + BlueTable retVal = new BlueTable (schema); + foreach (IBlueRow r in addedRows) + if (!deletedRows.containsID ((int)r["id"])) + retVal.addRow (r); + return retVal; + } + + public virtual IBlueRowCollection getTrulyDeletedRows () + { + BlueTable retVal = new BlueTable (schema); + foreach (IBlueRow r in deletedRows) + if (!addedRows.containsID ((int)r["id"])) + retVal.addRow (r); + return retVal; + } + + public virtual IBlueRowCollection getTrulyChangedRows () + { + BlueTable retVal = new BlueTable (schema); + foreach (IBlueRow r in addedRows) + if (deletedRows.containsID ((int)r["id"])) + retVal.addRow (r); + return retVal; + } + + + public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns) + { + BlueTable truly = new BlueTable(deletedRows.schema); + BlueRowIdComparator c = new BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); + + foreach(IBlueRow r in td) + { + truly.addRow(r); + } + + return truly; + } + + public IBlueRowCollection getTrulyChangedRows(params string[] idColumns) + { + + BlueTable truly = new BlueTable(addedRows.schema); + BlueRowIdComparator c = new BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); + ICollection tc = CollectionHelpers.difference(addedRows, ta, ed); + + foreach(IBlueRow r in tc) + { + truly.addRow(r); + } + + return truly; + } + + public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns) + { + BlueTable truly = new BlueTable(deletedRows.schema); + BlueRowIdComparator c = new BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); + ICollection tco = CollectionHelpers.difference(deletedRows, td, ed); + + foreach(IBlueRow r in tco) + { + truly.addRow(r); + } + + return truly; + } + + public TablePacket toTablePacket () + { + return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows), + BlueDataUtility.toDataTable (deletedRows)); + + } + + public IBlueTableSchema schema + { + get + { + IBlueTableSchema retVal = newRows.schema != null ? newRows.schema : + addedRows.schema != null ? addedRows.schema : deletedRows.schema; + return retVal; + } + } + + /* + status D total A total D this A This D' total A' Total + x 0 0 0 0 0 0 + 0 0 0 1 1 this+ + 0 0 1 0 1this + 0 0 1 1 1 this 1 this+ + x 0 1 0 0 + error 0 1 0 1 + 0 1 1 0 1 total- + 0 1 1 1 1 total-, 1 this + + x 1 0 0 0 + 1 0 0 1 1 this+ + error 1 0 1 0 + error 1 0 1 1 + x 1 1 0 0 + error 1 1 0 1 + 1 1 1 0 1 total -1 keep the old delete and remove the old add + 1 1 1 1 1 total-, 1 this + + + d' = dthis - (dthis intersect a) + d // keep the old delete + a' = a - (dthis intersect a) + athis // keep the new add + * */ + public static DataPacket combineWithIds(ICollection packets, string[] idColumns) + { + if ( packets.Count == 0 ) + { + return new DataPacket( + new BlueTable(new DataTable()), + new BlueTable(new DataTable()), + new BlueTable(new DataTable())); + } + + DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ArrayList added = new ArrayList(); + ArrayList deleted = new ArrayList(); + BlueTable dt = null; + BlueTable at = null; + IBlueRowCollection nt = null; + + foreach(DataPacket dp in packets) + { + ArrayList oldDeleted = deleted; + ArrayList oldAdded = added; + + added = new ArrayList(); + deleted = new ArrayList(); + if ( dt == null ) + { + dt = new BlueTable(dp.schema); + at = new BlueTable(dp.schema); + } + + deleted.AddRange(oldDeleted); + deleted.AddRange( CollectionHelpers.difference( + CollectionHelpers.difference(dp.deletedRows, oldAdded, ed), + oldDeleted, ed) ); + + added.AddRange(dp.addedRows); + added.AddRange( CollectionHelpers.difference( + CollectionHelpers.difference(oldAdded, dp.deletedRows, ed), + added, ed) ); + nt = dp.newRows; + } + + foreach ( IBlueRow dr in added ) + { + at.addRow(dr); + } + + foreach ( IBlueRow dr in deleted ) + { + dt.addRow(dr); + } + + return new DataPacket(nt,at,dt); + } + + public DataPacket deepCopy () + { + return new DataPacket (new OrderedBlueTable (newRows), + new OrderedBlueTable (newRows), + new BlueTable (deletedRows)); + } + + public virtual DataPacket createImmutable () + { + return + new DataPacket( + newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows), + addedRows, + deletedRows); + } + #region ICloneable Members + + public object Clone() + { + return MemberwiseClone(); + } + + #endregion + + public string toIncrementalIDString() + { + System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName); + sb.Append ("\ndeleted : "); + foreach (IBlueRow r in deletedRows) + { + sb.Append(r["id"]); + sb.Append(','); + } + sb.Append("\nadded : "); + foreach (IBlueRow r in addedRows) + { + sb.Append(r["id"]); + sb.Append(','); + } + sb.Append("\n"); + return sb.ToString(); + } + + public override string ToString() + { + return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Core/DataStream.cs b/src/DataStreamSuite/DataStream/Core/DataStream.cs new file mode 100644 index 000000000..1f14a3027 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/DataStream.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections; + +//using BCG.Utilities; +//using BCG.Utilities.General; +using mbts; + +namespace BCG +{ + /// + /// Base class for all the sreams + /// + public class DataStream + { + internal static event EventHandler onFireStart; + protected bool myTestMode = Convert.ToBoolean (DataStreamLoadFileConfig.instance.isTestMode); + protected bool myFirstSubImplemented = true, + myLastSubImplemented = true; + protected IDictionary myDelMap = new Hashtable (); + private Guid myID = Guid.Empty; + public DateTime last_fired_time = DateTime.MinValue; + public int n_updates = 0; + + public DataStream () + : + this (null) + {} + + public DataStream (DataStream realSource) + { + myRealSource = realSource; + myOnChangeDelegate = null; + myPendingAdds = new Stack(); + } + + public int numberOfSubscribers + { + get + { + return + myOnChangeDelegate != null? + myOnChangeDelegate.GetInvocationList().Length + : + 0; + } + } + + public delegate void ChangeDelegate(object newValue); + public event ChangeDelegate onChange + { + add + { + ChangeDelegate currValue = value; + if (myTestMode) + { + ChangeDelegateWrapper w = new ChangeDelegateWrapper (value); + currValue = new ChangeDelegate (w.wrapperFunction); + myDelMap.Add (value, currValue); + } + if (myRealSource != null) + myRealSource.onChange += currValue; + else + { + myOnChangeDelegate = + (ChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue); + + + if (myOnChangeDelegate.GetInvocationList().Length == 1) // first subscriber + { + myPendingAdds.Push(new ArrayList()); + myOnFirstSubscriber(); + ArrayList calledDelegates = (ArrayList) myPendingAdds.Pop(); + + object v = myGetValueForInitialSubscription(); + //BlueTrace.WriteViewStreamSubscribe(this.ID, v != null); + if (v != null && calledDelegates.IndexOf(currValue) == -1) + currValue (v); + } + else + { + object v = myGetValueForInitialSubscription(); + // BlueTrace.WriteViewStreamSubscribe(this.ID, v != null); + if (v != null) + currValue (v); + } + } + } + remove + { + ChangeDelegate currValue = value; + // BlueTrace.WriteViewStreamUnsubscribe(this.ID); + if (myTestMode) + { + currValue = myDelMap[value] as ChangeDelegate; + myDelMap.Remove (value); + } + if (myRealSource != null) + myRealSource.onChange -= currValue; + else + { + myOnChangeDelegate = + (ChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue); + + // this is need to avoid a callback in a reentrant add. + if (myPendingAdds.Count > 0) + ((ArrayList) (myPendingAdds.Peek())).Add(currValue); + + if (myOnChangeDelegate == null) // last subscriber + { + myOnLastSubscriber(); + myLastValue = null; + } + } + } + } + + protected virtual object myGetValueForInitialSubscription() + {return myLastValue;} + + protected void mySetRealSource (DataStream src) + {myRealSource = src;} + + protected virtual void myOnFirstSubscriber() {myFirstSubImplemented = false;} + protected virtual void myOnLastSubscriber () {myLastSubImplemented = false;} + protected virtual void myFireEvent(object o) + { + last_fired_time = DateTime.Now; + n_updates++; + // BlueEventLog.logAssert (myRealSource == null); + myLastValue = o; + if (myOnChangeDelegate != null) + { + // this is need to avoid a callback in a reentrant add. + if (myPendingAdds.Count > 0) + foreach (Delegate d in myOnChangeDelegate.GetInvocationList()) + ((ArrayList) (myPendingAdds.Peek())).Add(d); + if (onFireStart != null) + onFireStart(this, EventArgs.Empty); + myOnChangeDelegate (o); + } + } + + public Guid ID + { + get + { + if (this.myID == Guid.Empty) // && BlueTraceHelper.IsEnabled(BlueTraceLevel.Verbose, BlueTraceKeywords.Streams)) + { + // BlueTraceHelper.FastInitGuid(ref this.myID); + // BlueTrace.WriteViewStreamCreate(this.GetType().ToString(), this.myID); + } + return this.myID; + } + } + + // this was made internal for testing purposes only. + // don't mess with it. + internal ChangeDelegate myOnChangeDelegate; + protected object myLastValue; + protected DataStream myRealSource; + private Stack myPendingAdds; + } + + internal class ChangeDelegateWrapper + { + private HighPerformanceTimer myTimer = new HighPerformanceTimer (); + private DataStream.ChangeDelegate myDel; + + internal ChangeDelegateWrapper (DataStream.ChangeDelegate del) {myDel = del;} + + internal void wrapperFunction (object o) + { + myTimer.start (); + myDel (o); + myTimer.stop (); + string minfo = "Type = " + myDel.Method.DeclaringType.FullName + " , method = " + myDel.Method.Name; + // BlueEventLog.logEvent (minfo + " , time = " + myTimer.lastIntervalTime.ToString (), BlueEventLevel.Performance); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Core/IExportableDataStream.cs b/src/DataStreamSuite/DataStream/Core/IExportableDataStream.cs new file mode 100644 index 000000000..1116fe3eb --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/IExportableDataStream.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public delegate void ExportableChangeDelegate(object data_packet); + public interface IExportableDataStream + { + void begin_subscription(); + event ExportableChangeDelegate on_change; + } +} diff --git a/src/DataStreamSuite/DataStream/Core/LightTableStream.cs b/src/DataStreamSuite/DataStream/Core/LightTableStream.cs new file mode 100644 index 000000000..561dbd769 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/LightTableStream.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections; +using System.Data; + +using BCG.Utilities; + + +/* this class reprsents a table stream for use with CppDataStreams. It does not hold on to the last value fired. + * This way, the last value is purged from managed memory and only the slimmer, sexier Cpp implementation of the row remains. + * + */ +namespace BCG +{ + public class LightTableStream : TableStream + { + public LightTableStream(string tableName) + : + base(tableName, null, null) + { + } + + public LightTableStream(string tableName, ViewStreamDatabase db) + : + base(tableName, db, null) + {} + + public LightTableStream(string tableName, string colExp) + : + base(tableName, + null, + colExp) + {} + + + public LightTableStream(string ttableName, ViewStreamDatabase db, string colExp) : + base(ttableName,db,colExp) + { + } + + + protected override void myHandleUpdates(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) + { + + base.myHandleUpdates(newRows, addedRows, deletedRows); + //once you've fired the rows, delete them becuase we don't want to hold onto them in memory + myLastValue = null; + + } + + } +} diff --git a/src/DataStreamSuite/DataStream/Core/ProxyStream.cs b/src/DataStreamSuite/DataStream/Core/ProxyStream.cs new file mode 100644 index 000000000..9ec179502 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/ProxyStream.cs @@ -0,0 +1,526 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; +using System.Reflection; + +using BCG.Utilities; +using BCG.Utilities.General; + +namespace BCG +{ + /// Derive from this class to create a new proxy stream for a stream that + /// should be computed out of process. In the proxy class add constructors + /// that exactly mirror the ones you want to expose from the real stream. + /// Each of the proxy constructors should just call the base constructor + /// passing it a, possibly null VSDB, and the arguments it received in + /// exactly the same order. Only non nullparameters of types string, int, bool or + /// DateTime are supported. DateTime arguments will loose their time portion, + /// string arguments can only have digits or letters and ints have to be positive. + /// + /// For example, if you real stream is: + /// + /// class TradeLog : ViewStream + /// { + /// public TradeLog(string forAccount, string forTrader) {....} + /// } + /// + /// Then your proxy stream would like: + /// + /// class TradeLogProxy : ProxyStream + /// { + /// public TradeLogProxy(string forAccount, string forTrader) : base(null,forTrader, forAccount) {} + /// } + /// + /// Unfortunatly, PubSub tags have a very limited alphabet, one that does not even support upper and lower + /// case dinstinctions, so some special care needs to be taken by the proxy or the real stream when dealing + /// with case sensitive arguments or with string arguments that can contain _ (the only non alpha numeric + /// character allowed in a string). The proxy tries to help by supporting the following conventions: + /// + /// - When passing a symbol, mark the parameter with the kind of symbol you mean, option or + /// underlyier. Example: + /// public TheosProxy([Option] string symbol) : base(symbol) {} or + /// public FuturePositionProxy([Underlyer] string future) : base(symbol) {} + /// + /// - When passing an exchange mark it with the Exchange attribute. Example: + /// public QuoteWidthsProxy([Symbol] string symbol, + /// [Exchange] string e) : base(e) {} + /// + /// - When passing a group name mark it with the group. Since group names can have _ and presumably numbers + /// there can be only one group argument which has to be last one. + /// attribute. Ex: + /// public RiskTheosProxy([Group] string g) : base(g) {} + /// + /// - When passing an instrument type mark it as InstrumentType. Example: + /// public SymbolsOfTypeProxy([InstrumentType] string it) : base(it) {} + /// + /// - When passing a trader mark it as trader. Example: + /// public TraderTrades([trader] string joes) : base(joes) {} + /// + /// - When passing a string that can contain _, the string argument must + /// be the last one in the constructor and should be marked as [LongString] + /// public WorkerHealth([LongString] string machineName) : base(machineName) {} + /// + /// - When passing a symbol, instrument type combo combine both in one string + /// with the symbol:InstrumentType format and market it with BCGSym: + /// public PositionProxy([BCGSym] string bcgsym) : base(bcgsym) {} + /// More complex cases should be handled by the real stream. + + + public abstract class ProxyStream : ViewStream + { + public enum ProxyParameterType + { + clrType, + optionSymbol, + trader, + underlyerSymbol, + group, + exchange, + instrumentType, + bcgSym + } + + public class ProxyParameter + { + public ProxyParameterType type; + public object value; + } + + public class MarshallingException : SystemException + { + public MarshallingException (string format, params object [] args) + : base(String.Format(format,args) ) + {} + } + + [AttributeUsage(AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)] + protected class ProxyParameterAttribute : Attribute + {} + + protected class OptionAttribute : ProxyParameterAttribute + { } + + protected class LongStringAttribute : ProxyParameterAttribute + { } + + protected class UnderlyierAttribute : ProxyParameterAttribute + { } + + protected class BCGSymAttribute : ProxyParameterAttribute + { } + + protected class GroupAttribute : ProxyParameterAttribute + { } + + protected class ExchangeAttribute : ProxyParameterAttribute + { } + + protected class TraderAttribute: ProxyParameterAttribute + { } + + protected class InstrumentTypeAttribute : ProxyParameterAttribute + { } + + public string getTag() { return myTag; } + private string myTag; + + protected ProxyStream(params object[] args) + {myInit(null, args);} + + protected ProxyStream(ViewStreamDatabase db, params object[] args) + {myInit(db, args);} + + class MyException : System.Exception + { + public MyException (string msg) : base (msg) + { + } + } + + private static void myThrowIfFalse(bool condition, string msg) + { + if (!condition) + throw new MyException(msg); + } + + private void myInit(ViewStreamDatabase db, params object[] args) + { + try + { + ConstructorInfo ci = GetType().GetConstructor(args.Select(t => t.GetType()).ToArray()); + foreach (object o in args) + myThrowIfFalse(o != null, + "Null arguments are not supported."); + + myTag = myGetTag(ci, args, db != null); + mySetRealSource(new TableStream("prepub_" + myTag, db)); + } + catch (MyException e) + { + // BlueEventLog2.instance().logEvent(e.Message, BlueEventLog2Level.Fatal); + mySetRealSource(new TableStream("error_tag")); + } + } + + public static string getProxyName(string tag) + { return tag.Split('_')[0]; } + + public static List unmarshal(Type proxyType, string tag) + { + myThrowIfFalse (proxyType.IsSubclassOf(typeof(ProxyStream)), + String.Format("{0} is not a subclass of ProxyStream", proxyType.Name)); + + myThrowIfFalse (tag.ToLower().StartsWith(proxyType.Name.ToLower()), + String.Format("Tag {0} does not match proxy type {1}.", tag, proxyType.Name)); + + string encodedParameters = + tag.Length == proxyType.Name.Length ? "" : + tag.Substring((proxyType.Name + "_").Length); + + ConstructorInfo ci; + if (myConstructorsNeedToBeEnconded(proxyType)) + { + List signature = myUnmarshalConstructorSignature(ref encodedParameters); + ci = proxyType.GetConstructor(signature.ToArray()); + if (ci == null) + { + signature.Insert(0, typeof(ViewStreamDatabase)); + ci = proxyType.GetConstructor(signature.ToArray()); + if (ci == null) + throw new MarshallingException("Cannot find constructor for {0} matching signature {1}", + proxyType.Name, encodedParameters); + } + } + else + ci = proxyType.GetConstructors()[0]; + ParameterInfo[] parameters = ci.GetParameters(); + + List result = new List(); + + ParameterInfo [] pars = ci.GetParameters(); + // skip VSDB if first parameter + int i = pars.Length > 0 && pars[0].ParameterType.Equals(typeof(ViewStreamDatabase)) ? 1 : 0; + for (;i < pars.Length; i++) + result.Add(new ProxyParameter + { + type = myGetParameterType(pars[i]), + value = myGetParameterValue(pars[i], ref encodedParameters) + }); + return result; + } + + private static List myUnmarshalConstructorSignature(ref string encodedParameters) + { + List result = new List (); + for (int i = 0; i < encodedParameters.Length; i++) + if (encodedParameters[i] == 's') + result.Add(typeof(string)); + else if (encodedParameters[i] == 'd') + result.Add(typeof(DateTime)); + else if (encodedParameters[i] == 'i') + result.Add(typeof(Int32)); + else if (encodedParameters[i] == 'b') + result.Add(typeof(bool)); + else if (encodedParameters[i] == '_') + break; + else + throw new MarshallingException("'{0}' does not start with a valid constructor signature", + encodedParameters); + if (result.Count > 0) + encodedParameters = encodedParameters.Substring(result.Count + 1); + return result; + } + + private static ProxyParameterType myGetParameterType(ParameterInfo pi) + { + if (pi.ParameterType == typeof(string)) + { + ProxyParameterType result = ProxyParameterType.clrType; + object[] attr = pi.GetCustomAttributes(false); + if (attr.Length > 0) + { + foreach (Attribute a in attr) + if (a is OptionAttribute) + result = ProxyParameterType.optionSymbol; + else if (a is ExchangeAttribute) + result = ProxyParameterType.exchange; + else if (a is TraderAttribute) + result = ProxyParameterType.trader; + else if (a is GroupAttribute) + result = ProxyParameterType.group; + else if (a is InstrumentTypeAttribute) + result = ProxyParameterType.instrumentType; + else if (a is UnderlyierAttribute) + result = ProxyParameterType.underlyerSymbol; + else if (a is BCGSymAttribute) + result = ProxyParameterType.bcgSym; + else if (a is LongStringAttribute) + result = ProxyParameterType.clrType; + } + return result; + } + else if (pi.ParameterType == typeof(Int32) || + pi.ParameterType == typeof(DateTime) || + pi.ParameterType == typeof(bool)) + return ProxyParameterType.clrType; + else + { + myThrowIfFalse (false,String.Format( + "{0} is not a supported proxy parameter type", pi.ParameterType.Name)); + return ProxyParameterType.clrType; + } + } + + private static object myGetParameterValue(ParameterInfo pi, ref string encodedParameters) + { + if (pi.ParameterType == typeof(DateTime)) + { + string[] parts = encodedParameters.Split('_'); + if (parts.Length < 3) + throw new MarshallingException("{0} does not start with a valid date", encodedParameters); + + int year; + if (!Int32.TryParse(parts[0], out year)) + throw new MarshallingException("{0} is not a valid year string", parts[0]); + int month; + if (!Int32.TryParse(parts[1], out month)) + throw new MarshallingException("{0} is not a valid month string", parts[1]); + int day; + if (!Int32.TryParse(parts[2], out day)) + throw new MarshallingException("{0} is not a valid day string", parts[2]); + + DateTime result; + try { + result = new DateTime(year, month, day); + } + catch (ArgumentException) + { + throw new MarshallingException("{0}/{1}/{2} is not a valid date", year, month, day ); + } + + if (parts.Length > 3) + encodedParameters = encodedParameters.Substring(parts[0].Length + parts[1].Length + parts[2].Length + 3); + else + encodedParameters = ""; + return + result; + } + else + { + if (pi.ParameterType == typeof(string) && myIsBCGSym(pi)) + { + string[] parts = encodedParameters.Split('_'); + if (parts.Length < 2) + throw new MarshallingException("{0} does not start with a valid BCGSym string", encodedParameters); + else + { + if (parts.Length > 2) + encodedParameters = encodedParameters.Substring(parts[0].Length + parts[1].Length + 2); + else + encodedParameters = ""; + return parts[0] + ":" + parts[1]; + } + } + else if (pi.ParameterType == typeof(string) && myIsLongString(pi)) + { + string result = encodedParameters; + encodedParameters = ""; + return result; + } + else + { + string[] parts = encodedParameters.Split('_'); + object result = null; + if (pi.ParameterType == typeof(string)) + result = parts[0]; + else if (pi.ParameterType == typeof(Int32)) + { + int n; + if (!Int32.TryParse(parts[0], out n)) + throw new MarshallingException("{0} is not a valid int", parts[0]); + result = n; + } + else if (pi.ParameterType == typeof(Boolean)) + { + bool b; + if (!bool.TryParse(parts[0], out b)) + throw new MarshallingException("{0} is not a valid bool", parts[0]); + result = b; + } + else + myThrowIfFalse (false, + String.Format("{0} is not a supported proxy parameter type", pi.ParameterType.Name)); + if (parts.Length > 1) + encodedParameters = encodedParameters.Substring(parts[0].Length + 1); + else + encodedParameters = ""; + return result; + } + } + } + + private bool myValidateString(string s, bool allowUnderscore) + { + foreach (char c in s) + if (!Char.IsDigit(c) && !Char.IsLetter(c) && !(allowUnderscore && c.Equals('_'))) + return false; + return true; + } + + private static bool myHasAttribute(ParameterInfo pi, Type attrType) + { + object[] attr = pi.GetCustomAttributes(false); + if (attr.Length > 0) + foreach (Attribute at in attr) + if (at.GetType().Equals(attrType)) + return true; + return false; + } + + private static bool myIsBCGSym(ParameterInfo pi) + {return myHasAttribute(pi, typeof(BCGSymAttribute));} + + private static bool myIsLongString(ParameterInfo pi) + { return myHasAttribute(pi, typeof(LongStringAttribute)) || myHasAttribute(pi, typeof(GroupAttribute)); } + + private static bool myFirstParameterIsVSDB(ConstructorInfo ci) + { + return + ci.GetParameters().Length > 0 && + ci.GetParameters()[0].ParameterType.Equals(typeof(ViewStreamDatabase)); + } + + private static bool myIsSameModuloVSDB(ConstructorInfo ci0, ConstructorInfo ci1) + { + if ((ci1.GetParameters().Length - ci0.GetParameters().Length) != 1) + return false; + + if (!myFirstParameterIsVSDB(ci1)) + return false; + + for (int i = 0; i < ci0.GetParameters().Length; i++) + if (!ci1.GetParameters()[i + 1].ParameterType.Equals( + ci0.GetParameters()[i].ParameterType)) + return false; + return true; + } + + private static bool myConstructorsNeedToBeEnconded(Type t) + { + ConstructorInfo [] cis = t.GetConstructors(); + if (cis.Length == 1) + return false; + else if (cis.Length > 2) + return true; + else + { + ConstructorInfo ci0 = cis[0]; + ConstructorInfo ci1 = cis[1]; + + return !(myIsSameModuloVSDB(ci0, ci1) || myIsSameModuloVSDB(ci1, ci0)); + } + } + + private string myGetConstructorSignature(ConstructorInfo ci) + { + string result =""; + int i = myFirstParameterIsVSDB(ci) ? 1 : 0; + for (; i < ci.GetParameters().Length; i++) + { + Type t = ci.GetParameters()[i].ParameterType; + if (t.Equals(typeof(string))) + result += "s"; + else if (t.Equals(typeof(DateTime))) + result += "d"; + else if (t.Equals(typeof(bool))) + result += "b"; + else if (t.Equals(typeof(Int32))) + result += "i"; + else + myThrowIfFalse (false, + String.Format("{0} is not a supported proxy parameter type", t.Name)); + } + return result; + } + + private string myGetTag(ConstructorInfo ci, object[] args, bool firstArgIsVSDB) + { + Type t = ci.DeclaringType; + string tag = t.Name; + if (myConstructorsNeedToBeEnconded(t)) + tag += "_" + myGetConstructorSignature(ci); + + int i = myFirstParameterIsVSDB(ci) ? 1 : 0; + foreach (object a in args) + { + Debug.Assert(a != null); + if (a.GetType() == typeof(DateTime)) + { + DateTime dt = (DateTime)a; + tag += String.Format("_{0}_{1}_{2}", dt.Year, dt.Month, dt.Day); + } + else if (a.GetType() == typeof(string)) + { + if (myIsBCGSym(ci.GetParameters()[i])) + { + string[] parts = a.ToString().Split(':'); + myThrowIfFalse (parts.Length == 2, + String.Format("Invalid BCGSym string argument {0}. Use sym:InstrumentType", + a)); + if (!myValidateString(parts[0], false) || !myValidateString(parts[1], false)) + { + // BlueEventLog2.instance().logEvent( + // String.Format( + // "Invalid string argument {0}. Currently, only letters and digits are supported in string arguments", + // a),BlueEventLog2Level.Error); + return "error_tag"; + } + tag += String.Format("_{0}_{1}", parts[0], parts[1]); + } + else if (myIsLongString(ci.GetParameters()[i])) + { + // not a BCGSym + if (!myValidateString(a.ToString(), true)) + { + // BlueEventLog2.instance().logEvent( + // String.Format("Invalid string argument {0}. Currently, only letters, digits and _'s are supported in long string arguments", + // a),BlueEventLog2Level.Error); + return "error_tag"; + } + //can't do this-- ci.GetParameters() order is undefined + //myThrowIfFalse (i.Equals(ci.GetParameters().Length - 1), + // "Long string argument must be last"); + tag += "_" + a.ToString(); + } + else + { + // not a BCGSym + if (!myValidateString(a.ToString(), false)) + { + // BlueEventLog2.instance().logEvent(String.Format( + // "Invalid string argument {0}. Currently, only letters and digits are supported in string arguments", a), + // BlueEventLog2Level.Error); + return "error_tag"; + } + tag += "_" + a.ToString(); + } + } + else if (a.GetType() == typeof(Int32)) + { + myThrowIfFalse ((int) a >= 0, + String.Format("Invalid int argument {0}. Currently only positive integers are supported.", + a)); + tag += "_" + a.ToString(); + } else if (a.GetType() == typeof(bool)) + tag += "_" + a.ToString(); + else + myThrowIfFalse (false, + String.Format("{0} is not a supported proxy parameter type", a.GetType().Name)); + i++; + } + return tag; + } + } +} diff --git a/src/DataStreamSuite/DataStream/Core/TablePacket.cs b/src/DataStreamSuite/DataStream/Core/TablePacket.cs new file mode 100644 index 000000000..89a0e75d3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/TablePacket.cs @@ -0,0 +1,26 @@ +using System; +using System.Data; +using System.Collections; + +namespace BCG +{ + /// + /// Defines the type that will be passed as the callback from the TableStream class + /// + public class TablePacket + { + public DataTable newTable = new DataTable(); + public DataTable addedRows = new DataTable(); + public DataTable deletedRows = new DataTable(); + + public TablePacket () {} + + public TablePacket (DataTable nt, DataTable ar, DataTable dr) + { + newTable = nt; + addedRows = ar; + deletedRows = dr; + } + + } +} diff --git a/src/DataStreamSuite/DataStream/Core/TableStream.cs b/src/DataStreamSuite/DataStream/Core/TableStream.cs new file mode 100644 index 000000000..fef2b2524 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/TableStream.cs @@ -0,0 +1,184 @@ +using System; +using System.Collections; +using System.Data; + +using BCG.Utilities; + +namespace BCG +{ + public class TableStream : ViewStream + { + public string tableName; + private ITableAccumulator myTA; + private bool myGotData = false; + + public TableStream(string tableName) + : + this(tableName, null, null) + { + } + + public TableStream(string tableName, ViewStreamDatabase db) + : + this(tableName, db, null) + {} + + public TableStream(string tableName, string colExp) + : + this(tableName, + null, + colExp) + {} + + + public TableStream(string ttableName, ViewStreamDatabase db, string colExp) + { + tableName = PubStreamPrefixer.prefix (ttableName); + // if (colExp != null) + // mySetRealSource (new ProjectStream (new TableStream (tableName, db), colExp)); + // else + { + if (db == null) + myTA = ViewStreamDatabase.instance.getITableAccumulator (tableName); + else + myTA = db.getITableAccumulator (tableName); + } + } + + protected override void myOnFirstSubscriber() + { + myTA.onUpdate += new TableUpdateDelegate (myHandleUpdates); + } + + protected override void myOnLastSubscriber() + { + myTA.onUpdate -= new TableUpdateDelegate (myHandleUpdates); + } + + protected virtual void myHandleUpdates (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) + { + if (!myGotData) + { + myGotData = true; + // BlueEventLog.logEvent("Received data for " + tableName + " for the first time", BlueEventLevel.Information); + } + myFireEvent (new DataPacket (newRows, addedRows, deletedRows)); + } + + public override string ToString() + { + return GetType() + " table = " + tableName; + } + } + + public class ThreadedTableStream : TableStream, IExportableDataStream + { + static private Lazy _vsd = new Lazy(() => new ViewStreamDatabase(), true); + public ThreadedTableStream(string table) : base(table, _vsd.Value) + { + } + + public void begin_subscription() + { + this.onChange += _stream_onChange; + } + + public event ExportableChangeDelegate on_change; + void _stream_onChange(object newValue) + { + if(on_change != null) + on_change(newValue); + } + } + + // public class ExportableTableStream : IExportableDataStream + // { + // TableStream _stream; + // public ExportableTableStream(string table, IDataPump pump) + // { + // _stream = new TableStream(table, ((ExportableDataPump)pump).vsd.Value); + // } + + // public void begin_subscription() + // { + // _stream.onChange += _stream_onChange; + // } + + // public event ExportableChangeDelegate on_change; + // void _stream_onChange(object newValue) + // { + // if(on_change != null) + // on_change(newValue); + // } + // } + + // public class ExportableQuery : IExportableDataStream + // { + // ViewStream _stream; + // public ExportableQuery(string query, BCG.IDataPump _) + // { + // // BCG.Evaluator.DataStreamExpression.syntax parser = new BCG.Evaluator.DataStreamExpression.syntax(); + // // BCG.Evaluator.DataStreamExpression.DataStreamExpression exp = + // // (BCG.Evaluator.DataStreamExpression.DataStreamExpression)parser.Parse(query); + + // _stream = exp.getView(new System.Collections.Hashtable()); + // } + + // public void begin_subscription() + // { + // _stream.onChange += _stream_onChange; + // } + + // public event ExportableChangeDelegate on_change; + // void _stream_onChange(object newValue) + // { + // if(on_change != null) + // on_change(newValue); + // } + // } + + // public class ExportableDataPump : IDataPump + // { + // public Lazy vsd = new Lazy(() => new ViewStreamDatabase(), true); + // System.Threading.Thread _thread; + // bool _continue = true; + // public ExportableDataPump(string pumpname) + // { + // tag = pumpname; + // _thread = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart((_) => + // { + // while(_continue) + // { + // vsd.Value.doWork(); + // System.Threading.Thread.Sleep(50); + // } + // })); + // _thread.Name = "DataPump " + pumpname; + // _thread.IsBackground = true; + // } + // public string tag + // { + // get; + // private set; + // } + // public void start() + // { + // try + // { + // if(_thread.ThreadState != System.Threading.ThreadState.Running) + // _thread.Start(); + // } + // catch(System.Threading.ThreadStateException) { } + // } + // public void stop() + // { + // try + // { + // _continue = false; + // if(_thread.ThreadState == System.Threading.ThreadState.Running) + // _thread.Join(500); + // } + // catch(System.Threading.ThreadStateException) { } + // } + // } +} diff --git a/src/DataStreamSuite/DataStream/Core/ViewStream.cs b/src/DataStreamSuite/DataStream/Core/ViewStream.cs new file mode 100644 index 000000000..22084a5e4 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/ViewStream.cs @@ -0,0 +1,97 @@ +using System; +//using BCG.Utilities.BlueConfiguration; + +using BCG.Utilities; + +namespace BCG +{ + /// + /// Marker abstract class + /// + public abstract class ViewStream : DataStream + { + private IBlueRowCollection myLastValueForTestMode; + protected bool myUseUncopiedNewRowsInFireEvent = DataStreamLoadFileConfig.instance.useUncopiedNewRowsInFireEvent; + + public ViewStream (ViewStream sourceStream) + : + base(sourceStream) + {} + + public ViewStream() + {} + + public IBlueRowCollection rows + { + get + { + if (myRealSource != null) + { + // BlueEventLog.logAssert (myRealSource is ViewStream); + + return (myRealSource as ViewStream).rows; + } + else + { + // used to force a computation of a stream which + // happens to have no subscribers + onChange += new DataStream.ChangeDelegate(myDummy); + try + { + + if (myLastValue != null) + return ((DataPacket) myLastValue).newRows; + else + return null; + } + finally + { + onChange -= new DataStream.ChangeDelegate(myDummy); + } + } + } + } + + protected override object myGetValueForInitialSubscription() + { + if (myLastValue == null) + return null; + else + { + return new DataPacket( + (myLastValue as DataPacket).newRows, + (myLastValue as DataPacket).newRows, + new BlueTable((myLastValue as DataPacket).newRows.schema)); + } + + } + + protected override void myFireEvent(object o) + { + DataPacket dp = (DataPacket) o; + + // BlueTrace.WriteViewStreamFireEvent(this.ID, + // dp.newRows == null ? 0 : dp.newRows.Count, + // dp.addedRows == null ? 0 : dp.addedRows.Count, + // dp.deletedRows == null ? 0 : dp.deletedRows.Count); + + if (myTestMode) + { + // BlueEventLog.logAssert ((myFirstSubImplemented && myLastSubImplemented) | myRealSource != null, + // @"either myOnFirstSub/myOnLastSub should be implemented or mySetRealSource + // called with a non-null arg in " + this.GetType().FullName); + if (myLastValueForTestMode != null) + BlueDataUtility.checkConsistency(myLastValueForTestMode, + dp.newRows, dp.deletedRows, dp.addedRows); + myLastValueForTestMode = new ImmutableBlueTable(dp.newRows); + } + // the view already garantees that added & deleted are immutable. + // this garantess that new is also immutable + base.myFireEvent(myUseUncopiedNewRowsInFireEvent ? dp : dp.createImmutable()); + + } + + private void myDummy (object o) + {} + } +} diff --git a/src/DataStreamSuite/DataStream/Core/ViewStreamDatabase.cs b/src/DataStreamSuite/DataStream/Core/ViewStreamDatabase.cs new file mode 100644 index 000000000..1c5b3987b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Core/ViewStreamDatabase.cs @@ -0,0 +1,312 @@ +using System; +using System.Collections; +using System.IO; +using BCG.Utilities; +using BCG.Utilities.General; +//using BCG.Utilities.BlueConfiguration; +using System.Collections.Generic; +using System.Threading; +using mbts; + +namespace BCG +{ + public class ViewStreamDatabase : IWorker, IDisposable + { + internal delegate void TagFireDelegate(string tag); + public delegate void DoWorkDelegate (); + public event DoWorkDelegate onDoWorkStart, + onDoWorkEnd; + internal event TagFireDelegate onTagFireStart, + onTagFireEnd; + //currently onTagUpdate is usd by the DataStreamPerfRecorder to keep track of how often the updates for various streams come in + //and how big they are. + //public delegate void OnTagUpdateDelegate(PubMetaData metaData); + //public event OnTagUpdateDelegate onTagUpdate; + private IDictionary myITableAccumulatorMap = new Hashtable (); + private static ViewStreamDatabase mySelf; + private ISubscriberCore mySubscriberCore; + private ArrayList myWorkers = new ArrayList (); + private bool myIsDisposed = false; + private bool myIsInDoWork = false; + private static object myTimeInterval; + private HighPerformanceTimer myTimer = new HighPerformanceTimer(); + private double myLogTimeThreshold = 10000; + private readonly double _dowork_threshold = 500; + private Thread myThreadOwner = null; // ViewStreamDatabase takes a thread affinity when created. If called from a different thread it will assert. + + private static readonly object myInstanceLockObject = new object(); + private readonly object myStateLock = new object(); + + public ViewStreamDatabase () + : this (new SubscriberCore ()) + { + } + + public ViewStreamDatabase (ISubscriberCore subscriberCore) + { + mySubscriberCore = subscriberCore; + // if (DataStreamLoadFileConfig.instance.profileStreams) + // new DataStreamPerfRecorder(this); + } + + private static int myGetTimeInterval() + { + if (myTimeInterval == null) + { + // TODO + string timeInterval = String.Empty; //ConfigurationRequester.request("viewStreamDatabaseTimeOut"); + if (timeInterval == null || timeInterval == "") + myTimeInterval = 0; + else + myTimeInterval = Int32.Parse(timeInterval); + } + return (int) myTimeInterval; + } + + public static int instance_hashcode = -1; + public static ViewStreamDatabase instance + { + get + { + if(mySelf == null || mySelf.myIsDisposed) + lock(myInstanceLockObject) + { + if (mySelf == null || mySelf.myIsDisposed) + { + mySelf = new ViewStreamDatabase(); + instance_hashcode = mySelf.GetHashCode(); + } + } + return mySelf; + } + set + { + lock(myInstanceLockObject) + { + if (mySelf != null) + mySelf.mySubscriberCore.Dispose (); + mySelf = value; + instance_hashcode = mySelf.GetHashCode(); + } + } + } + + public ICollection getCachedTables() + { + List result = new List(); + + + foreach (string s in myITableAccumulatorMap.Keys) + if (((ITableAccumulator)myITableAccumulatorMap[s]).isActive()) + result.Add(s); + + return result; + } + + public ISubscriberCore subscriber{get{return mySubscriberCore;}} + + + public ITableAccumulator getITableAccumulator (string tableName) + { + // If theres a ITableAccumulator for the tableName its returned (to share subscriptions), otherwise + // a new one is obtained from the Subscriber factory + + myITableAccumulatorMap[tableName] = myITableAccumulatorMap[tableName] == null ? + AccumulatorFactory.getAccumulator(tableName, mySubscriberCore) : myITableAccumulatorMap[tableName]; + + return myITableAccumulatorMap [tableName] as ITableAccumulator; + } + + public void doWork () + { + verifyAccess(); + + if (myIsSuspended) + return; + if (myIsInDoWork) + return; + else + { + myIsInDoWork = true; // to allow reentrancy + try + { + if (onDoWorkStart != null) + onDoWorkStart(); + + ArrayList updateList = mySubscriberCore.getUpdates(myGetTimeInterval()); + IDictionary updateMap = myGetUpdateMap(updateList); + + foreach (DictionaryEntry de in updateMap) + { + Stream[] allUpdates = (de.Value as ArrayList).ToArray(typeof(Stream)) as Stream[]; + string tag = de.Key.ToString(); + if (myITableAccumulatorMap[tag] == null){} + // BlueEventLog.logEvent("Received a SubscriberCore's callback for a tag that was never subscribed", BlueEventLevel.Warning); + else + { + myTimer.start(); + + + if (onTagFireStart != null) + onTagFireStart(tag); + + (myITableAccumulatorMap[tag] as ITableAccumulator).update(allUpdates); + if (onTagFireEnd != null) + onTagFireEnd(tag); + myTimer.stop(); + if (myTimer.lastIntervalTime > myLogTimeThreshold) + { + // BlueEventLog.logEvent("callback on tag " + tag.ToUpper() + " took " + + // myTimer.lastIntervalTime + " msecs", BlueEventLevel.Warning); + } + } + } + + // to allow for registering of workers on the worker callback + myIsCallingWorkers = true; + myPendingOps = new ArrayList(); + foreach (IWorker w in myWorkers) + { + myTimer.start(); + w.doWork(); + myTimer.stop(); + if (myTimer.lastIntervalTime > _dowork_threshold){} + // mbts.Logger.log (bts.log.Level.Warn, bts.log.Subsystem.gen, String.Format ("VSD.dowork on worker {0} took {1}", w.GetType().ToString (), myTimer.lastIntervalTime)); + } + myIsCallingWorkers = false; + + foreach (PendingAddRemoveWorker op in myPendingOps) + if (op.isAdd) + addWorker(op.w); + else + removeWorker(op.w); + } + finally + { + myPendingOps = null; + myIsInDoWork = false; + myIsCallingWorkers = false; + if (onDoWorkEnd != null) + onDoWorkEnd (); + } + }//end if we're not in do work + + } + + private bool myIsSuspended = false; + + public void suspend() { myIsSuspended = true; } + + public void resume() { myIsSuspended = false; } + + public bool checkAccess() + { + if (this.myThreadOwner == null) this.myThreadOwner = Thread.CurrentThread; + return this.myThreadOwner == Thread.CurrentThread; + } + + public void verifyAccess() + { + if (!checkAccess()) + { + string errorMessage = string.Format("Invalid cross thread access on ViewStreamDatabase. Owning thread: {0}. Accessing thread: {1}", this.myThreadOwner.ManagedThreadId, Thread.CurrentThread.ManagedThreadId); + +//#if DEBUG +// BlueEventLog2.instance().assert(false, errorMessage); +//#else +// BlueEventLog2.instance().logEvent(errorMessage, BlueEventLog2Level.Fatal); +//#endif + + } + } + + private IDictionary myGetUpdateMap (ArrayList updates) + { + IDictionary retVal = new Hashtable (); + foreach (PubSubUpdate update in updates) + { + if (retVal[update.tag] == null) + retVal[update.tag] = new ArrayList (); + (retVal[update.tag] as ArrayList).Add (update.body); + } + if (retVal.Count > 0) + { + string tagString = ""; + foreach (string tag in retVal.Keys) + tagString += tag + " , "; + } + return retVal; + } + + public bool isConsistent{get {return true;}} + + public void addWorker(IWorker w) + { + verifyAccess(); + if (myIsCallingWorkers) + myPendingOps.Add(new PendingAddRemoveWorker(true,w)); + else + myWorkers.Add(w); + } + + + public void removeWorker(IWorker w) + { + verifyAccess(); + if (myIsCallingWorkers) + myPendingOps.Add(new PendingAddRemoveWorker(false,w)); + else + myWorkers.Remove(w); + } + + #region IDisposable Members + + public void Dispose() + { + if (!myIsDisposed) + { + myIsDisposed = true; + mySubscriberCore.Dispose (); + } + } + + private ArrayList myPendingOps; + private bool myIsCallingWorkers = false; + class PendingAddRemoveWorker + { + public PendingAddRemoveWorker(bool isAdd, IWorker w) + { + this.isAdd = isAdd; + this.w = w; + } + + public bool isAdd; + public IWorker w; + } + + #endregion + } + + /* + /// + /// Returns a subscriber based on the name of the tag. Hardcoded right now, will change to config + /// later + /// + public class SubscriberFactory + { + public static ITableAccumulator getITableAccumulator (string tag, SubscriberCore subCore) + { + string tableName = tag.Substring (3); + if (!DataStreamGatewaySettings.instance.isSerializableUnderNewScheme (tableName)) + return new TableSubscriber (tag, subCore); + else + { + string subName = DataStreamGatewaySettings.instance.getSubscriber (tableName); + if (subName.Equals ("BCG.NonIncrementalTableSubscriber")) + return new NonIncrementalTableSubscriber (tag, subCore); + else + return new TableSubscriber (tag, subCore); + } + } + }*/ +} diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj index 9d6b6f5de..db079e6d4 100644 --- a/src/DataStreamSuite/DataStream/DataStream.csproj +++ b/src/DataStreamSuite/DataStream/DataStream.csproj @@ -1,5 +1,7 @@ - + + + net7.0 enable diff --git a/src/DataStreamSuite/DataStream/DynamicLoading/DataStreamUserRespresentation.cs b/src/DataStreamSuite/DataStream/DynamicLoading/DataStreamUserRespresentation.cs new file mode 100644 index 000000000..86dda7cc9 --- /dev/null +++ b/src/DataStreamSuite/DataStream/DynamicLoading/DataStreamUserRespresentation.cs @@ -0,0 +1,24 @@ +using System; + +namespace BCG +{ + public class DataStreamUserRespresentation : Attribute + { + private bool myIsVisible = true; + private string myFriendlyName; + + public DataStreamUserRespresentation (string friendlyName) + { + myFriendlyName = friendlyName; + } + + public DataStreamUserRespresentation (bool isVisible) + { + myIsVisible = isVisible; + } + + public bool isVisible {get{return myIsVisible;}} + + public string friendlyName {get{return myFriendlyName;}} + } +} diff --git a/src/DataStreamSuite/DataStream/DynamicLoading/PrimitiveManager.cs b/src/DataStreamSuite/DataStream/DynamicLoading/PrimitiveManager.cs new file mode 100644 index 000000000..b39832da0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/DynamicLoading/PrimitiveManager.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections; +using System.Reflection; +using System.IO; + +//using BCG.Utilities.BlueConfiguration; +//using BCG.Utilities; + +namespace BCG +{ + public class PrimitiveManager + { + private IDictionary myTypeLookupTable = new Hashtable (); + private static PrimitiveManager mySelf; + private ArrayList myAssemblyNames = new ArrayList (); + + private PrimitiveManager() + { + myLoadAllTypes (); + } + + private void myLoadAllTypes () + { + DataStreamLoadFileConfig config = DataStreamLoadFileConfig.instance; + + AppDomain currentDomain = AppDomain.CurrentDomain; + currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver); + loadAssembly (this.GetType ().Assembly.FullName); + + foreach (string assemblyName in config.loadedAssemblies) + { + string resolvedAssemblyPath = assemblyName; + if (!System.IO.Path.IsPathRooted(assemblyName)) + { + string appPath = Path.GetDirectoryName( + Assembly.GetAssembly(typeof(PrimitiveManager)).CodeBase.Substring("file:///".Length)); + resolvedAssemblyPath = System.IO.Path.GetFullPath(System.IO.Path.Combine(appPath, assemblyName)); + } + loadAssembly(resolvedAssemblyPath); + } + } + + private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) + { + foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies()) + if (args.Name.Equals(a.FullName)) + return a; + return null; + } + + /// + /// Loads all the Stream types associated with the assembly + /// + /// + public void loadAssembly (string assemblyName) + { + if (myAssemblyNames.Contains (assemblyName)) + return; + myAssemblyNames.Add (assemblyName); + Assembly loadedAssembly = GetType ().Assembly.FullName.Equals (assemblyName) ? + GetType ().Assembly : + Assembly.LoadFrom (assemblyName); + + foreach (Type type in loadedAssembly.GetTypes()) + if (!type.IsInterface + && !type.IsAbstract + && type.IsSubclassOf (typeof(ViewStream)) + && type.IsPublic) + { + bool isVisible = true; + string userRep = createDefaultName (type.Name); + foreach (Attribute att in type.GetCustomAttributes (false)) + if (att is DataStreamUserRespresentation) + { + DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation; + isVisible = dsatt.isVisible; + userRep = isVisible ? + dsatt.friendlyName == null ? userRep : dsatt.friendlyName + : null; + break; + } + if (isVisible) + { + // BlueEventLog.logAssert(myTypeLookupTable [userRep] == null); + myTypeLookupTable[userRep] = type; + } + } + } + + public static string createDefaultName (string typeName) + { + if (!typeName.EndsWith ("Stream")) + return typeName; + string retVal = typeName.Remove (typeName.Length - "Stream".Length, "Stream".Length); + string startString = new String (new char[] {retVal[0]}).ToLower (); + retVal = retVal.Remove (0, 1); + retVal = retVal.Insert (0, startString); + return retVal; + } + + + + public static PrimitiveManager instance + { + get {return mySelf = mySelf == null ? new PrimitiveManager () : mySelf;} + } + + /// + /// Returns the Type name corresponding to the primitive name + /// + /// + /// + public string find (string friendlyName) + { + if (myTypeLookupTable[friendlyName] == null) + throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types"); + return (myTypeLookupTable[friendlyName] as Type).Name; + } + + /// + /// Constructs and returns a ViewStream of type corresponding to the friendly name + /// + /// friendly primitive name + /// args can be null or an empty ICollection if the paramterless constructor + /// is the desired one + /// + public ViewStream build (string friendlyName, ICollection args) + { + object[] constructorArgs = null; + if (args != null && args.Count > 0) + { + constructorArgs = new object [args.Count]; + args.CopyTo (constructorArgs, 0); + } + if (myTypeLookupTable[friendlyName] == null) + throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types"); + return + constructorArgs == null ? + Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type) as ViewStream : + Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream; + } + + /// + /// Returns the name of all the primitives + /// + /// + public string[] getPrimitiveNames () + { + string[] retVal = new string [myTypeLookupTable.Count]; + myTypeLookupTable.Keys.CopyTo (retVal, 0); + return retVal; + } + + /// + /// Return the types of all the streams + /// + /// + public Type[] getStreamTypes () + { + Type[] retVal = new Type [myTypeLookupTable.Count]; + myTypeLookupTable.Values.CopyTo (retVal, 0); + return retVal; + } + + public ICollection getConstructorSigns (string primitiveName) + { + ArrayList retVal = new ArrayList (); + // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null); + Type t = myTypeLookupTable[primitiveName] as Type; + foreach (ConstructorInfo ci in t.GetConstructors()) + { + string sign = ""; + sign = t.Name + " ("; + foreach (ParameterInfo pi in ci.GetParameters ()) + sign += pi.ParameterType.Name + + " " + + pi.Name + ", "; + sign = sign.Length == (t.Name + " (").Length ? sign + ")": sign.Remove (sign.Length - 2, 2) + ")"; + retVal.Add (sign); + } + return retVal; + } + + public string getAssemblyName (string primitiveName) + { + // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null); + return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name; + + } + + } +} diff --git a/src/DataStreamSuite/DataStream/PrimitiveManager.cs b/src/DataStreamSuite/DataStream/PrimitiveManager.cs deleted file mode 100644 index 8c986d696..000000000 --- a/src/DataStreamSuite/DataStream/PrimitiveManager.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BCG { - using System; - using System.Collections; - - public class PrimitiveManager { - public static PrimitiveManager instance => throw new NotImplementedException(); - - public ViewStream build(string friendlyName, ICollection args) => - throw new NotImplementedException(); - - public string find(string friendlyName) => - throw new NotImplementedException(); - } -} diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs new file mode 100644 index 000000000..73c3e72f2 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections; + +namespace BCG +{ + internal class AddColumnsRow : IBlueRow + { + private IBlueTableSchema mySchema; + private IBlueRow myRow; + //private object[] myCachedValues; + + #region Constructors + public AddColumnsRow (IBlueTableSchema schema, IBlueRow row) + { + mySchema = schema; + myRow = row; + //myCachedValues = new object[schema.columnNames.Length]; + } + #endregion + + #region IBlueRow members + public IBlueTableSchema schema{get{return mySchema;}} + #endregion + + #region ICollection members + public int Count {get{return mySchema.columnNames.Length;}} + + public bool IsSynchronized {get{return false;}} + + public object SyncRoot {get{return this;}} + + public void CopyTo (Array array, int index) + { + int i = index; + foreach (object o in this as IEnumerable) + array.SetValue (o, i++); + } + #endregion + + #region IDictionary members + public bool IsFixedSize {get{return true;}} + + public bool IsReadOnly {get{return true;}} + + /// + /// The key can either be the string corrsponding to the column name or + /// the order of the column + /// + public object this [object key] + { + get + { + if (key is string) + return this[(string)key]; + return this[(int)key]; + } + set {throw new NotSupportedException ("The IBlueRow is readonly");} + } + + public object this[int index] + { + get + { + //if (myCachedValues[index] == null) + // myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index]; + //return myCachedValues[index]; + return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index]; + } + } + + public object this[string colName] {get{return this[mySchema[colName].colNo];}} + + public ICollection Keys{get{return mySchema.columnNames;}} + + public ICollection Values + { + get + { + object[] retVal = new object[mySchema.columnNames.Length]; + CopyTo (retVal, 0); + return retVal; + } + } + + public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + + public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} + + public bool Contains (object key){return mySchema[(string)key] != null;} + + /// + /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to + /// iterate through the values, typecast this into an IEnumerable + /// + /// + public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);} + + IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);} + + public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");} + #endregion + + } + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs new file mode 100644 index 000000000..bb2500791 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs @@ -0,0 +1,13 @@ +// using System; +// using System.Data; +// using System.Collections; +// using BCG.Utilities; +// using BCG.Utilities.General; +// namespace BCG +// { + +// public class AddColumnsStream : MapStream +// { +// public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef)) {} +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddRegexColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddRegexColumnsStream.cs new file mode 100644 index 000000000..4f6492719 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddRegexColumnsStream.cs @@ -0,0 +1,13 @@ + +// using System; + +// namespace BCG { + +// public class AddRegexColumnsStream : ViewStream { + +// public AddRegexColumnsStream (ViewStream vs, string srcCol, string regex, string newColDefs) { +// IStreamMapper mapper = new RegexMatchAdder (srcCol, newColDefs, regex); +// mySetRealSource (new MapStream (vs, mapper)); +// } +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs new file mode 100644 index 000000000..ac1336fe6 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs @@ -0,0 +1,73 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; +// using BCG.Evaluator.AddColumnsExpression; + +// namespace BCG +// { +// /// +// /// Provides methods to add columns to the schema +// /// +// public class ColumnAdder : IStreamMapper +// { +// protected IDictionary myNewColumns; +// protected IBlueTableSchema mySchema; + +// static syntax myParser = +// new syntax(); + +// /// +// /// string represting the columns to be added to the output table. +// /// +// /// +// public ColumnAdder (string columnDef) +// { +// AddColumnsExpression ace; + +// if (DataStreamLoadFileConfig.instance.useStaticParser) +// { +// lock (myParser) +// { +// ace = +// (AddColumnsExpression)myParser.Parse(columnDef); +// } +// } +// else +// { +// syntax parser = new syntax(); +// ace = +// (AddColumnsExpression)parser.Parse(columnDef); +// } + +// if (ace == null) +// throw new Exception ("Invalid columnn expression: " + columnDef); +// myNewColumns = ace.columns; +// } + +// public virtual IBlueRow map (IBlueRow row) +// { +// if (mySchema == null) +// myConstructSchema (row.schema); +// return new AddColumnsRow (mySchema, row); +// } + +// public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema) +// { +// myConstructSchema (inputSchema); +// return mySchema; +// } + +// private void myConstructSchema (IBlueTableSchema schema) +// { +// ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count]; +// int ciIndex = 0; +// foreach (ColumnInfo ci in schema.columnInfo) +// cis[ciIndex++] = ci; +// foreach (DictionaryEntry de in myNewColumns) +// cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++, +// ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v); +// mySchema = new BlueTableSchema (cis, schema.tableName); +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/MathStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/MathStream.cs new file mode 100644 index 000000000..38ace9708 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/MathStream.cs @@ -0,0 +1,100 @@ +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; + +// namespace BCG +// { + +// public class BooleanStringReplaceStream : ViewStream, IStreamMapper +// { +// private BooleanExpressionFilter _bool; +// private string _column, _value; +// private IBlueTableSchema _schema; +// public BooleanStringReplaceStream(ViewStream vs, string boolean_expression, string column, string value) +// { +// _column = column; +// _value = value; +// if(!string.IsNullOrEmpty(boolean_expression)) +// _bool = new BooleanExpressionFilter(boolean_expression); +// mySetRealSource(new MapStream(vs, this)); +// } +// public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) +// { +// _schema = inputSchema; +// return inputSchema; +// } +// public IBlueRow map(IBlueRow row) +// { +// MutableBlueRow result = new MutableBlueRow(_schema, row); +// if(_bool == null || _bool.filter(row)) +// result.setValue(_column, _value); +// return result; +// } + +// } + +// public class MathStream : ViewStream, IStreamMapper +// { +// private ArithmeticExpressionEvaluator.ArithmeticExpression myExp; +// private BooleanExpressionFilter _bool; +// List myExtraColumns = new List(); +// BlueTableSchema mySchema; + + +// public MathStream(ViewStream vs, string mathExpression) +// : this(vs, mathExpression, string.Empty) +// { +// } + +// public MathStream(ViewStream vs, string mathExpression, string boolean_expression) +// { +// myExp = ArithmeticExpressionEvaluator.compile(mathExpression); +// myExtraColumns = ArithmeticExpressionEvaluator.getOutputIdentifiers(myExp); +// if(!string.IsNullOrEmpty(boolean_expression)) +// _bool = new BooleanExpressionFilter(boolean_expression); +// mySetRealSource(new MapStream(vs, this)); +// } + +// #region IStreamMapper Members + +// public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) +// { +// if (mySchema == null) +// { +// Dictionary colIdx = new Dictionary(); +// foreach (ColumnInfo ci in inputSchema.columnInfo) +// colIdx[ci.colName] = ci; + +// foreach (string ncn in myExtraColumns) +// { +// ColumnInfo newC; +// if (colIdx.ContainsKey(ncn)) +// newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double)); +// else +// newC = new ColumnInfo(ncn, colIdx.Count, typeof(double)); +// colIdx[ncn] = newC; +// } +// mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")"); +// } +// return mySchema; +// } + +// #endregion + +// #region IDynamicStreamMapper Members + +// public IBlueRow map(IBlueRow row) +// { +// MutableBlueRow result = new MutableBlueRow(mySchema, row); +// if(_bool == null || _bool.filter(row)) +// { +// foreach(var kvp in ArithmeticExpressionEvaluator.eval(myExp, (System.Collections.IDictionary)result)) +// result.setValue(kvp.Key, kvp.Value); +// } +// return result; +// } + +// #endregion +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/RegexMatchAdder.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/RegexMatchAdder.cs new file mode 100644 index 000000000..75e72a870 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/RegexMatchAdder.cs @@ -0,0 +1,48 @@ + +// using System; +// using System.Collections; +// using System.Collections.Generic; +// using System.Text.RegularExpressions; +// using System.ComponentModel; + +// using BCG.Utilities; + +// namespace BCG { + +// public class RegexMatchAdder : ColumnAdder { + +// private int[] myAddedColNos; +// private string mySrcCol; +// private Regex myRegex; + +// public RegexMatchAdder (string srcCol, string columnDef, string regex) : base (columnDef) { +// mySrcCol = srcCol; +// myRegex = new Regex (regex); +// } + +// public override IBlueRow map (IBlueRow row) { +// MutableBlueRow result = new MutableBlueRow (mySchema, row); + +// if (myAddedColNos == null) { +// myAddedColNos = new int[myNewColumns.Count]; +// for (int i = 0; i < myAddedColNos.Length; ++i) +// myAddedColNos[i] = i + row.Count; +// } + +// object[] values = new object[myNewColumns.Count]; +// string src = (string)row[mySrcCol]; +// // extract the requested regex substrings +// Match m = myRegex.Match (src); +// for (int icol = 0; icol < myAddedColNos.Length; ++icol) { +// int i = icol + row.Count; +// Group g = m.Groups[mySchema.columnNames[i]]; +// object val = TypeDescriptor.GetConverter (mySchema[i].colType) +// .ConvertFromString (g.Success ? g.Value: string.Empty); +// result.setValue (i, val); +// } + +// return result; +// } + +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/XYTranspositionStream.cs b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/XYTranspositionStream.cs new file mode 100644 index 000000000..9b31ce7c1 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/XYTranspositionStream.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public class XYTranspositionStream : ViewStream + { + private ViewStream _stream; + private IBlueTableSchema _schema; + private string[] _transposed_cols; + + public XYTranspositionStream(ViewStream stream, string transposed_cols) + { + _transposed_cols = transposed_cols.Split(' '); + _stream = stream; + } + + protected override void myOnFirstSubscriber() + { + _stream.onChange += _stream_onChange; + } + + protected override void myOnLastSubscriber() + { + if (_stream != null) + _stream.onChange -= _stream_onChange; + } + + private void _stream_onChange(object newValue) + { + DataPacket dp = newValue as DataPacket; + if (dp != null) + { + if (_schema == null) + _make_schema(dp.schema); + BlueTable result = new BlueTable(_schema); + int id = 1; + foreach (IBlueRow row in dp.newRows) + { + foreach (string transposed_col in _transposed_cols) + { + MutableBlueRow mbr = new MutableBlueRow(_schema); + for (int i = 0; i < _schema.columnNames.Length; i++) + { + if (_schema.columnNames[i] == "id") + mbr.setValue(i, id++); + else if (_schema.columnNames[i] == "param_name") + mbr.setValue(i, transposed_col); + else if (_schema.columnNames[i] == "param_value") + mbr.setValue(i, row[transposed_col]); + else + mbr.setValue(i, row[_schema.columnNames[i]]); + } + result.addRow(mbr); + } + } + myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(_schema) : (myLastValue as DataPacket).newRows)); + } + } + + private void _make_schema(IBlueTableSchema input_schema) + { + List info = new List(); + int col_counter = 0; + info.Add(new ColumnInfo("id", col_counter++, typeof(int))); + info.Add(new ColumnInfo("param_name", col_counter++, typeof(string))); + info.Add(new ColumnInfo("param_value", col_counter++, typeof(object))); + foreach (ColumnInfo ci in input_schema.columnInfo) + if (!_is_transposing_column(ci.colName) && ci.colName != "id") + info.Add(new ColumnInfo(ci.colName, col_counter++, ci.colType)); + _schema = new BlueTableSchema(info.ToArray(), String.Format("xy_transposed_{0}", input_schema.tableName)); + } + + private bool _is_transposing_column(string col_name) + { + foreach (string tc in _transposed_cols) + if (col_name == tc) + return true; + return false; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs new file mode 100644 index 000000000..9ef52fdfe --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs @@ -0,0 +1,244 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using mbts; + +namespace BCG +{ + public class YXTranspositionStream : ViewStream, INonIncGroupByEvaluator + { + private class OnlyFireOnceStream : ViewStream + { + private ViewStream _stream; + private bool _hooked = false; + + public OnlyFireOnceStream(ViewStream stream) + { + _stream = stream; + } + + protected override void myOnFirstSubscriber() + { + _stream.onChange += _stream_onChange; + _hooked = true; + } + + protected override void myOnLastSubscriber() + { + if (_hooked && _stream != null) + _stream.onChange -= _stream_onChange; + } + + private void _stream_onChange(object newValue) + { + DataPacket dp = newValue as DataPacket; + if (dp != null && dp.newRows.Count > 0) + { + _stream.onChange -= _stream_onChange; + myFireEvent(new DataPacket(dp.newRows, dp.newRows, new BlueTable(dp.newRows.schema))); + } + } + } + + public delegate void IncrementValues(object[] current_values, object[] values_to_inc); + public delegate string GetColumnPrefix(object[] transposing_col_values); + + public delegate string GetColumnSuffix(string col_name); + private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; }); + + private IncrementValues _increment_values; + private GetColumnPrefix _get_column_prefix; + + private IBlueTableSchema _schema; + + private string[] _group_by_cols; + private Pair[] _cols_per_transposed; + private string[] _transposing_cols; + + private int[] _transposing_col_indices; + + private Comparison> _transpsed_col_value_comparer_wrapper; + + public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols, + IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison transposed_col_value_comparer) + : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del, + get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true) + { } + + public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols, + IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison transposed_col_value_comparer, bool only_fire_once) + { + if (get_column_suffix_del != null) + _get_column_suffix = get_column_suffix_del; + if (transposed_col_value_comparer != null) + { + _transpsed_col_value_comparer_wrapper = new Comparison>((p1,p2) => + { + return transposed_col_value_comparer(p1.second, p2.second); + }); + } + _group_by_cols = group_by_cols.Split(' '); + _transposing_cols = transposed_cols_group.Split(' '); + string[] cols = cols_per_transposed.Split(' '); + _cols_per_transposed = new Pair[cols.Length]; + for (int i = 0; i < _cols_per_transposed.Length; i++) + { + string[] name_type = cols[i].Split('\\'); + // mbts.Logger.assert(name_type.Length == 2, "Expected cols_per_transposed to be in format name\\type"); + _cols_per_transposed[i] = new Pair(name_type[0], Type.GetType(name_type[1])); + } + _increment_values = increment_values_del; + _get_column_prefix = get_column_prefix_del; + if(only_fire_once) + mySetRealSource(new NonIncGroupByStream(new OnlyFireOnceStream(stream), this)); + else + mySetRealSource(new NonIncGroupByStream(stream, this)); + } + + private void _set_schema(List tc_prefixes, Pair[] group_by_col_info, string table_name) + { + int col_index = 0; + List col_info = new List(); + col_info.Add(new ColumnInfo("id", col_index++, typeof(int))); + foreach (Pair p in group_by_col_info) + col_info.Add(new ColumnInfo(p.first, col_index++, p.second)); + foreach (string tc_prefix in tc_prefixes) + foreach (Pair p in _cols_per_transposed) + col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second)); + IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name); + if (_schema == null) + _schema = schema; + } + + private string _get_full_column_name(string prefix, string col_name) + { + string suffix = _get_column_suffix(col_name); + if (suffix == string.Empty) + return prefix; + else + return prefix + "_" + suffix; + } + + #region INonIncGroupByEvaluator Members + + public string[] get_group_by_cols() + { + return _group_by_cols; + } + + public IBlueRowCollection evaluate(IDictionary> grouped_rows) + { + Dictionary> all_tc_infos_and_values = new Dictionary>(); + Dictionary group_by_cols_value_dict = new Dictionary(); + BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length); + Pair[] group_by_col_info = new Pair[_group_by_cols.Length]; + string table_name = string.Empty; + if (grouped_rows.Count > 0) + { + IBlueTableSchema schema = null; + foreach (KeyValuePair> kvp in grouped_rows) + { + schema = kvp.Value[0].schema; + break; + } + table_name = schema.tableName; + _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols); + int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols); + for (int i = 0; i < group_by_indices.Length; i++) + { + ColumnInfo ci = schema.columnInfo[group_by_indices[i]]; + group_by_col_info[i] = new Pair(ci.colName, ci.colType); + } + } + else + return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); + Dictionary tc_prefixes = new Dictionary(); + List> tc_values = new List>(); + foreach (KeyValuePair> kvp in grouped_rows) + { + //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values + Dictionary incrementing_values = new Dictionary(); + foreach (IBlueRow r in kvp.Value) + { + int hash = hasher.get_hash(r, _transposing_col_indices); + object[] values; + if (!incrementing_values.TryGetValue(hash, out values)) + { + incrementing_values.Add(hash, _get_per_transposing_col_values(r)); + string prefix = _get_column_prefix(_get_transposing_col_values(r)); + if (!tc_prefixes.ContainsKey(hash)) + { + tc_prefixes.Add(hash, prefix); + tc_values.Add(new Pair(hash, _get_transposing_col_values(r))); + } + } + else + _increment_values(values, _get_per_transposing_col_values(r)); + } + all_tc_infos_and_values.Add(kvp.Key, incrementing_values); + //An ugly way to extract group information needed to insert values later + foreach (IBlueRow r in kvp.Value) + { + group_by_cols_value_dict.Add(kvp.Key, _get_group_by_cols_values(r)); + break; + } + } + //Sort the transposed schema + if (_transpsed_col_value_comparer_wrapper != null) + tc_values.Sort(_transpsed_col_value_comparer_wrapper); + List prefixes = new List(); + foreach (Pair p in tc_values) + prefixes.Add(tc_prefixes[p.first]); + _set_schema(prefixes, group_by_col_info, table_name); + BlueTable result = new BlueTable(_schema); + int id = 1; + foreach (KeyValuePair> kvp in all_tc_infos_and_values) + { + MutableBlueRow mbr = new MutableBlueRow(_schema); + mbr.setValue("id", id++); + for (int i = 0; i < _group_by_cols.Length; i++) + mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]); + foreach (KeyValuePair tc_cols in kvp.Value) + { + string prefix = tc_prefixes[tc_cols.Key]; + for (int i = 0; i < _cols_per_transposed.Length; i++) + { + string column = _get_full_column_name(prefix, _cols_per_transposed[i].first); + if(mbr.schema.columnNames.Contains(column)) + mbr.setValue(column, tc_cols.Value[i]); + } + } + result.addRow(mbr); + } + return result; + } + + #endregion + + private object[] _get_transposing_col_values(IBlueRow row) + { + object[] values = new object[_transposing_cols.Length]; + for (int i = 0; i < _transposing_cols.Length; i++) + values[i] = row[_transposing_cols[i]]; + return values; + } + + private object[] _get_per_transposing_col_values(IBlueRow row) + { + object[] values = new object[_cols_per_transposed.Length]; + for (int i = 0; i < _cols_per_transposed.Length; i++) + values[i] = row[_cols_per_transposed[i].first]; + return values; + } + + private object[] _get_group_by_cols_values(IBlueRow row) + { + object[] values = new object[_group_by_cols.Length]; + for (int i = 0; i < values.Length; i++) + values[i] = row[_group_by_cols[i]]; + return values; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs b/src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs new file mode 100644 index 000000000..bb2647bfb --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs @@ -0,0 +1,22 @@ +// using System; +// using BCG; +// using BCG.Utilities; + +// namespace BCG +// { +// public class CartesianProductStream : ViewStream +// { +// public CartesianProductStream(ViewStream v1, ViewStream v2) +// { +// mySetRealSource( +// new JoinStream (new AddColumnsStream (v1, +// "hiddenCartesianColumn=1"), +// "hiddenCartesianColumn", +// "!id hiddenCartesianColumn dateTime:dateBoundary", +// new AddColumnsStream (v2, +// "hiddenCartesianColumn=1"), +// "hiddenCartesianColumn", +// "!id hiddenCartesianColumn")); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Case Changing/CaseChangeStream.cs b/src/DataStreamSuite/DataStream/Primitives/Case Changing/CaseChangeStream.cs new file mode 100644 index 000000000..8f128ca6f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Case Changing/CaseChangeStream.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class CaseChangeStream : ViewStream, IStreamMapper + { + + private string[] myCaseChangingColumns; + bool myIsUpperCase; + + public CaseChangeStream(ViewStream input, bool toUpper, params string[] columnsToChangeCase) + { + myCaseChangingColumns = columnsToChangeCase; + mySetRealSource(new SuperMapStream(input, this)); + myIsUpperCase = toUpper; + } + + + + + #region IStreamMapper Members + + public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) + { + return inputSchema; + } + + #endregion + + #region IDynamicStreamMapper Members + + public IBlueRow map(IBlueRow row) + { + object[] outValues = new object[row.Count]; + + row.CopyTo(outValues,0); + + foreach (string colName in myCaseChangingColumns) + { + ColumnInfo thisCol = row.schema[colName]; + object rowVal = outValues[thisCol.colNo]; + + if (rowVal is string && !string.IsNullOrEmpty((string)rowVal)) + outValues[thisCol.colNo] = myIsUpperCase ? ((string)rowVal).ToUpper() : ((string)rowVal).ToLower(); + } + + return new BlueRow(row.schema, outValues); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Case Changing/ToLowerStream.cs b/src/DataStreamSuite/DataStream/Primitives/Case Changing/ToLowerStream.cs new file mode 100644 index 000000000..37c2af935 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Case Changing/ToLowerStream.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class ToLowerStream : CaseChangeStream + { + public ToLowerStream(ViewStream input, params string[] colNames) + : base(input, false, colNames) { } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Case Changing/ToUpperStream.cs b/src/DataStreamSuite/DataStream/Primitives/Case Changing/ToUpperStream.cs new file mode 100644 index 000000000..96575276b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Case Changing/ToUpperStream.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class ToUpperStream : CaseChangeStream + { + public ToUpperStream(ViewStream input, params string[] colNames) + : base(input, true, colNames) { } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsMapper.cs b/src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsMapper.cs new file mode 100644 index 000000000..51151ded0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsMapper.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections; +using System.Text; + +using BCG; +using BCG.Utilities; + +namespace BCG +{ + public class ConcatenateColumnsMapper : IStreamMapper + { + protected IDictionary myTempColMap = new SortedList (); + protected ArrayList myColMap; + protected BlueTableSchema mySchema; + + public ConcatenateColumnsMapper(params string [] columnDefs) + { + foreach (string cd in columnDefs) + { + string outColumn = + cd.Substring(0,cd.IndexOf(":")); + + string [] inColumns = + cd.Substring(cd.IndexOf(":") + 1).Split(' '); + + string [] trimedIn = new string[inColumns.Length]; + int n = 0; + foreach (string s in inColumns) + trimedIn[n++] = s.Trim(); + + myTempColMap[outColumn.Trim()] = trimedIn; + } + } + #region IStreamMapper Members + + public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema) + { + if (mySchema == null) + { + /* + * The new columns form the first columns, and the existing columns in the input + * schema are appened. + * */ + ArrayList cis = new ArrayList (); + foreach (string cn in myTempColMap.Keys) + cis.Add (new ColumnInfo (cn, cis.Count, typeof(string))); + foreach (ColumnInfo ci in inputSchema.columnInfo) + cis.Add (new ColumnInfo (ci.colName, cis.Count, ci.colType)); + mySchema = new BlueTableSchema("ConcatenateColumns " + inputSchema.tableName, + cis.ToArray (typeof(ColumnInfo)) as ColumnInfo[]); + /* + * An arraylist is formed which indexes the new columns to the existing ones that form it. Hashtables + * are avoided wherever possible (which is natural as well, as the keys are ordered integers) + * */ + myColMap = new ArrayList (myTempColMap.Count); + foreach (DictionaryEntry de in myTempColMap) + { + string[] cols = de.Value as string[]; + ArrayList temp = new ArrayList (); + foreach (string col in cols) + temp.Add (inputSchema[col].colNo); + myColMap.Add (temp.ToArray(typeof(int)) as int[]); + } + myTempColMap.Clear (); + } + return mySchema; + } + + #endregion + + #region IDynamicStreamMapper Members + + public virtual IBlueRow map (IBlueRow row) + { + object[] retVal = new object[mySchema.columnInfo.Length]; + row.CopyTo (retVal, myColMap.Count); + for (int colNo = 0; colNo < myColMap.Count; colNo ++) + retVal[colNo] = myGetConcatValue (row, colNo); + return new BlueRow (mySchema, retVal); + } + + #endregion + + /// + /// Given a row and a column number, the method extracts the constituent columns indices + /// (from the colMap array) and form the resultant value corresponding to colNo and the row passed + /// + /// + /// + /// + protected virtual object myGetConcatValue (IBlueRow row, int colNo) + { + StringBuilder b = new StringBuilder(); + foreach (int cn in myColMap[colNo] as int[]) + b.Append(row[cn].ToString()); + return b.ToString (); + } + } + + public class ConcatenateColumnsMapperForSameDataSet : ConcatenateColumnsMapper + { + private HashTableND[] myDicts; + + public ConcatenateColumnsMapperForSameDataSet (params string [] columnDefs) + : base (columnDefs) + { + myDicts = new HashTableND [columnDefs.Length]; + int i = 0; + foreach (DictionaryEntry de in myTempColMap) + myDicts[i++] = new HashTableND ((de.Value as string[]).Length); + } + + protected override object myGetConcatValue (IBlueRow row, int colNo) + { + int[] cols = myColMap[colNo] as int[]; + object[] vals = new object[cols.Length]; + for (int i = 0; i < vals.Length; i++) + vals[i] = row[cols[i]]; + HashTableND ht = myDicts[colNo]; + object rv = ht.getValue (vals); + if (rv != null) + return rv; + else + { + rv = base.myGetConcatValue (row, colNo); + object[] args = new object[vals.Length + 1]; + vals.CopyTo (args, 0); + args[args.Length-1] = rv; + ht.add (args); + return rv; + } + } + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsStream.cs new file mode 100644 index 000000000..c250a9745 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsStream.cs @@ -0,0 +1,42 @@ +using System; +using BCG; +using BCG.Utilities; + +namespace BCG +{ + public class ConcatenateColumnsStream : ViewStream + { + // Concatenates the string representation of some of the fields + // in the input stream + // use the following syntax for the columnDef + // a:b c d + // where a is the name of the output column and b, c, and d are + // names of input colums to be concatenated + public ConcatenateColumnsStream( + ViewStream sourceStream, + params string [] columnDefs) + : this (sourceStream, true, columnDefs) + {} + + + + /// + /// + /// + /// + /// true when the data being appended does not change often + /// + public ConcatenateColumnsStream (ViewStream sourceStream, + bool isConcatenatedDataConfined, + params string[] columnDefs) + + { + mySetRealSource ( + new MapStream (sourceStream, + isConcatenatedDataConfined ? new ConcatenateColumnsMapperForSameDataSet (columnDefs) : + new ConcatenateColumnsMapper (columnDefs) + ) + ); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs b/src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs new file mode 100644 index 000000000..65b113dcb --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs @@ -0,0 +1,55 @@ +using System; + +namespace BCG +{ + [DataStreamUserRespresentation ("count")] + public class CountStream : ViewStream + { + private DataPacket myLastFiredDataPacket; + private IBlueTableSchema mySchema = new BlueTableSchema ("count", + new ColumnInfo ("id", 0, typeof(int)), + new ColumnInfo ("count", 1, typeof(int))); + private int myId = 0; + private ViewStream myVS; + + public CountStream (ViewStream vs) + { + myVS = vs; + } + + public CountStream (string tableName) + : this (new TableStream (tableName)) + {} + + protected override void myOnFirstSubscriber () { + myVS.onChange += new ChangeDelegate(myHandleChanges); + } + protected override void myOnLastSubscriber () { + myVS.onChange -= new ChangeDelegate(myHandleChanges); + } + + + private void myHandleChanges(object newValue) + { + myFireEvent ( + myLastFiredDataPacket = + myLastFiredDataPacket == null ? + new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), + new BlueTable (mySchema)) + : + new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), + myLastFiredDataPacket.newRows) + ); + } + + private IBlueRowCollection myGetRows (object dp, int id) + { + BlueTable retVal = new BlueTable (mySchema); + retVal.addRow + (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count)); + return retVal; + } + + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Crushing/AverageCollectorStream.cs b/src/DataStreamSuite/DataStream/Primitives/Crushing/AverageCollectorStream.cs new file mode 100644 index 000000000..ad81dd7a3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Crushing/AverageCollectorStream.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class AverageCollectorStream : ViewStream, ITableCrusher + { + public AverageCollectorStream(ViewStream stream, params string[] columnsToAverage) + { + myAveragingColumns = columnsToAverage; + mySetRealSource(new CrushingCollectorStream(stream, this)); + } + + private string[] myAveragingColumns; + + #region ITableCrusher Members + public IBlueRow crush(IBlueRowCollection rows, int idToUse) + { + IDictionary columnSums = new Dictionary(); + + IBlueRow resultRow = null; + foreach (IBlueRow row in rows) + { + resultRow = row; + + foreach (string colName in myAveragingColumns) + { + double sum = 0.0f; + columnSums.TryGetValue(colName, out sum); + sum += (double) row[colName]; + columnSums[colName] = sum; + } + } + + object[] resultValues = new object[rows.schema.columnNames.Length]; + + if (resultRow != null) + resultRow.CopyTo(resultValues, 0); + + resultValues[rows.schema["id"].colNo] = idToUse; + resultRow = new BlueRow(rows.schema, resultValues); + return resultRow; + } + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingCollectorStream.cs b/src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingCollectorStream.cs new file mode 100644 index 000000000..b7e6586e0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingCollectorStream.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class CrushingCollectorStream : ViewStream + { + + /* Arguments: a single + * Every time one of input the streams updates, it is crushed down to a single row, + * and added to the collection of output rows. + * + * The column 'tableIdentifierColumnName' is added to the schema, and each row from the viewstream + * is put into the + * + * */ + public CrushingCollectorStream(ViewStream toCrush, ITableCrusher crusher) + { + myToCrush = toCrush; + myRows = new List(); + myCrusher = crusher; + } + + private ViewStream myToCrush; + private List myRows; + private ITableCrusher myCrusher; + private int myNextId = 0; + + protected override void myOnFirstSubscriber() + { + myToCrush.onChange += new ChangeDelegate(myToCrush_onChange); + } + + protected override void myOnLastSubscriber() + { + myToCrush.onChange -= myToCrush_onChange; + } + + void myToCrush_onChange(object newValue) + { + DataPacket dp = (DataPacket)newValue; + + IBlueRow crushed = myCrusher.crush(dp.newRows, myNextId++); + + BlueTable newRows = new BlueTable(dp.schema); + newRows.addRows(myRows); + newRows.addRow(crushed); + myRows.Add(crushed); + + BlueTable added = new BlueTable(dp.schema); + added.addRow(crushed); + + myFireEvent(new DataPacket(newRows, added, new BlueTable(dp.schema))); + + } + + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingStream.cs b/src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingStream.cs new file mode 100644 index 000000000..7eefa3be1 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingStream.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class CrushingStream : ViewStream + { + public CrushingStream(ITableCrusher crusher, params ViewStream[] streamsToCrush) + { + myToCrush = streamsToCrush; + myCrusher = crusher; + myOldRows = new IBlueRow[myToCrush.Length]; + + //set up an anonymous delegate for each of the streams + //we do this so that we can unsubscribe + myDelegates = new ChangeDelegate[myToCrush.Length]; + for (int i = 0; i < myToCrush.Length; ++i) + { + int tempI = i; + myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); }); + } + } + + private IBlueRow[] myOldRows; + private ViewStream[] myToCrush; + private ITableCrusher myCrusher; + + private ChangeDelegate[] myDelegates; + + protected override void myOnFirstSubscriber() + { + for (int i = 0; i < myToCrush.Length; ++ i) + myToCrush[i].onChange += myDelegates[i]; + } + protected override void myOnLastSubscriber() + { + for (int i = 0; i < myToCrush.Length; ++i) + myToCrush[i].onChange -= myDelegates[i]; + } + + protected virtual void myHandleChange(int streamIndex, IBlueRowCollection newRows) + { + IBlueRow oldRow = myOldRows[streamIndex]; + + IBlueRow newRow = myCrusher.crush(newRows, streamIndex); + myOldRows[streamIndex] = newRow; + + BlueTable newState = new BlueTable(newRows.schema); + newState.addRows(myOldRows); + BlueTable addedRows = new BlueTable(newRows.schema); + addedRows.addRow(newRow); + BlueTable removedRows = new BlueTable(newRows.schema); + removedRows.addRow(oldRow); + + myFireEvent(new DataPacket(newState, addedRows, removedRows)); + + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Crushing/ITableCrusher.cs b/src/DataStreamSuite/DataStream/Primitives/Crushing/ITableCrusher.cs new file mode 100644 index 000000000..2e531fd45 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Crushing/ITableCrusher.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public interface ITableCrusher + { + IBlueRow crush(IBlueRowCollection rows, int idToUse); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs b/src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs new file mode 100644 index 000000000..23877d527 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class DerivativeStream : ViewStream + { + + private IBlueTableSchema mySchema; + private string myDerivandColumn; + private string myOutputColumn; + private string myKeyColumn; + private DateTime myLastPacketArrival = DateTime.MinValue; + private ViewStream myDataSource; + + private IDictionary myPreviousValues; + + + public DerivativeStream(ViewStream input, string keyColumn, string columnToDifferentiate, string outputColName) + { + myDerivandColumn = columnToDifferentiate; + myOutputColumn = outputColName; + myKeyColumn = keyColumn; + myDataSource = input; + } + + protected override void myOnFirstSubscriber() + { + myDataSource.onChange += new ChangeDelegate(myDataSource_onChange); + } + + protected override void myOnLastSubscriber() + { + myDataSource.onChange -= myDataSource_onChange; + } + + + void myDataSource_onChange(object newValue) + { + DataPacket dp = (DataPacket)newValue; + if (myLastPacketArrival == DateTime.MinValue) + mySchema = myBuildOutputSchema(dp.schema); + else + { + DateTime now = DateTime.Now; + BlueTable thisVersion = new BlueTable(mySchema); + double timeScale = 1.0 / (now - myLastPacketArrival).TotalSeconds; + foreach (IBlueRow r in dp.newRows) + thisVersion.addRow(myMakeRow(r, timeScale)); + + if (myLastValue == null) + myFireEvent(new DataPacket(thisVersion, new BlueTable(mySchema), new BlueTable(mySchema))); + else + { + IBlueRowCollection deleted, added; + BlueDataUtility.computeDifferences(thisVersion, ((DataPacket)myLastValue).newRows, out deleted, out added); + myFireEvent(new DataPacket(thisVersion, added, deleted)); + } + } + myStoreNewPacketInfo(dp); + } + + protected IBlueRow myMakeRow(IBlueRow r, double timeScale) + { + object[] outValues = new object[mySchema.columnInfo.Length]; + r.Values.CopyTo(outValues, 0); + + object derivand = r[myDerivandColumn]; + double lastValue = default(double); + if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value) + outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale; + else + outValues[mySchema.columnInfo.Length - 1] = 0.0; + return new BlueRow(mySchema, outValues); + + } + + protected IBlueTableSchema myBuildOutputSchema(IBlueTableSchema inSchema) + { + ColumnInfo[] cols = new ColumnInfo[inSchema.columnInfo.Length + 1]; + inSchema.columnInfo.CopyTo(cols, 0); + cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double)); + return new BlueTableSchema("derviative of " + inSchema.tableName, cols); + } + protected void myStoreNewPacketInfo(DataPacket dp) + { + myPreviousValues = new Dictionary(); + myLastPacketArrival = DateTime.Now; + foreach (IBlueRow row in dp.newRows) + { + object derivand = row[myDerivandColumn]; + if (derivand != null && derivand != DBNull.Value) + myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand); + } + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs b/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs new file mode 100644 index 000000000..b25e2aec0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using mbts; + +namespace BCG +{ + + + [DataStreamUserRespresentation("diff")] + public class DiffStream : ViewStream + { + ViewStream myLStream, myRStream; + + DataPacket myLastR = null, myLastL = null; + + BlueTable myLastFired = null; + + double myThreshold = .00005; + + string[] myColumnsToMatch; + public DiffStream(ViewStream v1, ViewStream v2) : this(null, v1, v2, "id") { } + public DiffStream(ViewStream v1, ViewStream v2, params string[] ColumnsToMatch) : this(null, v1, v2, ColumnsToMatch) { } + public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : + this(null, v1, v2, thresh, ColumnsToMatch) { } + + public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) + { + myThreshold = thresh; + myLStream = v1; + myRStream = v2; + myColumnsToMatch = columnsToMatch; + } + public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch) + : this(db,v1,v2,.00005,columnsToMatch) + { + } + + + protected override void myOnFirstSubscriber() + { + myLStream.onChange += myLChange; + myRStream.onChange += myRChange; + + } + + protected override void myOnLastSubscriber() + { + myLStream.onChange -= myLChange; + myRStream.onChange -= myRChange; + } + + struct ListComparer : IEqualityComparer> + { + public bool Equals(List x, List y) + { + if (x.Count != y.Count) + return false; + + for (int i = 0; i < x.Count; ++i) + if (!x[i].Equals(y[i])) + return false; + return true; + } + + public int GetHashCode(List obj) + { + int ret = 0; + for(int i = 0; i < obj.Count; ++i) + ret ^= obj[i].GetHashCode(); + return ret; + } + } + + void myHandleCallbacks() + { + DataPacket packet = null; + + if (myLastL == null || myLastR == null) + { + packet = myLastL == null ? myLastR : myLastL; + if (packet != null) + myFireEvent(packet); + + myLastFired = new BlueTable(packet.newRows); + return; + } + + BlueTable table = new BlueTable(myLastL.newRows.schema); + + Dictionary, Pair> rowPairs = new Dictionary, Pair>(new ListComparer()); + + foreach(BlueRow row in myLastL.newRows) + { + List hash = new List(); + foreach (string column in myColumnsToMatch) + { + try + { + hash.Add(row[column]); + } + catch (Exception) + { + } + } + + rowPairs[hash] = new Pair(row, null); + } + + foreach (BlueRow row in myLastR.newRows) + { + List hash = new List(); + foreach (string column in myColumnsToMatch) + { + try + { + hash.Add(row[column]); + } + catch (Exception) { } + } + + Pair pair; + rowPairs.TryGetValue(hash, out pair); + + + if (pair == null) + pair = new Pair(null, row); + else + pair.second = row; + + rowPairs[hash] = pair; + } + + int id = 0; + foreach (KeyValuePair, Pair> hashpair in rowPairs) + { + Pair pair = hashpair.Value; + + if (pair.first == null || pair.second == null) + { + BlueRow row = pair.first == null ? pair.second : pair.first; + + Dictionary newRow = new Dictionary(); + foreach (ColumnInfo info in row.schema.columnInfo) + { + newRow[info.colName] = row[info.colName]; + } + newRow["id"] = ++id; + table.addRow(new BlueRow(newRow)); + } + else + { + bool allZeros = true; + Dictionary newRow = new Dictionary(); + foreach (ColumnInfo info in pair.first.schema.columnInfo) + { + + if (info.colType != typeof(double)) + { + newRow[info.colName] = pair.first[info.colNo]; + continue; + } + + double l = (double)pair.first[info.colNo]; + double r = (double)pair.second[info.colNo]; + + double value = Math.Abs(l - r); + + if (value != 0) + { + double denom = Math.Max(Math.Abs(l), Math.Abs(r)); + + value /= denom; + + if (value > myThreshold) + { + allZeros = false; + } + else value = 0; + } + newRow[info.colName] = value; + } + if (allZeros) + continue; + newRow["id"] = ++id; + table.addRow(new BlueRow(newRow)); + } + + } + + myFireEvent(new DataPacket(table, table, myLastFired == null ? new BlueTable(table.schema) : myLastFired)); + myLastFired = table; + } + + + void myRChange(object newValue) + { + myLastR = (DataPacket)newValue; + + myHandleCallbacks(); + } + + + void myLChange(object newValue) + { + myLastL = (DataPacket)newValue; + + myHandleCallbacks(); + } + + + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs new file mode 100644 index 000000000..a1908da0b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs @@ -0,0 +1,36 @@ +// using System; +// using System.Collections; + +// using BCG.Utilities; + +// namespace BCG +// { +// public class DistinctRowEvaluator : GroupByEvaluator +// { + +// private int myIdGenerator = 0; +// public DistinctRowEvaluator (string[] columnNames) : base (null, columnNames) {} + +// public DistinctRowEvaluator(string colMapExpression, string[] columnNames) +// : base (colMapExpression, null, columnNames) {} + + +// public override IBlueRow evaluate (DataPacket packet) +// { +// myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable; +// string hashString = GroupByStream.getRepString (packet, myGroupByCols); +// if (packet.newRows.Count == 0) +// //if there are no rows for this group we remove the group from the output table +// myOutputTable.removeRow (hashString); +// else if (myOutputTable.getRow (hashString) == null) +// //a row is added for the first time (or first time after the removal of the group) +// myOutputTable.addRow ( +// BlueDataUtility.getNewReplacedRow ( +// BlueDataUtility.extractRow (packet.newRows, 0), +// myIdGenerator++, "id"), +// hashString); + +// return myProject (myOutputTable.getRow (hashString)); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs b/src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs new file mode 100644 index 000000000..dae2754b7 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs @@ -0,0 +1,23 @@ +// using System; +// using System.Collections; + +// namespace BCG +// { +// /// +// /// Provides a stream that with distinct rows with respect to the columns +// /// the user specifies in the argument +// /// +// [DataStreamUserRespresentation ("distinct")] +// public class DistinctRowStream : ViewStream +// { +// public DistinctRowStream (ViewStream stream, params string[] cols) +// { +// ArrayList groupByCols = new ArrayList (); +// foreach (string col in cols) +// foreach (string c in col.Split (' ')) +// groupByCols.Add (c); +// mySetRealSource (new GroupByStream (stream, +// new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false)); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs b/src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs new file mode 100644 index 000000000..ca7ba0707 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs @@ -0,0 +1,45 @@ +using System; +using System.IO; + +namespace BCG +{ + public class FileViewStream : ViewStream + { + private DataPacket myDp; + public FileViewStream (string dataFileName, + string schemaFileName) + : this (dataFileName, schemaFileName, Int32.MaxValue) + {} + + public FileViewStream (string dataFileName, + string schemaFileName, + int bufferSize) + { + IBlueRowCollection newRows = + new FileRowCollection (dataFileName, schemaFileName, bufferSize); + myFireEvent (new DataPacket (newRows, new BlueTable(newRows), new BlueTable (newRows.schema))); + } + + public FileViewStream (StreamReader dataReader, + StreamReader schemaReader) + : this (dataReader, schemaReader, Int32.MaxValue) + {} + + public FileViewStream (StreamReader dataReader, + StreamReader schemaReader, + int bufferSize) + { + IBlueRowCollection newRows = + new FileRowCollection (dataReader, schemaReader, bufferSize); + myDp = new DataPacket (newRows, new BlueTable(newRows), new BlueTable (newRows.schema)); + } + + protected override void myOnFirstSubscriber() + { + if (myDp != null) + myFireEvent (myDp); + } + + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/CaptureSnapshotStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/CaptureSnapshotStream.cs new file mode 100644 index 000000000..63e923291 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/CaptureSnapshotStream.cs @@ -0,0 +1,40 @@ +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; +// using System.IO; + +// namespace BCG +// { +// public class CaptureSnapshotStream : ViewStream +// { +// public CaptureSnapshotStream(string basePath,ViewStream sourceStream) +// { +// myBasePath = basePath; +// mySource = sourceStream; +// Directory.CreateDirectory(myBasePath); +// } + +// protected override void myOnFirstSubscriber() +// { +// mySource.onChange += new ChangeDelegate(mySource_onChange); +// } + +// protected override void myOnLastSubscriber() +// { +// mySource.onChange -= mySource_onChange; +// } + +// void mySource_onChange(object newValue) +// { +// DateTime now = DateTime.Now; +// new ViewStreamWaiter(new SaverStream(new BlueRowCollectionStaticStream(((DataPacket)newValue).newRows), +// Path.Combine(myBasePath, String.Format("{0}_{1}.ds", now.ToString("yyyyMMdd_HHmmss"), +// now.Millisecond)))).waitUntilArrived(); +// myFireEvent(newValue); +// } + +// private string myBasePath; +// private ViewStream mySource; +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs new file mode 100644 index 000000000..b9d7cea15 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs @@ -0,0 +1,49 @@ +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.IO; + +// using System.Runtime.Serialization.Formatters.Binary; + +// namespace BCG +// { +// public class DumpLoaderStream : ViewStream, ISideEffectStream +// { +// //wtf +// //public class Exception : System.Exception +// //{public Exception(string msg) : base(msg) { }} + +// public DumpLoaderStream(string fileName, int secondsBetweenPolls) +// { +// mySetRealSource( +// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, +// ViewStreamDatabase.instance)); +// } + +// public DumpLoaderStream(string fileName) +// { +// if (fileName == null) +// throw new LoaderStream.Exception("File name cannot be null"); + +// for (int i = 0; i < 5; i++) +// try +// { +// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read)) +// { +// IClientDeserializer deser = new ClientUniversalDeserializer(); +// byte[] the_ops; +// Array the_rows; +// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows); + +// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0))); +// } +// break; +// } +// catch (IOException) +// { +// System.Threading.Thread.Sleep(50); +// } + +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs new file mode 100644 index 000000000..b2d3b576f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs @@ -0,0 +1,46 @@ +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.IO; + +// using System.Runtime.Serialization.Formatters.Binary; + +// namespace BCG +// { +// [DataStreamUserRespresentation("loader")] +// public class LoaderStream : ViewStream, ISideEffectStream +// { +// public class Exception : System.Exception +// {public Exception(string msg) : base(msg) { }} + +// public LoaderStream(string fileName, int secondsBetweenPolls) +// { +// mySetRealSource( +// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, +// ViewStreamDatabase.instance)); +// } + +// public LoaderStream(string fileName) +// { +// if (fileName == null) +// throw new LoaderStream.Exception("File name cannot be null"); + +// for (int i = 0; i < 5; i++) +// try +// { +// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read)) +// { +// BinaryFormatter bf = new BinaryFormatter(); +// BlueTable bt = (BlueTable)bf.Deserialize(fs); +// mySetRealSource(new BlueRowCollectionStaticStream(bt)); +// } +// break; +// } +// catch (IOException) +// { +// System.Threading.Thread.Sleep(50); +// } + +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs new file mode 100644 index 000000000..629fc149f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Runtime.Serialization.Formatters.Binary; +using BCG.Utilities.General; +using BCG.Utilities; + +namespace BCG +{ + public delegate void PlaybackStreamProgress(long readFar, long totalBytesRead); + public class PlaybackStream : ViewStream, IWorker + { + + public PlaybackStream(string fileName) : this(fileName, 1.0) { } + + private string myFileName; + private FileStream myFileReader; + private double myPlaybackSpeed; + private DateTime myPlaybackStart; + private DateTime myNextPacketTime; + private IClientDeserializer myDeserializer; + private ITableAccumulator myAccumulator; + + private BlueTable myLastTableVersion; + + public PlaybackStream(string fileName, double playbackSpeed) + { + myFileName = fileName; + myPlaybackSpeed = playbackSpeed; + } + + protected override void myOnFirstSubscriber() + { myPlaybackStart = DateTime.Now; + myNextPacketTime = DateTime.MinValue; + myFileReader = new FileStream(myFileName, FileMode.Open, FileAccess.Read, FileShare.Read); + myReadNextPacketTime(); + ViewStreamDatabase.instance.addWorker(this); + } + + protected override void myOnLastSubscriber() + { + if (myFileReader != null) + myFileReader.Close(); + ViewStreamDatabase.instance.removeWorker(this); + } + + protected void myReadNextPacketTime() + { + if (myFileReader.Position == myFileReader.Length) + { + myNextPacketTime = DateTime.MaxValue; + myFileReader.Dispose(); + if (onFileExhausted != null) + onFileExhausted(); + } + else + { + double secondsElapsed = new BinaryReader(myFileReader).ReadDouble(); + myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed)); + if (onProgress != null) + onProgress(myFileReader.Position, myFileReader.Length); + } + } + + public event PlaybackStreamProgress onProgress; + public event Action onFileExhausted; + + + #region IWorker Members + + public void doWork() + { + if (myNextPacketTime <= DateTime.Now) + { + if (myDeserializer == null) + { + myDeserializer = new ClientUniversalDeserializer(); + myAccumulator = new TableAccumulator(myFileName, null); + } + + BinaryReader reader = new BinaryReader(myFileReader); + int numBytes = reader.ReadInt32(); + byte[] thisData = reader.ReadBytes(numBytes); + byte[] ops; + Array resultRowCollections; + myDeserializer.deserialize(new Stream[] { new MemoryStream(thisData) }, out ops, out resultRowCollections); + + //there should only be one op in there becuae we just read a single update. + // BlueEventLog.logAssert(ops.Length == 1,"Playback stream got " +ops.Length.ToString() + " op codes in one update, from stream file "+myFileName); + //there shoudl only be one set of collections becaue there was just a single op code + //(those two arrays are parallel) + // BlueEventLog.logAssert(resultRowCollections.Length == 1, "StreamOpCode was create, serializer should return exactly 1 set of rows"); + + if (ops[0] == (byte)StreamOpCode.create) + { + myLastTableVersion = new BlueTable((resultRowCollections.GetValue(0) as IBlueRowCollection[])[0]); + myFireEvent(new DataPacket(myLastTableVersion, new BlueTable(myLastTableVersion.schema), new BlueTable(myLastTableVersion.schema))); + } + else + { + // BlueEventLog.logAssert(ops[0] == (byte)StreamOpCode.multipleRemoveAdd, "Playback stream got unexpected open code " + ops[0].ToString() + " from file " + myFileName); + // BlueEventLog.logAssert(myLastTableVersion != null, "Playback stream for file "+myFileName + "got a non-create before a create"); + IBlueRowCollection[] deletedAndAdded = resultRowCollections.GetValue(0) as IBlueRowCollection[]; + BlueDataUtility.absorbChanges(myLastTableVersion, deletedAndAdded[0], deletedAndAdded[1]); + myFireEvent(new DataPacket(myLastTableVersion, deletedAndAdded[1], deletedAndAdded[0])); + } + + myReadNextPacketTime(); + } + + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs new file mode 100644 index 000000000..14993d6de --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace BCG +{ + public class RecorderStream : ViewStream + { + private ViewStream mySource; + private DateTime myStartTime; + private FileStream myOutFile; + private string myFileName; + private IBlueDataSerializer mySerializer; + + public RecorderStream(ViewStream src, string recordFile) + { + mySource = src; + myFileName = recordFile; + + myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); + } + + protected override void myOnFirstSubscriber() + { + myStartTime = DateTime.Now; + mySource.onChange += new ChangeDelegate(mySource_onChange); + myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate); + } + + protected override void myOnLastSubscriber() + { + mySource.onChange -= mySource_onChange; + myOutFile.Close(); + } + + void mySource_onChange(object newValue) + { + DataPacket dp = (DataPacket)newValue; + + BlueTable toWrite = new BlueTable(dp.schema); + + if (mySerializer == null) + { + mySerializer = new UniversalSerializer(dp.schema); + } + foreach (IBlueRow r in dp.newRows) + toWrite.addRow(r); + + byte[] result; + + if (myLastValue == null) + mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result); + else + mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result); + + BinaryWriter bw = new BinaryWriter(myOutFile); + //write down what time this happened + bw.Write((DateTime.Now - myStartTime).TotalSeconds); + bw.Write(result.Length); + bw.Write(result); + myOutFile.Flush(); + + myFireEvent(dp); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs new file mode 100644 index 000000000..11bae65c1 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs @@ -0,0 +1,64 @@ +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.IO; + +// using System.Runtime.Serialization.Formatters.Binary; + +// namespace BCG +// { +// [DataStreamUserRespresentation("saver")] +// public class SaverStream : ViewStream, ISideEffectStream +// { +// private ViewStream myStream; +// private string myFileName; + +// public class Exception : System.Exception +// { +// public Exception(string msg) : base(msg) { } +// } + +// public SaverStream(ViewStream vs, string fileName) +// { +// if (fileName == null) +// throw new SaverStream.Exception("File name cannot be null"); +// myStream = vs; +// myFileName = fileName; + +// } + +// protected override void myOnFirstSubscriber() +// { myStream.onChange += new ChangeDelegate(myHandleChanges); } + +// void myHandleChanges(object newValue) +// { +// DataPacket dp = newValue as DataPacket; +// mySave(dp); +// myFireEvent(newValue); +// } + +// protected override void myOnLastSubscriber() +// { myStream.onChange -= new ChangeDelegate(myHandleChanges); } + +// protected void mySave(DataPacket dp) +// { +// for (int i = 0; i < 5 ; i++) +// try +// { +// using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write)) +// { +// BinaryFormatter bf = new BinaryFormatter(); +// BlueTable t = new BlueTable(dp.schema); +// foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable +// t.addRow(new BlueRow(r)); +// bf.Serialize(fs, t); +// } +// break; +// } +// catch (IOException) +// { +// System.Threading.Thread.Sleep(5); +// } +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/BinaryFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/BinaryFilter.cs new file mode 100644 index 000000000..11c0d71a7 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/BinaryFilter.cs @@ -0,0 +1,15 @@ +using System; + +namespace BCG +{ + public class BinaryFilter : IStreamFilter + { + public BinaryFilter(bool filterValue) + {pass = filterValue;} + + public bool filter (IBlueRow dr) + {return pass;} + + bool pass = true; + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/BooleanExpressionFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/BooleanExpressionFilter.cs new file mode 100644 index 000000000..2b2214363 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/BooleanExpressionFilter.cs @@ -0,0 +1,18 @@ +// using System; +// using System.Collections; + +// using BCG.Utilities; + +// namespace BCG +// { +// public class BooleanExpressionFilter : IStreamFilter +// { +// public BooleanExpressionFilter(string exp) +// {myInterpreter = new BooleanExpressionsInterpreter(exp);} + +// public bool filter(IBlueRow dr) +// { return myInterpreter.eval(dr); } + +// private BooleanExpressionsInterpreter myInterpreter; +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs new file mode 100644 index 000000000..dec70b4de --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using BCG.Utilities.General; +namespace BCG +{ + public class DateBasedFilter: IDynamicStreamFilter, IWorker + { + private ViewStreamDatabase myVSD; + private string myDateColumn; + private int myNDays; + private DateTime myReference; + private DateTime myTodaysDate; + + public DateBasedFilter(string dateColumn, int nDays) + : this(dateColumn, nDays, ViewStreamDatabase.instance) { } + + public DateBasedFilter(string dateColumn, int nDays, ViewStreamDatabase vsd) + { + myDateColumn = dateColumn; + myNDays = nDays; + myVSD = vsd; + myVSD.addWorker(this); + myTodaysDate = DateTime.Today; + myReference = myTodaysDate.AddDays(-1 * myNDays); + } + + public event FilterChangeDelegate onFilterChange; + + public bool filter(IBlueRow r) + { + if (DBNull.Value.Equals(r[myDateColumn])) + { + //return false; + return true; //it can be considered as our assumption whenever datecolumn is db null we return true + } + DateTime t = (DateTime)r[myDateColumn]; + bool returnVallue = t < myReference ? false : true; + return returnVallue; + } + + + public void doWork() + { + if (DateTime.Now >= myTodaysDate.AddDays(1)) + { + myTodaysDate = DateTime.Today; + myReference = myTodaysDate.AddDays(-1 * myNDays); + onFilterChange(); + } + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilterStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilterStream.cs new file mode 100644 index 000000000..66847e218 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilterStream.cs @@ -0,0 +1,15 @@ +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; + +// namespace BCG +// { +// public class DateBasedFilterStream: ViewStream +// { +// public DateBasedFilterStream(ViewStream vs, string dateColName, int pastTodayDays) +// { +// mySetRealSource(new FilterStream(vs, new DateBasedFilter(dateColName, pastTodayDays))); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs new file mode 100644 index 000000000..8512bb50d --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using BCG.Utilities.General; + +namespace BCG +{ + public class DayRollingFilter : IDynamicStreamFilter, IWorker + { + private int myNPastTodayDays; + private string myDateColName; + private DateTime myLastCheckedTime = DateTime.Now; + /// + /// filters based on how 'old' the date in the dateCol are. + /// + /// + /// specifies the colName from which the date is retrieved + /// rows that removed from today by 'pastTodayDays' will be filtered + public DayRollingFilter(ViewStreamDatabase vsd, string dateColName, int pastTodayDays) + { + myNPastTodayDays = pastTodayDays; + myDateColName = dateColName; + vsd.addWorker(this); + } + + #region IDynamicStreamFilter Members + + public event FilterChangeDelegate onFilterChange; + + #endregion + + #region IStreamFilter Members + + public bool filter(IBlueRow r) + { + DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date; + DateTime today = DateTime.Now.Date; + return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays); + } + + #endregion + + #region IWorker Members + + public void doWork() + { + if (DateTime.Now.Date != myLastCheckedTime.Date) + { + myLastCheckedTime = DateTime.Now; + onFilterChange(); + } + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingStream.cs new file mode 100644 index 000000000..abf34eeb4 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingStream.cs @@ -0,0 +1,15 @@ +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; + +// namespace BCG +// { +// public class DayRollingStream : ViewStream +// { +// public DayRollingStream(ViewStream vs, string dateColName, int pastTodayDays) +// { +// mySetRealSource(new FilterStream(vs, new DayRollingFilter(ViewStreamDatabase.instance, dateColName, pastTodayDays))); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs new file mode 100644 index 000000000..9dd0c5097 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs @@ -0,0 +1,306 @@ +// using BCG; +// using System; +// using System.Data; +// using System.Collections; +// using BCG.Utilities; +// using System.Text.RegularExpressions; +// using mbts; + +// namespace BCG.StreamWorld.Addition +// { +// [DataStreamUserRespresentation (false)] +// public class FilterStream : ViewStream +// { +// private ViewStream myBaseStream; +// private BlueTable myBaseTable, myStateTable; +// private IStreamFilter myFilter; + +// public FilterStream(ViewStream vs) +// : this(vs, "1=1") +// { } + +// public FilterStream(ViewStream vs, string filterExp) +// : this(vs, new BooleanExpressionFilter(filterExp)) +// { } + +// public FilterStream(ViewStream vs, IStreamFilter filter) +// { +// myBaseStream = vs; +// myFilter = filter; +// if (filter is IDynamicStreamFilter) +// (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange); +// } + + +// protected override void myOnFirstSubscriber() +// { +// myBaseStream.onChange += new ChangeDelegate(myHandleChanges); +// } + +// protected override void myOnLastSubscriber() +// { +// myBaseStream.onChange -= new ChangeDelegate(myHandleChanges); +// myBaseTable.clear(); +// myStateTable.clear(); +// } + +// private void myHandleChanges(object newValue) +// { +// DataPacket dp = newValue as DataPacket; +// if (myStateTable == null) +// { +// BlueEventLog.logAssert(myBaseTable == null); +// myStateTable = new BlueTable(dp.schema); +// myBaseTable = new BlueTable(dp.schema); +// } +// BlueTable dels = new BlueTable(dp.schema), +// adds = new BlueTable(dp.schema); +// foreach (IBlueRow r in dp.deletedRows) +// { +// myBaseTable.deleteRow(r); +// if (myStateTable.containsID((int)r["id"])) +// { +// dels.addRow(r); +// myStateTable.deleteRow(r); +// } +// } +// foreach (IBlueRow r in dp.addedRows) +// { +// myBaseTable.addRow(r); +// if (myFilter.filter(r)) +// { +// adds.addRow(r); +// myStateTable.addRow(r); +// } +// } +// if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null)) +// myFireEvent(new DataPacket(myStateTable, adds, dels)); +// } + +// public IStreamFilter filter +// { +// set +// { +// myFilter = value; +// if (myBaseTable != null) +// { +// IBlueRowCollection oldState = new BlueTable(myStateTable); +// myStateTable.clear(); +// foreach (IBlueRow r in myBaseTable) +// if (myFilter.filter(r)) +// myStateTable.addRow(r); +// IBlueRowCollection dels, adds; +// BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds); +// myFireEvent(new DataPacket(myStateTable, adds, dels)); +// } +// } +// get { return myFilter; } +// } + +// //public string filterString { set { filter = new BooleanExpressionFilter(value); } } + +// private void myHandleFilterChange() { filter = myFilter; } +// } +// } + +// namespace BCG +// { +// public class RegExpFilter : IStreamFilter +// { +// private Regex[] myRegExps; +// private string[] myCols; + +// public RegExpFilter(string col, string regExp) +// : this (new string[]{col}, new string[] {regExp}) +// {} + +// public RegExpFilter (string[] cols, string[] regExps) +// { +// myCols = cols; +// myRegExps = new Regex[regExps.Length]; +// for (int i = 0; i < regExps.Length; i++) +// myRegExps[i] = new Regex(regExps[i]); +// } + +// #region IStreamFilter Members +// public bool filter(IBlueRow dr) +// { +// bool rv = true; +// for (int i = 0; i < myCols.Length; i++) +// { +// rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString()); +// if (!rv) +// return rv; +// } +// return rv; +// } +// #endregion +// } + +// public class FilterStream : ViewStream +// { +// private BlueTable myLastFiredRows; +// private IBlueRowCollection myLastReceivedRows; +// private IStreamFilter myFilter; +// private ViewStream myParentStream; +// private string myFilterString; +// private BCG.StreamWorld.Addition.FilterStream myFilterStream; +// private bool myUseNewFilter = DataStreamLoadFileConfig.instance.useNewFilter; +// private HighPerformanceTimer _timer = new HighPerformanceTimer(); + +// #region constructors +// public delegate bool OnFilter(IBlueRow r); +// private class OnFilterWrapper : IStreamFilter +// { +// public OnFilterWrapper(OnFilter f) +// { this.f = f; } + +// public bool filter(IBlueRow dr) +// { return f(dr); } + +// private OnFilter f; +// } +// public FilterStream(ViewStream vs, OnFilter f) +// : this(vs, new OnFilterWrapper(f)) +// { } + + +// public FilterStream(ViewStream vs, IStreamFilter filter) +// { +// if (myUseNewFilter) +// { +// myFilterStream = new BCG.StreamWorld.Addition.FilterStream(vs, filter); +// mySetRealSource(myFilterStream); +// } +// else +// { +// myParentStream = vs; +// myFilter = filter; +// myFilterString = filter.ToString(); +// if (filter is IDynamicStreamFilter) +// (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange); +// } +// } + +// public FilterStream(ViewStream vs, string filterExpression) +// : this(vs, new BooleanExpressionFilter(filterExpression)) +// { +// myFilterString = filterExpression; +// } + +// public FilterStream(ViewStream vs, string col, string regExp) +// : this(vs, new RegExpFilter(col, regExp)) +// { +// myFilterString = col + " , " + regExp; +// } + +// public FilterStream(ViewStream vs) +// : this(vs, new BinaryFilter(true)) +// { +// myFilterString = true.ToString(); +// } +// #endregion + +// #region overridden methods +// protected override void myOnFirstSubscriber() +// { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); } + +// protected override void myOnLastSubscriber() +// { +// myLastFiredRows = null; +// myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks); +// } + +// protected override void myFireEvent(object o) +// { +// DataPacket dp = o as DataPacket; +// if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0) +// base.myFireEvent(o); +// } +// #endregion + +// private void myHandleCallbacks(object newValue) +// { +// _timer.start(); +// DataPacket retVal; +// DataPacket arrivedPacket = newValue as DataPacket; +// myLastReceivedRows = arrivedPacket.newRows; +// if (myLastFiredRows != null) +// { +// retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema), +// new BlueTable(myLastFiredRows.schema)); +// foreach (IBlueRow r in arrivedPacket.deletedRows) +// if (myLastFiredRows.containsID((int)r["id"])) +// { +// myLastFiredRows.deleteRow(r); +// (retVal.deletedRows as BlueTable).addRow(r); +// } +// foreach (IBlueRow r in arrivedPacket.addedRows) +// if (myFilter.filter(r)) +// { +// myLastFiredRows.addRow(r); +// (retVal.addedRows as BlueTable).addRow(r); +// } +// } +// else +// { +// myLastFiredRows = new BlueTable(arrivedPacket.schema); +// retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema)); +// foreach (IBlueRow r in arrivedPacket.newRows) +// if (myFilter.filter(r)) +// myLastFiredRows.addRow(r); +// } +// _timer.stop(); +// myFireEvent(retVal); +// } + +// public IStreamFilter filter +// { +// set +// { +// if (myUseNewFilter) +// { +// myFilterStream.filter = filter; +// } +// else +// { +// myFilter = value; +// if (numberOfSubscribers > 0) +// { +// if (myLastReceivedRows != null) +// { +// BlueTable nRows = new BlueTable(myLastReceivedRows.schema); +// foreach (IBlueRow r in myLastReceivedRows) +// if (myFilter.filter(r)) +// nRows.addRow(r); +// IBlueRowCollection dRows = new BlueTable(nRows.schema), +// aRows = new BlueTable(nRows.schema); +// BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows); +// myLastFiredRows = nRows; +// myFireEvent(new DataPacket(nRows, aRows, dRows)); +// } +// } +// } +// } +// get +// { +// if (myUseNewFilter) +// return myFilterStream.filter; +// else +// return myFilter; +// } +// } + +// //public string filterString { set { filter = new BooleanExpressionFilter(value); } } + +// public override string ToString() +// { +// return GetType().ToString() + " " + myFilterString; +// } + +// private void myHandleFilterChange() +// { +// filter = myFilter; +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/IDynamicStreamFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/IDynamicStreamFilter.cs new file mode 100644 index 000000000..ee7296a01 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/IDynamicStreamFilter.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public delegate void FilterChangeDelegate(); + + public interface IDynamicStreamFilter : IStreamFilter + { + event FilterChangeDelegate onFilterChange; + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/IStreamFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/IStreamFilter.cs new file mode 100644 index 000000000..88611f561 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/IStreamFilter.cs @@ -0,0 +1,16 @@ +using System; +using System.Data; + +namespace BCG +{ + /// + /// Exposes the function, filter(DataTable) that decides whether a row passes some + /// filter criteria or not. This interface is passed as an argument to the FilterStream + /// as the filtering criteria + /// + /// + public interface IStreamFilter + { + bool filter (IBlueRow dr); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs new file mode 100644 index 000000000..0d902f6cc --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using BCG.Utilities.General; + +namespace BCG +{ + public class SlidingWindowFilter: IDynamicStreamFilter, IWorker + { + private ViewStreamDatabase myVSD; + private string myDateColumn; + private int myNSecs; + private DateTime myReference; + private DateTime myCurrentTime; + private TimeSpan mySpan = new TimeSpan(0, 0, 0, 0, 500); + private DateTime myLastTimeFilteredApplied; + + public SlidingWindowFilter(string dateColumn, int nSecs) + : this(dateColumn, nSecs, ViewStreamDatabase.instance) { } + + public SlidingWindowFilter(string dateColumn, int nSecs, ViewStreamDatabase vsd) + { + myDateColumn = dateColumn; + myNSecs = nSecs; + myVSD = vsd; + myVSD.addWorker(this); + myCurrentTime = DateTime.Now; + myReference = myCurrentTime.AddSeconds(-1 * myNSecs); + } + + public event FilterChangeDelegate onFilterChange; + + public bool filter(IBlueRow r) + { + if (DBNull.Value.Equals(r[myDateColumn])) + { + //it can be considered as our assumption whenever datecolumn is db null we return true + return true; //dont use this for volparamsched use true + } + DateTime t = (DateTime)r[myDateColumn]; + bool returnVallue = t < myReference ? false : true; + return returnVallue; + } + + + public void doWork() + { + if (DateTime.Now - myLastTimeFilteredApplied > mySpan) + { + myCurrentTime = DateTime.Now; + myReference = myCurrentTime.AddSeconds(-1 * myNSecs); + myLastTimeFilteredApplied = DateTime.Now; + onFilterChange(); + + } + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilterStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilterStream.cs new file mode 100644 index 000000000..1f343a470 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilterStream.cs @@ -0,0 +1,16 @@ +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; + +// namespace BCG +// { +// public class SlidingWindowFilterStream: ViewStream +// { +// //use this with caution ask Vamsee +// public SlidingWindowFilterStream(ViewStream vs, string dateColName, int secs) +// { +// mySetRealSource(new FilterStream(vs, new SlidingWindowFilter(dateColName, secs))); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs new file mode 100644 index 000000000..4b12556a1 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using BCG.Utilities.General; + +namespace BCG +{ + public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable + { + #region constructors + public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive) + : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) + { + } + + public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) + : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) + { + } + + public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive) + : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) + { + } + + public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) + { + myVsd = vsd; + myRowTimeColumnName = rowTimeColumnName; + myRowTimeToLive = rowTimeToLive; + myFilterFrequency = filterFrequency; + myNextFilter = DateTime.MinValue; + + // add worker + myVsd.addWorker(this); + } + #endregion + + public event FilterChangeDelegate onFilterChange; + + #region IStreamFilter + public bool filter(IBlueRow row) + { + DateTime now = DateTime.Now; + DateTime timeStamp = (DateTime)row[myRowTimeColumnName]; + + if (now > timeStamp) + { + TimeSpan span = now - timeStamp; + + if (span >= myRowTimeToLive) + { + return (false); + } + } + + return (true); + } + #endregion + + #region IWorker + public void doWork() + { + DateTime now = DateTime.Now; + + if (now > myNextFilter) + { + // fire event + onFilterChange(); + + myNextFilter = now.Add(myFilterFrequency); + } + } + #endregion + + #region IDisposable + public void Dispose() + { + // remove worker + myVsd.removeWorker(this); + } + #endregion + + #region state + private ViewStreamDatabase myVsd; + private string myRowTimeColumnName; + private TimeSpan myRowTimeToLive; + private TimeSpan myFilterFrequency; + private DateTime myNextFilter; + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Freeze/FreezeDataStream.cs b/src/DataStreamSuite/DataStream/Primitives/Freeze/FreezeDataStream.cs new file mode 100644 index 000000000..a6174b256 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Freeze/FreezeDataStream.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections; + +using BCG.Utilities; + +namespace BCG +{ + public class FreezeDataStream : DataStream + { + public FreezeDataStream(DataStream streamToFreeze) + : this (streamToFreeze, false) + {} + + public FreezeDataStream(DataStream streamToFreeze, bool freezeNow) + { + myIsFrozen = freezeNow; + myStreamToFreeze = streamToFreeze; + myPendingChanges = new ArrayList(); + } + + + public bool frozen + { + set + { + if (myIsFrozen != value) + { + myIsFrozen = value; + if (value) // not frozen to frozen + myPendingChanges.Clear(); + else // frozen to unfrozen + foreach (object o in myPendingChanges) + myFireEvent(o); + } + } + get{return myIsFrozen;} + } + + protected override void myOnFirstSubscriber () + {myStreamToFreeze.onChange += new DataStream.ChangeDelegate (myCallBack);} + + protected override void myOnLastSubscriber () + {myStreamToFreeze.onChange -= new DataStream.ChangeDelegate (myCallBack);} + + internal void myCallBack(object o) + { + if (!myIsFrozen) + myFireEvent(o); + else { + if (o is ICloneable) + o = (o as ICloneable).Clone(); + myPendingChanges.Add(o); + } + } + + internal ArrayList myPendingChanges; + internal DataStream myStreamToFreeze; + internal bool myIsFrozen; + + public void myCallBackTest(object o) + { + myCallBack(o); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Freeze/FreezeViewStream.cs b/src/DataStreamSuite/DataStream/Primitives/Freeze/FreezeViewStream.cs new file mode 100644 index 000000000..280ddaa66 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Freeze/FreezeViewStream.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections; + +using BCG.Utilities; + +namespace BCG +{ + public class FreezeViewStream : ViewStream + { + public FreezeViewStream(ViewStream streamToFreeze) + : this (streamToFreeze, false) + {} + + public FreezeViewStream (ViewStream viewToFreeze, bool isFrozen) + { + myFDS = new FreezeDataStream(viewToFreeze,isFrozen); + mySetRealSource(myFDS); + } + + public bool frozen + { + set + {myFDS.frozen = value;} + get + {return myFDS.frozen ;} + } + + internal FreezeDataStream myFDS; + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs b/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs new file mode 100644 index 000000000..ef4ba2e1b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs @@ -0,0 +1,447 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using BCG.Utilities.General; + +using FuzzyRowComparer = System.Func; +using System.Diagnostics; + +namespace BCG +{ + /// + /// This Implements a fuzzy join over two streams across a time window. This works by keeping a buffer + /// specified of a few seconds. When a row expires from one buffer + /// (after timeMS) it is compared to all of the entries currently in the other buffer using the supplied + /// fuzzy compare function. The best (closest to 0) non-infinity match is chosen and the two rows are combined into one + /// and fired. Any rows that expire without a match are still fired but with no value in the other columns. + /// + /// Requirements: + /// * Both streams have a date/time column for sorting and expiring from the match buffer + /// * FuzzyCompare must be implemented so that rows that match have a value close to 0. + /// + /// 1/10/2011 - Changed the code to ensure that the right hand side is only matched once. + /// This was a fairly significant logical re-architecture and has caused some pollution (side-effects) in some of the matching and merging functions. :( + /// Probably need to go back and refactor some of the helper methods to make the code easier to read. + /// + public class FuzzyJoinOverTimeStream : ViewStream, IWorker + { + private const string NEW_TIME_COLUMN = "FuzzyJoinTimeColumn"; + protected ViewStream myLeftSource; + protected ViewStream myRightSource; + protected ViewStreamDatabase myVsd; + + protected List myLeftRows = new List(); + protected List myRightRows = new List(); + private Dictionary myBestMatch = new Dictionary(); + private string myLeftTimeColumn; + private string myRightTimeColumn; + private IComparer myLeftCompare; + private IComparer myRightCompare; + protected FuzzyRowComparer myFuzzyCompare; + protected TimeSpan myTimeWindow; + protected IBlueTableSchema myLeftSchema; + protected IBlueTableSchema myRightSchema; + protected IBlueTableSchema myOutputSchema; + private int myID = 0; + + protected TimeWindowTable myStateRows; + + private DateTime myNextCull; + const long MinCullIntervalMS = 1000; + + // #constructor + public FuzzyJoinOverTimeStream(ViewStream left, string leftTimeColumn, ViewStream right, string rightTimeColumn, int timeMS, FuzzyRowComparer fuzzyCompare, ViewStreamDatabase vsd) + { + this.myLeftSource = left; + this.myRightSource = right; + this.myLeftTimeColumn = leftTimeColumn; + this.myRightTimeColumn = rightTimeColumn; + this.myLeftCompare = new BlueRowComparer(true, leftTimeColumn, "id"); + this.myRightCompare = new BlueRowComparer(true, rightTimeColumn, "id"); + this.myNextCull = DateTime.Now.AddMilliseconds(MinCullIntervalMS); + this.myTimeWindow = TimeSpan.FromMilliseconds(timeMS); + this.myVsd = vsd; + this.myFuzzyCompare = fuzzyCompare; + } + + public virtual IBlueTableSchema OutputSchema + { + get + { + if (this.myOutputSchema == null) + { + if (this.myLeftSchema != null && this.myRightSchema != null) + { + ColumnInfo[] leftColumns = this.myLeftSchema.columnInfo; + ColumnInfo[] rightColumns = this.myRightSchema.columnInfo; + ColumnInfo[] addColumns = new ColumnInfo[] { new ColumnInfo(NEW_TIME_COLUMN, 0, typeof(DateTime)) }; + IEnumerable newColumns = leftColumns.Concat(rightColumns).GroupBy(c => c.colName).Select(s => s.First()); + newColumns = newColumns.Concat(addColumns); + + // BlueTableSchema constructor assumes that each column has a valid non overlapping column number + ColumnInfo[] newColumnsWithIndex = new ColumnInfo[newColumns.Count()]; + int x = 0; + foreach (ColumnInfo ci in newColumns) + { + newColumnsWithIndex[x] = new ColumnInfo(ci.colName, x, ci.colType); + x++; + } + this.myOutputSchema = new BlueTableSchema(newColumnsWithIndex, "FuzzyJoin"); + } + } + return this.myOutputSchema; + } + } + + protected override void myOnFirstSubscriber() + { + this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange); + this.myRightSource.onChange += new ChangeDelegate(myRight_onChange); + this.myVsd.addWorker(this); + } + + protected override void myOnLastSubscriber() + { + this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange); + this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange); + this.myVsd.removeWorker(this); + } + + void myRight_onChange(object newValue) + { + // Collect all right hand columns + DataPacket dp = (DataPacket)newValue; + AddNewRows(dp.addedRows, this.myRightRows, this.myRightCompare); + this.myRightSchema = dp.schema; + } + + void myLeft_onChange(object newValue) + { + DataPacket dp = (DataPacket)newValue; + AddNewRows(dp.addedRows, this.myLeftRows, this.myLeftCompare); + this.myLeftSchema = dp.schema; + } + + void AddNewRows(IBlueRowCollection rows, List list, IComparer compare) + { + // Rows tend to come in in reverse order since they were added as they arrived + // To avoid unnecessary memcpy in array resize add them to our collection backwards + IBlueRow[] rowsArray = new IBlueRow[rows.Count]; + rows.CopyTo(rowsArray, 0); + for (int x = rowsArray.Length - 1; x >= 0; x--) + { + SortedInsert(rowsArray[x], list, compare); + } + } + + private void SortedInsert(IBlueRow row, List list, IComparer compare) + { + int index = list.BinarySearch(row, compare); + if (index < 0) + { + list.Insert(~index, row); + } + } + + private IBlueRow[] PruneExpiredRows(List list, string timeColumn, DateTime expiry) + { + // Linear search is fine here b/c we don't expect the data to be very large + int count; + for (count = 0; count < list.Count; count++) + { + if ((DateTime)list[count][timeColumn] > expiry) + { + // Stop when we find the first row that we can keep. Since this is sorted all others are also kept. + break; + } + } + + IBlueRow[] expiredRows = new IBlueRow[count]; + list.CopyTo(0, expiredRows, 0, count); + list.RemoveRange(0, count); + + return expiredRows; + } + + class RowMatch : IComparable + { + public RowMatch(IBlueRow row, double fitness) + { + this.myRightRow = row; + this.myFitness = fitness; + } + public IBlueRow myRightRow { get; private set; } + public IBlueRow myLeftRow { get; set; } + public double myFitness { get; private set; } + public IBlueRow myMergedRow { get; set; } + + #region IComparable Members + + public int CompareTo(RowMatch other) + { + return myFitness.CompareTo(other.myFitness); + } + + #endregion + } + + public void doWork() + { + DateTime now = DateTime.Now; + if (now < this.myNextCull) return; + + this.myNextCull = now.AddMilliseconds(MinCullIntervalMS); + + // Create a BlueTable for sending out. Determine schema + + DateTime expirationCutoff = now.Subtract(this.myTimeWindow); + + // If we haven't heard from one or both of the input streams then we can't formulate a valid output row + if (this.OutputSchema == null) return; + + // Get the removed rows from left + IBlueRow[] leftExpired = PruneExpiredRows(this.myLeftRows, this.myLeftTimeColumn, expirationCutoff); + IEnumerable[] rightMatches = new IEnumerable[leftExpired.Length]; + + + List corrections = new List(); + + /* Try to find the best match without making this an NP problem + * 1. Find a list of best matches for each expiring trade + * 2. Mark the right row with the 'best' match that was found for it + * 3. When we emit a trade only include the right hand side if we are the best match + * 4. If later a better match is found !! emit a remove row for the merged row and then a new row for the unmatched left hand side + */ + + // Search for matches in right + for (int x = 0; x < leftExpired.Length; x++) + { + IBlueRow row = leftExpired[x]; + var matches = FindMatches(row, this.myRightRows, corrections); + if (matches != null) + { + rightMatches[x] = matches; + } + } + + // get the removed rows from right -- + IBlueRow[] rightExpired = PruneExpiredRows(this.myRightRows, this.myRightTimeColumn, expirationCutoff.Subtract(this.myTimeWindow)); + + // If there's nothing to do go home early + // note: This test has to happen after searching for matches or myRightRows will not be complete + if (rightExpired.Length == 0 && leftExpired.Length == 0) return; + + // Lazy initialize of myStateRows now that we know the full schema + if (this.myStateRows == null) + { + this.myStateRows = new TimeWindowTable(this.OutputSchema, (int)TimeSpan.FromHours(12).TotalMilliseconds, NEW_TIME_COLUMN); + } + + BlueTable adds; + MergeExpiredRows(leftExpired, rightMatches, rightExpired, out adds); + + // Add adds to state + foreach (IBlueRow row in adds) + { + this.myStateRows.addRow(row); + } + BlueTable dels = (BlueTable)this.myStateRows.moveWindow(); + + HandleCorrection(corrections, adds, dels); + + // Fire event + this.myFireEvent(new DataPacket(this.myStateRows, adds, dels)); + + // remove matched flags for expired rows + foreach (IBlueRow row in rightExpired) + { + this.myBestMatch.Remove(row); + } + } + + private void HandleCorrection(List corrections, BlueTable adds, BlueTable dels) + { + // We already emitted correction.MergedRow so we need to redact that and + // then emit correction.left as an add since it is no longer matched + foreach (RowMatch correction in corrections.Distinct()) + { + Debug.Assert(correction.myMergedRow != null); + this.myStateRows.RemoveRow(correction.myMergedRow); + adds.addRow(this.MergeRows(correction.myLeftRow, null)); + dels.addRow(correction.myMergedRow); + } + } + + private BlueTable MergeExpiredRows(IBlueRow[] leftExpired, IEnumerable[] rightMatches, IBlueRow[] rightExpired, out BlueTable adds) + { + adds = new BlueTable(this.OutputSchema); + + // + // Loop over both left and right expired rows and emit them in order. + // (If they're going to get shuffled on the wire anyway why bother?) + // + int l = 0; + int r = 0; + for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; ) + { + if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn]) + { + // Advance Left + IEnumerable rm = rightMatches[l]; + IBlueRow leftRow = leftExpired[l]; + + IBlueRow merged = MergeLeftWithBestRight(leftRow, rm); + + // Increment and fire + l++; + adds.addRow(merged); + } + else + { + IBlueRow rightRow = rightExpired[r]; + + // If a match was found this row will already be emitted with it's left hand counterpart + if (!this.myBestMatch.ContainsKey(rightRow)) + { + IBlueRow merged = this.MergeRows(null, rightRow); + adds.addRow(merged); + } + r++; + } + } + + // handle remaining rows. Only one of the next two loops will execute + for (; l < leftExpired.Length; l++) + { + IBlueRow merged = this.MergeLeftWithBestRight(leftExpired[l], rightMatches[l]); + adds.addRow(merged); + } + + for (; r < rightExpired.Length; r++) + { + if (!this.myBestMatch.ContainsKey(rightExpired[r])) + { + IBlueRow merged = this.MergeRows(null, rightExpired[r]); + adds.addRow(merged); + } + } + + return adds; + } + + private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable rightMatches) + { + IBlueRow merged = null; + if (rightMatches != null) + { + foreach (RowMatch match in rightMatches) + { + RowMatch bestMatch = this.myBestMatch[match.myRightRow]; + //if (match.myFitness == bestMatch.myFitness && bestMatch.myLeftRow == null) + if (match == bestMatch) + { + merged = this.MergeRows(leftRow, match.myRightRow); + + // store these for later corrections + match.myMergedRow = merged; + match.myLeftRow = leftRow; + + // Make sure that we're updating this.myBestMatch[match.myRightRow] + Debug.Assert(match == this.myBestMatch[match.myRightRow]); + break; + } + } + } + + // If none of the matches was 'best' emit with empty right hand side + if (merged == null) + { + merged = this.MergeRows(leftRow, null); + } + return merged; + } + + /// + /// Override this function to provide a custom row merge. + /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting. + /// + protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right) + { + Dictionary newRow = new Dictionary(); + + if (left == null && right == null) throw new ArgumentException("oh come on."); + + if (left != null) + { + foreach (string columnName in left.schema.columnNames) + { + if (columnName == "id") continue; + newRow[columnName] = left[columnName]; + } + } + + if (right != null) + { + foreach (string columnName in right.schema.columnNames) + { + if (columnName == "id") continue; + if (!newRow.ContainsKey(columnName)) + { + newRow[columnName] = right[columnName]; + } + } + } + newRow["id"] = this.myID++; + newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn]; + + return new BlueRow(this.OutputSchema, newRow); + } + + private IEnumerable FindMatches(IBlueRow row, List list, List corrections) + { + List matches = null; + + // Search through all rows in list and compare with the fuzzy comparer + for (int x = 0; x < list.Count; x++) + { + IBlueRow match = list[x]; + double fitness = Math.Abs(this.myFuzzyCompare(row, match)); + + if (!double.IsInfinity(fitness)) + { + // Lazy allocate list - not having any matches will be fairly common + if (matches == null) + { + matches = new List(); + } + + RowMatch currentMatch = new RowMatch(match, fitness); + int insertIndex = matches.BinarySearch(currentMatch); + matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch); + + // Mark the best match for this particular right hand match + RowMatch oldMatch; + if (this.myBestMatch.TryGetValue(match, out oldMatch)) + { + if (currentMatch.myFitness < oldMatch.myFitness) + { + this.myBestMatch[match] = currentMatch; + if (oldMatch.myMergedRow != null) + { + // We already emitted oldMatch.MergedRow so we need to fix it + corrections.Add(oldMatch); + } + } + } + else + { + this.myBestMatch[match] = currentMatch; + } + } + } + + return matches; + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GridBindable/GridBindableViewStream.cs b/src/DataStreamSuite/DataStream/Primitives/GridBindable/GridBindableViewStream.cs new file mode 100644 index 000000000..a68c805c2 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GridBindable/GridBindableViewStream.cs @@ -0,0 +1,42 @@ +// using System; +// using System.Collections; + +// namespace BCG +// { +// public class GridBindableViewStream : DataStream +// { +// private ViewStream myStream; +// private GridBindableBlueTable myList; + +// public GridBindableViewStream (ViewStream stream) +// { +// myStream = stream; +// myStream.onChange += new ChangeDelegate(myHandleCallbacks); +// } + +// private void myHandleCallbacks (object newValue) +// { +// DataPacket dp = newValue as DataPacket; +// if (myList != null) +// { +// foreach (IBlueRow r in dp.deletedRows) +// myList.Remove (r); +// foreach (IBlueRow r in dp.addedRows) +// myList.Add (r); +// } +// else +// { +// myList = myGetBindableCollection (dp.newRows); +// } +// myFireEvent ( +// new DataPacket(myList, +// myGetBindableCollection (dp.addedRows), +// myGetBindableCollection (dp.deletedRows))); +// } + +// private GridBindableBlueTable myGetBindableCollection (IBlueRowCollection col) +// { +// return new GridBindableBlueTable (col.schema, col); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs new file mode 100644 index 000000000..c9ccf45a5 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + + public class AverageGroupsStream : ViewStream + { + public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn) + { + + myColToAverage = colToAverage; + myInput = input; + myKeyColumn = keyColumn; + } + + private ViewStream myInput; + private string myColToAverage; + private BlueTableSchema mySchema; + private string myKeyColumn; + + protected override void myOnFirstSubscriber() + { + myInput.onChange += new ChangeDelegate(myInput_onChange); + } + protected override void myOnLastSubscriber() + { + myInput.onChange -= myInput_onChange; + } + private void myInput_onChange(object newValue) + { + if (mySchema == null) + myBuildSchema(((DataPacket)newValue).schema); + + IBlueRowCollection newRows = myEvaluate((DataPacket)newValue); + + if (myLastValue == null) + myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema))); + else + { + IBlueRowCollection added, deleted; + BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added); + myFireEvent(new DataPacket(newRows, added, deleted)); + } + } + + private void myBuildSchema(IBlueTableSchema schema) + { + List columns = new List(); + + foreach (ColumnInfo ci in schema.columnInfo) + columns.Add(ci); + + columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double))); + mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray()); + } + + + + private IBlueRowCollection myEvaluate(DataPacket packet) + { + + IDictionary colToRunningTotal = new Dictionary(); + IDictionary colToCount = new Dictionary(); + IDictionary colToLastRow = new Dictionary(); + + foreach (IBlueRow r in packet.newRows) + { + double thisVal = Convert.ToDouble(r[myColToAverage]); + + string keyVal = r[myKeyColumn].ToString(); + if (!colToCount.ContainsKey(keyVal)) + { + colToRunningTotal[keyVal] = thisVal; + colToCount[keyVal] = 1; + } + else + { + colToRunningTotal[keyVal] += thisVal; + colToCount[keyVal] += 1; + } + colToLastRow[keyVal] = r; + } + + BlueTable result = new BlueTable(mySchema); + foreach (string key in colToCount.Keys) + { + object[] values = new object[mySchema.columnInfo.Length]; + + colToLastRow[key].CopyTo(values, 0); + values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0; + result.addRow(new BlueRow(mySchema, values)); + } + + return result; + } + + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs new file mode 100644 index 000000000..73a2929af --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs @@ -0,0 +1,82 @@ +// using System; + +// namespace BCG +// { +// /// +// /// Serves as an abtraction for some of the functionalities needed in implementations +// /// of IGroupByEvaluator +// /// +// public abstract class GroupByEvaluator : IGroupByEvaluator +// { +// protected string[] myGroupByCols; +// protected string myOutputColumn; +// protected GroupByOutputTable myOutputTable; +// protected string myColMapExpression; + +// #region constructors +// public GroupByEvaluator (string outputCol, string[] groupByCols) +// { +// myOutputColumn = outputCol; +// myGroupByCols = groupByCols; +// } + +// public GroupByEvaluator (string colMapExpression, string outputCol, string[] groupByCols) +// : this (outputCol, groupByCols) {myColMapExpression = colMapExpression;} +// #endregion + +// #region IGroupByEvaluator methods +// public virtual string[] groupByColumns {get{return myGroupByCols;}} + +// public virtual void clear () +// { +// if ( myOutputTable != null ) +// myOutputTable.clear (); +// } + +// public virtual IBlueTableSchema getOutputSchema (IBlueTableSchema schema) +// { +// IBlueTableSchema retVal = +// myColMapExpression == null || myColMapExpression.Length < 1 ? +// schema : +// new ProjectHelper (schema, myColMapExpression, schema.tableName).schema; +// return retVal; +// } + +// public abstract IBlueRow evaluate (DataPacket dp) ; +// #endregion + +// #region Helper methods +// /// +// /// Projects the r, replacing the id of the returned rows with the id provided in the arg +// /// +// /// +// /// +// /// +// protected IBlueRow myProject (IBlueRow r, int id) +// { +// return r == null ? null : +// myColMapExpression == null || myColMapExpression.Length < 1 ? +// BlueDataUtility.getNewReplacedRow (r, id, "id") : +// BlueDataUtility.getNewReplacedRow(myProjectRow (r, myColMapExpression), +// id, "id"); +// } + +// /// +// /// Projects the id, conserving the id +// /// +// /// +// /// +// protected IBlueRow myProject (IBlueRow r) {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);} +// #endregion + +// private IBlueRow myProjectRow (IBlueRow r, string colMapExpression) +// { +// BlueTable auxTable = new BlueTable (r.schema); +// auxTable.addRow (r); +// BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable); +// ProjectStream projStream = new ProjectStream (staticStream, colMapExpression); +// return BlueDataUtility.extractRow (projStream.rows, 0); +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs new file mode 100644 index 000000000..cd747823e --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs @@ -0,0 +1,77 @@ +using System; +using BCG.Utilities; +using System.Collections; + +namespace BCG +{ + /// + /// GroupByOutputTable offers a conveniant way to manage a table, where theres one row per hash + /// + public class GroupByOutputTable : IBlueRowCollection + { + private IDictionary myDict = new Hashtable (); + private BlueTable myTable; + + public GroupByOutputTable (IBlueTableSchema schema) {myTable = new BlueTable (schema);} + + /// + /// Adds a row to table, with the hash string defined in the second arg. If theres a row + /// already with the hash code, its replaced (always one row per group) + /// + /// + /// + public void addRow (IBlueRow row, string groupRepString) + { + if (myDict[groupRepString] != null) + myTable.deleteRow ((IBlueRow)myDict[groupRepString]); + myTable.addRow (row); + myDict[groupRepString] = myTable[(int)row["id"]]; + } + + /// + /// Removes the row present in the hash slot defined by the arg + /// + /// + public void removeRow (string groupRepString) + { + // BlueEventLog.logAssert (myDict[groupRepString] != null); + myTable.deleteRow ((IBlueRow)myDict[groupRepString]); + myDict.Remove (groupRepString); + } + + /// + /// Removes all rows from the table, the schema is retained. + /// + public void clear () + { + myTable.clear (); + myDict.Clear (); + } + + public IBlueRow getRow (string groupRepString) + { + return myDict[groupRepString] == null ? null : + myDict[groupRepString] as IBlueRow; + } + + #region IBlueRowCollection members + public IBlueRow this[int id] {get{return myTable[id];}} + public IBlueTableSchema schema {get{return myTable.schema;}} + public bool containsID (int id) {return myTable.containsID (id);} + + #endregion + + #region ICollection members + public int Count {get{return myTable.Count;}} + public bool IsSynchronized {get{return false;}} + public object SyncRoot {get{return this;}} + public void CopyTo (Array array, int index) {myTable.CopyTo (array, index);} + #endregion + + #region IEnumerable members + public IEnumerator GetEnumerator() {return myTable.GetEnumerator ();} + #endregion + + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs new file mode 100644 index 000000000..0d520578c --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs @@ -0,0 +1,261 @@ +using System; +using System.Collections; +using BCG.Utilities; +using System.Collections.Generic; +using System.Linq; + +namespace BCG +{ + public class GroupByStream : MethodDrivenStream + { + private class MyFuncGroupByEvaluator : IGroupByEvaluator + { + private string[] _groupby_cols; + private Func _groupby_func; + public MyFuncGroupByEvaluator(IEnumerable groupby_cols, Func groupby_func) + { + _groupby_cols = groupby_cols.ToArray(); + _groupby_func = groupby_func; + } + public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); } + + public string[] groupByColumns { get { return _groupby_cols; } } + + public void clear() { } + + public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; } + } + /// + /// Serves as a quick way to access all rows belonging to the same group + /// + private bool myIdGuard = true; + private RowDictionaryTable myBookKeepingTable ; + private IGroupByEvaluator myGroupByEvaluator; + private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's + //can map a schema to another + private IBlueTableSchema myOutputSchema; + private int myIdGenerator = 0; + + public GroupByStream(ViewStream stream, IEnumerable groupby_cols, Func groupby_func) + : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) + { } + + public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator) + : this (stream, groupByEvaluator, true) + {} + + public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard) + : base (stream) + { + myGroupByEvaluator = groupbyEvaluator; + myIdGuard = idGuard; + } + + protected override void myHandleFirstCall () + { + mySchema = myLastArrivedPacket.schema; + myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema); + myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, + myGroupByEvaluator.groupByColumns); + GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema); + foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) + { + IBlueRow newRow = myEvaluate (new DataPacket (rowGroup, + rowGroup, new BlueTable (mySchema))); + if (newRow != null) + temp.addRow (newRow, myGetGroupString (rowGroup)); + } + myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema)); + } + + /// + /// Adding need not to anything as deletion already takes care + /// of addition + /// + protected override void myHandleIncrementalAdd() {} + + /// + /// Both deletes and adds are handled here to make the intermediate results not + /// appear in the spitted fired DataPacket (note this is done only in GroupByStream + /// because of the nature of it) + /// + protected override void myHandleIncrementalDelete() + { + myFireSwitch = false; + RowDictionaryTable delGroups, addGroups; + //form groups for deleted and added rows + myInitIncrementalChangeData (out delGroups, out addGroups); + ArrayList handledAddGroups = new ArrayList (); + string groupRepString; + + foreach (IBlueRowCollection delGroup in delGroups.getGroups ()) + { + // BlueEventLog.logAssert (delGroup.Count > 0); + groupRepString = myGetGroupString (delGroup); + IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ? + new BlueTable (mySchema) : addGroups.getGroup (groupRepString); + handledAddGroups.Add (addGroup); + IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ? + new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString); + IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup)); + myFireSwitch |= myMergeRow (newRow, groupRepString); + } + + foreach (IBlueRowCollection addGroup in addGroups.getGroups ()) + { + // BlueEventLog.logAssert (addGroup.Count > 0); + if (!handledAddGroups.Contains (addGroup)) + { + //this is the case when there are rows added to a group and none + //have been removed, hence it 'escapes' from being dealt with in + //the loop above + + groupRepString = myGetGroupString (addGroup); + IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString); + // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0); + IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, + new BlueTable (aggGroup.schema))); + myFireSwitch |= myMergeRow (newRow, groupRepString); + } + } + } + + private bool myMergeRow (IBlueRow newRow, string groupRepString) + { + IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString); + //BlueEventLog.logAssert (newRow != null || existingRow != null); + if (existingRow == null && newRow == null) + return false; + else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow)) + return false; + else if (existingRow == null) + { + //row added for the first time + ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString); + ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow); + return true; + } + else + { + ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString); + ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow); + if (newRow != null) + { + ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString); + ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow); + } + return true; + } + } + + /// + /// Divides the added and deletd rows of the input packet into groups, initializes the added and removed rows + /// of the output packet, andupdates the book keeping tables + /// + /// + /// + private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) + { + delGroups = + new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns); + addGroups = + new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns); + foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows) + myBookKeepingTable.removeRow (deletedRow); + foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows) + myBookKeepingTable.addRow (addedRow); + myLastFiredPacket.addedRows = new BlueTable (myOutputSchema); + myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema); + } + + protected override void myOnTableClear () + { + if (myBookKeepingTable.Count == 0) + return; //ie there arent any rows, so no need to optimize the 'cleansing' process + myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows); + myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema); + (myLastFiredPacket.newRows as GroupByOutputTable).clear (); + myBookKeepingTable.clear (); + myGroupByEvaluator.clear (); + } + + private IBlueRow myEvaluate (DataPacket dp) + { + IBlueRow evalRow = myGroupByEvaluator.evaluate (dp); + return evalRow == null ? null : + myIdGuard ? + BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow; + } + + private string myGetGroupString (IBlueRowCollection group) + { + return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns); + } + + protected override void myOnLastSubscriber () + { + //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream + //gets resubscribed, we start the process of building data again, and treat the callback as the first + //callback. + myInit (); + myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); + } + + protected override void myInit () + { + base.myInit (); + if (myBookKeepingTable != null) + { + //to avoid calling init before the first callback + myBookKeepingTable.clear (); + myGroupByEvaluator.clear (); + } + } + + public static string getRepString (DataPacket dp, string[] colNames) + { + return BlueDataUtility.rowToString (getRepRow (dp), colNames); + } + + public static IBlueRow getRepRow (DataPacket dp) + { + if (dp.newRows.Count > 0) + return BlueDataUtility.extractRow (dp.newRows, 0); + else if (dp.addedRows.Count > 0) + return BlueDataUtility.extractRow (dp.addedRows, 0); + else if (dp.deletedRows.Count > 0) + return BlueDataUtility.extractRow (dp.deletedRows, 0); + // BlueEventLog.logAssert (false, "Illegal Data Packet"); + return null; + } + + /// + /// performs an add with the 2 arguments representing the last and new values and boolean indicating + /// an add or remove. If the + /// + /// + /// + /// + /// + public static object incrementalAdd (object lastValue, object newValue, bool isAdd) + { + if (lastValue == null || lastValue.Equals (DBNull.Value)) + return newValue; + // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) || + // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) || + // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal))); + Object retVal = null; + if (newValue.GetType().Equals (typeof(int))) + retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue; + else if (newValue.GetType().Equals (typeof(Double))) + retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue; + else if (lastValue.GetType().Equals (typeof(Decimal))) + retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue; + else{} + // BlueEventLog.logAssert (false, "Illegal type"); + return retVal; + } + + } + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/IGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/IGroupByEvaluator.cs new file mode 100644 index 000000000..365211dff --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/IGroupByEvaluator.cs @@ -0,0 +1,36 @@ +using System; + +namespace BCG +{ + /// + /// Used by the GroupByStream to evaluate representative rows for groups (ie IGroupByEvaluator will group + /// rows with same values for specified columns in one group, the same as a SQL GroupBy operation) + /// + public interface IGroupByEvaluator + { + /// + /// Crunches the group into one row which is the returned value. Here the DataPacket + /// represents a group, with the newRows being the final group (after the additions and + /// deletetion) and the added and deletedRows being the increments. If a group has + /// to be totally removed then a null should be returned + /// + IBlueRow evaluate (DataPacket packet); + + /// + /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator + /// distinguishes groups + /// + string[] groupByColumns {get;} + + /// + /// Clears the Evaluator structure. This is for optimization pusposes, where the reboot of the server + /// is treated as a "clear rows" operation rather than a "delete-all" operation + /// + void clear (); + + /// + /// Returns the schema into which the this IGroupByEvaluator would map rows into + /// + IBlueTableSchema getOutputSchema (IBlueTableSchema schema); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/IncrementalGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/IncrementalGroupByEvaluator.cs new file mode 100644 index 000000000..9b9b60bd4 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/IncrementalGroupByEvaluator.cs @@ -0,0 +1,79 @@ +// using System; +// using BCG.Utilities; + +// namespace BCG +// { +// /// +// /// Abstracts the functionality used to GroupBy rows in an incremental fation, +// /// governed by the delegates passed in the argument +// /// +// public class IncrementalGroupByEvaluator : GroupByEvaluator +// { +// /// +// /// Delegate supposed to act incrementally and produce an output object given the last value +// /// of the data, new value and add/remove boolean. If its the first data, then null would be passed +// /// on as the last value. Also if the user wants the group to be removed than null should be returned +// /// +// public delegate object IncrementalComputeDelegate (object prevData, object newData, bool isAdd); +// private IncrementalGroupByEvaluator.IncrementalComputeDelegate myCompute; +// private int myOutputColIndex = -1, myIdColIndex = -1, myIdGenerator = 0; + +// #region constructors +// public IncrementalGroupByEvaluator (IncrementalComputeDelegate compute, string outputCol, string[] groupByCols) +// : base (outputCol, groupByCols) +// {myCompute = compute;} + +// public IncrementalGroupByEvaluator (IncrementalComputeDelegate compute, string colMapExpression, +// string outputCol, string[] groupByCols) : base (colMapExpression, outputCol, groupByCols) +// {myCompute = compute;} +// #endregion + +// public override IBlueRow evaluate (DataPacket packet) +// { +// myCheckTableState (packet);//initialization, if required +// string hashString = GroupByStream.getRepString (packet, myGroupByCols);//hash of the group repsented by the packet +// foreach (IBlueRow delRow in packet.deletedRows) +// { +// IBlueRow existantRow = myOutputTable.getRow (hashString); +// BlueEventLog.logAssert (existantRow != null);//if rows are to be deleted, there ought to be a rep row for this group +// object newValue = myCompute (existantRow[myOutputColIndex], delRow[myOutputColIndex], false); +// myOutputTable.addRow ( +// BlueDataUtility.getNewReplacedRow ( +// existantRow, +// new object[]{myIdGenerator++, newValue}, new int[]{myIdColIndex, myOutputColIndex}), +// //BlueDataUtility.getNewReplacedRow (existantRow, newValue, myOutputColIndex), +// hashString); +// } +// foreach (IBlueRow addedRow in packet.addedRows) +// { +// IBlueRow existantRow = myOutputTable.getRow (hashString); +// object newValue = myCompute (existantRow == null ? null : existantRow[myOutputColIndex], +// addedRow[myOutputColIndex], true); +// myOutputTable.addRow ( +// //BlueDataUtility.getNewReplacedRow (addedRow, newValue, myOutputColIndex), hashString); +// BlueDataUtility.getNewReplacedRow ( +// addedRow, +// new object[]{myIdGenerator++, newValue}, new int[]{myIdColIndex, myOutputColIndex}), +// hashString); +// } +// IBlueRow res = myOutputTable.getRow (hashString); +// //ref to deleted group is kept in myOutputTable +// return res[myOutputColIndex] == null ? null : myProject(res); +// } + +// /// +// /// Initalizes the state of the class, ie by evaluating the output Col Index and initialzing the table +// /// +// /// +// private void myCheckTableState (DataPacket packet) +// { +// if (myOutputTable == null) +// { +// myOutputTable = new GroupByOutputTable (packet.schema); +// myOutputColIndex = packet.schema [myOutputColumn].colNo; +// myIdColIndex = packet.schema["id"].colNo; +// } +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs new file mode 100644 index 000000000..0ae40b979 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs @@ -0,0 +1,47 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; + +// namespace BCG +// { +// /// +// /// Computes the max for a given column +// /// +// public class MaxGroupByEvaluator : GroupByEvaluator +// { + +// public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols) +// {} + +// public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) +// : base (colMapEpression, outputCol, groupByCols) {} + +// public override IBlueRow evaluate (DataPacket dp) +// { +// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : +// myOutputTable; +// string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols); +// IBlueRow currMaxRow = myOutputTable.getRow (hashString); +// IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn); +// if (maxRow != null) +// { +// BlueEventLog.logAssert (currMaxRow != null); +// BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0); +// if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0) +// { +// maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn); +// if (maxRow == null) +// myOutputTable.removeRow (hashString); +// else +// myOutputTable.addRow (maxRow, hashString); +// } +// } +// maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn); +// // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true, +// // otherwise if maxRow > currMaxRow then its replaced +// if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0)) +// myOutputTable.addRow (maxRow, hashString); +// return myProject (myOutputTable.getRow (hashString)); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs new file mode 100644 index 000000000..5ab927f41 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs @@ -0,0 +1,20 @@ +// using System; + +// namespace BCG +// { +// public class MaxStream : ViewStream +// { +// public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) +// { +// } +// public MaxStream (ViewStream stream, string outputCol, string[] inputCols) +// { +// mySetRealSource (new GroupByStream (stream, +// new MaxGroupByEvaluator (outputCol, inputCols))); +// } + +// public MaxStream (ViewStream stream, string outputCol) +// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"}) +// {} +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MethodDrivenStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MethodDrivenStream.cs new file mode 100644 index 000000000..b509224f5 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MethodDrivenStream.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections; +using BCG.Utilities; +using System.IO; + +namespace BCG +{ + /// + /// Some basic event handling functionality factorred out of any Stream whose behaviour + /// is driven by some interface passed on by the user + /// + public abstract class MethodDrivenStream : ViewStream + { + protected ViewStream myStream; + protected DataPacket myLastArrivedPacket; + protected DataPacket myLastFiredPacket; + protected bool myFireSwitch = true; + + public MethodDrivenStream(ViewStream vs) {myStream = vs;} + + /// + /// Calls the base class's "change handlers". Sets the fireSwitch to true, so if + /// the base class does not want to fire the packet, it should set the switch to false + /// + /// + internal void myChangeHandler (object o) + { + myFireSwitch = true; + myLastArrivedPacket = (DataPacket)o; + if (myLastFiredPacket == null) + { + myHandleFirstCall (); + myFireEvent (myLastFiredPacket.Clone ()); + return; + } + if (myLastArrivedPacket.newRows.Count == 0) + myOnTableClear (); + else + { + myHandleIncrementalDelete (); + myHandleIncrementalAdd (); + } + if (myFireSwitch) + myFireEvent (myLastFiredPacket.Clone()); + } + + /// + /// Used for Unit Testing since handler is interal + /// + /// + /// + public void myChangeHandlerTest(object o) + { + myChangeHandler(o); + } + + + /// + /// To handle the first call, the LastFiredPacket is supposed to be initialized and filled + /// with data. + /// + protected abstract void myHandleFirstCall (); + + /// + /// Handle incremenetal adds, and myLastFiredPacket filled with the resulting data + /// + protected abstract void myHandleIncrementalAdd(); + + + /// + /// Handle incremenetal deletes, and myLastFiredPacket filled with the resulting data + /// + protected abstract void myHandleIncrementalDelete(); + + protected virtual void myInit () + { + myLastFiredPacket = null; + myLastArrivedPacket = null; + } + + protected abstract void myOnTableClear (); + + protected override void myOnFirstSubscriber() + { + myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); + } + + protected override void myOnLastSubscriber () + { + myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs new file mode 100644 index 000000000..dc6700e22 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs @@ -0,0 +1,40 @@ +// using System; +// using BCG.Utilities; + +// namespace BCG +// { +// public class MinGroupByEvaluator : MaxGroupByEvaluator +// { +// public MinGroupByEvaluator (string outputCol, string[] groupByCols) +// : base (outputCol, groupByCols) {} + +// public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) : +// base (colMapEpression, outputCol, groupByCols) {} + +// public override IBlueRow evaluate (DataPacket dp) +// { +// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable; +// string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols); +// IBlueRow currMinRow = myOutputTable.getRow (stringRep); +// IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn); +// if (minRow != null) +// { +// // BlueEventLog.logAssert (currMinRow != null); +// // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0); +// if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0) +// { +// minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn); +// if (minRow == null) +// myOutputTable.removeRow (stringRep); +// else +// myOutputTable.addRow (minRow, stringRep); +// } +// } +// minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn); +// if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0)) +// myOutputTable.addRow (minRow, stringRep); +// return myOutputTable.getRow (stringRep); +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs new file mode 100644 index 000000000..674c5b0de --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs @@ -0,0 +1,17 @@ +// using System; + +// namespace BCG +// { +// public class MinStream : ViewStream +// { +// public MinStream (ViewStream stream, string outputCol, string[] inputCol) +// { +// mySetRealSource (new GroupByStream (stream, +// new MinGroupByEvaluator (outputCol, inputCol))); +// } + +// public MinStream (ViewStream stream, string outputCol) +// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"}) +// {} +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs new file mode 100644 index 000000000..3b4bfffdf --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public interface INonIncGroupByEvaluator + { + IBlueRowCollection evaluate(IDictionary> grouped_rows); + string[] get_group_by_cols(); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs new file mode 100644 index 000000000..ca44d0a3e --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using mbts; + +namespace BCG +{ + public class NonIncGroupByStream : ViewStream + { + private ViewStream _main_stream; + private INonIncGroupByEvaluator _groupby_func; + private BlueRowHasher _row_hasher; + private string[] _groupby_cols; + private HighPerformanceTimer _timer = new HighPerformanceTimer(); + private int _n_total_fires = 0, _n_expensive_fires = 0; + + public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) + { + _main_stream = main_stream; + _groupby_func = groupby_func; + _groupby_cols = _groupby_func.get_group_by_cols(); + _row_hasher = new BlueRowHasher(_groupby_cols.Length); + } + + protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } + + protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } + + private void _main_stream_onchange(object nv) + { + DataPacket dp = nv as DataPacket; + ///grouping together + _timer.start(); + IDictionary> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher); + _timer.stop(); + // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime)); + /// reducing + _timer.start(); + IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); + _timer.stop(); + _n_total_fires++; + if (_timer.lastIntervalTime > 50) + { + _n_expensive_fires++; + // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires)); + } + myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs new file mode 100644 index 000000000..286d25c21 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs @@ -0,0 +1,127 @@ +using System; +using BCG.Utilities; +using System.Collections; + +namespace BCG +{ + /// + /// All rows with the same values for designated columns are stored in + /// the same IBlueRowCollection. Such a structure will be useful for operations + /// such as GroupBy, Union and finding distinct rows in a table + /// + public class RowDictionaryTable : IBlueRowCollection + { + protected IDictionary myRowCollections = new Hashtable (); + protected BlueTable myLinearCollection; + protected string[] myCols; + protected IBlueTableSchema mySchema; + + protected RowDictionaryTable () {} + + public RowDictionaryTable (IBlueRowCollection rows, params string[] cols) + : this (rows.schema, cols) + { + foreach (IBlueRow row in rows) + addRow (row); + } + + public RowDictionaryTable (IBlueTableSchema schema, params string[] cols) + { + mySchema = schema; + myCols = cols; + foreach (string col in myCols) + { + if (mySchema[col] == null){} + // BlueEventLog.logAssert (false,"Unknown col " + col + " in " + mySchema.tableName); + } + myLinearCollection = new BlueTable (mySchema); + } + + + public virtual void addRow (IBlueRow row) + { + BlueTable rows = (BlueTable)getGroup (row); + if (rows == null) + { + //i.e we dont have any group for this row => we need to create a new group + rows = new BlueTable (mySchema); + myRowCollections [myRowToString (row)] = rows; + } + rows.addRow (row); + myLinearCollection.addRow (row); + } + + public virtual void removeRow (IBlueRow row) + { + BlueTable rowGroup = (BlueTable)getGroup (row); +// BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0); + rowGroup.deleteRow (row); + myLinearCollection.deleteRow (row); + if (rowGroup.Count == 0) + myRowCollections.Remove (myRowToString (row)); + //we dont keep any groups with no rows, if row count goes to 0, we delete the group + //in its entirety + } + + public void removeGroup (IBlueRow representativeRow) + { + IBlueRowCollection rowGroup = getGroup (representativeRow); + // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0); + foreach (IBlueRow row in rowGroup) + myLinearCollection.deleteRow (row); + myRowCollections.Remove (myRowToString (representativeRow)); + } + + public void clear () + { + myLinearCollection.clear (); + myRowCollections.Clear (); + } + + /// + /// Retuns a group to which the row belong. If theres no such group a null is returned + /// + /// + /// + //public IBlueRowCollection getGroup (IBlueRow row) {return getGroup ( + + public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));} + + public ICollection getGroups () {return myRowCollections.Values;} + + private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2) + { + bool retVal = true; + foreach (string col in myCols) + retVal &= r1[col].Equals(r2[col]); + return retVal; + } + + public IBlueRowCollection getGroup (string repString) + { + IBlueRowCollection retVal = myRowCollections[repString] != null ? + (IBlueRowCollection) myRowCollections[repString] : null; + return retVal; + } + + private string myRowToString (IBlueRow r) {return BlueDataUtility.rowToString (r, myCols);} + + #region IBlueRowCollection members + public IBlueRow this[int id] {get{return myLinearCollection[id];}} + public IBlueTableSchema schema {get{return mySchema;}} + public bool containsID (int id) {return myLinearCollection.containsID (id);} + #endregion + + #region ICollection members + public int Count {get{return myLinearCollection.Count;}} + public bool IsSynchronized {get{return false;}} + public object SyncRoot {get{return this;}} + public void CopyTo (Array array, int index) {myLinearCollection.CopyTo (array, index);} + #endregion + + #region IEnumerable members + public IEnumerator GetEnumerator() {return myLinearCollection.GetEnumerator ();} + #endregion + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs new file mode 100644 index 000000000..562e04799 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs @@ -0,0 +1,94 @@ +// using System; + +// namespace BCG +// { +// /// +// /// Computes the sum for a given column +// /// +// public class SumGroupByEvaluator : GroupByEvaluator +// { +// public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols) +// {} + +// public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) +// : base (colMapEpression, outputCol, groupByCols) {} + +// public override IBlueRow evaluate (DataPacket dp) +// { +// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable; + +// IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn); +// return sumRow; +// } +// } + + +// public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator +// { +// private string[] myOutputCols, myGroupByCols; +// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) +// { +// myOutputCols = spaceSeperatedOutputCols.Split(' '); +// myGroupByCols = spaceSeperatedGroupByCols.Split(' '); +// } + +// public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) +// { +// myOutputCols = output_columns; +// myGroupByCols = groupby_columns; +// } + +// #region IGroupByEvaluator Members + +// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);} + +// public string[] groupByColumns { get { return myGroupByCols; } } + +// public void clear() {/*stateless*/} + +// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;} + +// #endregion +// } + +// public class SumGroupByEvaluator : GroupByEvaluator +// { +// public SumGroupByEvaluator(string outputCol, string[] groupByCols) +// : base(outputCol, groupByCols) +// { } + +// public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) +// : base(colMapEpression, outputCol, groupByCols) { } + +// public override IBlueRow evaluate(DataPacket dp) +// { +// myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; + +// IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn); +// return sumRow; +// } +// } + +// public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator +// { +// private string[] myOutputCols, myGroupByCols; +// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) +// { +// myOutputCols = spaceSeperatedOutputCols.Split(' '); +// myGroupByCols = spaceSeperatedGroupByCols.Split(' '); +// } + +// #region IGroupByEvaluator Members + +// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);} + +// public string[] groupByColumns { get { return myGroupByCols; } } + +// public void clear() {/*stateless*/} + +// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;} + +// #endregion +// } + +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs new file mode 100644 index 000000000..238cbd27a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs @@ -0,0 +1,107 @@ +// using System; +// using System.Collections.Generic; + +// namespace BCG +// { +// public class NullMapper : IStreamMapper +// { +// private Dictionary _valid_cols = new Dictionary(); + +// public NullMapper(string[] valid_cols) +// { +// foreach (string c in valid_cols) +// _valid_cols[c] = true; +// _valid_cols["id"] = true; +// } + +// public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; } + +// public IBlueRow map(IBlueRow row) +// { +// MutableBlueRow out_row = new MutableBlueRow(row); +// foreach (string c in row.schema.columnNames) +// if (!_valid_cols.ContainsKey(c)) +// out_row.setValue(c, DBNull.Value); +// return out_row; +// } +// } + +// public class SumStream : ViewStream +// { +// public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) +// { +// string[] outputCols = spaceSeparatedOutputCols.Split(' '); +// string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); +// mySetRealSource(new GroupByStream(stream, +// outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator : +// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); +// } + +// public SumStream(ViewStream stream, string[] output_columns) +// { +// ViewStream rs = new AddColumnsStream(stream, "fake_col=0"); +// rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false); +// rs = new ProjectStream(rs, "!fake_col"); +// mySetRealSource(rs); +// } + +// public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) +// { +// string[] output_columns = space_seperated_output_columns.Split(' '); +// string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' '); +// mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns)); +// } + +// public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) +// { +// ViewStream rs = null; +// groupby_columns = groupby_columns ?? new string[] {}; +// if (groupby_columns.Length == 0) +// { +// rs = new AddColumnsStream(stream, "fake_col=0"); +// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"})); +// rs = new ProjectStream(rs, "!fake_col"); +// } +// else +// { +// rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns)); +// } +// if (force_null_into_other_columns) +// { +// List valid_cols = new List(output_columns); +// valid_cols.AddRange(groupby_columns); +// rs = new MapStream(rs, new NullMapper(valid_cols.ToArray())); +// } +// mySetRealSource(rs); +// } + +// public SumStream (ViewStream stream, string outputCol) +// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol") +// { +// } +// } + +// public class IntSumStream : ViewStream +// { +// public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) +// { +// string[] outputCols = spaceSeparatedOutputCols.Split(' '); +// string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); +// mySetRealSource(new GroupByStream(stream, +// outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator: +// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); +// } + +// } + + +// public class SumStream : ViewStream +// { +// public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) +// { +// string[] inputCols = spaceSeparatedInputCols.Split(' '); +// mySetRealSource(new GroupByStream(stream, +// new SumGroupByEvaluator(outputCol, inputCols))); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs new file mode 100644 index 000000000..31a17fc3a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs @@ -0,0 +1,31 @@ +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; + +// namespace BCG +// { +// public class FullJoinStream : ViewStream +// { +// private ViewStream myMainStream; +// public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2, string cmExp2, string empty_col) +// { +// myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2); +// ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1); +// filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col)); +// myMainStream = new AppendStream(myMainStream, filtered); +// } + +// protected override void myOnFirstSubscriber() +// { +// myMainStream.onChange += myHandleMainStreamChange; +// } + +// private void myHandleMainStreamChange(object newData) +// { +// IBlueRowCollection result = (newData as DataPacket).newRows; +// myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows)); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs new file mode 100644 index 000000000..3a842b994 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs @@ -0,0 +1,35 @@ +// using System; + +// namespace BCG +// { +// [DataStreamUserRespresentation (false)] +// public class InternalJoinForeignStream : InternalJoinStream +// { +// public InternalJoinForeignStream (ViewStream vsLeft, +// string jcLeft, +// ViewStream vsRight, +// string jcRight) +// : base (vsLeft, jcLeft, vsRight, jcRight) {} + +// public InternalJoinForeignStream (ViewStream vsLeft, +// string jcLeft, +// ViewStream vsRight) : this (vsLeft, jcLeft, vsRight, "id") {} + +// protected override void myInitFiredPacket () +// { +// myLastFiredPacket = new DataPacket (); +// myLastFiredPacket.newRows = new JoinForeignTable (mySchema); +// myLastFiredPacket.addedRows = new BlueTable (mySchema); +// myLastFiredPacket.deletedRows = new BlueTable (mySchema); +// } + +// public override void onTableClear (bool isLeft) +// { +// //incase of natural join, left or right wont matter, the end result zeroes out +// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; +// myLastFiredPacket.newRows = new JoinForeignTable (mySchema); +// myLastFiredPacket.addedRows = new BlueTable (mySchema); +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs new file mode 100644 index 000000000..d8edb70b7 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs @@ -0,0 +1,183 @@ +// using System; +// using System.Collections; + +// namespace BCG +// { +// /// +// /// Does all the computations required for performing the Joins. The JoinStream is a wrapper around +// /// InternalJoinStreaming taking caring of the column namings etc +// /// +// [DataStreamUserRespresentation (false)] +// public class InternalJoinStream : ViewStream +// { +// protected JoinOperand myLeftOperand, myRightOperand; +// protected int myLeftJoinColOrdinal, myRightJoinColOrdinal; +// protected ViewStream myLeftStream, myRightStream; +// protected JoinTableSchema mySchema; +// protected DataPacket myLastFiredPacket; +// protected bool myOnceFired = false; + +// public IBlueTableSchema schema {get{return mySchema;}} + +// public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) +// { +// myLeftStream = vsLeft; +// myRightStream = vsRight; +// myLeftOperand = new JoinOperand (jcLeft, true); +// myRightOperand = new JoinOperand (jcRight, false); +// } + +// protected override void myOnFirstSubscriber() +// { +// myLeftOperand.master = this; +// myRightOperand.master = this; +// myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftOperand.changeHandler); +// myRightStream.onChange += new DataStream.ChangeDelegate (myRightOperand.changeHandler); +// } + +// protected override void myOnLastSubscriber () +// { +// myInit (); +// myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftOperand.changeHandler); +// myRightStream.onChange -= new DataStream.ChangeDelegate (myRightOperand.changeHandler); +// } + +// public JoinOperand getOtherOperand (JoinOperand jo) +// { +// JoinOperand retVal = jo == myLeftOperand ? myRightOperand : myLeftOperand; +// return retVal; +// } + +// public void startSession () +// { +// if (myLastFiredPacket != null) +// { +// //reset add and deletes +// myLastFiredPacket.addedRows = new BlueTable (mySchema); +// myLastFiredPacket.deletedRows = new BlueTable (mySchema); +// return; +// } +// //first time case, need to initialize everything +// else if (myLeftOperand.hasReceivedData && myRightOperand.hasReceivedData) +// { +// myLeftJoinColOrdinal = myLeftOperand.joinColOridinal; +// myRightJoinColOrdinal = myRightOperand.joinColOridinal; +// mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol, +// myRightOperand.operandSchema, myRightOperand.joinCol); +// myInitFiredPacket (); +// } +// } + +// protected virtual void myInitFiredPacket () +// { +// myLastFiredPacket = new DataPacket (); +// myLastFiredPacket.newRows = new JoinTable (mySchema); +// myLastFiredPacket.addedRows = new BlueTable (mySchema); +// myLastFiredPacket.deletedRows = new BlueTable (mySchema); +// } + +// public IBlueTableSchema getSchema (bool isLeft) +// { +// IBlueTableSchema retVal; +// retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema; +// return retVal; +// } + +// public void endSession () +// { +// if (!myOnceFired) +// { +// myOnceFired = true; +// myFireEvent (myLastFiredPacket.Clone()); +// } +// else if (myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count != 0) +// myFireEvent (myLastFiredPacket.Clone()); +// } + + +// protected virtual void myInit () +// { +// myLeftOperand.init (); +// myRightOperand.init (); +// myLastFiredPacket = null; +// } + +// public virtual void onTableClear (bool isLeft) +// { +// //incase of natural join, left or right wont matter, the end result zeroes out +// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; +// myLastFiredPacket.newRows = new JoinTable (mySchema); +// myLastFiredPacket.addedRows = new BlueTable (mySchema); +// } + +// public virtual void deleteRows (IBlueRowCollection rows, bool isLeft) +// { +// JoinOperand thisOp, otherOp; +// int joinColOrdinal; +// myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft); +// foreach (IBlueRow row in rows) +// { +// thisOp.deleteRow (row); +// if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0) +// continue; //row does not participate in join +// ((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket); +// } +// } + +// public virtual void addRows (IBlueRowCollection rows, bool isLeft) +// { +// JoinOperand thisOp, otherOp; +// int joinColOrdinal; +// myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft); +// if (!isLeft) +// { +// thisOp = myRightOperand; +// otherOp = myLeftOperand; +// joinColOrdinal = myRightJoinColOrdinal; +// } +// foreach (IBlueRow row in rows) +// { +// thisOp.addRow (row); +// RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]); +// if (matchedNodes.Count == 0) +// continue; //this row did not participate in the join +// if (isLeft) +// foreach (RowNode matchedNode in matchedNodes) +// ((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket); +// else +// foreach (RowNode matchedNode in matchedNodes) +// ((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket); +// } +// } + +// private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft) +// { +// if (isLeft) +// { +// thisOp = myLeftOperand; +// otherOp = myRightOperand; +// joinColOrdinal = myLeftJoinColOrdinal; +// } +// else +// { +// thisOp = myRightOperand; +// otherOp = myLeftOperand; +// joinColOrdinal = myRightJoinColOrdinal; +// } +// } + +// /**************************************TEST METHOD(S)*********************************************/ +// /// +// /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer, +// /// BaseServer etc), by directly calling the test method and passing data +// /// +// /// +// /// +// public void test (DataPacket dp, bool isLeft) +// { +// JoinOperand jo = isLeft ? myLeftOperand : myRightOperand; +// jo.changeHandler (dp); +// } +// /***********************************************************************************************/ +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs new file mode 100644 index 000000000..b2a1099e8 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs @@ -0,0 +1,64 @@ +// using System; +// using System.Collections; + +// namespace BCG +// { +// /// +// /// Hashtable of DoubleyLinkedList tailored for the Join deletions +// /// +// internal class JoinDeletionIndex +// { +// IDictionary myIndex = new Hashtable (); +// private string myOpIdName ; +// private bool myIsLeft; + +// /// +// /// +// /// opIdName represents the name of the column corresponding to id of the operand table +// /// Indicator for left or right operand table +// public JoinDeletionIndex(string opIdName, bool isLeft) +// { +// myOpIdName = opIdName; +// myIsLeft = isLeft; +// } + +// /// +// /// Adds a joinRow to the hashtable; in addition the joinRow is made to point to the node added +// /// +// /// JoinRow to add +// public void add (JoinRow row) +// { +// object o = row[myOpIdName]; +// if (myIndex[o] == null) +// myIndex[o] = new RowList (); +// RowNode nodeAdded = ((RowList)myIndex[o]).add (row); +// if (myIsLeft) +// row.leftNode = nodeAdded; +// else +// row.rightNode = nodeAdded; + +// } + +// /// +// /// To remove a node, if node does not exist exception is thrown +// /// +// /// +// public void remove (RowNode node) {((RowList)myIndex[node.row[myOpIdName]]).remove (node);} + +// public RowList getMatchingRows (object key) { +// RowList retVal = (RowList)myIndex[key]; +// if (retVal == null) +// return new RowList (); +// return retVal; +// } + +// /// +// /// Empties the entry for the key provided +// /// +// /// +// public void removeAll (object obj) {myIndex.Remove(obj);} + +// public void clear () {myIndex.Clear ();} + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignStream.cs new file mode 100644 index 000000000..a28fb7090 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignStream.cs @@ -0,0 +1,59 @@ +// using System; +// using System.Data; +// using System.Collections; +// using System.Text.RegularExpressions; +// using BCG.Utilities; +// using BCG.Utilities.General; + +// namespace BCG +// { +// public class ForeignJoinStream : JoinStream +// { + +// public ForeignJoinStream (ViewStream v1, string joinCol1, ColumnMap cm1, +// ViewStream v2, string joinCol2, ColumnMap cm2) +// : base (v1, joinCol1, cm1, v2, joinCol2, cm2) +// {} + +// public ForeignJoinStream (ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2, string cmExp2) +// : base (v1, joinCol1, cmExp1, v2, joinCol2, cmExp2) +// {} + +// public ForeignJoinStream (ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2) +// : +// this (v1,joinCol1, new ColumnMap(cmExp1), v2, joinCol2) +// {} + +// public ForeignJoinStream (ViewStream v1, string joinCol1, +// ViewStream v2, string joinCol2, string cmExp2) +// : +// this (v1,joinCol1, v2, joinCol2, new ColumnMap(cmExp2)) +// {} + + +// public ForeignJoinStream (ViewStream v1, string joinCol1, ViewStream v2, string joinCol2) +// : base (v1, joinCol1, v2, joinCol2) +// {} + +// public ForeignJoinStream (ViewStream v1, string joinCol1, ColumnMap cm1, ViewStream v2, string joinCol2) +// : base (v1, joinCol1, cm1, v2, joinCol2) +// {} + + +// public ForeignJoinStream (ViewStream v1, string joinCol1, ViewStream v2, string joinCol2, ColumnMap cm2) +// : base (v1, joinCol1, v2, joinCol2, cm2) +// {} + +// protected override InternalJoinStream myConstructInternalJoinStream ( +// ViewStream vs1, +// string jc1, +// ViewStream vs2, +// string jc2) +// { +// return new InternalJoinForeignStream (vs1, jc1, vs2, jc2); +// } +// } +// } + diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs new file mode 100644 index 000000000..ec7fda2d8 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs @@ -0,0 +1,20 @@ +// using System; +// using BCG.Utilities; + +// namespace BCG +// { +// internal class JoinForeignTable : JoinTable +// { +// public JoinForeignTable (JoinTableSchema schema) : base (schema) {} + +// public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp) +// { +// JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema); +// BlueEventLog.logAssert (myIdIndex[leftRow["id"]] == null); +// myIdIndex[leftRow["id"]] = newRow; +// ((BlueTable)(dp.addedRows)).addRow (newRow); +// myLeftIndex.add (newRow); +// myRightIndex.add (newRow); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs new file mode 100644 index 000000000..62fb042f3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs @@ -0,0 +1,211 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; +// using BCG.Evaluator.ColumnMapExpression; + +// namespace BCG +// { +// /// +// /// JoinNameResolver is used to bring together the 2 naming processes, ie the one done ny Join to resolve +// /// clashes and the second, that the user does to rename columns +// /// +// public class JoinNameResolver +// { +// /// +// /// Internal class to represent a column (ie name) and its association, left or right +// /// +// internal class OriginalNameInfo +// { +// public bool isLeft; +// public string colName; +// public OriginalNameInfo (string colName, bool isLeft) {this.colName = colName; this.isLeft = isLeft;} +// public override int GetHashCode () +// { +// string prependString = isLeft ? "left" : "right"; +// return String.Concat (prependString, colName).GetHashCode (); +// } +// public override bool Equals (object obj) {return GetHashCode () == obj.GetHashCode ();} +// } + + +// private ArrayList myJoinNames; +// private JoinNameResolver.OriginalNameInfo[] myOriginalColInfo; +// private JoinTableSchema myJoinSchema; +// private IBlueTableSchema myLeftSchema, myRightSchema; +// private string myLeftColExpression, myRightColExpression; +// private string[] myUserRules, myResolvedNames; +// private IDictionary myFinalMap = new Hashtable (); + + +// public JoinNameResolver (JoinTableSchema joinSchema, IBlueTableSchema leftSchema, string leftColMapExpression, +// IBlueTableSchema rightSchema, string rightColMapExpression) +// { +// myJoinSchema = joinSchema; +// myLeftSchema = leftSchema; +// myRightSchema = rightSchema; +// myLeftColExpression = leftColMapExpression; +// myRightColExpression = rightColMapExpression; +// /*initialize the joinNames one-D structure. The id is removed as it does not +// * take part in the renaming process */ +// myJoinNames = new ArrayList (joinSchema.columnNames); +// /*initialize the originalInfo one-D structure. Its an array of OriginalNameInfo, ie name and a bool indicating that +// * it belongs to the left or right table */ +// myOriginalColInfo = new OriginalNameInfo [joinSchema.columnNames.Length - 1]; +// myUserRules = new String[myOriginalColInfo.Length]; +// myFormOriginalNameInfo (true); +// myFormOriginalNameInfo (false); +// myFormUserRules (); +// myFormResolvedNames (); +// myFormFinalMap (); +// } + +// /// +// /// Forms the array representing the name as they appear in the JoinOperand. +// /// This is done so that its alligned with the myJoinNames +// /// +// /// indicating whether its the left operand or the right +// private void myFormOriginalNameInfo (bool isLeft) +// { +// IBlueTableSchema operandSchema = isLeft ? myLeftSchema : myRightSchema; +// foreach (string colName in operandSchema.columnNames) +// { +// string joinName = myJoinSchema.getRenamedName (colName, isLeft); +// //joinName is bound to be unique +// int index = myJoinNames.IndexOf (joinName); +// BlueEventLog.logAssert (index >= 0 && index < myOriginalColInfo.Length); +// myOriginalColInfo[index] = new OriginalNameInfo(colName, isLeft); +// } +// } +// static syntax myParser = +// new syntax(); + +// /// +// /// Creates the user mapping from the grammer and then actually call the methods that form the array of +// /// user defined names +// /// +// private void myFormUserRules () +// { +// if (DataStreamLoadFileConfig.instance.useStaticParser) +// { +// lock (myParser) +// { +// myFormUserRules(true, (ColumnMapExpression) +// myParser.Parse(myLeftColExpression)); +// myFormUserRules(false, (ColumnMapExpression) +// myParser.Parse(myRightColExpression)); +// } +// } +// else +// { +// syntax parser = new syntax(); +// myFormUserRules(true, (ColumnMapExpression) +// parser.Parse(myLeftColExpression)); +// myFormUserRules(false, (ColumnMapExpression) +// parser.Parse(myRightColExpression)); +// } + +// } + +// /// +// /// Forms the array of user defined names +// /// +// /// +// /// +// private void myFormUserRules (bool isLeft, ColumnMapExpression cme) +// { +// IDictionary userMap = cme.columns; +// if (cme.isInclusive) +// for(int i = 0; i < myOriginalColInfo.Length ; i++) +// { +// OriginalNameInfo oni = myOriginalColInfo[i]; +// if (oni.isLeft == isLeft) +// { +// if (userMap[oni.colName] != null) +// { +// if ((string)userMap[oni.colName] != oni.colName) +// myUserRules[i] = (string)userMap[oni.colName]; +// else +// myUserRules[i] = oni.colName; +// } +// } +// } +// else +// { +// for (int i = 0; i < myOriginalColInfo.Length; i ++) +// { +// OriginalNameInfo oni = myOriginalColInfo[i]; +// if (oni.isLeft == isLeft) +// { +// if (userMap[oni.colName] == null) +// myUserRules[i] = oni.colName; +// else if ((string)userMap[oni.colName] != oni.colName) +// myUserRules[i] = (string)userMap[oni.colName]; +// } +// } +// } +// } + +// private void myFormResolvedNames() +// { +// string prependString; +// myResolvedNames = new string[myUserRules.Length]; +// Array.Copy (myUserRules, myResolvedNames, myUserRules.Length); +// string[] temp = myUserRules; +// myUserRules = new String[temp.Length + 1]; +// Array.Copy (temp, 0, myUserRules, 0, temp.Length); +// myUserRules[myUserRules.Length - 1] = "id"; + +// for (int i = 0; i < myResolvedNames.Length; i ++) +// { +// ArrayList occurances = myFindAllInCollection (myUserRules, myResolvedNames[i]); +// while (occurances.Count > 1) +// { +// prependString = i < myLeftSchema.columnNames.Length ? "L." : "R."; +// myResolvedNames[i] = prependString + myResolvedNames[i]; +// occurances = myFindAllInCollection (myUserRules, myResolvedNames[i]); +// occurances.AddRange (myFindAllInCollection (myResolvedNames, myResolvedNames[i])); +// } +// } +// } + +// private void myFormFinalMap () +// { +// foreach (string resolvedName in myResolvedNames) +// if (resolvedName != null) +// myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)]; +// } + + +// /// +// /// Utility method to invert a hashtable +// /// +// /// +// /// + +// private IDictionary reciprocateDictionary (IDictionary arg) +// { +// IDictionary retVal = new Hashtable (); +// foreach (DictionaryEntry de in arg) +// retVal[de.Value] = de.Key; +// return retVal; +// } + +// /// +// /// Utility method to find all occurances of an obkect in an IList +// /// +// /// +// /// +// /// +// private ArrayList myFindAllInCollection (IList list, object obj) +// { +// ArrayList retVal = new ArrayList (); +// for (int i = 0; i < list.Count; i++) +// if (list[i] != null && list[i].Equals (obj)) +// retVal.Add (i); +// return retVal; +// } + +// public IDictionary getMap () {return myFinalMap;} + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs new file mode 100644 index 000000000..b1576801f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs @@ -0,0 +1,113 @@ +// // using System; +// // using System.Collections; +// // using BCG.Utilities; + +// // namespace BCG +// // { +// // /// +// // /// Represents each operand of the Join Operations. Does some bookkeeping like mantaining indexes on +// // /// rows for fast retrieval and row matching +// // /// +// // public class JoinOperand +// // { +// // private string myJoinCol; +// // private DataPacket myLastArrivedPacket; +// // private JoinOperand myOtherOperand; +// // private bool myIsLeft; +// // private JoinOperandRowIndex myRowIndex; +// // private InternalJoinStream myMaster; +// // private IBlueTableSchema myOpSchema; + +// // public JoinOperand (string joinCol, bool isLeft) +// // { +// // myJoinCol = joinCol; +// // myIsLeft = isLeft; +// // } + +// // internal void changeHandler (object o) +// // { +// // DataPacket temp = (DataPacket)o; +// // if (myLastArrivedPacket != null) +// // mySubsequentChangeHandler ((DataPacket)o); +// // else +// // myFirstChangeHandler ((DataPacket)o); +// // } + +// // private void myFirstChangeHandler (DataPacket arrivedDp) +// // { +// // myLastArrivedPacket = arrivedDp; +// // myOpSchema = myLastArrivedPacket.newRows.schema; +// // myOtherOperand = myMaster.getOtherOperand (this); +// // myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo); +// // if (myOtherOperand.hasReceivedData) +// // { +// // //the other operand has already received data, so the join would +// // //actually be performed now, note, this is the first and only time +// // //the whole join is being performed. All other joins would be +// // //incremental +// // myMaster.startSession (); +// // myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft); +// // myMaster.endSession (); +// // } +// // else +// // myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows, +// // myOpSchema[myJoinCol].colNo, +// // myOpSchema["id"].colNo); +// // } + +// // private void mySubsequentChangeHandler (DataPacket arrivedDp) +// // { +// // myLastArrivedPacket = arrivedDp; +// // if (!myOtherOperand.hasReceivedData) +// // { +// // //no reason to call the master to actually perform +// // //the join, as the other operand is not there. Though +// // //this case has not been witnessed as yet, but +// // //theoriticaly its possible +// // foreach (IBlueRow row in myLastArrivedPacket.deletedRows) +// // deleteRow (row); +// // foreach (IBlueRow row in myLastArrivedPacket.addedRows) +// // addRow (row); +// // return; +// // } +// // myMaster.startSession (); +// // if (myLastArrivedPacket.newRows.Count == 0) +// // { +// // myRowIndex.clear (); +// // myMaster.onTableClear (myIsLeft); +// // } +// // else +// // { +// // //the only condition when the join will actually take place +// // myMaster.deleteRows (myLastArrivedPacket.deletedRows, myIsLeft); +// // myMaster.addRows (myLastArrivedPacket.addedRows, myIsLeft); +// // } +// // myMaster.endSession (); +// // } + +// // internal void addRow (IBlueRow row) {myRowIndex.add (row);} + +// // internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);} + +// // internal IBlueTableSchema operandSchema {get{return myOpSchema;}} + +// // internal InternalJoinStream master {set{myMaster = value;}} + +// // internal string joinCol {get{return myJoinCol;}} + +// // internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}} + +// // internal RowList getMatchingRows (object key) {return (RowList) myRowIndex.getMatchingRows (key);} + +// // internal bool hasReceivedData {get{return myLastArrivedPacket != null;}} + +// // internal void init () +// // { +// // ///should be brought back to the the state before the first callback +// // ///(ie the state after construction) +// // myRowIndex = null; +// // myLastArrivedPacket = null; +// // } + +// // } +// // } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs new file mode 100644 index 000000000..3c22cd209 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Datastructure used to mantain the join rows, there would be 2 of those, for right and left operands + /// respectivly. Objective of this data structure is to get all rows belonging to a joinCol value and to + /// delete rows provided their id's. + /// + internal class JoinOperandRowIndex + { + private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable (); + private int myJoinColOrdinal, myIdColOrdinal; + + /// + /// + /// + /// index of the joinCol + /// index of the id + public JoinOperandRowIndex (int joinColIndex, int idColindex) + { + myJoinColOrdinal = joinColIndex; + myIdColOrdinal = idColindex; + } + + public JoinOperandRowIndex(IBlueRowCollection rows, int joinColIndex, int idColIndex) : + this(joinColIndex, idColIndex) + { + foreach (IBlueRow row in rows) + add (row); + } + + public void add (IBlueRow row) + { + RowList list; + if (myJoinColDict[row[myJoinColOrdinal]] == null) + myJoinColDict[row[myJoinColOrdinal]] = list = new RowList (); + else + list = (RowList)myJoinColDict[row[myJoinColOrdinal]]; + myIdDict[row[myIdColOrdinal]] = list.add (row); + } + + public void remove (IBlueRow row) + { + RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]]; + ((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node); + myIdDict.Remove (row[myIdColOrdinal]); + } + + public RowList getMatchingRows (object key) + { + RowList retVal = (RowList)myJoinColDict[key]; + if (retVal == null) + return new RowList (); + return retVal; + } + + public void clear () + { + myJoinColDict.Clear (); + myIdDict.Clear (); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs new file mode 100644 index 000000000..96a7d3be1 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs @@ -0,0 +1,154 @@ +// using System; +// using BCG.Utilities; +// using System.Collections; + +// namespace BCG +// { +// internal class JoinRow : IBlueRow +// { +// private RowNode myLeftNode, myRightNode; +// protected IBlueRow myLeftRow, myRightRow; +// protected int myId; +// protected JoinTableSchema mySchema; +// private object[] myValues; + +// #region Constructor(s) + +// /// +// /// Constructs a row, assigning it the same ID as that of the left row. +// /// +// /// left row of the join +// /// right row of the join +// /// schema of the joined row +// public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) +// : this ((int)leftRow["id"], leftRow, rightRow, schema) {} + +// /// +// /// Constructs a row assigning it the ID provioded as the first argument +// /// +// /// id of the joined row +// /// left row of the join +// /// right row of the join +// /// schema of the joined row +// public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) +// { +// myId = id; +// myLeftRow = leftRow; +// myRightRow = rightRow; +// mySchema = schema; +// myValues = new object [schema.columnNames.Length]; +// } + +// #endregion + +// #region IBlueRow members +// public IBlueTableSchema schema {get{return mySchema;}} +// #endregion + +// public bool isPartialJoin{get{return myRightRow == null;}} + +// public RowNode leftNode +// { +// get{return myLeftNode;} +// set{myLeftNode = value;} +// } + +// public RowNode rightNode +// { +// get{return myRightNode;} +// set{myRightNode = value;} +// } + +// #region ICollection members +// public int Count{get{return mySchema.columnNames.Length;}} + +// public bool IsSynchronized {get{return false;}} + +// public object SyncRoot{get{return this;}} + +// public void CopyTo (Array array, int index) +// { +// int i = 0; +// foreach (object o in this as IEnumerable) +// array.SetValue (o, i++); +// } + +// #endregion + +// #region IDictionary members +// public bool IsFixedSize {get{return true;}} + +// public bool IsReadOnly {get{return true;}} + +// /// +// /// The key can either be the string corrsponding to the column name or +// /// the order of the column +// /// +// public virtual object this [object key] +// { +// get +// { +// if (key is string) +// return this[(string)key]; +// return this[(int)key]; +// } +// set {throw new NotSupportedException ("The IBlueRow is read only");} +// } + +// public object this [int index] +// { +// get +// { +// if (index == mySchema.columnNames.Length - 1) +// return myId; +// else if (index < myLeftRow.Count) +// { +// if (myValues[index] == null) +// myValues[index] = myLeftRow[index]; +// return myValues[index]; +// } +// else if (index >= myLeftRow.Count && index < mySchema.columnNames.Length - 1) +// { +// if (myValues[index] == null) +// myValues[index] = (myRightRow == null) ? DBNull.Value : myRightRow[index - myLeftRow.Count]; +// return myValues[index]; +// } +// BlueEventLog.logAssert (false, "Incorrect index in JoinRow"); +// return null; +// } +// } + +// public object this [string colName] {get{return this[mySchema[colName].colNo];}} + +// public ICollection Keys{get{return mySchema.columnNames;}} + +// public ICollection Values +// { +// get +// { +// object[] retVal = new object[mySchema.columnNames.Length]; +// CopyTo (retVal, 0); +// return retVal; +// } +// } + +// public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + +// public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} + +// public bool Contains (object key){return mySchema[(string)key] != null;} + +// /// +// /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to +// /// iterate through the values, typecast this into an IEnumerable +// /// +// /// +// public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);} + +// IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);} + +// public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");} +// #endregion +// } + +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs new file mode 100644 index 000000000..60daab27f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs @@ -0,0 +1,165 @@ +// using System; +// using System.Data; +// using System.Collections; +// using System.Text.RegularExpressions; +// using BCG.Utilities; +// using BCG.Utilities.General; + +// namespace BCG +// { +// public class JoinStream : ViewStream +// { +// protected InternalJoinStream myIjs; +// protected ProjectStream myPs; +// protected ColumnMap myLeftCm, myRightCm; +// protected bool myIsProject = true; +// private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; + +// protected JoinStream () {} +// public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1, +// ViewStream v2, string joinCol2, ColumnMap cm2) +// { +// if (myUseSuperJoin) +// { +// ViewStream rs = this is LeftJoinStream ? +// new SuperLeftJoinStream(v1, joinCol1, cm1.columnExpression, v2, joinCol2, cm2.columnExpression) as ViewStream : +// new SuperJoinStream(v1, joinCol1, cm1.columnExpression, v2, joinCol2, cm2.columnExpression) as ViewStream; +// mySetRealSource(rs); +// } +// else +// { +// myLeftCm = cm1; +// myRightCm = cm2; +// //myIjs = new InternalJoinStream (v1, joinCol1, v2, joinCol2); +// myIjs = myConstructInternalJoinStream(v1, joinCol1, v2, joinCol2); +// } +// } + +// public JoinStream (ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2, string cmExp2) +// : +// this ( +// v1, +// joinCol1, +// (cmExp1 == null ? new ColumnMap("!"):new ColumnMap(cmExp1)), +// v2, +// joinCol2, +// (cmExp2 == null ? new ColumnMap("!"):new ColumnMap(cmExp2))) +// {} + +// public JoinStream (ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2) +// : +// this (v1,joinCol1, new ColumnMap(cmExp1), v2, joinCol2) +// {} + +// public JoinStream (ViewStream v1, string joinCol1, +// ViewStream v2, string joinCol2, string cmExp2) +// : +// this (v1,joinCol1, v2, joinCol2, new ColumnMap(cmExp2)) +// {} + + +// public JoinStream (ViewStream v1, string joinCol1, ViewStream v2, string joinCol2) +// { +// if (myUseSuperJoin) +// { +// ViewStream rs = this is LeftJoinStream ? +// new SuperLeftJoinStream(v1, joinCol1, v2, joinCol2) as ViewStream : +// new SuperJoinStream(v1, joinCol1, v2, joinCol2) as ViewStream; +// mySetRealSource(rs); +// } +// else +// { +// //myIjs = new InternalJoinStream (v1, joinCol1, v2, joinCol2); +// myIjs = myConstructInternalJoinStream(v1, joinCol1, v2, joinCol2); +// myIsProject = false; +// } +// } + +// public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1, ViewStream v2, string joinCol2) +// { +// myLeftCm = cm1; +// myRightCm = new ColumnMap ("!"); +// //myIjs = new InternalJoinStream (v1, joinCol1, v2, joinCol2); +// myIjs = myConstructInternalJoinStream (v1, joinCol1, v2, joinCol2); +// } + + +// public JoinStream (ViewStream v1, string joinCol1, ViewStream v2, string joinCol2, ColumnMap cm2) +// { +// myLeftCm = new ColumnMap ("!"); +// myRightCm = cm2; +// //myIjs = new InternalJoinStream (v1, joinCol1, v2, joinCol2); +// myIjs = myConstructInternalJoinStream (v1, joinCol1, v2, joinCol2); + +// } + +// protected virtual InternalJoinStream myConstructInternalJoinStream ( +// ViewStream vs1, +// string joinCol1, +// ViewStream vs2, +// string joinCol2) +// { +// return new InternalJoinStream (vs1, joinCol1, vs2, joinCol2); +// } + +// protected void myInternalJoinChangeHandler (object o) +// { +// if (myIsProject) +// { +// if (myPs == null) +// { +// JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression, +// myIjs.getSchema (false), myRightCm.columnExpression); +// IDictionary map = resolver.getMap (); +// map["id"] = "id"; +// myPs = new ProjectStream (myIjs, map); +// myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler); +// return; +// } +// else +// BlueEventLog.logAssert (false, "This call back should not have been called"); +// } +// else +// myFireEvent ((o as ICloneable).Clone()); +// } + +// private void myProjectChangeHandler (object o) +// { +// if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1) +// myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler); +// myFireEvent ((o as ICloneable).Clone()); + +// } + +// protected override void myOnFirstSubscriber () +// { +// myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler); +// } + +// protected override void myOnLastSubscriber () +// { +// // how does this relate to the -= in myProjectChangeHandler? +// myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler); + +// // if there is a project after the ijs we need to +// // unsubscribe to it otherwise references will be kept and +// // a memory leak will occur +// if (myPs != null) +// myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler); +// myPs = null; +// } + +// /**************************************TEST METHOD(S)*********************************************/ +// /// +// /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer, +// /// BaseServer etc), by directly calling the test method and passing data +// /// +// /// +// /// +// public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);} +// /***********************************************************************************************/ +// } +// } + diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs new file mode 100644 index 000000000..d7c99d48f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs @@ -0,0 +1,84 @@ +// // using System; +// // using System.Collections; +// // using System.IO; +// // using BCG.Utilities.General; + +// // namespace BCG +// // { +// // internal class JoinTable : IBlueRowCollection +// // { +// // protected JoinTableSchema mySchema; +// // protected IDictionary myIdIndex = new Hashtable (); +// // protected JoinDeletionIndex myLeftIndex, myRightIndex; +// // private int myNewRowId; + +// // public JoinTable (JoinTableSchema schema) +// // { +// // mySchema = schema; +// // myLeftIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", true), true); +// // myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false); +// // } + +// // public IBlueRow this [int id] +// // { +// // get +// // { +// // if (myIdIndex[id] == null) +// // throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); +// // return (IBlueRow)myIdIndex[id]; +// // } +// // } + +// // #region JoinTable members +// // public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp) +// // { +// // JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema); +// // myIdIndex[myNewRowId ++] = newRow; +// // ((BlueTable)(dp.addedRows)).addRow (newRow); +// // myLeftIndex.add (newRow); +// // if (rightRow != null) //in case of left joins when right row is null +// // myRightIndex.add (newRow); +// // } + +// // public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, DataPacket dp) +// // { +// // JoinDeletionIndex thisIndex = myRightIndex, otherIndex = myLeftIndex; +// // if (isLeft) +// // { +// // thisIndex = myLeftIndex; +// // otherIndex = myRightIndex; +// // } +// // RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]); +// // foreach (RowNode node in matchingNodes) +// // { +// // JoinRow row = (JoinRow)node.row; +// // RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode; +// // if (correspondingNode != null) +// // otherIndex.remove (correspondingNode); //in case of partial left joinss +// // myIdIndex.Remove (row["id"]); +// // ((BlueTable)dp.deletedRows).addRow (row); +// // } +// // thisIndex.removeAll (deletedRow["id"]); +// // } +// // #endregion +// // #region ICollection members +// // public int Count{get{return myIdIndex.Count;}} + +// // public bool IsSynchronized {get{return false;}} + +// // public object SyncRoot{get{return this;}} + +// // public void CopyTo (Array array, int index) {myIdIndex.Values.CopyTo (array, index);} +// // #endregion + +// // #region IEnumerable members +// // public IEnumerator GetEnumerator () {return myIdIndex.Values.GetEnumerator ();} +// // #endregion + +// // #region IBlueRowCollection members +// // public IBlueTableSchema schema {get{return mySchema;}} +// // public bool containsID (int id) {return myIdIndex[id] != null;} +// // #endregion +// // } + +// // } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs new file mode 100644 index 000000000..3126d28db --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs @@ -0,0 +1,107 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; + +// namespace BCG +// { +// /// +// /// Represents the schema of the Join Table. This schema is formed by combining the respective +// /// schemas of the operands. +// /// +// public class JoinTableSchema : BlueTableSchema +// { +// private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = ""; +// private IDictionary myLeftNameToAlias = new Hashtable (); +// private IDictionary myRightNameToAlias = new Hashtable (); +// private IDictionary myAliasToLeftName = new Hashtable(), +// myAliasToRightName = new Hashtable(); +// /// +// /// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of +// /// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id" +// /// is the last column in the schema +// /// +// /// +// /// +// /// +// /// +// public JoinTableSchema (IBlueTableSchema leftSchema, string leftJc, +// IBlueTableSchema rightSchema, string rightJc) +// { +// myIndexByColOrder = new ColumnInfo[leftSchema.columnNames.Length + rightSchema.columnNames.Length + 1]; +// myTableName = leftSchema.tableName + " J " + rightSchema.tableName; +// ColumnInfo[] cis = leftSchema.columnInfo; +// string assignedColName; +// bool renamed = false; +// foreach (ColumnInfo ci in cis) +// { +// assignedColName = ci.colName; +// while (rightSchema[assignedColName] != null) +// { +// assignedColName = "L." + assignedColName; +// renamed = true; +// while (leftSchema[assignedColName] != null) +// assignedColName = "L." + assignedColName; +// } +// ColumnInfo newCi = new ColumnInfo (assignedColName, ci.colNo, ci.colType); +// BlueEventLog.logAssert (myIndexByColName[newCi.colName] == null); +// BlueEventLog.logAssert (myIndexByColOrder[newCi.colNo] == null); +// if (renamed) +// BlueEventLog.logAssert (leftSchema[assignedColName] == null); +// BlueEventLog.logAssert (rightSchema[assignedColName] == null); +// myIndexByColName [newCi.colName] = newCi; +// myIndexByColOrder [newCi.colNo] = newCi; +// myLeftNameToAlias[ci.colName] = newCi.colName; +// myLeftJc = ci.colName == leftJc ? assignedColName : myLeftJc; +// myLeftId = ci.colName == "id" ? assignedColName : myLeftId; +// renamed = false; +// } +// BlueEventLog.logAssert (myLeftJc != "" && myLeftId != ""); +// int n = leftSchema.columnNames.Length; +// cis = rightSchema.columnInfo; +// renamed = false; +// foreach (ColumnInfo ci in cis) +// { +// assignedColName = ci.colName; +// while (leftSchema[assignedColName] != null) +// { +// assignedColName = "R." + assignedColName; +// renamed = true; +// while(rightSchema[assignedColName] != null) +// assignedColName = "R." + assignedColName; +// } +// ColumnInfo newCi = new ColumnInfo (assignedColName, ci.colNo + n, ci.colType); +// BlueEventLog.logAssert (myIndexByColName[assignedColName] == null); +// BlueEventLog.logAssert (myIndexByColOrder[newCi.colNo] == null); +// BlueEventLog.logAssert (leftSchema[assignedColName] == null); +// if (renamed) +// BlueEventLog.logAssert (rightSchema[assignedColName] == null); +// myIndexByColName [newCi.colName] = newCi; +// myIndexByColOrder [newCi.colNo] = newCi; +// myRightNameToAlias[ci.colName] = newCi.colName; +// myRightJc = ci.colName == rightJc ? assignedColName : myRightJc; +// myRightId = ci.colName == "id" ? assignedColName : myLeftJc; +// renamed = false; +// } +// myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int)); +// myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"]; +// myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias); +// myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias); +// } + +// public string leftJoinCol {get{return myLeftJc;}} + +// public string rightJoinCol {get{return myRightJc;}} + +// public string getRenamedName (string origName, bool isLeft) +// { +// if (isLeft) +// return (string)myLeftNameToAlias[origName]; +// return (string)myRightNameToAlias[origName]; +// } + +// public string getOriginalName(string rename, bool isLeft) +// { +// return isLeft ? myAliasToLeftName[rename].ToString() : myAliasToRightName[rename].ToString(); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/MultiColumnJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/MultiColumnJoinStream.cs new file mode 100644 index 000000000..b6747817a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/MultiColumnJoinStream.cs @@ -0,0 +1,54 @@ +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; + +// namespace BCG +// { +// /// +// /// Same as a join except can join based on +// /// multiple columns (this abstracts what we +// /// have been doing for ages - concatenating, +// /// joining and then projecting the concatanted +// /// columns out) +// /// +// public class MultiColumnJoinStream : ViewStream +// { +// public MultiColumnJoinStream(ViewStream leftStream, string leftJoinColumns, string leftProject, +// ViewStream rightStream, string rightJoinColumns, string rightProject) +// : this(leftStream, leftJoinColumns, leftProject, +// rightStream, rightJoinColumns, rightProject, false) +// { } + +// public MultiColumnJoinStream(ViewStream leftStream, string leftJoinColumns, string leftProject, +// ViewStream rightStream, string rightJoinColumns, string rightProject, bool leftJoin) +// { +// string[] leftJoinColArray = leftJoinColumns.Split(' ', ',', ';'), +// rightJoinColArray = rightJoinColumns.Split(' ', ',', ';'); +// if (leftJoinColArray.Length == 1 && rightJoinColArray.Length == 1) +// /// so that this handles regular joins too +// mySetRealSource(leftJoin ? new SuperLeftJoinStream(leftStream, leftJoinColArray[0], leftProject, +// rightStream, rightJoinColArray[0], rightProject) : +// new SuperJoinStream(leftStream, leftJoinColArray[0], leftProject, +// rightStream, rightJoinColArray[0], rightProject)); +// else +// { +// string leftCols = "ljc:"; +// foreach (string c in leftJoinColArray) +// leftCols += c + " "; +// leftCols = leftCols.Trim(); +// string rightCols = "rjc:"; +// foreach (string c in rightJoinColArray) +// rightCols += c + " "; +// rightCols = rightCols.Trim(); +// ViewStream res = +// leftJoin ? new SuperLeftJoinStream(new ConcatenateColumnsStream(leftStream, leftCols), "ljc", leftProject, +// new ConcatenateColumnsStream(rightStream, rightCols), "rjc", rightProject) : +// new SuperJoinStream(new ConcatenateColumnsStream(leftStream, leftCols), "ljc", leftProject, +// new ConcatenateColumnsStream(rightStream, rightCols), "rjc", rightProject); +// res = new ProjectStream(res, "!ljc rjc"); +// mySetRealSource(res); +// } +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/INonIncJoinInlineMapper.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/INonIncJoinInlineMapper.cs new file mode 100644 index 000000000..3e4e33102 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/INonIncJoinInlineMapper.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public interface INonIncJoinInlineMapper + { + IBlueTableSchema get_output_schema(IBlueTableSchema schema); + + IBlueRowCollection map_rows(IBlueRowCollection rows); + + event MapperChangeDelegate onMapperChange; + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs new file mode 100644 index 000000000..d30d99def --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using mbts; + +namespace BCG +{ + public class NonIncJoinBookkeeper : IDisposable + { + public Action on_new_data; + private readonly NonIncJoinSpec _join_spec; + private readonly int _join_position; + private readonly int _start_index_in_output; + private readonly BlueRowHasher _row_hasher; + private readonly bool _is_left_most; + private readonly bool _is_tag_along; + private int _n_total_fires = 0, _n_expensive_fires = 0; + + private IBlueTableSchema _schema = null; + private IDictionary _row_map = new Dictionary(); + private List> _row_list = new List>(); + private IBlueRow _solitary_row = null; + + private int[] _join_col_indices = null, + _data_col_indices = null; + + public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) + { + _join_spec = the_join_spec; + _join_position = join_position; + _start_index_in_output = start_index_in_output; + _row_hasher = row_hasher; + _is_left_most = _join_position == 0; + _is_tag_along = _join_spec.join_col_names.Length == 0; + _join_spec.stream.onChange += _on_stream_cb; + } + + public void _on_stream_cb(object dp_obj) + { + HighPerformanceTimer timer = new HighPerformanceTimer(); + timer.start(); + DataPacket dp = (DataPacket)dp_obj; + if (_schema == null) + { + _schema = dp.schema; + _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names); + _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names); + } + + if (_is_left_most) + { + _row_list.Clear(); + foreach (IBlueRow r in dp.newRows) + _row_list.Add(new Pair(_row_hasher.get_hash(r, _join_col_indices), r)); + } + else + { + if (_is_tag_along) + _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0); + else + { + _row_map.Clear(); + foreach (IBlueRow r in dp.newRows) + _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r; + } + } + timer.stop(); + _n_total_fires++; + if (timer.lastIntervalTime > 50) + { + _n_expensive_fires++; + string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires); + // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg); + } + if (on_new_data != null) + on_new_data(); + } + + public bool is_ready { get { return _schema != null; } } + + public void constribute_cols_to_schema(List cis) + { + for (int i = 0; i < _join_spec.data_col_names.Length; i++) + cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); + } + + public List> get_list() { return _row_list; } + + public void copy_master_row_to_output_row(int row_index, object[] output_row) + { + IBlueRow this_row = _row_list[row_index].second; + BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); + } + + public void copy_matching_row_to_output_row(int row_hash, object[] output_row) + { + if (_is_tag_along) + { + if (_solitary_row != null) + BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output); + else + copy_dbnull_to_output_row(output_row); + } + else + { + IBlueRow this_row = null; + if (_row_map.TryGetValue(row_hash, out this_row)) + BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); + else + copy_dbnull_to_output_row(output_row); + } + } + + private void copy_dbnull_to_output_row(object[] output_row) + { + for (int i = 0; i < _data_col_indices.Length; i++) + output_row[i + _start_index_in_output] = DBNull.Value; + } + + #region IDisposable + + public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs new file mode 100644 index 000000000..c6cb15fb6 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using mbts; + +namespace BCG +{ + public class NonIncJoinSpec + { + public ViewStream stream; + public string[] join_col_names, data_col_names, projected_col_names; + + public NonIncJoinSpec() { } + + public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names) + : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names) + { } + + public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) + { + stream = the_stream; + join_col_names = the_join_col_names; + data_col_names = the_data_col_names; + projected_col_names = the_projected_col_names; + } + + public static NonIncJoinSpec[] from_triplet(params Triplet[] join_specs) + { + List rv = new List(); + foreach (Triplet r in join_specs) + { + string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' '); + string[] visible_cols = r.third.Split(' '); + List data_col_names = new List(), mapped_col_names = new List(); + foreach (string s in visible_cols) + { + string[] toks = s.Split(':'); + if (toks.Length == 1) + { + data_col_names.Add(toks[0]); + mapped_col_names.Add(toks[0]); + } + else if (toks.Length == 2) + { + data_col_names.Add(toks[0]); + mapped_col_names.Add(toks[1]); + } + } + rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray())); + } + return rv.ToArray (); + } + } + + //[DataStreamUserRespresentation("ninctest")] + //public class NonIncJoinTestStream : ViewStream + //{ + // public NonIncJoinTestStream (string key_cols, string data_cols) + // { + // NonIncJoinSpec.from_triplet(new Triplet(null, key_cols, data_cols)); + // } + //} +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs new file mode 100644 index 000000000..6c52ce66e --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs @@ -0,0 +1,183 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using mbts; + +namespace BCG +{ + public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker + { + private readonly NonIncJoinSpec[] _join_specs; + private readonly INonIncJoinInlineMapper _output_mapper; + private readonly BlueRowHasher _row_hasher; + private readonly TimeSpan _throttle_period; + private readonly ViewStreamDatabase _vsd; + + private List _book_keeping_data = new List(); + private IBlueTableSchema _output_schema = null; + private DateTime _last_fire_time = DateTime.MinValue; + private bool _is_dirty = false; + + public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj, + ViewStream right, string right_join_cols, string right_proj) + : this(new Triplet[] + { + new Triplet (left, left_join_cols, left_proj), + new Triplet (right, right_join_cols, right_proj) + }) + { } + + public NonIncJoinStream(params Triplet[] join_specs) + : this (NonIncJoinSpec.from_triplet (join_specs), null, 0) + { } + + public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms, + params Triplet[] join_specs) + : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) + { } + + public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) + : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) + { } + + public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) + { + _join_specs = join_specs; + _output_mapper = output_mapper; + _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length); + _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms); + _vsd = vsd; + } + + protected override void myOnFirstSubscriber() + { + int data_col_no = 1; + for (int jp = 0; jp < _join_specs.Length; jp++) + { + NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher); + data_col_no += _join_specs[jp].data_col_names.Length; + s.on_new_data += _stream_callback; + _book_keeping_data.Add(s); + } + if (_output_mapper != null) + _output_mapper.onMapperChange += _on_mapper_change; + _vsd.addWorker(this); + } + + protected override void myOnLastSubscriber() + { + foreach (NonIncJoinBookkeeper d in _book_keeping_data) + d.Dispose(); + if (_output_mapper != null) + _output_mapper.onMapperChange -= _on_mapper_change; + _vsd.removeWorker(this); + } + + private void _stream_callback() + { + _is_dirty = true; + doWork(); + } + + private void _on_mapper_change() + { + _is_dirty = true; + doWork(); + } + + private bool _is_ready() + { + bool rv = true; + foreach (NonIncJoinBookkeeper d in _book_keeping_data) + rv &= d.is_ready; + return rv; + } + + private void _construct_schema() + { + if (_output_schema == null) + { + List cis = new List(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); + foreach (NonIncJoinBookkeeper d in _book_keeping_data) + d.constribute_cols_to_schema (cis); + _output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray()); + } + } + + private int _n_total_fires = 0, _n_expensive_fires = 0; + private void _form_and_fire() + { + if (_is_ready()) + { + _construct_schema(); + HighPerformanceTimer timer = new HighPerformanceTimer(); + timer.start(); + IList> left_row_list = _book_keeping_data[0].get_list(); + List output_rows = new List(); + + for (int i = 0; i < left_row_list.Count; i++) + { + object[] or = new object[_output_schema.columnInfo.Length]; + or[0] = i; + output_rows.Add(or); + } + + for (int ji = 0; ji < _book_keeping_data.Count; ji++) + { + NonIncJoinBookkeeper bk_data = _book_keeping_data[ji]; + for (int i = 0; i < left_row_list.Count; i++) + { + if (ji == 0) + bk_data.copy_master_row_to_output_row(i, output_rows[i]); + else + bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]); + } + } + IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); + timer.stop(); + _n_total_fires++; + if (timer.lastIntervalTime >= 250) + { + _n_expensive_fires++; + string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires); + // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg); + } + myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows)); + } + } + + private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) + { + if (_output_mapper != null) + { + HighPerformanceTimer timer = new HighPerformanceTimer(); + IBlueRowCollection output_rows; + timer.start(); + output_rows = _output_mapper.map_rows(unmapped_rows); + timer.stop(); + if (timer.lastIntervalTime >= 50) + { + // string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime); + // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg); + } + return output_rows; + } + else + { + return unmapped_rows; + } + } + + public void doWork() + { + if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) + { + _is_dirty = false; + _last_fire_time = DateTime.Now; + _form_and_fire(); + } + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/ParallelJoinTest.cs b/src/DataStreamSuite/DataStream/Primitives/Join/ParallelJoinTest.cs new file mode 100644 index 000000000..8d356c2b6 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/ParallelJoinTest.cs @@ -0,0 +1,371 @@ +// using System; +// using System.Collections; +// using System.Collections.Generic; +// using System.Text; +// using BCG.Utilities.General; +// using BCG.Utilities; + +// namespace BCG +// { + +// /// +// /// This class is used to effect a mapping between values in a join column and entries in three different tables: +// /// the left and right tables of a join, as well as the output table (the join table). +// /// +// class ParallelJoinTestTablePointer +// { +// int myLeftTableId = -1; +// public int LeftTableId +// { +// get { return myLeftTableId; } +// set { myLeftTableId = value; } +// } + +// int myRightTableId = -1; +// public int RightTableId +// { +// get { return myRightTableId; } +// set { myRightTableId = value; } +// } +// int myJoinTableId = -1; +// public int JoinTableId +// { +// get { return myJoinTableId; } +// set { myJoinTableId = value; } +// } + +// public ParallelJoinTestTablePointer(int leftId, int rightId, int joinId) +// { +// myLeftTableId = leftId; +// myRightTableId = rightId; +// myJoinTableId = joinId; +// } +// public ParallelJoinTestTablePointer() : this(-1,-1,-1) +// { + +// } +// } + + +// #region IBlueRow implementation for join and leftJoin + +// public class ParallelJoinTestRow : IBlueRow +// { +// private JoinProjectHelper myHelper; +// private IBlueRow myLeftRow, myRightRow; +// int myID; + +// public ParallelJoinTestRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper) +// { +// myID = id; +// myLeftRow = leftRow; +// myRightRow = rightRow; +// myHelper = joinProjectHelper; +// } + +// public int leftID { get { return (int)myLeftRow["id"]; } } + +// public int rightID { get { return (int)myRightRow["id"]; } } + +// public IBlueRow leftRow { get { return myLeftRow; } } + +// public IBlueRow rightRow { get { return myRightRow; } } + +// #region IBlueRow Members + +// public IBlueTableSchema schema { get { return myHelper.schema; } } + +// public object this[int index] { get { return this[schema[index].colName]; } } + +// public object this[string colName] +// { +// get +// { +// if (colName == "id") +// return myID; +// else +// { +// bool isLeft; +// string col = myHelper.getOriginalCol(colName, out isLeft); +// return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]); +// } +// } +// } + +// #endregion + +// #region IDictionary Members + +// public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } } + +// public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); } + +// public bool Contains(object key) { throw new Exception("The method or operation is not implemented."); } + +// public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + +// public bool IsFixedSize { get { return true; } } + +// public bool IsReadOnly {get { return true; }} + +// public System.Collections.ICollection Keys { get { return schema.columnNames; } } + +// public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); } + +// public System.Collections.ICollection Values +// { +// get +// { +// List rv = new List (); +// foreach (string col in Keys) +// rv.Add(this[col]); +// return rv; +// } +// } + +// public object this[object key] +// { +// get { return key is string ? this[key.ToString()] : this[(int)key]; } +// set { throw new NotSupportedException("IBlueRow is readonly"); } +// } + +// #endregion + +// #region ICollection Members + +// public void CopyTo(Array array, int index) { Values.CopyTo(array, index); } + +// public int Count { get { return schema.columnNames.Length; } } + +// public bool IsSynchronized { get { return false; } } + +// public object SyncRoot { get { return this; } } + +// #endregion + +// #region IEnumerable Members + +// System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + +// #endregion +// } + +// #endregion +// /// +// /// This class represents a join on two streams. It works by storing a table with the result of the join. +// /// +// public class ParallelJoinTestStream : ViewStream +// { +// BlueTable myJoinedTable; +// BlueTable myLeftTable; +// BlueTable myRightTable; +// Hashtable myKeyToTablePointerMap; +// ViewStream myVLeft, myVRight; + +// int myIdToUse; +// string myLeftKey, myRightKey, myLeftProj, myRightProj; +// public ParallelJoinTestStream(ViewStream leftStream, string leftKey, ViewStream rightStream, string rightKey) +// : this(leftStream, leftKey, "!", rightStream, rightKey, "!") { } +// public ParallelJoinTestStream(ViewStream leftStream, string leftKey, string leftProj, ViewStream rightStream, string rightKey, string rightProj) +// { +// myVLeft = leftStream; +// myVRight = rightStream; + +// myLeftProj = leftProj; +// myRightProj = rightProj; + +// myLeftKey = leftKey; +// myRightKey = rightKey; + +// myKeyToTablePointerMap = new Hashtable(); +// myIdToUse = 0; +// } + +// protected override void myOnFirstSubscriber() +// { +// myVLeft.onChange += new ChangeDelegate(myVLeft_onChange); +// myVRight.onChange += new ChangeDelegate(myVRight_onChange); +// } + +// void myVRight_onChange(object newValue) +// { +// myHandleChange(newValue as DataPacket, false); + +// } + +// void myVLeft_onChange(object newValue) +// { +// myHandleChange(newValue as DataPacket, true); + +// } + +// void myHandleChange(DataPacket dp, bool isLeft) +// { + +// int i = 0; // for going throuhg loops and such +// BlueTable addedRows = null, droppedRows = null; + +// if (myJoinedTable == null) +// { +// if (isLeft && myLeftTable == null) +// { +// myLeftTable = new BlueTable(dp.addedRows); + +// //go through the list of added rows and get the id of each row, storing this in the TablePointer map +// foreach (IBlueRow row in myLeftTable) +// { +// if (myKeyToTablePointerMap.ContainsKey(row[myLeftKey])) +// { +// ((ParallelJoinTestTablePointer)myKeyToTablePointerMap[row[myLeftKey]]).LeftTableId = (int)row["id"]; +// } +// else +// { +// myKeyToTablePointerMap[row[myLeftKey]] = new ParallelJoinTestTablePointer((int)row["id"],-1,-1); + +// } + +// } + +// } +// else if (!isLeft && myRightTable == null) +// { +// myRightTable = new BlueTable(dp.addedRows); +// //go through the list of added rows and get the id of each row, storing this in the TablePointer map +// foreach (IBlueRow row in myRightTable) +// { +// if (myKeyToTablePointerMap.ContainsKey(row[myRightKey])) +// { +// ((ParallelJoinTestTablePointer)myKeyToTablePointerMap[row[myRightKey]]).RightTableId = (int)row["id"]; +// } +// else +// { +// myKeyToTablePointerMap[row[myRightKey]] = new ParallelJoinTestTablePointer(-1,(int)row["id"],-1); + +// } + +// } +// } + +// if (myLeftTable != null && myRightTable != null) +// { +// //construct the schema for the table based upon the schemas of the two tables that are going to be joined together +// int numColumns = myLeftTable.schema.columnInfo.Length; + + +// ColumnInfo[] newColumnInfoArray = new ColumnInfo[numColumns * 2 + 1]; +// for (i = 0; i < numColumns; i++) +// { +// newColumnInfoArray[i] = new ColumnInfo("L." + myLeftTable.schema.columnNames[i], i, myLeftTable.schema.columnInfo[i].colType); +// newColumnInfoArray[numColumns + i] = new ColumnInfo("R." + myRightTable.schema.columnNames[i], i + numColumns, myRightTable.schema.columnInfo[i].colType); +// } +// newColumnInfoArray[numColumns + i] = new ColumnInfo("id", numColumns + i, (typeof(System.Int64))); + +// IBlueTableSchema newSchema = new BlueTableSchema("Joined Table", newColumnInfoArray); + +// myJoinedTable = new BlueTable(newSchema); +// addedRows = new BlueTable(newSchema); +// droppedRows = new BlueTable(newSchema); + + +// //now that we've created the table, we should have some leftover stuff from the side tables +// //that we were storing until now. Add that stuff to the table. +// BlueTable rowsToAdd = isLeft ? myRightTable : myLeftTable; + +// foreach (BlueRow row in rowsToAdd) +// { +// Object[] newRowObjects = new Object[newSchema.columnInfo.Length]; + +// if (isLeft) +// { +// for (i = 0; i < numColumns; i++) +// { +// newRowObjects[i] = row[i]; +// newRowObjects[numColumns + i] = null; + +// } +// newRowObjects[numColumns + i] = myIdToUse; +// //store the fact that the row with this key value went to this id + +// ((ParallelJoinTestTablePointer)myKeyToTablePointerMap[row[myLeftKey]]).JoinTableId = myIdToUse; +// myIdToUse++; + +// } +// else +// { +// for (i = 0; i < numColumns; i++) +// { +// newRowObjects[i] = null; +// newRowObjects[numColumns + i] = row[i]; + +// } +// newRowObjects[numColumns + i] = myIdToUse; +// //store the fact that the row with this key value went to this id +// ((ParallelJoinTestTablePointer)myKeyToTablePointerMap[row[myRightKey]]).JoinTableId = myIdToUse; +// myIdToUse++; +// } + +// BlueRow newRow = new BlueRow(newSchema, newRowObjects); +// addedRows.addRow(newRow); +// myJoinedTable.addRow(newRow); + +// } + + +// } +// } +// else +// { +// addedRows = new BlueTable(myJoinedTable.schema); +// droppedRows = new BlueTable(myJoinedTable.schema); +// } + + + + +// //setup is complete. At this point, we have to delete all the rows that were deleted, then add in the rows that were added in. + +// foreach (BlueRow row in dp.addedRows) +// { +// //check to see if this row has a match in the table already. +// //If it does, get rid of that row in the table because it's OUT OF DATE +// string joinColumn = isLeft ? myLeftKey : myRightKey; + +// if (myKeyToTablePointerMap.ContainsKey(row[joinColumn])) +// { +// //the value is already in the hashtable. +// ParallelJoinTestTablePointer tp = (ParallelJoinTestTablePointer) myKeyToTablePointerMap[row[joinColumn]]; +// int rowOfExistingGuy = tp.JoinTableId; + +// if (rowOfExistingGuy != -1) +// { + +// //remove this guy from the table +// IBlueRow rowToDelete = myJoinedTable[rowOfExistingGuy]; +// myJoinedTable.deleteRow(rowToDelete); +// droppedRows.addRow(rowToDelete); +// tp.JoinTableId = -1; +// } + + + +// } +// } + + +// //finally, fire off a packet, but only if you have enough information about the table to do so. + +// if (myJoinedTable != null) +// { +// DataPacket outPacket = new DataPacket(myJoinedTable,addedRows, new BlueTable(myJoinedTable.schema)); +// myFireEvent(outPacket); +// } + + + +// } + + +// } + + +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs b/src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs new file mode 100644 index 000000000..c79697571 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Implemented as a doubley linked list, it helps in Join operations, where O(1) deletions + /// are required + /// + internal class RowList : ICollection + { + private RowNode myFirstPtr; + private int myCount = 0; + + #region ICollection methods + public int Count{get{return myCount;}} + public bool IsSynchronized {get{return false;}} + public object SyncRoot {get{return this;}} + public void CopyTo (Array array, int index) + { + int i = 0; + foreach (object o in this) + array.SetValue (o, i++); + } + public IEnumerator GetEnumerator () {return new RowListEnumerator (this);} + #endregion + + /// + /// Nodes are added at the start of the collection + /// + /// + /// + public RowNode add (IBlueRow row) + { + if (myFirstPtr != null) //list is not empty + { + RowNode temp = new RowNode (row, this); + myFirstPtr.previous = temp; + temp.next = myFirstPtr; + myFirstPtr = temp; + } + else //i.e. if the list is empty + { + myFirstPtr = new RowNode (row, this); + } + myCount ++; + return myFirstPtr; + } + + public void clear() + { + myCount = 0; + myFirstPtr = null; + } + + /// + /// If node to be removed does not belong to the List, exception is thrown + /// + /// + public void remove (RowNode node) + { + myCount --; + if (node.isFirst && node.isLast) + { + clear (); + return; + } + else if (node.isFirst) + { + myFirstPtr = myFirstPtr.next; + myFirstPtr.previous = null; + return; + } + else if (node.isLast) + { + node.previous.next = null; + return; + } + else if (!node.isFirst && !node.isLast) + { + node.previous.next = node.next; + node.next.previous = node.previous; + return; + } + else + { + myCount ++; + throw new System.ApplicationException ("node does not belong to the list"); + } + } + + internal class RowListEnumerator : IEnumerator + { + private RowNode myPointer; + private RowList myList; + private int myPos = -1; + public RowListEnumerator (RowList list) {myList = list;} + public object Current {get{return myPointer;}} + public bool MoveNext() + { + myPointer = myPos == -1 ? myList.myFirstPtr : myPointer.next; + myPos ++; + return myPointer != null; + } + public void Reset() {myPos = -1;} + } + } + + internal class RowNode + { + private IBlueRow myRow; + private RowNode myNext; + private RowNode myPrevious; + private RowList myList; + + public RowNode (IBlueRow row){myRow = row;} + public RowNode (IBlueRow row, RowList list) : this (row) {myList = list;} + public bool isLast {get{return myNext == null;}} + public bool isFirst {get{return myPrevious == null;}} + public RowList list {get{return myList;}} + public IBlueRow row + { + get{return myRow;} + set{myRow = value;} + } + public RowNode next { + get{return myNext;} + set{myNext = value;} + } + public RowNode previous { + get{return myPrevious;} + set{myPrevious = value;} + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs new file mode 100644 index 000000000..9bf978c73 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs @@ -0,0 +1,570 @@ +// using System; +// using System.Collections; +// using System.Collections.Generic; +// using System.Text; +// using BCG.Utilities.General; +// using BCG.Utilities; +// using mbts; + +// namespace BCG +// { +// internal class SingleRowJoinDicionary : IDictionary +// { +// private TKey myKey; +// private TValue myValue; + +// #region IDictionary Members + +// public void Add(TKey key, TValue value) +// { +// myKey = key; +// myValue = value; +// } + +// public bool ContainsKey(TKey key) +// { +// return myKey.Equals(key); +// } + +// public ICollection Keys +// { +// get { return new TKey[] { myKey }; } +// } + +// public bool Remove(TKey key) +// { +// bool rv = myKey.Equals(key); +// myKey = default(TKey); +// return rv; +// } + +// public bool TryGetValue(TKey key, out TValue value) +// { +// bool rv = false; +// value = (rv = myKey.Equals(key)) ? myValue : default(TValue); +// return rv; +// } + +// public ICollection Values +// { +// get { throw new Exception("The method or operation is not implemented."); } +// } + +// public TValue this[TKey key] +// { +// get +// { +// throw new Exception("The method or operation is not implemented."); +// } +// set +// { +// throw new Exception("The method or operation is not implemented."); +// } +// } + +// #endregion + +// #region ICollection> Members + +// public void Add(KeyValuePair item) +// { +// throw new Exception("The method or operation is not implemented."); +// } + +// public void Clear() +// { +// throw new Exception("The method or operation is not implemented."); +// } + +// public bool Contains(KeyValuePair item) +// { +// throw new Exception("The method or operation is not implemented."); +// } + +// public void CopyTo(KeyValuePair[] array, int arrayIndex) +// { +// throw new Exception("The method or operation is not implemented."); +// } + +// public int Count +// { +// get { throw new Exception("The method or operation is not implemented."); } +// } + +// public bool IsReadOnly +// { +// get { throw new Exception("The method or operation is not implemented."); } +// } + +// public bool Remove(KeyValuePair item) +// { +// throw new Exception("The method or operation is not implemented."); +// } + +// #endregion + +// #region IEnumerable> Members + +// public IEnumerator> GetEnumerator() +// { +// throw new Exception("The method or operation is not implemented."); +// } + +// #endregion + +// #region IEnumerable Members + +// IEnumerator IEnumerable.GetEnumerator() +// { +// throw new Exception("The method or operation is not implemented."); +// } + +// #endregion +// } + +// #region Helper class to deal with stream projections + +// public class JoinProjectHelper +// { +// private IBlueTableSchema mySchema; +// private Dictionary> myAliasToOriginal = new Dictionary>(); + +// public JoinProjectHelper(IBlueTableSchema leftSchema, +// string leftProj, +// string leftJoinCol, +// IBlueTableSchema rightSchema, +// string rightProj, +// string rightJoinCol) +// { + +// JoinTableSchema joinedSchema = new JoinTableSchema(leftSchema, leftJoinCol, rightSchema, rightJoinCol); +// JoinNameResolver resolver = new JoinNameResolver(joinedSchema, leftSchema, leftProj, rightSchema, rightProj); +// IDictionary map = resolver.getMap(); +// map["id"] = "id"; +// ProjectHelper helper = new ProjectHelper(joinedSchema, ProjectHelper.invertDic(map), +// String.Format("({0} J {1})", leftSchema.tableName, rightSchema.tableName)); + +// mySchema = helper.schema; +// for (int aliasCol = 0; aliasCol < helper.colMap.Length; aliasCol++) +// { +// string aliasColName = mySchema[aliasCol].colName; +// if (aliasColName != "id") +// { +// int joinedColIndex = helper.colMap[aliasCol]; +// bool isLeft = joinedColIndex < leftSchema.columnInfo.Length; +// string originalCol = joinedSchema.getOriginalName(joinedSchema[joinedColIndex].colName, isLeft); +// myAliasToOriginal.Add(aliasColName, new Pair(originalCol, isLeft)); +// } +// } +// } + +// public IBlueTableSchema schema +// { +// get { return mySchema; } +// } + +// public string getOriginalCol(string col, out bool isLeft) +// { +// Pair p = myAliasToOriginal[col]; +// isLeft = p.second; +// return p.first; +// } +// } + +// #endregion + +// #region IBlueRow implementation for join and leftJoin + +// public class SuperJoinRow : IBlueRow +// { +// private JoinProjectHelper myHelper; +// private IBlueRow myLeftRow, myRightRow; +// int myID; + +// public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper) +// { +// myID = id; +// myLeftRow = leftRow; +// myRightRow = rightRow; +// myHelper = joinProjectHelper; +// } + +// public int leftID { get { return (int)myLeftRow["id"]; } } + +// public int rightID { get { return (int)myRightRow["id"]; } } + +// public IBlueRow leftRow { get { return myLeftRow; } } + +// public IBlueRow rightRow { get { return myRightRow; } } + +// #region IBlueRow Members + +// public IBlueTableSchema schema { get { return myHelper.schema; } } + +// public object this[int index] { get { return this[schema[index].colName]; } } + +// public object this[string colName] +// { +// get +// { +// if (colName == "id") +// return myID; +// else +// { +// bool isLeft; +// string col = myHelper.getOriginalCol(colName, out isLeft); +// return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]); +// } +// } +// } + +// #endregion + +// #region IDictionary Members + +// public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } } + +// public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); } + +// public bool Contains(object key) +// { +// return myLeftRow.Contains(key) || myRightRow.Contains(key); +// } + +// public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + +// public bool IsFixedSize { get { return true; } } + +// public bool IsReadOnly { get { return true; } } + +// public System.Collections.ICollection Keys { get { return schema.columnNames; } } + +// public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); } + +// public System.Collections.ICollection Values +// { +// get +// { +// List rv = new List(); +// foreach (string col in Keys) +// rv.Add(this[col]); +// return rv; +// } +// } + +// public object this[object key] +// { +// get { return key is string ? this[key.ToString()] : this[(int)key]; } +// set { throw new NotSupportedException("IBlueRow is readonly"); } +// } + +// #endregion + +// #region ICollection Members + +// public void CopyTo(Array array, int index) { Values.CopyTo(array, index); } + +// public int Count { get { return schema.columnNames.Length; } } + +// public bool IsSynchronized { get { return false; } } + +// public object SyncRoot { get { return this; } } + +// #endregion + +// #region IEnumerable Members + +// System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + +// #endregion +// } + +// #endregion +// /// +// /// this class is written 3-4 yrs after the original +// /// join, and learning from the past mistakes, is +// /// made super fast and mem efficent +// /// +// public class SuperJoinStream : ViewStream +// { +// protected Dictionary myLeftUnmatchedRowsAdds = new Dictionary(), +// myLeftUnmatchedRowsDels = new Dictionary(); +// private bool myManageUnmappedLeftRows = false; +// protected HighPerformanceTimer myTimer = new HighPerformanceTimer(); +// protected string myLeftProj = String.Empty, myRightProj = String.Empty; +// protected IBlueTableSchema myLeftSchema, myRightSchema; +// protected JoinProjectHelper myJoinProjectHelper; +// protected BlueTable myState; +// protected int myID; +// protected ViewStream myLeftStream, myRightStream; +// protected string myLeftKeyCol, myRightKeyCol; +// protected Dictionary> myLeftKeyToLeftRows = new Dictionary>(), +// myRightKeyToRightRows = new Dictionary>(); + +// protected Dictionary> myLeftIDToResultRows = new Dictionary>(), +// myRightIDToResultRows = new Dictionary>(); + +// public SuperJoinStream(ViewStream leftStream, string leftKey, ViewStream rightStream, string rightKey) +// : this(leftStream, leftKey, "!", rightStream, rightKey, "!") +// { } + +// public SuperJoinStream(ViewStream leftStream, string leftKey, string leftProj, +// ViewStream rightStream, string rightKey, string rightProj) +// : this(leftStream, leftKey, leftProj, rightStream, rightKey, rightProj, false) +// { +// } + +// internal SuperJoinStream(ViewStream leftStream, string leftKey, string leftProj, +// ViewStream rightStream, string rightKey, string rightProj, bool manageUnmappedLeftRows) +// { +// myLeftStream = leftStream; +// myRightStream = rightStream; +// myLeftKeyCol = leftKey; +// myRightKeyCol = rightKey; +// myLeftProj = leftProj; +// myRightProj = rightProj; +// myManageUnmappedLeftRows = manageUnmappedLeftRows; +// } + +// protected override void myOnFirstSubscriber() +// { +// myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges); +// myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges); +// } + +// protected override void myOnLastSubscriber() +// { +// myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges); +// myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges); +// myLeftKeyToLeftRows.Clear(); +// myRightKeyToRightRows.Clear(); +// myLeftIDToResultRows.Clear(); +// myRightIDToResultRows.Clear(); +// myState = null; +// myLeftSchema = null; +// myRightSchema = null; +// } + +// private void myHandleLeftStreamChanges(object newValue) +// { +// myHandleStreamChanges(newValue as DataPacket, true); +// } + +// private void myHandleRightStreamChanges(object newValue) +// { +// myHandleStreamChanges(newValue as DataPacket, false); +// } + +// private void myHandleStreamChanges(DataPacket dp, bool isLeft) +// { +// myTimer.start(); +// myInitSchema(dp.schema, isLeft); +// BlueTable dels = null, adds = null; +// if (myState != null) +// { +// dels = new BlueTable(myState.schema); +// adds = new BlueTable(myState.schema); +// } +// myDeleteRows(dp.deletedRows, isLeft, myState, dels); +// myAddRows(dp.addedRows, isLeft, myState, adds); +// myTimer.stop(); +// if (myState != null) +// { +// myFireEvent(new DataPacket(myState, adds, dels)); +// myLeftUnmatchedRowsAdds.Clear(); +// myLeftUnmatchedRowsDels.Clear(); +// } +// } + +// private void myInitSchema(IBlueTableSchema schema, bool isLeft) +// { +// if (myState == null) +// { +// if (isLeft) +// { +// myLeftSchema = schema; +// if (myRightSchema != null) +// { +// myJoinProjectHelper = new JoinProjectHelper(myLeftSchema, myLeftProj, myLeftKeyCol, +// myRightSchema, myRightProj, myRightKeyCol); +// myState = new BlueTable(myJoinProjectHelper.schema); +// } +// } +// else +// { +// myRightSchema = schema; +// if (myLeftSchema != null) +// { +// myJoinProjectHelper = new JoinProjectHelper(myLeftSchema, myLeftProj, myLeftKeyCol, +// myRightSchema, myRightProj, myRightKeyCol); +// myState = new BlueTable(myJoinProjectHelper.schema); +// } +// } +// } +// } + +// private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut) +// { +// object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol, +// otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol; +// Dictionary> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows, +// otherSideKeyToRows = isLeft ? myRightKeyToRightRows : myLeftKeyToLeftRows; +// Dictionary> thisIDToResultRows = isLeft ? myLeftIDToResultRows : myRightIDToResultRows, +// otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows; +// foreach (IBlueRow r in addedRows) +// { +// object currentKey = r[thisKeyCol]; +// int currentID = (int)r["id"]; +// ///add it to 1. +// if (!thisKeyToRows.ContainsKey(currentKey)) +// thisKeyToRows.Add(currentKey, new Dictionary()); +// thisKeyToRows[currentKey].Add(currentID, r); +// ///find matching rows, and form result rows +// if (otherSideKeyToRows.ContainsKey(currentKey)) +// { +// ICollection matchingRows = otherSideKeyToRows[currentKey].Values; +// foreach (IBlueRow mr in matchingRows) +// { +// int mrID = (int)mr["id"]; +// SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper); +// state.addRow(rr); +// addedRowsOut.addRow(rr); +// ///add to 3. +// if (!thisIDToResultRows.ContainsKey(currentID)) +// thisIDToResultRows.Add(currentID, new Dictionary()); +// thisIDToResultRows[currentID].Add((int)rr["id"], rr); +// ///add to 4. +// if (!otherIDToResultRows.ContainsKey(mrID)) +// { +// otherIDToResultRows.Add(mrID, new Dictionary()); +// if (myManageUnmappedLeftRows && !isLeft) +// myTakeCareOfUnmatachedLeftRows(mr, false); +// } +// otherIDToResultRows[mrID].Add((int)rr["id"], rr); +// } +// } +// else +// { +// if (myManageUnmappedLeftRows && isLeft) +// myTakeCareOfUnmatachedLeftRows(r, true); +// } +// } +// } + +// private void myDeleteRows(IBlueRowCollection deletedRows, bool isLeft, BlueTable state, BlueTable delRowsOut) +// { +// string thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol, +// otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol; +// Dictionary> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows, +// otherSideKeyToRows = isLeft ? myRightKeyToRightRows : myLeftKeyToLeftRows; +// Dictionary> thisIDToResultRows = isLeft ? myLeftIDToResultRows : myRightIDToResultRows, +// otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows; +// foreach (IBlueRow r in deletedRows) +// { +// int id = (int)r["id"]; +// object key = r[thisKeyCol]; +// ///delete it from 1. +// thisKeyToRows[key].Remove(id); +// if (thisKeyToRows[key].Count == 0) +// thisKeyToRows.Remove(key); +// ///form return rows and delete from 3. +// if (thisIDToResultRows.ContainsKey(id)) +// { +// ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection; +// thisIDToResultRows.Remove(id); +// foreach (SuperJoinRow rr in removedResultRows) +// { +// ///delete from 4. +// int otherId = isLeft ? rr.rightID : rr.leftID; +// otherIDToResultRows[otherId].Remove((int)rr["id"]); +// if (otherIDToResultRows[otherId].Count == 0) +// otherIDToResultRows.Remove(otherId); +// delRowsOut.addRow(rr); +// state.deleteRow(rr); +// if (myManageUnmappedLeftRows && !isLeft) +// if (!thisKeyToRows.ContainsKey(key)) +// myTakeCareOfUnmatachedLeftRows(rr.leftRow, true); +// } +// } +// else +// { +// if (myManageUnmappedLeftRows && isLeft) +// myTakeCareOfUnmatachedLeftRows(r, false); +// } +// } +// } + +// private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add) +// { +// int lid = (int)leftRow["id"]; +// if (add) +// myLeftUnmatchedRowsAdds.Add(lid, leftRow); +// else +// { +// if (myLeftUnmatchedRowsAdds.ContainsKey(lid)) +// myLeftUnmatchedRowsAdds.Remove(lid); +// else +// myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow); +// } +// } +// } + +// public class SuperLeftJoinStream : SuperJoinStream +// { +// private Dictionary myLeftIDToHalfRows = new Dictionary(); + +// public SuperLeftJoinStream(ViewStream leftStream, string leftJoinCol, +// ViewStream rightStream, string rightJoinCol) +// : this(leftStream, leftJoinCol, "!", rightStream, rightJoinCol, "!") +// { } + +// public SuperLeftJoinStream(ViewStream leftStream, string leftJoinCol, string leftProjExp, +// ViewStream rightStream, string rightJoinCol, string rightProjExp) +// : base(leftStream, leftJoinCol, leftProjExp, rightStream, rightJoinCol, rightProjExp, true) +// { } + +// protected override void myFireEvent(object o) +// { +// myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket); +// } + +// private void myHandleJoinResults(Dictionary unmatchedLeftRowsDels, +// Dictionary unmatchedLeftRowAdds, DataPacket resultPacket) +// { +// BlueTable dels = resultPacket.deletedRows as BlueTable, +// adds = resultPacket.addedRows as BlueTable; + +// foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) +// { +// ///an lr will be here if its actually deleted or +// ///a right row is added, so now it has a partner +// ///and is no longer unmatched. +// ///in both cases the half row that this lr +// ///created needs to be removed from this +// ///class's state and added to dels +// ///it means a right is added +// int lid = (int)lr["id"]; +// SuperJoinRow rr = myLeftIDToHalfRows[lid]; +// myLeftIDToHalfRows.Remove(lid); +// myState.deleteRow(rr); +// dels.addRow(rr); +// } + +// foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) +// { +// ///an lr will be here if its added with no +// ///matching right, or a right matching row +// ///is gone so it is unmatched now. +// ///in both cases a half row needs to be +// ///added to this class's state and added +// ///to adds +// int lid = (int)lr["id"]; +// SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper); +// myLeftIDToHalfRows.Add(lid, rr); +// myState.addRow(rr); +// adds.addRow(rr); +// } +// base.myFireEvent(new DataPacket(myState, adds, dels)); +// } +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/KeyBasedMath/KeyBasedMathStream.cs b/src/DataStreamSuite/DataStream/Primitives/KeyBasedMath/KeyBasedMathStream.cs new file mode 100644 index 000000000..a15a44a68 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/KeyBasedMath/KeyBasedMathStream.cs @@ -0,0 +1,57 @@ +// // // using System; + +// // // namespace BCG +// // // { + +// // // public class KeyBasedAddStream : KeyBasedMathStream +// // // { +// // // public KeyBasedAddStream(ViewStream vs1, ViewStream vs2, string keyCol, string colToAdd) +// // // : base(vs1, vs2, keyCol, colToAdd, (d1, d2) => { return d1 + d2; }) { } +// // // } + +// // // public class KeyBasedSubtractStream : KeyBasedMathStream +// // // { +// // // public KeyBasedSubtractStream(ViewStream vs1, ViewStream vs2, string keyCol, string colToSubtract) +// // // : base(vs1, vs2, keyCol, colToSubtract, (d1, d2) => { return d1 - d2; }) { } +// // // } + +// // // public class KeyBasedMultiplyStream : KeyBasedMathStream +// // // { +// // // public KeyBasedMultiplyStream(ViewStream vs1, ViewStream vs2, string keyCol, string colToMultiply) +// // // : base(vs1, vs2, keyCol, colToMultiply, (d1, d2) => { return d1 * d2; }) { } +// // // } + +// // // public class KeyBasedDivideStream : KeyBasedMathStream +// // // { +// // // public KeyBasedDivideStream(ViewStream vs1, ViewStream vs2, string keyCol, string colToDivide) +// // // : base(vs1, vs2, keyCol, colToDivide, (d1, d2) => { return d1 / d2; }) { } +// // // } + + + +// // // public abstract class KeyBasedMathStream : ViewStream +// // // { + +// // // public KeyBasedMathStream(ViewStream first, ViewStream second, string keyColName, string valueColumn, Func math) +// // // { +// // // ViewStream j; +// // // ViewStream m; + + +// // // string lValueTempName = "keysbasedmathtempleft"+valueColumn; +// // // string rValueTempName = "keysbasedmathtempright"+valueColumn; +// // // j = new JoinStream (first, +// // // keyColName, +// // // string.Format("!{0}:{1} {2} id", valueColumn, lValueTempName, keyColName), +// // // second, +// // // keyColName, +// // // string.Format("!{0}:{1} id", valueColumn, rValueTempName)); +// // // m = new MathStream (j, +// // // string.Format("{0} = {1} - {2}", valueColumn, lValueTempName, rValueTempName)); + +// // // m = new ProjectStream(m, string.Format("!{0} {1}", lValueTempName, rValueTempName)); + +// // // mySetRealSource(m); +// // // } +// // // } +// // // } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinForeignStream.cs b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinForeignStream.cs new file mode 100644 index 000000000..b680644f3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinForeignStream.cs @@ -0,0 +1,38 @@ +// using System; + +// namespace BCG +// { +// [DataStreamUserRespresentation (false)] +// public class LeftInternalJoinForeignStream : LeftInternalJoinStream +// { +// public LeftInternalJoinForeignStream (ViewStream vsLeft, +// string jcLeft, +// ViewStream vsRight, +// string jcRight) : +// base (vsLeft, jcLeft, vsRight, jcRight) {} + +// public LeftInternalJoinForeignStream (ViewStream vsLeft, +// string jcLeft, +// ViewStream vsRight) : +// this (vsLeft, jcLeft, vsRight, "id") {} + +// public override void onTableClear (bool isLeft) +// { +// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; +// myLastFiredPacket.newRows = new JoinForeignTable (mySchema); +// myLastFiredPacket.addedRows = new BlueTable (mySchema); +// if (!isLeft) +// foreach (IBlueRow row in myLeftDataPacket.newRows) +// ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket); +// } + +// protected override void myInitFiredPacket () +// { +// myLastFiredPacket = new DataPacket (); +// myLastFiredPacket.newRows = new JoinForeignTable (mySchema); +// myLastFiredPacket.addedRows = new BlueTable (mySchema); +// myLastFiredPacket.deletedRows = new BlueTable (mySchema); +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs new file mode 100644 index 000000000..88511470d --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs @@ -0,0 +1,182 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; + +// namespace BCG +// { +// /// +// /// Stream to internally handle Left Joins. The only difference between this and the left Join is the waty the +// /// DataPacket thats fired is initialized. +// /// +// [DataStreamUserRespresentation (false)] +// public class LeftInternalJoinStream : InternalJoinStream +// { +// protected DataPacket myLeftDataPacket, myRightDataPacket; +// //Container to contain all the left rows that are orphaned during a deletion operation +// private IDictionary myLeftOrphanedRows; +// public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) : +// base (vsLeft, jcLeft, vsRight, jcRight) {} + +// protected override void myOnFirstSubscriber() +// { +// myLeftOperand.master = this; +// myRightOperand.master = this; +// //the events are not hooked onto the operands delegate, but rather a level of indirection +// //added in the form on myLeftChangeHandler +// myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftChangeHandler); +// myRightStream.onChange += new DataStream.ChangeDelegate (myRightChangeHandler); +// } + +// protected override void myOnLastSubscriber () +// { +// myInit (); +// myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftChangeHandler); +// myRightStream.onChange -= new DataStream.ChangeDelegate (myRightChangeHandler); +// } + +// protected override void myInit() +// { +// base.myInit (); +// myLeftOrphanedRows = null; +// myLeftDataPacket = myRightDataPacket = null; +// } + + +// private void myLeftChangeHandler (object o) +// { +// myLeftDataPacket = (DataPacket)o; +// myTestAndFire (true); +// } + +// private void myRightChangeHandler (object o) +// { +// myRightDataPacket = (DataPacket)o; +// myTestAndFire (false); +// } + +// /// +// /// enforces the order of firing the events, this is to enforce that first the right operand +// /// has data (possibly 0 rows) before we the join is actually calculated => Performance boost +// /// +// /// +// private void myTestAndFire (bool isLeft) +// { +// JoinOperand jo = isLeft ? myLeftOperand : myRightOperand; +// DataPacket dp = isLeft ? myLeftDataPacket : myRightDataPacket; +// //to test whether both have received call backs +// if (myLeftOperand.hasReceivedData) +// jo.changeHandler (dp); +// else if (myLeftDataPacket != null && myRightDataPacket != null) +// { +// myRightOperand.changeHandler (myRightDataPacket); +// myLeftOperand.changeHandler (myLeftDataPacket); +// } +// } + +// /// +// /// Called by JoinOperand +// /// +// /// +// /// +// public override void addRows (IBlueRowCollection rows, bool isLeft) +// { +// foreach (IBlueRow row in rows) +// myAddRow (row, isLeft); +// if (!isLeft) +// foreach (RowNode node in myLeftOrphanedRows.Values) +// ((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket); +// } + +// /// +// /// called by JoinOperand +// /// +// /// +// /// +// public override void deleteRows (IBlueRowCollection rows, bool isLeft) +// { +// myLeftOrphanedRows = new Hashtable (); +// foreach (IBlueRow row in rows) +// myDeleteRow (row, isLeft); +// } + +// private void myAddRow (IBlueRow row, bool isLeft) +// { +// if (isLeft) +// { +// myLeftOperand.addRow (row); +// RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]); +// if (matchedRows.Count == 0) +// ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket); +// else +// foreach (RowNode matchedNode in matchedRows) +// ((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket); +// } +// else +// { +// RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]); +// int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; +// myRightOperand.addRow (row); +// int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; +// if (matchedRows.Count != 0){ +// foreach (RowNode matchedNode in matchedRows) +// { +// //if the left rows were orphaned in this transaction (ie in this callback), they should +// //just be removed from the orphanedlist and we dont need to worry about the consistency of +// //the jointable as a new row will replace it +// if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0) +// { +// if (myLeftOrphanedRows[matchedNode.row["id"]] == null) +// ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket); +// else +// myLeftOrphanedRows.Remove (matchedNode.row["id"]); +// } +// //they were orphaned by a previous transaction and hence the orphans need to be +// //removed from the Jointable and the complete row added +// ((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket); + +// } +// } +// } +// } + +// private void myDeleteRow (IBlueRow row, bool isLeft) +// { +// if (isLeft) +// { +// myLeftOperand.deleteRow (row); +// ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, row, myLastFiredPacket); +// } +// else +// { +// RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]); +// int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; +// myRightOperand.deleteRow (row); +// int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; +// if (matchedRows.Count != 0) +// { +// if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) +// //ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to +// //be handled specially, ie joined with nulls, if there are no right rows with this +// //joincolumn value being inserted. +// foreach (RowNode matchedNode in matchedRows) +// { +// // BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null); +// myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode; +// } +// ((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket); +// } +// } +// } + +// public override void onTableClear (bool isLeft) +// { +// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; +// myLastFiredPacket.newRows = new JoinTable (mySchema); +// myLastFiredPacket.addedRows = new BlueTable (mySchema); +// if (!isLeft) +// foreach (IBlueRow row in myLeftDataPacket.newRows) +// ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket); +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftJoinForeignStream.cs b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftJoinForeignStream.cs new file mode 100644 index 000000000..436281cc0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftJoinForeignStream.cs @@ -0,0 +1,59 @@ +// using System; +// using System.Data; +// using System.Collections; +// using System.Text.RegularExpressions; +// using BCG.Utilities; +// using BCG.Utilities.General; + +// namespace BCG +// { +// public class LeftJoinForeignStream : JoinStream +// { + +// public LeftJoinForeignStream (ViewStream v1, string joinCol1, ColumnMap cm1, +// ViewStream v2, string joinCol2, ColumnMap cm2) +// : base (v1, joinCol1, cm1, v2, joinCol2, cm2) +// {} + +// public LeftJoinForeignStream (ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2, string cmExp2) +// : base (v1, joinCol1, cmExp1, v2, joinCol2, cmExp2) +// {} + +// public LeftJoinForeignStream (ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2) +// : +// this (v1,joinCol1, new ColumnMap(cmExp1), v2, joinCol2) +// {} + +// public LeftJoinForeignStream (ViewStream v1, string joinCol1, +// ViewStream v2, string joinCol2, string cmExp2) +// : +// this (v1,joinCol1, v2, joinCol2, new ColumnMap(cmExp2)) +// {} + + +// public LeftJoinForeignStream (ViewStream v1, string joinCol1, ViewStream v2, string joinCol2) +// : base (v1, joinCol1, v2, joinCol2) +// {} + +// public LeftJoinForeignStream (ViewStream v1, string joinCol1, ColumnMap cm1, ViewStream v2, string joinCol2) +// : base (v1, joinCol1, cm1, v2, joinCol2) +// {} + + +// public LeftJoinForeignStream (ViewStream v1, string joinCol1, ViewStream v2, string joinCol2, ColumnMap cm2) +// : base (v1, joinCol1, v2, joinCol2, cm2) +// {} + +// protected override InternalJoinStream myConstructInternalJoinStream ( +// ViewStream vs1, +// string jc1, +// ViewStream vs2, +// string jc2) +// { +// return new LeftInternalJoinForeignStream (vs1, jc1, vs2, jc2); +// } +// } +// } + diff --git a/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftJoinStream.cs new file mode 100644 index 000000000..1011bc112 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftJoinStream.cs @@ -0,0 +1,58 @@ +// using System; +// using System.Data; +// using System.Collections; +// using System.Text.RegularExpressions; +// using BCG.Utilities; +// using BCG.Utilities.General; + +// namespace BCG +// { +// public class LeftJoinStream : JoinStream +// { +// public LeftJoinStream (ViewStream v1, string joinCol1, ColumnMap cm1, +// ViewStream v2, string joinCol2, ColumnMap cm2) +// : base (v1, joinCol1, cm1, v2, joinCol2, cm2) +// {} + +// public LeftJoinStream (ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2, string cmExp2) +// : base (v1, joinCol1, cmExp1, v2, joinCol2, cmExp2) +// {} + +// public LeftJoinStream (ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2) +// : +// this (v1,joinCol1, new ColumnMap(cmExp1), v2, joinCol2) +// {} + +// public LeftJoinStream (ViewStream v1, string joinCol1, +// ViewStream v2, string joinCol2, string cmExp2) +// : +// this (v1,joinCol1, v2, joinCol2, new ColumnMap(cmExp2)) +// {} + + +// public LeftJoinStream (ViewStream v1, string joinCol1, ViewStream v2, string joinCol2) +// : base (v1, joinCol1, v2, joinCol2) +// {} + +// public LeftJoinStream (ViewStream v1, string joinCol1, ColumnMap cm1, ViewStream v2, string joinCol2) +// : base (v1, joinCol1, cm1, v2, joinCol2) +// {} + + +// public LeftJoinStream (ViewStream v1, string joinCol1, ViewStream v2, string joinCol2, ColumnMap cm2) +// : base (v1, joinCol1, v2, joinCol2, cm2) +// {} + +// protected override InternalJoinStream myConstructInternalJoinStream ( +// ViewStream vs1, +// string jc1, +// ViewStream vs2, +// string jc2) +// { +// return new LeftInternalJoinStream (vs1, jc1, vs2, jc2); +// } +// } +// } + diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ForeignJoinCollectionEnumerator.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ForeignJoinCollectionEnumerator.cs new file mode 100644 index 000000000..7f6c09db7 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ForeignJoinCollectionEnumerator.cs @@ -0,0 +1,79 @@ +// using System; +// using System.Collections; + +// namespace BCG +// { +// internal class ForeignJoinCollectionEnumerator : IEnumerator +// { +// protected LightForeignJoinRowList.LightForeignJoinRowListEnumerator myListEnum; +// protected IDictionaryEnumerator myDictEnum; +// protected IDictionary myJoinColMap; +// protected LightForeignJoinSchema mySchema; + +// public ForeignJoinCollectionEnumerator (IDictionary joinColMap, LightForeignJoinSchema schema) +// { +// mySchema = schema; +// myJoinColMap = joinColMap; +// } + +// #region IEnumerator Members + +// public void Reset() +// { +// myListEnum = null; +// myDictEnum = null; +// } + +// public virtual object Current +// { +// get +// { +// return new LightForeignJoinRow (myListEnum.list.referencedRow, myListEnum.Current as IBlueRow, mySchema); +// } +// } + +// /// +// /// Gets the ILightForeignJoinRowList corresponding to the current position of the 'MyDictEnum', and moves +// /// the 'myListEnum' to the correct position, ie first referencing/left row. If the list does not have any such +// /// rows a false is returned +// /// +// /// +// protected virtual bool myMoveListEnumToRightPos () +// { +// myListEnum = (myDictEnum.Value as ILightForeignJoinRowList).GetEnumerator () +// as LightForeignJoinRowList.LightForeignJoinRowListEnumerator; +// if (myListEnum.list.count > 1 && myListEnum.list.referencedRow != null) +// { +// myListEnum.MoveNext (); +// return myListEnum.MoveNext (); +// } +// return false; +// } + +// public virtual bool MoveNext() +// { +// if (myDictEnum == null) //never been traversed +// { +// myDictEnum = myJoinColMap.GetEnumerator (); +// while (myDictEnum.MoveNext ()) +// if (myMoveListEnumToRightPos ()) +// return true; +// return false; +// } +// else +// { +// if (myListEnum.MoveNext()) +// return true; +// else +// while (myDictEnum.MoveNext()) +// if (myMoveListEnumToRightPos ()) +// return true; +// } +// return false; +// } + +// #endregion + +// } + +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs new file mode 100644 index 000000000..5766330e6 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// List where all the rows are mantained, both the referencing and the referced tables. + /// + public interface ILightForeignJoinRowList : IEnumerable + { + /// + /// Adds the row, and returns the node associated with the row (in the most simple of implementations + /// it should be the previous row, to make deletions easier) + /// + /// row to be added + /// boolean indicating if its a referenceD row or referncinG + /// Associated node (can be the node to which the row was added, previous etc, depending on implementaion) + INode add (IBlueRow r, bool isReD); + /// + /// Removes the row from the row, id is becomes irrelavant when isRed is true, as theres only one row of that kind + /// in the list + /// + /// Node associated with the row + /// id of the row to be deleted + void remove (INode n, bool isReD, int id); + /// + /// returns the number of elements in the list + /// + int count {get;} + /// + /// Node containing the referencedRow + /// + IBlueRow referencedRow {get;} + + /// + /// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row + /// + /// + void removeAll (bool isReD); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs new file mode 100644 index 000000000..7adab3364 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs @@ -0,0 +1,31 @@ +using System; + +namespace BCG +{ + /// + /// Respresents the node for the linked list. We can have as many rows in a node as possible, and hence + /// decrease the number of nodes and hence the memory, (at the cost of a linear search within a node). For big join + /// operations (a big trade log, for instance), where the rate of data refreshing is slow, we may use an impelentation + /// of INode that can accomodate a sizeable number of rows, decreasing the memory (8 bytes associated with each object) + /// + public interface INode + { + /// + /// Gets the row with the sepcific id, from within the node. Throws an ApplicationException if id does not exist + /// + /// + /// + IBlueRow getRow (int id); + /// + /// Gets all the rows in the node + /// + /// + IBlueRow[] getRows (); + /// + /// Replaces the row with the given id, with the row passed as the second arg + /// + /// New row + /// id of the row that will be replaced + void replace (int id, IBlueRow nr); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LeftForeignJoinCollectionEnumerator.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LeftForeignJoinCollectionEnumerator.cs new file mode 100644 index 000000000..e6fc9363a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LeftForeignJoinCollectionEnumerator.cs @@ -0,0 +1,38 @@ +// using System; +// using System.Collections; + +// namespace BCG +// { +// internal class LeftForeignJoinCollectionEnumerator : ForeignJoinCollectionEnumerator +// { +// public LeftForeignJoinCollectionEnumerator (IDictionary joinColMap, LightForeignJoinSchema schema) +// : base (joinColMap, schema) +// {} + +// public override object Current +// { +// get +// { +// return new LightLeftForeignJoinRow (myListEnum.list.referencedRow, myListEnum.Current as IBlueRow, mySchema); +// } +// } + +// protected override bool myMoveListEnumToRightPos() +// { +// if (mySchema != null) +// { +// myListEnum = (myDictEnum.Value as ILightForeignJoinRowList).GetEnumerator () +// as LightForeignJoinRowList.LightForeignJoinRowListEnumerator; +// if (myListEnum.list.count - (myListEnum.list.referencedRow == null ? 0 : 1) > 0) +// { +// myListEnum.MoveNext (); +// return myListEnum.MoveNext (); +// } +// } +// return false; +// } + + +// } + +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinCollection.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinCollection.cs new file mode 100644 index 000000000..e916dcbd2 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinCollection.cs @@ -0,0 +1,336 @@ +// // using System; +// // using System.Collections; +// // using BCG.Utilities; + +// // namespace BCG +// // { +// // /// +// // /// The central class that does maintans the data-structures associated with the join. +// // /// Being an implementation of the IBlueRowCollection, its passed as newRows to the DataPacet. +// // /// The seulting joined data is not cached, but on request the appropriate the IEnumerator +// // /// is returned, which has the logic to make joinedRows +// // /// +// // public class LightForeignJoinCollection : IBlueRowCollection +// // { +// // /* +// // * There are 2 Hashtables in the class, one indexed byt the join column value +// // * and the other indexed on IDs of the referencing table for quick accesses. +// // * */ +// // protected IDictionary myJoinKeyIndex = new Hashtable (), +// // myRinGIDIndex = new Hashtable (); +// // protected IBlueTableSchema myJoinSchema, myReDSchema, myRinGSchema; +// // protected string myReDCol, myRinGCol; + +// // public LightForeignJoinCollection (string rinGCol, string reDCol) +// // { +// // myReDCol = reDCol; +// // myRinGCol = rinGCol; +// // } + +// // private int myGetCount() +// // { +// // int rv = 0; +// // IEnumerator e = this.GetEnumerator (); +// // while (e.MoveNext()) +// // rv++; +// // return rv; +// // } + +// // protected DataPacket myGetImmutableDataPacket (DataPacket dp) +// // { +// // return dp == null ? null : +// // new DataPacket (dp.newRows is IImmutableRowCollection ? dp.newRows : new ImmutableBlueTable (dp.newRows), +// // dp.addedRows is IImmutableRowCollection ? dp.addedRows : new ImmutableBlueTable (dp.addedRows), +// // dp.deletedRows is IImmutableRowCollection ? dp.deletedRows : new ImmutableBlueTable (dp.deletedRows)); +// // } + + +// // public virtual DataPacket mergeRows (DataPacket dp, bool isReD) +// // { +// // return myGetImmutableDataPacket (isReD ? myMergeReDRows (dp) : myMergeRinGRows (dp)); +// // } + +// // /// +// // /// Handles changes in the right side when the left side (referencing side) has +// // /// not yet been received. Resets myJoinKeyIndex, hence crushing the old data. +// // /// +// // /// +// // /// +// // protected virtual DataPacket myHandleRightNoLeft (DataPacket dp) +// // { +// // myJoinKeyIndex = new Hashtable (); +// // foreach (IBlueRow redRow in dp.newRows) +// // myJoinKeyIndex.Add (redRow[myReDCol], new LightForeignJoinRowList (redRow)); +// // return null; +// // } + +// // /// +// // /// Handles the case when the added row count equals the new row count. This may +// // /// happen when +// // /// i) first callback is received OR +// // /// ii) the entire right side is deleted (special case of iii) OR +// // /// iii) the enitire right side is changed +// // /// +// // /// +// // /// +// // protected virtual DataPacket myHandleRightAllChanged (DataPacket dp) +// // { +// // bool isFirstTime = myJoinSchema == null; +// // DataPacket retVal = new DataPacket (); +// // //schema might be created for the first time here (might because it can ne on the anologous +// // //callback for he left side) +// // myJoinSchema = myJoinSchema == null ? +// // new LightForeignJoinSchema (myReDSchema, myReDCol, myRinGSchema, myRinGCol) : myJoinSchema; +// // retVal.deletedRows = isFirstTime ? +// // new LightForeignJoinSimpleCollection (myJoinSchema) as IBlueRowCollection : +// // new ImmutableBlueTable (this) as IBlueRowCollection; +// // myRemoveAll (true); +// // foreach (IBlueRow redRow in dp.newRows) +// // myAddReferencedRow (redRow); +// // retVal.addedRows = retVal.newRows = this; +// // return retVal; +// // } + +// // /// +// // /// Handles the changes in the right sides under the normal course of events, ie >= 0 rows added +// // /// and >= rows deleted, where the enitre right side is not wipped out or changed +// // /// +// // /// +// // /// +// // protected virtual DataPacket myHandleRightNormalCourse (DataPacket dp) +// // { +// // BlueEventLog.logAssert (myJoinSchema != null); +// // LightForeignJoinSimpleCollection dRows = new LightForeignJoinSimpleCollection (myJoinSchema), +// // aRows = new LightForeignJoinSimpleCollection (myJoinSchema); +// // foreach (IBlueRow redRow in dp.deletedRows) +// // { +// // ILightForeignJoinRowList list = myJoinKeyIndex[redRow[myReDCol]] as ILightForeignJoinRowList; +// // dRows.addRow (myGetJoinedRows (list)); +// // list.removeAll (true); +// // if (list.count == 0) +// // myJoinKeyIndex.Remove (redRow[myReDCol]); +// // } +// // foreach (IBlueRow redRow in dp.addedRows) +// // { +// // aRows.addRow (myGetJoinedRows (myAddReferencedRow (redRow))); +// // } +// // return new DataPacket (this, aRows, dRows); +// // } + +// // protected ILightForeignJoinRowList myAddReferencedRow (IBlueRow redRow) +// // { +// // ILightForeignJoinRowList list = (myJoinKeyIndex.Contains (redRow[myReDCol]) ? +// // myJoinKeyIndex[redRow[myReDCol]] : +// // myJoinKeyIndex[redRow[myReDCol]] = new LightForeignJoinRowList ()) +// // as ILightForeignJoinRowList; +// // list.add (redRow, true); +// // return list; +// // } + +// // protected virtual DataPacket myMergeReDRows (DataPacket dp) +// // { +// // myReDSchema = dp.schema; +// // if (myRinGSchema == null) +// // { +// // return myHandleRightNoLeft (dp); +// // } +// // else if (dp.newRows.Count == dp.addedRows.Count) +// // { +// // return myHandleRightAllChanged (dp); +// // } +// // else +// // { +// // return myHandleRightNormalCourse (dp); +// // } +// // } + +// // /// +// // /// Returns a collection of joined rows that correspond to one ILightForeignJoinRowList +// // /// (ie correspond to one joined col val). In case of an absence of either the referencedRow, +// // /// or the referencingRow or both, an empty collection is returned +// // /// +// // /// +// // /// +// // protected virtual IBlueRowCollection myGetJoinedRows (ILightForeignJoinRowList list) +// // { +// // BlueEventLog.logAssert (myJoinSchema != null); +// // if (list.referencedRow == null || list.referencedRow.Count == 1) +// // return new LightForeignJoinSimpleCollection (myJoinSchema); +// // int i = 0; +// // LightForeignJoinSimpleCollection retVal = new LightForeignJoinSimpleCollection (myJoinSchema); +// // foreach (IBlueRow ringRow in list) +// // if (i++ > 0) +// // retVal.addRow (new LightForeignJoinRow (list.referencedRow, ringRow, myJoinSchema)); +// // return retVal; +// // } + +// // /// +// // /// Removes either all the referenced or the referencing rows, depending on the bool arg. In case +// // /// of referencing rows, the dictionary of IDs is also cleared +// // /// +// // /// +// // protected void myRemoveAll (bool isReD) +// // { +// // ArrayList delList = new ArrayList (); +// // foreach (DictionaryEntry de in myJoinKeyIndex) +// // { +// // ILightForeignJoinRowList list = de.Value as ILightForeignJoinRowList; +// // list.removeAll (isReD); +// // if (list.count == 0) +// // delList.Add (de.Key); +// // } +// // foreach (object key in delList) +// // myJoinKeyIndex.Remove (key); +// // myRinGIDIndex = isReD ? myRinGIDIndex : new Hashtable (); +// // } + + +// // /// +// // /// Handled the case when the right side (referenced side) is absent and the +// // /// left side receives data. In this case the earler data (if any) is crushed and +// // /// the datastructures overwritten +// // /// +// // /// +// // /// +// // protected virtual DataPacket myHandleLeftNoRight (DataPacket dp) +// // { +// // myJoinKeyIndex = new Hashtable (); +// // myRinGIDIndex = new Hashtable (); +// // myAddReferencingRows (dp.newRows); +// // return null; +// // } + +// // /// +// // /// Anologous method for the right hand side +// // /// +// // /// +// // /// +// // protected virtual DataPacket myHandleLeftAllChanged (DataPacket dp) +// // { +// // DataPacket retVal = new DataPacket (); +// // //could be the first time the schema is fored +// // myJoinSchema = myJoinSchema == null ? +// // new LightForeignJoinSchema (myReDSchema, myReDCol, myRinGSchema, myRinGCol) : myJoinSchema; +// // retVal.deletedRows = new ImmutableBlueTable (this); +// // myRemoveAll (false); +// // myAddReferencingRows (dp.newRows); +// // retVal.addedRows = retVal.newRows = this; +// // return retVal; +// // } + +// // /// +// // /// Anonolgous to the right hand side method. However, in an effort to optimize, the rows +// // /// that are changed w.r.t to the non joining rows are not physically deleted and added, but +// // /// their referenced change. +// // /// +// // /// +// // /// +// // protected virtual DataPacket myHandleLeftNormalCourse (DataPacket dp) +// // { +// // // BlueEventLog.logAssert (myJoinSchema != null); +// // LightForeignJoinSimpleCollection dRows = new LightForeignJoinSimpleCollection (myJoinSchema), +// // aRows = new LightForeignJoinSimpleCollection (myJoinSchema); +// // IDictionary aux = new Hashtable (); +// // //for the deleted rows, its figured if they are added again with the joining column value not +// // //being changed, then merely the references need to be changed (ie physical deletion of nodes +// // //in the list not required => a performance boost) +// // foreach (IBlueRow ringRow in dp.deletedRows) +// // { +// // IBlueRow cr = dp.addedRows.containsID((int)ringRow["id"]) ? dp.addedRows[(int)ringRow["id"]] : null; +// // INode n = myRinGIDIndex[ringRow["id"]] as INode; +// // ILightForeignJoinRowList l = myJoinKeyIndex[ringRow[myRinGCol]] as ILightForeignJoinRowList; +// // if (cr != null && ringRow[myRinGCol].Equals (cr[myRinGCol])) +// // { +// // aux.Add (cr["id"], true); +// // if (l.referencedRow != null) +// // { +// // dRows.addRow (new LightForeignJoinRow (l.referencedRow, ringRow, myJoinSchema)); +// // aRows.addRow (new LightForeignJoinRow (l.referencedRow, cr, myJoinSchema)); +// // } +// // n.replace ((int)ringRow["id"], cr); +// // } +// // else +// // { +// // if (l.referencedRow != null) +// // dRows.addRow (new LightForeignJoinRow (l.referencedRow, ringRow, myJoinSchema)); +// // l.remove (n, false, (int)ringRow["id"]); +// // if (l.count == 0) +// // myJoinKeyIndex.Remove (ringRow[myRinGCol]); +// // myRinGIDIndex.Remove (ringRow["id"]); +// // } +// // } +// // foreach (IBlueRow ringRow in dp.addedRows) +// // { +// // if (!aux.Contains (ringRow["id"])) +// // { +// // ILightForeignJoinRowList l = (myJoinKeyIndex.Contains (ringRow[myRinGCol]) ? myJoinKeyIndex[ringRow[myRinGCol]] : +// // (myJoinKeyIndex[ringRow[myRinGCol]] = new LightForeignJoinRowList ())) as ILightForeignJoinRowList; +// // myRinGIDIndex.Add (ringRow["id"], l.add (ringRow, false)); +// // if (l.referencedRow != null) +// // aRows.addRow (new LightForeignJoinRow (l.referencedRow, ringRow, myJoinSchema)); +// // } +// // } +// // return new DataPacket (this, aRows, dRows); +// // } + +// // protected virtual DataPacket myMergeRinGRows (DataPacket dp) +// // { +// // myRinGSchema = dp.schema; +// // if (myReDSchema == null) +// // { +// // return myHandleLeftNoRight (dp); +// // } +// // else if (dp.newRows.Count == dp.addedRows.Count) +// // { +// // return myHandleLeftAllChanged (dp); +// // } +// // else +// // return myHandleLeftNormalCourse (dp); +// // } + +// // protected void myAddReferencingRows (IBlueRowCollection rows) +// // { +// // foreach (IBlueRow ringRow in rows) +// // myRinGIDIndex.Add (ringRow["id"], +// // ((myJoinKeyIndex.Contains (ringRow[myRinGCol]) ? myJoinKeyIndex[ringRow[myRinGCol]] : +// // (myJoinKeyIndex[ringRow[myRinGCol]] = new LightForeignJoinRowList ())) as ILightForeignJoinRowList).add (ringRow, false)); +// // } + +// // #region IBlueRowCollection Members + +// // public IBlueRow this[int id] {get {throw new NotSupportedException ("");}} + +// // public bool containsID (int id) {throw new NotSupportedException ("");} + +// // public IBlueTableSchema schema {get {return myJoinSchema;}} + +// // #endregion + +// // #region ICollection Members + +// // public bool IsSynchronized {get {return false;}} + +// // public int Count +// // { +// // get +// // { +// // return myGetCount (); +// // //BlueEventLog.logAssert (myCount == myGetCount ()); +// // //return myCount; +// // } +// // } + +// // public void CopyTo(Array array, int index) {throw new NotSupportedException ("");} + +// // public object SyncRoot {get {return null;}} + +// // #endregion + +// // #region IEnumerable + +// // public virtual System.Collections.IEnumerator GetEnumerator() +// // {return new ForeignJoinCollectionEnumerator (myJoinKeyIndex, myJoinSchema as LightForeignJoinSchema);} + +// // #endregion +// // } +// // } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs new file mode 100644 index 000000000..598d8d6bc --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections; +using BCG.Utilities; +using BCG; + +namespace BCG +{ + /// + /// Representation of each row formed by this Join. The Enumerator for the LightForeignJoinCollection + /// returns object of this type. Contains 3 pointers, to the left row, right row and schema respectively. + /// So each row is expected to have 12 bytes worth of pointers and 8 byte overhead for itself (as is the + /// case for all objects), so its a 20 byte package. + /// + public class LightForeignJoinRow : IBlueRow + { + protected IBlueRow myReDRow, myRingGRow; + protected IBlueTableSchema mySchema; + + #region Constructor(s) + + public LightForeignJoinRow (IBlueRow reDRow, IBlueRow rinGRow, IBlueTableSchema schema) + { + myReDRow = reDRow; + myRingGRow = rinGRow; + mySchema = schema; + } + + #endregion + + #region IBlueRow members + public IBlueTableSchema schema {get{return mySchema;}} + #endregion + + #region ICollection members + public int Count{get{return mySchema.columnNames.Length;}} + + public bool IsSynchronized {get{return false;}} + + public object SyncRoot{get{return this;}} + + public void CopyTo (Array array, int index) + { + int i = 0; + foreach (object o in this as IEnumerable) + array.SetValue (o, i++); + } + + #endregion + + #region IDictionary members + public bool IsFixedSize {get{return true;}} + + public bool IsReadOnly {get{return true;}} + + /// + /// The key can either be the string corrsponding to the column name or + /// the order of the column + /// + public virtual object this [object key] + { + get {return key is string ? this [key as string] : this [(int)key];} + set {throw new NotSupportedException ("The IBlueRow is read only");} + } + + public virtual object this [int index] + { + get + { + if (index == mySchema.columnNames.Length - 1) + return myRingGRow["id"]; + if (index < myRingGRow.Count) + return myRingGRow[index]; + else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1) + return myReDRow[index - myRingGRow.Count]; + // BlueEventLog.logAssert (false, "Incorrect index in JoinRow"); + return null; + } + } + + public object this [string colName] {get{return this[mySchema[colName].colNo];}} + + public ICollection Keys{get{return mySchema.columnNames;}} + + public ICollection Values + { + get + { + object[] retVal = new object[mySchema.columnNames.Length]; + CopyTo (retVal, 0); + return retVal; + } + } + + public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + + public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} + + public bool Contains (object key){return mySchema[(string)key] != null;} + + /// + /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to + /// iterate through the values, typecast this into an IEnumerable + /// + /// + public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);} + + IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);} + + public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");} + #endregion + } + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs new file mode 100644 index 000000000..fb88d8ebf --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections; +using BCG.Utilities; + +namespace BCG +{ + /// + /// The simplest implementation of ILightForeignJoinRowList with SimpleNode used for the nodes, hence one row + /// represented in each node + /// + public class LightForeignJoinRowList : ILightForeignJoinRowList + { + private SimpleFirstNode myFirst; + private SimpleNode myLast; + + #region constructors + /// + /// Default constructor, just sets up an empty first node + /// + public LightForeignJoinRowList() {myLast = myFirst = new SimpleFirstNode ();} + + public LightForeignJoinRowList (IBlueRowCollection rows, bool isReD) + : this () + { + foreach (IBlueRow r in rows) + add (r, isReD); + } + + public LightForeignJoinRowList (IBlueRow referencedRow) + : this() + { + add (referencedRow, true); + } + #endregion + + #region ILightForeignJoinRowList Members + + /// + /// In case of referenced rows its assumed that the row does not exist originally + /// + /// + /// + /// + public INode add (IBlueRow r, bool isReD) + { + myFirst.listCount ++; + if (isReD) + { + // BlueEventLog.logAssert (myFirst.row == null); + myFirst.row = r; + } + else + myLast = myLast.next = new SimpleNode (r, null, myLast); + return myLast; + } + + /// + /// Always assumes that the row node/row removes is already present. + /// + /// + /// + /// + public void remove (INode n, bool isReD, int id) + { + myFirst.listCount --; + if (isReD) + { + // BlueEventLog.logAssert ((n as SimpleNode).row != null); + myFirst.row = null; + } + else + { + SimpleNode sn = n as SimpleNode; + sn.prev.next = sn.next; + if (sn.next != null) + sn.next.prev = sn.prev; + else + myLast = sn.prev; + } + } + + public int count {get {return myFirst.listCount;}} + + public IBlueRow referencedRow {get {return myFirst.row;}} + + public void removeAll (bool isReD) + { + if (isReD) + { + myFirst.listCount = myFirst.listCount - (myFirst.row == null ? 0 : 1); + myFirst.row = null; + + } + else + { + (myLast = myFirst).next = null; + myFirst.listCount = myFirst.row == null ? 0 : 1; //list is empty now + } + } + + #endregion + + #region IEnumerable + + public IEnumerator GetEnumerator() + { + return new LightForeignJoinRowListEnumerator (this, myFirst); + } + + public class LightForeignJoinRowListEnumerator : IEnumerator + { + private ILightForeignJoinRowList myList; + private SimpleNode myPtr, myFirst; + public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first) + {myList = list; myFirst = first;} + public object Current {get{return myPtr.row;}} + public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;} + public void Reset() {myPtr = null;} + public ILightForeignJoinRowList list {get {return myList;}} + } + + #endregion + + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSchema.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSchema.cs new file mode 100644 index 000000000..7688de271 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSchema.cs @@ -0,0 +1,23 @@ +// using System; +// using BCG; + +// namespace BCG +// { +// /// +// /// Schema of the joined row. Only one instance of this type will be created shared by the rows +// /// +// public class LightForeignJoinSchema : JoinTableSchema +// { +// public IBlueTableSchema reDSchema, rinGSchema; +// public LightForeignJoinSchema (IBlueTableSchema reDSchema, +// string reDCol, +// IBlueTableSchema rinGSchema, +// string rinGCol) +// : base (rinGSchema, rinGCol, reDSchema, reDCol) +// { +// this.reDSchema = reDSchema; +// this.rinGSchema = rinGSchema; +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs new file mode 100644 index 000000000..3f25fd7c0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// A simple implementation of IBlueRowCollection which allows for the ImmutableBlueTable to contruct + /// itself from iterating through this class. The property 'schema' and the GetEnumerator are the + /// only 2 members that need implementation. This collection is used for the addedRows and deletedRows + /// + public class LightForeignJoinSimpleCollection : IBlueRowCollection + { + private IBlueTableSchema mySchema; + private ArrayList myRows = new ArrayList (); + + public LightForeignJoinSimpleCollection (IBlueTableSchema schema) {mySchema = schema;} + + public LightForeignJoinSimpleCollection (IBlueRowCollection rows) + : this (rows.schema) + { + foreach (IBlueRow r in rows) + addRow (r); + } + + public void addRow (IBlueRow r) + { + myRows.Add (r); + } + + public void addRow (IBlueRowCollection rs) + { + foreach (IBlueRow r in rs) + addRow (r); + } + + #region IBlueRowCollection Members + + public IBlueRow this[int id] {get{throw new NotSupportedException ("");}} + + public bool containsID (int id) {throw new NotSupportedException ("");} + + public IBlueTableSchema schema {get {return mySchema;}} + + #endregion + + #region ICollection Members + + public bool IsSynchronized {get {return false;}} + + public int Count {get {return myRows.Count;}} + + public void CopyTo (Array array, int index) {throw new NotSupportedException ("");} + + public object SyncRoot {get{return null;}} + + #endregion + + #region IEnumerable Members + + public IEnumerator GetEnumerator() {return myRows.GetEnumerator ();} + + #endregion + } + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinStream.cs new file mode 100644 index 000000000..ae327c2d8 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinStream.cs @@ -0,0 +1,215 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; + +// namespace BCG +// { +// public class LightForeignJoinStreamInternal : ViewStream +// { +// protected ViewStream myREdStream, myRErStream; +// protected LightForeignJoinCollection myCollection; +// protected int myReDCBCount, myRinGCBCount, myTotalCount = 0; + +// protected LightForeignJoinStreamInternal () {} +// public LightForeignJoinStreamInternal (ViewStream referencingStream, +// string referencingCol, +// ViewStream referencedStream, +// string referencedCol) +// { +// myREdStream = referencedStream; +// myRErStream = referencingStream; +// myCollection = new LightForeignJoinCollection (referencingCol, referencedCol); +// } + +// protected override void myOnFirstSubscriber() +// { +// myREdStream.onChange += new ChangeDelegate(myHandleReferencedChanges); +// myRErStream.onChange += new ChangeDelegate(myHandleReferencerChanges); +// } + +// protected override void myOnLastSubscriber() +// { +// myREdStream.onChange -= new ChangeDelegate(myHandleReferencedChanges); +// myRErStream.onChange -= new ChangeDelegate(myHandleReferencerChanges); +// } + + +// private void myHandleReferencedChanges (object newValue) +// { +// myHandleChanges (newValue as DataPacket, ref myReDCBCount, true); +// } + +// private void myHandleReferencerChanges (object newValue) +// { +// myHandleChanges (newValue as DataPacket, ref myRinGCBCount, false); +// } + +// private void myHandleChanges (DataPacket dp, ref int count, bool isReD) +// { +// DataPacket r = myCollection.mergeRows (count == 0 ? +// new DataPacket (dp.newRows, dp.newRows, dp.deletedRows) : +// dp, isReD); +// count ++; +// if (r != null && (myTotalCount == 0 || r.addedRows.Count + r.deletedRows.Count > 0)) +// { +// myTotalCount ++; +// myFireEvent (new DataPacket (r.newRows, +// new ImmutableBlueTable (r.addedRows), +// new ImmutableBlueTable (r.deletedRows))); +// } +// } + +// public LightForeignJoinSchema schema +// {get{return myCollection.schema as LightForeignJoinSchema;}} + +// } + + +// public class LightForeignJoinStream : ViewStream +// { +// protected LightForeignJoinStreamInternal myBasicStream; +// protected ViewStream myProjectStream; +// protected ColumnMap myReDColMap, myRinGColMap; +// protected bool myIsProject = true; +// private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; + +// protected LightForeignJoinStream () {} +// public LightForeignJoinStream(ViewStream referencingStream, +// string referencingCol, +// ColumnMap referencingColMap, +// ViewStream referencedStream, +// string referencedCol, +// ColumnMap referencedColMap) +// { +// if (myUseSuperJoin) +// { +// mySetRealSource(new SuperJoinStream(referencingStream, referencingCol, referencingColMap.columnExpression, +// referencedStream, referencedCol, referencedColMap.columnExpression)); +// } +// else +// { +// myReDColMap = referencedColMap; +// myRinGColMap = referencingColMap; +// myBasicStream = new LightForeignJoinStreamInternal(referencingStream, +// referencingCol, +// referencedStream, +// referencedCol); +// } +// } + +// public LightForeignJoinStream(ViewStream referencingStream, +// string referencingCol, +// ViewStream referencedStream, +// string referencedCol) +// { +// if (myUseSuperJoin) +// { +// mySetRealSource(new SuperJoinStream(referencingStream, referencingCol, referencedStream, referencedCol)); +// } +// else +// { +// myBasicStream = new LightForeignJoinStreamInternal(referencingStream, +// referencingCol, +// referencedStream, +// referencedCol); +// myIsProject = false; +// } +// } + +// public LightForeignJoinStream (ViewStream referencingStream, +// string referencingCol, +// string referencingColExp, +// ViewStream referencedStream, +// string referencedCol, +// string referencedColExp) +// : +// this (referencingStream, +// referencingCol, +// (referencingColExp == null ? new ColumnMap("!"):new ColumnMap(referencingColExp)), +// referencedStream, +// referencedCol, +// (referencedColExp == null ? new ColumnMap("!") : new ColumnMap(referencedColExp))) +// {} + +// public LightForeignJoinStream (ViewStream referencingStream, +// string referencingCol, +// ViewStream referencedStream, +// string referencedCol, +// string referencedColExp +// ) +// : +// this (referencingStream, +// referencingCol, +// "!", +// referencedStream, +// referencedCol, +// referencedColExp) +// {} + +// public LightForeignJoinStream (ViewStream referencingStream, +// string referencingCol, +// string referencingColExp, +// ViewStream referencedStream, +// string referencedCol) +// : +// this (referencingStream, +// referencingCol, +// referencingColExp, +// referencedStream, +// referencedCol, +// "!") +// {} + +// protected void myInternalJoinChangeHandler (object o) +// { +// if (myIsProject) +// { +// if (myProjectStream == null) +// { +// JoinNameResolver resolver = +// new JoinNameResolver (myBasicStream.schema, +// myBasicStream.schema.rinGSchema, +// myRinGColMap.columnExpression, +// myBasicStream.schema.reDSchema, +// myReDColMap.columnExpression +// ); +// IDictionary map = resolver.getMap (); +// map["id"] = "id"; +// myProjectStream = new ProjectStream (myBasicStream, map); +// myProjectStream.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler); +// return; +// } +// else +// BlueEventLog.logAssert (false, "This call back should not have been called"); +// } +// else +// myFireEvent ((o as ICloneable).Clone()); +// } + +// private void myProjectChangeHandler (object o) +// { +// if (myBasicStream.myOnChangeDelegate.GetInvocationList().Length > 1) +// myBasicStream.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler); +// myFireEvent ((o as ICloneable).Clone()); + +// } + +// protected override void myOnFirstSubscriber () +// { +// myBasicStream.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler); +// } + +// protected override void myOnLastSubscriber () +// { +// // how does this relate to the -= in myProjectChangeHandler? +// myBasicStream.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler); + +// // if there is a project after the ijs we need to +// // unsubscribe to it otherwise references will be kept and +// // a memory leak will occur +// if (myProjectStream != null) +// myProjectStream.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler); +// myProjectStream = null; +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinCollection.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinCollection.cs new file mode 100644 index 000000000..42d6a555c --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinCollection.cs @@ -0,0 +1,140 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; + +// namespace BCG +// { +// public class LightLeftForeignJoinCollection : LightForeignJoinCollection +// { +// public LightLeftForeignJoinCollection (string reDCol, string rinGCol) +// : base (reDCol, rinGCol) +// {} + +// /// +// /// Handles the changes in the right sides under the normal course of events, ie some >= 0 rows added +// /// and >= rows deleted, where the enitre right side is not wipped out or changed +// /// +// /// +// /// +// protected override DataPacket myHandleRightNormalCourse (DataPacket dp) +// { +// BlueEventLog.logAssert (myJoinSchema != null); +// BlueTable dRows = new BlueTable (myJoinSchema), +// aRows = new BlueTable (myJoinSchema); +// IDictionary aux = new Hashtable (); +// foreach (IBlueRow redRow in dp.deletedRows) +// { +// ILightForeignJoinRowList list = myJoinKeyIndex[redRow[myReDCol]] as ILightForeignJoinRowList; +// IBlueRow cr = dp.addedRows.containsID ((int)redRow["id"]) ? dp.addedRows [(int)redRow["id"]] : null; +// if (cr != null && cr[myReDCol].Equals (redRow[myReDCol])) +// { +// //i.e this referenced row is being modified but its 'place' w.r.t the join is not changing +// aux.Add (cr["id"], true); +// dRows.addRows (myGetJoinedRows (list)); +// list.removeAll (true); +// list.add (cr, true); +// aRows.addRows (myGetJoinedRows (list)); +// } +// else +// { +// // row is being deleted +// dRows.addRows (myGetJoinedRows (list)); +// list.removeAll (true); +// aRows.addRows (myGetJoinedRows (list)); +// if (list.count == 0) +// myJoinKeyIndex.Remove (redRow[myReDCol]); +// } +// } +// foreach (IBlueRow aRow in dp.addedRows) +// { +// if (!aux.Contains (aRow["id"])) +// { +// ILightForeignJoinRowList list = (myJoinKeyIndex.Contains (aRow[myReDCol]) ? myJoinKeyIndex[aRow[myReDCol]] +// : myJoinKeyIndex[aRow[myReDCol]] = new LightForeignJoinRowList ()) +// as ILightForeignJoinRowList; +// foreach (IBlueRow djr in myGetJoinedRows (list)) +// if (!dRows.containsID ((int)djr["id"])) +// dRows.addRow (djr); +// list.add (aRow, true); +// foreach (IBlueRow ajr in myGetJoinedRows (list)) +// { +// if (aRows.containsID ((int)ajr["id"])) +// aRows.deleteRow ((int)ajr["id"]); +// aRows.addRow (ajr); +// } +// } +// } +// return new DataPacket (this, aRows, dRows); +// } + +// protected override DataPacket myHandleLeftNormalCourse (DataPacket dp) +// { +// //the most common condition, when a few rows are deleted/added +// LightForeignJoinSimpleCollection dRows = new LightForeignJoinSimpleCollection (myJoinSchema), +// aRows = new LightForeignJoinSimpleCollection (myJoinSchema); +// IDictionary aux = new Hashtable (); +// //for the deleted rows, its figured if they are added again with the joining column value not +// //being changed, then merely the references need to be changed (ie physcial deletion of nodes +// //in the list not required, hence incresing performance +// foreach (IBlueRow ringRow in dp.deletedRows) +// { +// IBlueRow cr = dp.addedRows.containsID((int)ringRow["id"]) ? dp.addedRows[(int)ringRow["id"]] : null; +// INode n = myRinGIDIndex[ringRow["id"]] as INode; +// BlueEventLog.logAssert (n.getRow ((int)ringRow["id"])["id"].Equals (ringRow["id"])); +// ILightForeignJoinRowList l = myJoinKeyIndex[ringRow[myRinGCol]] as ILightForeignJoinRowList; +// if (cr != null && ringRow[myRinGCol].Equals (cr[myRinGCol])) +// { +// aux.Add (cr["id"], true); +// dRows.addRow (new LightLeftForeignJoinRow (l.referencedRow, ringRow, myJoinSchema)); +// aRows.addRow (new LightLeftForeignJoinRow (l.referencedRow, cr, myJoinSchema)); +// n.replace ((int)ringRow["id"], cr); +// } +// else +// { +// dRows.addRow (new LightLeftForeignJoinRow (l.referencedRow, ringRow, myJoinSchema)); +// l.remove (n, false, (int)ringRow["id"]); +// if (l.count == 0) +// myJoinKeyIndex.Remove (ringRow[myRinGCol]); +// myRinGIDIndex.Remove (ringRow["id"]); +// } +// } +// foreach (IBlueRow ringRow in dp.addedRows) +// { +// if (!aux.Contains (ringRow["id"])) +// { +// ILightForeignJoinRowList l = (myJoinKeyIndex.Contains (ringRow[myRinGCol]) ? myJoinKeyIndex[ringRow[myRinGCol]] : +// (myJoinKeyIndex[ringRow[myRinGCol]] = new LightForeignJoinRowList ())) as ILightForeignJoinRowList; +// myRinGIDIndex.Add (ringRow["id"], l.add (ringRow, false)); +// aRows.addRow (new LightLeftForeignJoinRow (l.referencedRow, ringRow, myJoinSchema)); +// } +// } +// return new DataPacket (this, aRows, dRows); +// } + + + +// /// +// /// Returns a row of left-joined rows that correspond to one ILightForeignJoinRowList (ie correspond to one joined col val) +// /// In case of an absence of the referencedRow, semi formed joined rows are returned with the left side being empty (provided +// /// that the right side has rows) +// /// +// /// +// /// +// protected override IBlueRowCollection myGetJoinedRows (ILightForeignJoinRowList list) +// { +// // BlueEventLog.logAssert (myJoinSchema != null); +// LightForeignJoinSimpleCollection retVal = new LightForeignJoinSimpleCollection (myJoinSchema); +// foreach (IBlueRow r in list) +// if (r != list.referencedRow) +// retVal.addRow (new LightLeftForeignJoinRow (list.referencedRow, r, myJoinSchema)); +// return retVal; +// } + +// public override IEnumerator GetEnumerator() +// { +// return new LeftForeignJoinCollectionEnumerator (myJoinKeyIndex, myJoinSchema as LightForeignJoinSchema); +// } + + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs new file mode 100644 index 000000000..27d4ff0df --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs @@ -0,0 +1,31 @@ +using System; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Serves as the row for left join operations. Differs the regular joinrow as the referencedRow can be + /// null resulting in DBNulls to be returned when indeces corresponding to the referenced rows are accessed + /// + public class LightLeftForeignJoinRow : LightForeignJoinRow + { + public LightLeftForeignJoinRow (IBlueRow reDRow, IBlueRow rinGRow, IBlueTableSchema schema) + : base (reDRow, rinGRow, schema) + {} + + public override object this [int index] + { + get + { + if (index == mySchema.columnNames.Length - 1) + return myRingGRow["id"]; + if (index < myRingGRow.Count) + return myRingGRow[index]; + else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1) + return myReDRow == null ? DBNull.Value : myReDRow[index - myRingGRow.Count]; + // BlueEventLog.logAssert (false, "Incorrect index in JoinRow"); + return null; + } + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinStream.cs new file mode 100644 index 000000000..d00499a90 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinStream.cs @@ -0,0 +1,114 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; + +// namespace BCG +// { +// public class LightLeftForeignJoinStreamInternal : LightForeignJoinStreamInternal +// { +// public LightLeftForeignJoinStreamInternal (ViewStream referencingStream, +// string referencingCol, +// ViewStream referencedStream, +// string referencedCol) +// { +// myREdStream = referencedStream; +// myRErStream = referencingStream; +// myCollection = new LightLeftForeignJoinCollection (referencingCol, referencedCol); +// } + +// } + + +// public class LightLeftForeignJoinStream : LightForeignJoinStream +// { +// private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; + +// public LightLeftForeignJoinStream (ViewStream referencingStream, +// string referencingCol, +// ColumnMap referencingColMap, +// ViewStream referencedStream, +// string referencedCol, +// ColumnMap referencedColMap) +// { +// if (myUseSuperJoin) +// { +// mySetRealSource (new SuperLeftJoinStream (referencingStream, referencingCol, referencingColMap.columnExpression, +// referencedStream, referencedCol, referencedColMap.columnExpression)); +// } +// else +// { +// myReDColMap = referencedColMap; +// myRinGColMap = referencingColMap; +// myBasicStream = new LightLeftForeignJoinStreamInternal(referencingStream, +// referencingCol, +// referencedStream, +// referencedCol); +// } +// } + +// public LightLeftForeignJoinStream (ViewStream referencingStream, +// string referencingCol, +// ViewStream referencedStream, +// string referencedCol) +// { +// if (myUseSuperJoin) +// { +// mySetRealSource(new SuperLeftJoinStream(referencingStream, referencingCol, +// referencedStream, referencedCol)); +// } +// else +// { +// myBasicStream = new LightLeftForeignJoinStreamInternal(referencingStream, +// referencingCol, +// referencedStream, +// referencedCol); +// myIsProject = false; +// } +// } + +// public LightLeftForeignJoinStream (ViewStream referencingStream, +// string referencingCol, +// string referencingColExp, +// ViewStream referencedStream, +// string referencedCol, +// string referencedColExp) +// : +// this (referencingStream, +// referencingCol, +// (referencingColExp == null ? new ColumnMap("!"):new ColumnMap(referencingColExp)), +// referencedStream, +// referencedCol, +// (referencedColExp == null ? new ColumnMap("!") : new ColumnMap(referencedColExp))) +// {} + +// public LightLeftForeignJoinStream (ViewStream referencingStream, +// string referencingCol, +// ViewStream referencedStream, +// string referencedCol, +// string referencedColExp +// ) +// : +// this (referencingStream, +// referencingCol, +// "!", +// referencedStream, +// referencedCol, +// referencedColExp) +// {} + +// public LightLeftForeignJoinStream (ViewStream referencingStream, +// string referencingCol, +// string referencingColExp, +// ViewStream referencedStream, +// string referencedCol) +// : +// this (referencingStream, +// referencingCol, +// referencingColExp, +// referencedStream, +// referencedCol, +// "!") +// {} + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleFirstNode.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleFirstNode.cs new file mode 100644 index 000000000..5f28d7bcf --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleFirstNode.cs @@ -0,0 +1,19 @@ +using System; + +namespace BCG +{ + /// + /// The first elements of the list. This node being always remains in the list. + /// + public class SimpleFirstNode : SimpleNode + { + private int myListCount; //the count for the list in kept in the first element, so as to avoid the integer overhead + public SimpleFirstNode (IBlueRow r) : base (r) {} + public SimpleFirstNode () : base (null, null, null) {} + public int listCount + { + get {return myListCount;} + set {myListCount = value;} + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs new file mode 100644 index 000000000..5166c1243 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs @@ -0,0 +1,37 @@ +using System; + +namespace BCG +{ + /// + /// The simplest implementation of the INode, with just one row in the node (the least memory effecient). + /// The memory requirement of each node is 8 (next and prev pointers) + 4 (row) + 8 (overhead for the object) = 20 bytes. + /// Should be used in conjunction with LightFriegnJoinRowList + /// + public class SimpleNode : INode + { + public IBlueRow row; + public SimpleNode next, prev; + + public SimpleNode (IBlueRow r) + : this (r, null, null) + {} + + public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv) + { + row = r; + next = nx; + prev = pv; + } + + #region INode Members + + public IBlueRow getRow (int id) {return row;} + + public IBlueRow[] getRows() {return new IBlueRow[] {row};} + + public void replace (int id, IBlueRow nr) {row = nr;} + + #endregion + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs new file mode 100644 index 000000000..fa49a7f14 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections; +using BCG; +using BCG.Utilities; + +namespace BCG +{ + public class LiteralStream : ViewStream + { + public LiteralStream(string tableName, string schema, string value) + { + string [] columns = schema.Split(' '); + + + ColumnInfo [] ci = new ColumnInfo[columns.Length + 1]; + ci[0] = new ColumnInfo("id",0,typeof(int)); + int n = 1; + foreach (string c in columns) + { + string [] columnAndType = c.Split('/'); + if (columnAndType.Length == 1) + ci[n] = new ColumnInfo(c,n,typeof(double)); + else + ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true)); + n++; + } + + + BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci)); + string [] rows = value.Split(';'); + n = 0; + foreach (string r in rows) + try + { + if(r == string.Empty) + continue; + t.addRow( + BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n)); + } + catch (Exception e) + { + throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message)); + } + + mySetRealSource(new BlueRowCollectionStaticStream(t)); + } + + public LiteralStream(string schema, string value) : this ("literal",schema, value) + {} + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs b/src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs new file mode 100644 index 000000000..c7eeb2855 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs @@ -0,0 +1,24 @@ +using System; + +namespace BCG +{ + public class DynamicUntypedToTypedMapper : IDynamicStreamMapper + { + IBlueTableSchema mySchema; + + public IBlueRow map (IBlueRow row) + { + if (mySchema == null) + { + ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length]; + int i = 0; + foreach (ColumnInfo ci in row.schema.columnInfo) + cis[i++] = ci.colType == null ? + new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) : + new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue); + mySchema = new BlueTableSchema (row.schema.tableName, cis); + } + return new BlueRow (mySchema, row); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/IChangeableStreamMapper.cs b/src/DataStreamSuite/DataStream/Primitives/Map/IChangeableStreamMapper.cs new file mode 100644 index 000000000..140d2c67b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/IChangeableStreamMapper.cs @@ -0,0 +1,12 @@ +using System; +using System.Data; + +namespace BCG +{ + public delegate void MapperChangeDelegate (); + + public interface IChangeableStreamMapper : IStreamMapper + { + event MapperChangeDelegate onMapperChange; + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/IDynamicStreamMapper.cs b/src/DataStreamSuite/DataStream/Primitives/Map/IDynamicStreamMapper.cs new file mode 100644 index 000000000..4f8b8ec89 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/IDynamicStreamMapper.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Maps a row to another row + /// + public interface IDynamicStreamMapper + { + /// + /// does a row |-> row map, mantaining the id + /// + IBlueRow map (IBlueRow row); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/IStreamMapper.cs b/src/DataStreamSuite/DataStream/Primitives/Map/IStreamMapper.cs new file mode 100644 index 000000000..a702785e3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/IStreamMapper.cs @@ -0,0 +1,13 @@ +using System; +using System.Data; + +namespace BCG +{ + public interface IStreamMapper : IDynamicStreamMapper + { + /// + /// Returns the schema into which the rows are mapped + /// + IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/ISuperStreamMapper.cs b/src/DataStreamSuite/DataStream/Primitives/Map/ISuperStreamMapper.cs new file mode 100644 index 000000000..079c99374 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/ISuperStreamMapper.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public interface ISuperStreamMapper + { + /// + /// returns output schema for the + /// rows that will be outputted + /// + /// + /// + IBlueTableSchema getOutputSchema(IBlueTableSchema schema); + /// + /// maps one row to multiple rows + /// + /// + /// + IBlueRow[] map(IBlueRow inputRow); + /// + /// fires an event when the mapper + /// changes itself + /// + event MapperChangeDelegate onMapperChange; + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs b/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs new file mode 100644 index 000000000..049faa6e0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs @@ -0,0 +1,209 @@ +using System; +using System.Data; +using System.Collections; +using mbts; + +namespace BCG +{ + + + /// + /// Provides a live Map-Stream, ie maps a IBlueRowCollection to another. The Mapping function + /// is provided by the user, in the forst of the interface IStreamMapper and maps a row to the + /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id + /// is preserved across the rows, ie row["id"] = f(row)["id"] + /// + [DataStreamUserRespresentation (false)] + public class MapStream : MethodDrivenStream + { + private HighPerformanceTimer t = new HighPerformanceTimer (); + protected IStreamMapper myMapper; + protected IChangeableStreamMapper myChangeableMapper = null; + private bool myUseNewMap = DataStreamLoadFileConfig.instance.useNewMap; + + public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs) + { + if (myUseNewMap) + mySetRealSource (new SuperMapStream (vs, mapper)); + else + myMapper = mapper; + } + public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper) + { + if (myUseNewMap) + mySetRealSource(new SuperMapStream(vs, mapper)); + else + { + mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged); + myChangeableMapper = mapper; + } + } + + /// + /// Gets or sets the value of the mapper. Setting the value will result in re + /// + private void myMapperChanged () + { + if (myLastArrivedPacket != null) + myChangeHandler (new DataPacket (myLastArrivedPacket.newRows, + myLastArrivedPacket.newRows, + myLastArrivedPacket.newRows)); + } + + + protected override void myHandleFirstCall () + { + + IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema); + myLastFiredPacket = new DataPacket (); + myLastFiredPacket.newRows = new BlueTable (outputSchema); + myLastFiredPacket.deletedRows = new BlueTable (outputSchema); + foreach (IBlueRow row in myLastArrivedPacket.newRows) + ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row)); + myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows); + } + + protected override void myHandleIncrementalAdd() + { + //t.start (); + myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema); + foreach (IBlueRow row in myLastArrivedPacket.addedRows) + { + IBlueRow newRow = myMapper.map (row); + (myLastFiredPacket.newRows as BlueTable).addRow (newRow); + (myLastFiredPacket.addedRows as BlueTable).addRow (newRow); + } + //t.stop (); + //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime); + //System.Diagnostics.Trace.WriteLine (" "); + } + + protected override void myHandleIncrementalDelete() + { + //t.start (); + myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema); + foreach (IBlueRow row in myLastArrivedPacket.deletedRows) + { + IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]]; + (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow); + (myLastFiredPacket.newRows as BlueTable).deleteRow (row); + } + //t.stop (); + //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime); + } + + + protected override void myOnFirstSubscriber() + { + myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); + } + + protected override void myOnLastSubscriber () + { + myInit (); + if (myStream != null) + myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); + } + + protected override void myOnTableClear () + { + myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; + myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema); + myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema); + } + + + } + + /* + public class DistinctColumnSplitterStream : ViewStream + { + private string _split_col; + private ViewStream _stream; + + public DistinctColumnSplitterStream(ViewStream vs, string split_col) + { + _split_col = split_col; + _stream = vs; + } + + protected override void myOnFirstSubscriber() + { + _stream.onChange += _stream_onChange; + } + + protected override void myOnLastSubscriber() + { + _stream.onChange -= _stream_onChange; + } + + private void _stream_onChange(object newValue) + { + DataPacket dp = newValue as DataPacket; + System.Collections.Generic.Dictionary dict = new System.Collections.Generic.Dictionary(); + if (dp != null) + { + foreach (IBlueRow r in dp.newRows) + { + object split_val = r[_split_col]; + if (!dict.ContainsKey(split_val)) + dict.Add(split_val, _make_table(split_val, r.schema.columnInfo)); + add_row(r, dict[split_val]); + } + } + } + + private IBlueTableSchema add_id_col(BlueTable[] without_ids) + { + //ColumnInfo[] ci = new ColumnInfo[without_id.columnInfo.Length + 1]; + //Array.Copy(without_id.columnInfo, 0, ci, 1, without_id.columnInfo.Length); + //ci[0] = new ColumnInfo("id", 0, typeof(int)); + System.Collections.Generic.List cols = new System.Collections.Generic.List(); + int place_holder = 0; + foreach (BlueTable bt in without_ids) + for (int i = 0; i < bt.schema.columnInfo.Length; i++) + cols.Add(new ColumnInfo(bt.schema.columnNames[i], place_holder++, bt.schema.columnInfo[i].colType); + ColumnInfo[] res_ci = new ColumnInfo[cols.Count + 1]; + } + + private BlueTable _make_aggregate_table(System.Collections.Generic.Dictionary dict) + { + int size = 0; + foreach (System.Collections.Generic.KeyValuePair kvp in dict) + { + if (kvp.Value.Count > size) + size = kvp.Value.Count; + } + BlueTable res = new BlueTable + for (int i = 0; i < size; i++) + { + + } + } + + private BlueTable _make_table(object split_col_val, ColumnInfo[] base_ci) + { + string split_col_prefix = String.Format("{0}-{1}:", _split_col, split_col_val.ToString()); + ColumnInfo[] split_ci = new ColumnInfo[base_ci.Length - 1]; + for (int i = 0; i < base_ci.Length; i++) + { + if(base_ci[i].colName != "id") + split_ci[i] = new ColumnInfo(split_col_prefix + base_ci[i].colName, i, base_ci[i].colType); + } + return new BlueTable("", split_ci); + } + + private void add_row(IBlueRow r, BlueTable t) + { + object[] vals = new object[r.schema.columnInfo.Length - 1]; + int corrector = 0; + for (int i = 0; i < vals.Length; i++) + { + if (r.schema.columnNames[i] == "id") + corrector = 1; + vals[i] = r[i + corrector]; + } + t.addRow(new BlueRow(t.schema, vals)); + } + }*/ +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/MapperBase.cs b/src/DataStreamSuite/DataStream/Primitives/Map/MapperBase.cs new file mode 100644 index 000000000..ea10ecb86 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/MapperBase.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class MapperBase : IStreamMapper + { + #region IStreamMapper Members + + public virtual IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) + { + return inputSchema; + } + + #endregion + + #region virtual IDynamicStreamMapper Members + + public virtual IBlueRow map(IBlueRow row) + { + return row; + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs b/src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs new file mode 100644 index 000000000..ee067b228 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.Text; +using BCG.Utilities; +using BCG.Utilities.General; +using mbts; + +namespace BCG +{ + /// + /// SuperMap gives the user the ability + /// to map one row to one or more rows + /// (as opposed to MapStream where there + /// is a one to one mapping) + /// It takes in an ISuperMapper which does + /// the mapping. Additinally the ISuperMapper + /// can chage itself and raise an event causing + /// a remapping to happen + /// + public class SuperMapStream : ViewStream + { + #region super mapper that wraps around stream mapper + private class MyMappingWrapper : ISuperStreamMapper + { + private IStreamMapper myMapper; + + public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; } + + #region ISuperStreamMapper Members + + public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return myMapper.getOutputSchema(schema); } + + public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; } + + public event MapperChangeDelegate onMapperChange + { + add { } + remove { } + } + + #endregion + } + #endregion + + private class MyFuncWrapper : ISuperStreamMapper + { + private Func _get_schema; + private Func> _map_rows; + public MyFuncWrapper(Func get_schema, + Func> map_rows) + { + _get_schema = get_schema; + _map_rows = map_rows; + } + + public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); } + + public IBlueRow[] map(IBlueRow r) + { + IEnumerable rows = _map_rows(r); + return rows == null ? new IBlueRow[] { } : new List(rows).ToArray(); + } + + public event MapperChangeDelegate onMapperChange; + private void _fake() { onMapperChange(); } + } + + //maps the id of the underlying stream to the + //rows of the resulting stream + private Dictionary> myMap = new Dictionary>(); + private int myID = 0; + private BlueTable myRows, myBaseRows; + private ViewStream myStream; + private ISuperStreamMapper myMapper; + public bool generate_ids = true; + + public SuperMapStream(ViewStream vs, Func get_schema, + Func> map_rows) + : this(vs, new MyFuncWrapper(get_schema, map_rows)) + { } + + public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func> map_rows) + : this(vs, s => out_schema, map_rows) + { } + + public SuperMapStream(ViewStream vs, IStreamMapper mapper) + : this(vs, new MyMappingWrapper(mapper)) + { } + + public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) + { + myStream = vs; + myMapper = mapper; + myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange); + } + + protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); } + + protected override void myOnLastSubscriber() + { + myStream.onChange -= new ChangeDelegate(myHandleData); + myRows = null; + myBaseRows = null; + myMap.Clear(); + //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange); + } + + private void myHandleData(object newValue) + { + DataPacket idp = newValue as DataPacket; + //HighPerformanceTimer timer = new HighPerformanceTimer(); + //timer.start(); + DataPacket dp = newValue as DataPacket; + myRows = myRows ?? new BlueTable(myMapper.getOutputSchema(dp.schema)); + myBaseRows = myBaseRows ?? new BlueTable(dp.schema); + BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows); + BlueTable dels = new BlueTable(myRows.schema), + adds = new BlueTable(myRows.schema); + + foreach (IBlueRow r in dp.deletedRows) + { + List mappedRows = myMap[(int)r["id"]]; + foreach (IBlueRow mr in mappedRows) + { + dels.addRow(mr); + myRows.deleteRow(mr); + } + myMap.Remove((int)r["id"]); + } + foreach (IBlueRow r in dp.addedRows) + { + IBlueRow[] mappedRows = myMapper.map(r); + foreach (IBlueRow mr in mappedRows) + { + IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr; + if (!myMap.ContainsKey((int)r["id"])) + { + myMap.Add((int)r["id"], new List()); + } + myMap[(int)r["id"]].Add(idChangedMapRow); + myRows.addRow(idChangedMapRow); + adds.addRow(idChangedMapRow); + } + } + //timer.stop(); + // if (timer.lastIntervalTime > 20) + // {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime)); + // myFireEvent(new DataPacket(myRows, adds, dels)); + } + + private void myHandleMapChange() + { + if (myBaseRows != null) + myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows))); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Mux/MuxStream.cs b/src/DataStreamSuite/DataStream/Primitives/Mux/MuxStream.cs new file mode 100644 index 000000000..3eef03727 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Mux/MuxStream.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + /// + /// Maps (multiplexes) columns values which belong to a given set into a single string + /// value in a new column. Values that are not in the set are converted to strings + /// and copied to the output column. + /// + public class MuxStream : ViewStream, IStreamMapper + { + private string myOutColumn; + private string myInColumn; + private string myOutValue; + private object[] myMuxedValues; + private BlueTableSchema mySchema; + + public MuxStream(ViewStream inStream, string outColumn, string outValue, string inColumn, params object[] muxedValues) + { + myOutColumn = outColumn; + myOutValue = outValue; + + myInColumn = inColumn; + myMuxedValues = (object[])muxedValues.Clone(); + mySetRealSource(new MapStream(inStream, this)); + } + + #region IStreamMapper Members + + IBlueTableSchema IStreamMapper.getOutputSchema(IBlueTableSchema inputSchema) + { + ColumnInfo[] cis = new ColumnInfo[inputSchema.columnInfo.Length + 1]; + Array.Copy(inputSchema.columnInfo, cis, inputSchema.columnInfo.Length); + cis[cis.Length - 1] = new ColumnInfo(myOutColumn, cis.Length - 1, typeof(string)); + mySchema = new BlueTableSchema(cis, String.Format("muxed {0} to {1} in ({2})", myInColumn, myOutColumn, inputSchema.tableName)); + return mySchema; + } + + #endregion + + #region IDynamicStreamMapper Members + + IBlueRow IDynamicStreamMapper.map(IBlueRow row) + { + MutableBlueRow r = new MutableBlueRow(mySchema, row); + if (Array.IndexOf(myMuxedValues, r[myInColumn]) != -1) // muxable value? + r.setValue(myOutColumn, myOutValue); + else + r.setValue(myOutColumn, row[myInColumn].ToString()); + return r; + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantMapper.cs b/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantMapper.cs new file mode 100644 index 000000000..3be4d3ba9 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantMapper.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections; + +using BCG; +using BCG.Utilities; +using System.Collections.Generic; +using mbts; + +namespace BCG +{ + public class NullToConstantMapper : IStreamMapper + { + public NullToConstantMapper(string field, object constant) + { + myField = field; + myConstant = constant; + } + + #region IStreamMapper Members + + public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) + { + return inputSchema; + } + + #endregion + + #region IDynamicStreamMapper Members + + public IBlueRow map(IBlueRow row) + { + if(row[myField].Equals(DBNull.Value)) + { + Hashtable newRow = new Hashtable(row); + newRow[myField]= myConstant; + return new BlueRow(row.schema, newRow); + } + else + return row; + } + + #endregion + + private string myField; + private object myConstant; + } + + public class AllNullToConstantMapper : IStreamMapper + { + public AllNullToConstantMapper(object constant) + { + myConstant = constant; + } + + #region IStreamMapper Members + + public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) + { + return inputSchema; + } + + #endregion + + #region IDynamicStreamMapper Members + + public IBlueRow map(IBlueRow row) + { + bool change = false; + Hashtable newRow = new Hashtable(row); + foreach(ColumnInfo info in row.schema.columnInfo) + { + if(row[info.colName].Equals(DBNull.Value)) + { + change = true; + newRow[info.colName] = Convert.ChangeType(myConstant, info.colType); + } + } + if(change) + return new BlueRow(row.schema, newRow); + else + return row; + } + + #endregion + private object myConstant; + } + + public class SpecifiedNullToConstantMapper : IStreamMapper + { + private Pair[] _col_val_pair_list; + public SpecifiedNullToConstantMapper(params Pair[] col_val_pair_list) + { + _col_val_pair_list = col_val_pair_list; + } + + public IBlueTableSchema getOutputSchema(IBlueTableSchema s) {return s;} + + public IBlueRow map(IBlueRow r) + { + MutableBlueRow rv = new MutableBlueRow(getOutputSchema(r.schema), r); + foreach (Pair p in _col_val_pair_list) + { + object v = BlueDataUtility.get_row_value(r, p.first, DBNull.Value); + if (v == null || v == DBNull.Value) + rv.setValue(p.first, p.second); + } + return rv; + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantStream.cs b/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantStream.cs new file mode 100644 index 000000000..3bb71696a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantStream.cs @@ -0,0 +1,21 @@ +using System; +using BCG; +using BCG.Utilities; + +namespace BCG +{ + public class NullToConstantStream : ViewStream + { + public NullToConstantStream(ViewStream v, string field, object constant) + { + if(string.IsNullOrEmpty(field)) + mySetRealSource(new MapStream(v, new AllNullToConstantMapper(constant))); + else + mySetRealSource (new MapStream(v,new NullToConstantMapper(field, constant))); + } + public NullToConstantStream(ViewStream v, object constant) + { + mySetRealSource(new MapStream(v, new AllNullToConstantMapper(constant))); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/OuterJoin/OuterJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/OuterJoin/OuterJoinStream.cs new file mode 100644 index 000000000..0b2f76856 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/OuterJoin/OuterJoinStream.cs @@ -0,0 +1,70 @@ +// using System; + +// namespace BCG +// { +// /// +// /// Given two view streams, vs1 and vs2, as follows: +// /// +// /// vs1: A B C1 D1 +// /// vs2: C2 D2 E F +// /// +// /// where the rows C1, C2, D1 and D2 intersect the resulting stream is: +// /// +// /// A B C1 D1 E F +// /// +// public class OuterJoinStream : ViewStream +// { +// public OuterJoinStream(ViewStream vs1, string joinCol1, ViewStream vs2, string joinCol2) +// : this (vs1, joinCol2, new ColumnMap(null), vs2, joinCol2, new ColumnMap(null)) +// { +// } + +// public OuterJoinStream(ViewStream vs1, string joinCol1, string cmExp1, +// ViewStream vs2, string joinCol2, string cmExp2) +// : this (vs1, joinCol1, new ColumnMap(cmExp1), vs2, joinCol2, new ColumnMap(cmExp2)) +// { +// } + +// public OuterJoinStream(ViewStream vs1, string joinCol1, ColumnMap cm1, +// ViewStream vs2, string joinCol2, ColumnMap cm2) +// { +// #region set source +// ViewStream temp = new AppendStream +// ( +// new ProjectStream +// ( +// new FilterStream +// ( +// new LeftJoinStream +// ( +// vs2, joinCol2, cm1, new AddColumnsStream(vs1, "filterCol='filtered'"), joinCol1, cm2 +// ), +// "filterCol=null" +// ), +// "!filterCol" +// ), +// new JoinStream(vs1, joinCol1, cm1, vs2, joinCol2, cm2) +// ); + +// temp = new AppendStream +// ( +// temp, +// new ProjectStream +// ( +// new FilterStream +// ( +// new LeftJoinStream +// ( +// vs1, joinCol1, cm1, new AddColumnsStream(vs2, "filterCol='filtered'"), joinCol2, cm2 +// ), +// "filterCol=null" +// ), +// "!filterCol" +// ) +// ); + +// mySetRealSource(temp); +// #endregion +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Poll/PollStream.cs b/src/DataStreamSuite/DataStream/Primitives/Poll/PollStream.cs new file mode 100644 index 000000000..2642c3b53 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Poll/PollStream.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using BCG.Utilities.General; + +namespace BCG +{ + public class PollStream : ViewStream, IWorker + { + public delegate IBlueRowCollection RowGetter(); + public PollStream(RowGetter g, int secondsBetweenPolls, ViewStreamDatabase db) + { + myGetter = g; + mySecondsBetweenPolls = secondsBetweenPolls; + myDB = db; + myLastChecked = DateTime.Now; + } + + protected override void myOnFirstSubscriber() + { + myDB.addWorker(this); + } + + protected override void myOnLastSubscriber() + { + myDB.removeWorker(this); + } + + public void doWork() + { + if ((DateTime.Now - myLastChecked).TotalSeconds > mySecondsBetweenPolls) + { + myLastChecked = DateTime.Now; + if (myLastValue != null) + { + DataPacket previous = (DataPacket)myLastValue; + IBlueRowCollection newRows = myGetter(); + IBlueRowCollection a, d; + if (newRows == null) + return; + BlueDataUtility.computeDifferences(newRows, previous.newRows, out d, out a); + if (d.Count > 0 || a.Count > 0) + myFireEvent(new DataPacket(newRows, a, d)); + } + else + { + IBlueRowCollection newRows = myGetter(); + if (newRows == null) + { + return; + } + new BlueTable(newRows.schema); + myFireEvent(new DataPacket(newRows, newRows, new BlueTable(newRows.schema))); + } + } + } + + DateTime myLastChecked; + RowGetter myGetter; + int mySecondsBetweenPolls; + ViewStreamDatabase myDB; + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ChangeTableNameStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ChangeTableNameStream.cs new file mode 100644 index 000000000..ef684ceec --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ChangeTableNameStream.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections; + +namespace BCG +{ + public class ChangeTableNameStream : ViewStream, IStreamMapper + { + private class MyRow : IBlueRow + { + private IBlueRow myRow; + private IBlueTableSchema mySchema; + internal MyRow (IBlueTableSchema s, IBlueRow r) + { + mySchema = s; + myRow = r; + } + + #region IBlueRow Members + + public IBlueTableSchema schema { get { return mySchema; } } + + public object this[int index] { get { return myRow[index]; } } + + public object this[string colName] { get { return myRow[colName]; } } + + #endregion + + #region IDictionary Members + + public void Add(object key, object value) + { + throw new Exception("The method or operation is not implemented."); + } + + public void Clear() + { + throw new Exception("The method or operation is not implemented."); + } + + public bool Contains(object key) { return myRow.Contains(key); } + + public IDictionaryEnumerator GetEnumerator() { return myRow.GetEnumerator(); } + + public bool IsFixedSize { get { return true; } } + + public bool IsReadOnly { get { return true; } } + + public ICollection Keys { get { return myRow.Keys; } } + + public void Remove(object key) + { + throw new Exception("The method or operation is not implemented."); + } + + public ICollection Values { get { return myRow.Values; } } + + public object this[object key] + { + get { return myRow[key]; } + set { throw new Exception("The method or operation is not implemented."); } + } + + #endregion + + #region ICollection Members + + public void CopyTo(Array array, int index) { myRow.CopyTo(array, index); } + + public int Count { get { return myRow.Count; } } + + public bool IsSynchronized { get { return myRow.IsSynchronized; } } + + public object SyncRoot { get { return myRow.SyncRoot; } } + + #endregion + + #region IEnumerable Members + + IEnumerator IEnumerable.GetEnumerator() { return myRow.GetEnumerator(); } + + #endregion + } + + private string myNewName; + private IBlueTableSchema mySchema; + + public ChangeTableNameStream (ViewStream vs, string name) + { + myNewName = name; + mySetRealSource (new MapStream (vs, this)); + } + + #region IStreamMapper Members + + public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema) + { + if (mySchema == null) + { + ArrayList cis = new ArrayList (); + foreach (ColumnInfo ci in inputSchema.columnInfo) + cis.Add (new ColumnInfo (ci.colName, ci.colNo, ci.colType)); + mySchema = new BlueTableSchema (myNewName, cis.ToArray (typeof(ColumnInfo)) as ColumnInfo[]); + } + return mySchema; + } + + #endregion + + #region IDynamicStreamMapper Members + + public IBlueRow map (IBlueRow row) + { + return new MyRow (getOutputSchema (row.schema), row); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ColumnMap.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ColumnMap.cs new file mode 100644 index 000000000..0123c7e68 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ColumnMap.cs @@ -0,0 +1,68 @@ +// using System; +// using System.Collections; +// using System.Data; +// using System.Text.RegularExpressions; +// using BCG.Utilities; +// using BCG.Evaluator.ColumnMapExpression; + +// namespace BCG +// { +// /// +// /// ColumnMap governs what columns that are to be projected, hidden or aliased +// /// +// public class ColumnMap +// { +// private string myColMapExpression; + +// /// +// /// ColumnMap governs what columns that are to be projected, hidden or aliased. If all columns +// /// are to be preserved with their names, ColumnMap("", false) should be passed +// /// +// /// +// /// Includes the mapping of the columns to their respective aliases as well as the columns that +// /// need to be included/excluded depending on the second parameter. e.g. "a:b c" means to map +// /// a onto b, and include or exclude c depending on the value of the second parameter +// /// +// /// +// /// isInclusive decides whether to include the unmapped columns in argument one or to exclude it +// /// from the resulting table +// /// +// public ColumnMap (string colMapExpression) +// { +// myColMapExpression = colMapExpression == null ? "!" : colMapExpression; +// } + +// public IDictionary getColumnMap (IBlueTableSchema schema) +// { +// IDictionary retVal = new Hashtable (); +// syntax parser = +// new syntax(); + +// ColumnMapExpression e = (ColumnMapExpression) +// parser.Parse(this.myColMapExpression); + +// if (e.isInclusive) +// { +// retVal = new Hashtable (e.columns); +// retVal["id"] = "id"; +// } +// else +// foreach (ColumnInfo ci in schema.columnInfo) +// if (e.columns[ci.colName] == null) +// retVal[ci.colName] = ci.colName; +// else if ((string)e.columns[ci.colName] != ci.colName) +// retVal[ci.colName] = e.columns[ci.colName]; +// return invertDictionary (retVal); +// } + +// public static IDictionary invertDictionary (IDictionary dic) +// { +// IDictionary retVal = new Hashtable (); +// foreach (DictionaryEntry de in dic) +// retVal[de.Value] = de.Key; +// return retVal; +// } + +// public string columnExpression {get{return myColMapExpression;}} +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs new file mode 100644 index 000000000..e4f31f5f5 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs @@ -0,0 +1,115 @@ +// using System; +// using System.Collections; +// using BCG.Evaluator.ColumnMapExpression; + +// namespace BCG +// { +// /// +// /// Provides the place holder for the int array that maps aliased column indices to the original +// /// ones (and the algorithms to compute them) +// /// +// internal class ProjectHelper +// { +// internal int[] colMap; +// internal IBlueTableSchema schema; +// internal string[] relevantOrigCols; + +// /// +// /// +// /// original(unprojected) schema +// /// expression defining the mapping +// /// table name of the new schema +// internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName) +// : this (isch, myGetDictionary (isch, exp), tableName) +// {} + +// /// +// /// +// /// original (unprojected) schema +// /// dictionary mapping unprojected names to their projected names, dropped cols should not be there +// /// table name of the new schema +// internal ProjectHelper (IBlueTableSchema isch, IDictionary origToAlias, string tableName) +// { +// ArrayList colMapAl = new ArrayList (), +// colInfoAl = new ArrayList (); +// relevantOrigCols = new string[origToAlias.Count-1]; +// int c = 0; +// ArrayList iColNames = new ArrayList (isch.columnNames); +// iColNames.Sort (); +// foreach (string cn in iColNames) +// { +// if (origToAlias.Contains (cn)) +// { +// ColumnInfo ci = isch[cn]; +// colInfoAl.Add (new ColumnInfo (origToAlias[ci.colName] as string, +// colInfoAl.Count, +// ci.colType) +// ); +// colMapAl.Add (ci.colNo); +// if (ci.colName != "id") +// relevantOrigCols[c++] = ci.colName; +// } +// } + +// schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName, +// colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]); +// colMap = colMapAl.ToArray (typeof(int)) as int[]; +// } + +// /// +// /// Gets an IDictionary mapping the original names to their aliased names. (The columns +// /// dropped do not appear in the returned value) +// /// +// /// +// /// +// static syntax myParser = +// new syntax(); +// /// +// private static IDictionary myGetDictionary (IBlueTableSchema isch, string exp) +// { +// IDictionary origToAlias = new Hashtable (); + +// ColumnMapExpression e; +// if (DataStreamLoadFileConfig.instance.useStaticParser) +// { +// lock (myParser) +// { +// e = (ColumnMapExpression) +// myParser.Parse(exp); +// } +// } +// else +// { +// syntax parser = new syntax(); +// e = (ColumnMapExpression) +// myParser.Parse(exp); + +// } +// if (e.isInclusive) +// { +// origToAlias = new Hashtable (e.columns); +// origToAlias["id"] = "id"; +// } +// else +// foreach (ColumnInfo ci in isch.columnInfo) +// if (e.columns[ci.colName] == null) +// origToAlias[ci.colName] = ci.colName; +// else if ((string)e.columns[ci.colName] != ci.colName) +// origToAlias[ci.colName] = e.columns[ci.colName]; +// return origToAlias; +// } + +// /// +// /// Returns a new dictionary with the keys and values interchanging positions +// /// +// /// +// /// +// internal static IDictionary invertDic (IDictionary dic) +// { +// IDictionary retVal = new Hashtable (dic.Count); +// foreach (DictionaryEntry de in dic) +// retVal.Add (de.Value, de.Key); +// return retVal; +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs new file mode 100644 index 000000000..a0f4fa96a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs @@ -0,0 +1,98 @@ +// using System; +// using System.Collections; + +// namespace BCG +// { +// public class ProjectStream : ViewStream +// { +// private ProjectHelper myHelper; +// private string myColExp, myTableName; +// private ViewStream myVs; +// private IDictionary myDic; +// private BCG.Utilities.General.HighPerformanceTimer _timer = new Utilities.General.HighPerformanceTimer(); +// /// +// /// +// /// stream to susbsribe to as the input stream +// /// expression defining the column mapping +// public ProjectStream (ViewStream vs, string colExp) +// { +// myVs = vs; +// myColExp = colExp; +// } + +// /// +// /// +// /// stream to subscribe to as the input stream +// /// expression defining the column mapping +// /// name of the mapped table +// public ProjectStream (ViewStream vs, string colExp, string tableName) +// { +// if (colExp == "!") +// mySetRealSource (new ChangeTableNameStream (vs, tableName)); +// else +// { +// myVs = vs; +// myColExp = colExp; +// myTableName = tableName; +// } +// } + +// /// +// /// +// /// stream to subscribe to as the input stream +// /// dictionary mapping the aliased to the original +// public ProjectStream (ViewStream vs, IDictionary dic) +// { +// myVs = vs; +// myDic = ProjectHelper.invertDic (dic); +// } + +// protected override void myOnFirstSubscriber() +// { +// myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange); +// } + +// protected override void myOnLastSubscriber() +// { +// myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);; +// } + +// internal void myHandleChange (object newValue) +// { +// DataPacket dp = newValue as DataPacket; +// if (myHelper != null) +// { +// _timer.start(); +// IBlueRowCollection dels, adds; +// BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols); +// _timer.stop(); +// if (dels.Count + adds.Count > 0) +// myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper), +// new ProjectedTable (adds, myHelper), +// new ProjectedTable (dels, myHelper) +// ) +// ); +// } +// else +// { //first time a callback is received +// myHelper = myDic == null ? +// new ProjectHelper (dp.schema, myColExp, myTableName) : +// new ProjectHelper (dp.schema, myDic, myTableName); + +// IBlueRowCollection temp; +// myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper), +// temp, +// new BlueTable (temp.schema) +// ) +// ); + +// } +// } +// public void myHandleChangeTest(object newValue) +// { +// myHandleChange(newValue); +// } + +// } + +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedRow.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedRow.cs new file mode 100644 index 000000000..136128087 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedRow.cs @@ -0,0 +1,94 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; + +// namespace BCG +// { +// /// +// /// Projected row just keeps a pointer to the original row its projecting and to the ProjectHelper +// /// (which provided the projected schema and the mapping). All indexing to it, are redirected (though +// /// the colMapper in ProjectHelper) to the original row +// /// +// internal class ProjectedRow : IBlueRow +// { +// private IBlueRow myRow; +// private ProjectHelper myHelper; + +// internal ProjectedRow (IBlueRow row, ProjectHelper helper) +// { +// myRow = row; +// myHelper = helper; +// } + +// #region IBlueRow Members + +// public object this [string colName] {get {return this[myHelper.schema[colName].colNo];}} + +// public object this [int index] {get {return myRow[myHelper.colMap[index]];}} + +// public IBlueTableSchema schema {get {return myHelper.schema;}} + +// #endregion + +// #region IDictionary Members + +// public bool IsReadOnly {get {return true;}} + +// public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator (this);} + +// object System.Collections.IDictionary.this[object key] +// { +// get {return key is int ? this[(int)key] : this[key as string];} +// set {throw new ApplicationException ("Is read only");} +// } + +// public void Remove(object key) {throw new ApplicationException ("Is read only");} + +// public bool Contains (object key) {return myHelper.schema [key as string] != null;} + +// public void Clear() {throw new ApplicationException ("Is read only");} + +// public ICollection Values +// { +// get +// { +// object[] retVal = new object[Count]; +// CopyTo (retVal, 0); +// return retVal; +// } +// } + +// public void Add (object key, object value) {throw new ApplicationException ("Is read only");} + +// public ICollection Keys {get {return myHelper.schema.columnNames;}} + +// public bool IsFixedSize {get {return true;}} + +// #endregion + +// #region ICollection Members + +// public bool IsSynchronized {get {return false;}} + +// public int Count {get {return myHelper.schema.columnNames.Length;}} + +// public void CopyTo (Array array, int index) +// { +// foreach (ColumnInfo ci in myHelper.schema.columnInfo) +// array.SetValue (this[ci.colNo], index ++); +// } + +// public object SyncRoot {get {return null;}} + +// #endregion + +// #region IEnumerable Members + +// IEnumerator System.Collections.IEnumerable.GetEnumerator() {return new BlueRowEnumerator (this);} + +// #endregion + +// } + + +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs new file mode 100644 index 000000000..bde6a3926 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs @@ -0,0 +1,95 @@ +// using System; +// using System.Collections; + +// namespace BCG +// { +// /// +// /// All what this type does is to keep a reference to the original rows, and all accessors +// /// to the row, return a ProjectedRow (which is basically a wrapping around the original row) +// /// +// internal class ProjectedTable : IImmutableRowCollection +// { +// private IBlueRowCollection myRows; +// private ProjectHelper myHelper; + +// internal ProjectedTable (IBlueRowCollection rows, ProjectHelper helper) +// { +// if (helper == null) throw new ArgumentException("helper"); + +// myRows = rows ?? new BlueTable(helper.schema); +// myHelper = helper; +// } + +// #region IBlueRowCollection Members + +// public IBlueRow this [int id] +// { +// get {return new ProjectedRow (myRows[id], myHelper);} +// } + +// public bool containsID(int id) {return myRows.containsID (id);} + +// public IBlueTableSchema schema {get {return myHelper.schema;}} + +// #endregion + +// #region ICollection Members + +// public bool IsSynchronized {get {return myRows.IsSynchronized;}} + +// public int Count {get {return myRows.Count;}} + +// public void CopyTo (Array array, int index) +// { +// foreach (IBlueRow r in myRows) +// array.SetValue (this[(int)r["id"]], index++); +// } + + +// public object SyncRoot {get {return myRows.SyncRoot;}} + +// #endregion + +// #region IEnumerable Members + +// public IEnumerator GetEnumerator () +// { +// return new MyEnumerator (myRows, myHelper); +// } + +// private class MyEnumerator : IEnumerator +// { +// #region IEnumerator Members + +// private IEnumerator myMaster; +// private ProjectHelper myHelper; +// public MyEnumerator (IBlueRowCollection master, ProjectHelper helper) +// { +// myMaster = master.GetEnumerator (); +// myHelper = helper; +// } + + +// public void Reset() {myMaster.Reset();} + +// public object Current +// { +// get +// { +// return myMaster.Current == null ? null : +// new ProjectedRow (myMaster.Current as IBlueRow, myHelper); +// } +// } + +// public bool MoveNext() {return myMaster.MoveNext ();} + +// #endregion + +// } + + +// #endregion + +// } + +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SqlStream.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SqlStream.cs new file mode 100644 index 000000000..e772d0b48 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SqlStream.cs @@ -0,0 +1,164 @@ +// using System; +// using System.Data; +// using System.Data.SqlClient; +// using BCG.Utilities.BlueConfiguration; + +// namespace BCG +// { +// public class SqlStream : ViewStream +// { + +// protected string myQuery; +// protected string myConnectionString; + +// #region constructors + +// public SqlStream(string connectionString, string query) +// { +// myConnectionString = connectionString; +// myQuery = query; +// mySQLCommandTimeout = DEFAULT_SQL_COMMAND_TIMEOUT; +// myInitConnection(query, false); +// } + +// public SqlStream (string query) +// : this (ConfigurationRequester.request ("persistenceServerSQLServer"), +// ConfigurationRequester.request ("persistenceServerSQLDatabase"), +// query, false, DEFAULT_SQL_COMMAND_TIMEOUT) +// { +// } + +// public SqlStream(string server, string database, string query) +// : this(server, database, query, false, DEFAULT_SQL_COMMAND_TIMEOUT) +// { +// } + +// public SqlStream(string server, string database, string query, int timeout) +// : this (server, database, query, false, timeout) +// { +// } + +// public SqlStream(string server, string database, string query, bool createId) +// : this(server, database, query, createId, DEFAULT_SQL_COMMAND_TIMEOUT) +// { +// } + +// public SqlStream(string server, string database, string query, bool createId, int timeout) +// { +// mySQLCommandTimeout = timeout; +// myConnectionString = "server=" + server + ";Integrated Security=SSPI;database=" + database; +// myInitConnection(query, createId); +// } + +// private void myInitConnection(string query, bool createId) +// { +// using (SqlConnection theConnection = new SqlConnection(myConnectionString)) +// { + +// myQuery = query; +// theConnection.Open(); +// myInitialize(createId, theConnection); +// } +// } + +// public SqlStream (SqlConnection c, string query) +// : this (c, query, false, DEFAULT_SQL_COMMAND_TIMEOUT) +// { +// } + +// public SqlStream (SqlConnection c, string query, bool createId) +// : this (c, query, createId, DEFAULT_SQL_COMMAND_TIMEOUT) +// { +// } + +// public SqlStream(SqlConnection c, string query, bool createId, int timeout) +// { +// mySQLCommandTimeout = timeout; +// myQuery = query; +// myInitialize(createId,c); +// } + +// public SqlStream (DataTable dt) +// : this (dt, false, DEFAULT_SQL_COMMAND_TIMEOUT) +// { +// } + +// public SqlStream (DataTable dt, bool createId) +// : this (dt, createId, DEFAULT_SQL_COMMAND_TIMEOUT) +// { +// } + +// public SqlStream(DataTable dt, bool createId, int timeout) +// { +// mySQLCommandTimeout = timeout; +// myInitialize(dt, createId); +// } +// #endregion + +// protected void myInitialize (bool createId,SqlConnection c) +// { +// myInitialize (myQuery, createId, c); +// } + +// protected void myInitialize (string query, bool createId,SqlConnection c) +// { +// DataSet ds = new DataSet(); +// using(SqlCommand sc = new SqlCommand(query, c)) +// { +// sc.CommandTimeout = mySQLCommandTimeout; +// new SqlDataAdapter(sc).Fill (ds); +// myInitialize (ds.Tables[0], createId); +// } +// } + +// protected virtual void myInitialize (DataTable dt, bool createId) +// { +// if (createId) +// { +// int i = 0; +// foreach (DataRow dr in dt.Rows) +// dr["id"] = ++i; +// } + +// DataPacket dp = new DataPacket(); +// dp.newRows = new BlueTable(dt); +// dp.addedRows = dp.newRows; +// if (myLastValue == null) +// dp.deletedRows = new BlueTable(dp.newRows.schema); +// else +// dp.deletedRows = new BlueTable((myLastValue as DataPacket).newRows); +// myFireEvent(dp.Clone()); +// } + +// public virtual string query +// { +// set{ +// myQuery = value; +// using (SqlConnection c = new SqlConnection(myConnectionString)) +// { +// myInitialize(false, c); +// } +// } +// } + +// protected override void myOnFirstSubscriber() +// { +// if (myLastDP != null) +// myFireEvent((myLastDP as ICloneable).Clone()); +// } + +// protected override void myOnLastSubscriber () +// { +// // this value will be cleared by the base class +// // but in this particular stream this behaviour +// // is not usefull since any new subscribers +// // would force the query to be recreated +// myLastDP = myLastValue; +// } + +// private static readonly int DEFAULT_SQL_COMMAND_TIMEOUT = 30; + +// private object myLastDP; +// private int mySQLCommandTimeout; +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs new file mode 100644 index 000000000..cfe814f39 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs @@ -0,0 +1,263 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using BCG.Utilities; + +namespace BCG +{ + [DataStreamUserRespresentation("csvSaver")] + public class CSVSaverStream : ViewStream, ISideEffectStream + { + private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)), + new ColumnInfo("fileName", 1, typeof(string)), + new ColumnInfo("nWrites", 2, typeof(int)), + new ColumnInfo("nLastRows", 3, typeof(int))); + + private string myFullyQualifiedFileName; + private ViewStream myStream; + private bool myOverwrite, myFireResultSnapshot; + private int myNWrites = 0, myNMaxWrites = Int32.MaxValue; + + public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName) + : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) + { } + + public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) + { + myStream = vs; + myFullyQualifiedFileName = fullyQualifiedFileName; + myOverwrite = overwrite; + myNMaxWrites = nMaxWrites; + myFireResultSnapshot = fireResultSnapshot; + } + + protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleChanges); } + + protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); } + + private void myHandleChanges(object newValue) + { + IBlueRowCollection totalState = (newValue as DataPacket).newRows; + if (totalState.Count > 0 && myNWrites < myNMaxWrites) + { + myNWrites++; + int i = 0; + StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write)); + foreach (IBlueRow r in totalState) + { + if (i++ == 0) + { + string header = String.Empty; + foreach (string col in r.schema.columnNames) + if (col != "id") + header += (header == String.Empty ? col : "," + col); + writer.WriteLine(header); + writer.Flush(); + } + writer.WriteLine(myGetCSVRow(r)); + writer.Flush(); + } + writer.Close(); + if (myFireResultSnapshot) + { + IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count); + BlueTable nRows = new BlueTable(myResultSchema); + nRows.addRow(nr); + myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows))); + } + else + myFireEvent(newValue); + } + } + + private string myGetNewFileName() + { + string[] tags = myFullyQualifiedFileName.Split('.'); + if (myOverwrite) + return tags[0] + ".csv"; + else + return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv"; + } + + private static string myGetCSVRow(IBlueRow r) + { + string rv = String.Empty; + int i = 0; + foreach (string col in r.schema.columnNames) + { + if (col != "id") + { + string colText = String.Empty; + if (r[col] != null && r[col] != DBNull.Value) + colText = r[col].ToString(); + if (r.schema[col].colType == typeof(string)) + colText = "\"" + colText + "\""; + rv += (i++ == 0 ? colText : "," + colText); + } + } + return rv; + } + + private string myFormateDateTime(DateTime dt) + { + return + string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}", + dt.Year.ToString("0000"), + dt.Month.ToString("00"), + dt.Day.ToString("00"), + dt.Hour.ToString("00"), + dt.Minute.ToString("00"), + dt.Second.ToString("00"), + dt.Millisecond.ToString("0000")); + } + } + + [DataStreamUserRespresentation("CSVReader")] + public class CSVReaderStream : ViewStream + { + private IBlueRowCollection myRows; + + public CSVReaderStream(string fileAddress) + { + myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress); + } + + protected override void myOnFirstSubscriber() + { + myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema))); + } + } + + [DataStreamUserRespresentation ("CSVReaderStreamMapper")] + public class CSVReaderStreamMapperStream : ViewStream + { + public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) + { + mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema))); + } + } + + public class CSVReaderStreamMapper : IStreamMapper + { + private IBlueTableSchema mySchema; + private readonly string[] myOutputSchemaStringArray; + + public CSVReaderStreamMapper(string outputSchema) + { + myOutputSchemaStringArray = outputSchema.Split(' '); + } + + public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) + { + if (mySchema == null) + { + ColumnInfo[] ci = inputSchema.columnInfo; + foreach (string col in myOutputSchemaStringArray) + { + string colName = col.Substring(0, col.IndexOf('\\')).ToLower(); + for (int i = 0; i < ci.Length; i++) + { + if (ci[i].colName.ToLower().Equals(colName)) + { + Type t = myGetType(col.Substring(col.IndexOf('\\') + 1)); + ci[i] = new ColumnInfo(ci[i].colName, i, t); + break; + } + } + } + mySchema = new BlueTableSchema(ci, "CSVReader"); + } + return mySchema; + } + + public IBlueRow map(IBlueRow row) + { + object[] vals = new object[row.schema.columnInfo.Length]; + row.Values.CopyTo(vals, 0); + for (int i = 0; i < mySchema.columnInfo.Length; i++) + { + Type t = mySchema.columnInfo[i].colType; + try + { + vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture); + } + catch + { + if (t.Equals(typeof(Int16))) + { + Int16 result; + if (Int16.TryParse((string)vals[i], out result)) + { + vals[i] = result; + continue; + } + double r2; + if (Double.TryParse((string)vals[i], out r2)) + { + try + { + vals[i] = (Int16)r2; + continue; + } + catch { } + } + } + else if (t.Equals(typeof(Int32))) + { + Int32 result; + if (Int32.TryParse((string)vals[i], out result)) + { + vals[i] = result; + continue; + } + double r2; + if (Double.TryParse((string)vals[i], out r2)) + { + try + { + vals[i] = (Int32)r2; + continue; + } + catch { } + } + } + else if (t.Equals(typeof(Int64))) + { + Int64 result; + if (Int64.TryParse((string)vals[i], out result)) + { + vals[i] = result; + continue; + } + double r2; + if (Double.TryParse((string)vals[i], out r2)) + { + try + { + vals[i] = (Int64)r2; + continue; + } + catch { } + } + } + vals[i] = DBNull.Value; + } + } + return new BlueRow(mySchema, vals); + } + + private Type myGetType(string type) + { + type = type.ToLower(); + if (type.Contains("int")) + return typeof(int); + else if (type.Contains("double")) + return typeof(double); + else if (type.Contains("date")) + return typeof(DateTime); + else + return typeof(string); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/ISideEffectStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/ISideEffectStream.cs new file mode 100644 index 000000000..9a50514b4 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/ISideEffectStream.cs @@ -0,0 +1,8 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public interface ISideEffectStream { } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/PersisterStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/PersisterStream.cs new file mode 100644 index 000000000..c644c7603 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/PersisterStream.cs @@ -0,0 +1,74 @@ +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; +// using System.Threading.Tasks; +// using System.IO; +// using System.Runtime.Serialization.Formatters.Binary; + +// namespace BCG +// { +// public class PersisterStream : ViewStream +// { +// private readonly ViewStream _main_stream; +// private readonly string _file_name; +// private const string _sub_directory = "Streams"; +// private int _n_updates = 0; + +// public PersisterStream(string table_name) +// : this(new TableStream (table_name), _get_file_path (table_name)) +// { } + +// public PersisterStream(ViewStreamDatabase vsd, string table_name) +// : this(new TableStream(table_name, vsd), _get_file_path(table_name)) +// { } + +// public PersisterStream(ViewStream main_stream, string directory, string name) +// : this(new ChangeTableNameStream(main_stream, name), String.Format("{0}{1}{2}", directory, @"\", name)) { } + +// private PersisterStream(ViewStream main_stream, string file_name) +// { +// _file_name = file_name; +// _main_stream = new SaverStream(main_stream, file_name); +// } + +// protected override void myOnFirstSubscriber() +// { +// IBlueRowCollection rows = new LoaderStream(_file_name).rows; +// if (rows != null) +// myFireEvent(new DataPacket(rows, rows, myLastValue == null ? new BlueTable(rows.schema) : (myLastValue as DataPacket).newRows)); +// _main_stream.onChange += _main_stream_onchange; +// } + +// protected override void myOnLastSubscriber() +// { +// _n_updates = 0; +// _main_stream.onChange -= _main_stream_onchange; +// } + +// private void _main_stream_onchange(object nv) +// { +// DataPacket dp = nv as DataPacket; +// if (_n_updates++ == 0) +// /// to make it incrementally consistent with the bootstap possible fire +// myFireEvent(new DataPacket(dp.newRows, dp.newRows, myLastValue == null ? new BlueTable(dp.newRows) : (myLastValue as DataPacket).newRows)); +// else +// myFireEvent(dp); +// } + +// private static string _get_default_path() +// { +// string rv = String.Format("{0}\\{1}"/*, mbts.config.Configuration.get().data_path, _sub_directory*/); +// if (!Directory.Exists(rv)) +// { +// try { Directory.CreateDirectory(rv); } +// catch (Exception ex) { /*mbts.Logger.log(bts.log.Level.Error, bts.log.Subsystem.Persist, String.Format("Could not create directory {0}, exception {1}", rv, ex.Message));*/ } +// } +// return rv; +// } + +// public string get_file_name() { return _file_name; } + +// private static string _get_file_path(string file_name) { return String.Format("{0}\\{1}", _get_default_path(), file_name); } +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs new file mode 100644 index 000000000..4fbc9a476 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs @@ -0,0 +1,115 @@ +// using System; +// using System.Collections; +// using BCG.Utilities; +// using BCG.Evaluator.SortExpression; + + +// namespace BCG +// { +// public class SortRow : IComparable, IBlueRow +// { +// private IBlueRow myRow; +// private IEnumerable myColumns; +// private IComparer myComparer; + +// public SortRow (IBlueRow row, SortExpression sortExpression) +// { +// myRow = row; +// myColumns = sortExpression.columns; +// } + +// public SortRow (IBlueRow row, IEnumerable columns) +// { +// myRow = row; +// myColumns = columns; +// } + +// public SortRow (IBlueRow row, IComparer comparer) +// { +// myRow = row; +// myComparer = comparer; +// } + +// public IBlueRow internal_row { get { return myRow; } } + +// #region IComparable method +// public int CompareTo (object obj) +// { +// IBlueRow otherRow = (IBlueRow)obj; +// if (myComparer == null) +// { +// foreach (SortExpression.ColumnCriteria cc in myColumns) +// { +// int retVal = myIsNull (this[cc.columnName]) ? +// myIsNull (otherRow[cc.columnName]) ? 0 : -1 : +// myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]); +// retVal *= cc.ascending ? 1 : -1; +// if (retVal != 0) +// return retVal; +// } +// return 0; +// } +// else +// return myComparer.Compare (this, otherRow); +// } +// #endregion + +// #region IBlueRow members +// public IBlueTableSchema schema{get{return myRow.schema;}} +// #endregion + +// #region ICollection members +// public int Count {get{return myRow.Count;}} + +// public bool IsSynchronized {get{return false;}} + +// public object SyncRoot {get{return this;}} + +// public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);} +// #endregion + +// #region IDictionary members +// public bool IsFixedSize {get{return true;}} + +// public bool IsReadOnly {get{return true;}} + +// /// +// /// The key can either be the string corrsponding to the column name or +// /// the order of the column +// /// +// public object this [object key] +// { +// get +// { +// if (key is string) +// return this[(string)key]; +// return this[(int)key]; +// } +// set {throw new NotSupportedException ("The IBlueRow is readonly");} +// } + +// public object this [int index] {get{return myRow[index];}} + +// public object this [string colName] {get{return myRow[colName];}} + +// public ICollection Keys{get{return myRow.Keys;}} + +// public ICollection Values{get{return myRow.Values;}} + +// public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + +// public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} + +// public bool Contains (object key){return myRow.Contains (key);} + +// IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();} + +// IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();} + +// public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");} +// #endregion + +// private bool myIsNull (object o) {return o == null || o is DBNull;} + +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs new file mode 100644 index 000000000..eee047756 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs @@ -0,0 +1,129 @@ +// using System; +// using System.Collections; +// using BCG.Evaluator.SortExpression; + + +// namespace BCG +// { +// /// +// /// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort +// /// to merge the additions to the accumulates table +// /// +// public class SortStream : ViewStream +// { +// private ViewStream myStream; +// private IEnumerable mySortCols; +// private IComparer myComparer; +// private DataPacket myLastFiredPacket; +// private DataPacket myLastRecievedPacket; + +// public SortStream (ViewStream stream, string sortExpression) +// { +// myStream = stream; +// mySetSortExpression (sortExpression); +// } + +// public SortStream (ViewStream vs, string colName, bool isAsc) +// { +// myStream = vs; +// string sortExpression = isAsc ? "+" + colName : "-" + colName; +// mySetSortExpression (sortExpression); +// } + +// public SortStream (ViewStream vs, IComparer comparer) +// { +// myStream = vs; +// myComparer = comparer; +// } + +// public string sortExpression +// { +// set +// { +// myComparer = null; +// myLastFiredPacket = null; +// mySetSortExpression (value); +// myChangeHandler (myLastRecievedPacket); +// } +// } + +// public IComparer comparer +// { +// set +// { +// myComparer = value; +// myLastFiredPacket = null; +// myChangeHandler(myLastRecievedPacket); +// } +// } + +// private void mySetSortExpression (string sortExpression) +// { +// syntax parser = +// new syntax(); + +// mySortCols = ((SortExpression) +// parser.Parse (sortExpression)).columns; +// } + +// protected override void myOnFirstSubscriber() +// { +// myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); +// } + +// protected override void myOnLastSubscriber() +// { +// myInit (); +// myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); +// } + +// internal void myChangeHandler (object o) +// { +// myLastRecievedPacket = (DataPacket)o; +// //first callback, so we need to sort the whole table, this is the +// //only time we will do that for all the other sorting operations would +// //be sorting the increments and inserting them into the whole list +// if (myLastFiredPacket == null) +// { +// myLastFiredPacket = new DataPacket (); +// myLastFiredPacket.newRows = +// myComparer == null ? +// new SortedTable (myLastRecievedPacket.newRows, mySortCols) : +// new SortedTable (myLastRecievedPacket.newRows, myComparer); +// myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows); +// myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema); +// myFireEvent (myLastFiredPacket.Clone()); +// return; +// } +// myHandleDeletes (myLastRecievedPacket); +// myHandleAdds (myLastRecievedPacket); +// myFireEvent (myLastFiredPacket.Clone()); +// } + +// public void myChangeHandlerTest(object o) +// { +// myChangeHandler(o); +// } + +// private void myHandleDeletes (DataPacket dp) +// { +// myLastFiredPacket.deletedRows = +// myComparer == null ? +// new SortedTable (dp.deletedRows, mySortCols) : +// new SortedTable (dp.deletedRows, myComparer); +// ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows); +// } + +// private void myHandleAdds (DataPacket dp) +// { +// myLastFiredPacket.addedRows = new BlueTable (dp.addedRows); +// ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows); +// } + +// private void myInit () +// { +// myLastFiredPacket = null; +// myLastRecievedPacket = null; +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs new file mode 100644 index 000000000..e5348e0d0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs @@ -0,0 +1,136 @@ +// using System; +// using System.Collections; +// using BCG.Evaluator.SortExpression; + +// namespace BCG +// { +// /// +// /// Represents an IBlueRowCollection with the rows sorted according to the +// /// +// /// +// public class SortedTable : IBlueRowCollection +// { +// private ArrayList myRows = new ArrayList (); +// private Hashtable myIndex = new Hashtable (); +// private IBlueTableSchema mySchema; +// private IEnumerable mySortCols; +// private IComparer myComparer; + +// #region SortedTable method(s) +// public void addRows (IBlueRowCollection rows) +// { +// //requires merger of the addedRows to the already existing ones +// if (rows.Count == 0) +// //at times there are calls with 0 additions, in that case +// //we ofcourse need no processing +// return; +// SortedTable addedRows = +// myComparer == null ? new SortedTable (rows, mySortCols) : +// new SortedTable (rows, myComparer); +// int addedRowsIndex = 0; +// SortRow newRow = addedRows.getRowByIndex (addedRowsIndex); +// for (int i = 0; i < myRows.Count; i ++) +// if (newRow.CompareTo(myRows[i]) <= 0) +// { +// myRows.Insert (i, newRow); +// myIndex[newRow["id"]] = newRow; +// if (++addedRowsIndex == addedRows.Count) +// return; +// newRow = addedRows.getRowByIndex (addedRowsIndex); +// } +// if (addedRowsIndex < addedRows.Count) +// for (int i = addedRowsIndex; i < addedRows.Count; i++) +// { +// newRow = addedRows.getRowByIndex (i); +// myRows.Add (newRow); +// myIndex[newRow["id"]] = newRow; +// } +// } + +// public void deleteRows (IBlueRowCollection deletedRows) +// { +// foreach (IBlueRow deletedRow in deletedRows) +// { +// IBlueRow row = (SortRow)myIndex[deletedRow["id"]]; +// myRows.Remove (row); +// myIndex.Remove (row["id"]); +// } +// } + +// public SortRow getRowByIndex (int index) {return (SortRow)myRows[index];} +// #endregion + +// #region Constructor(s) +// /// +// /// Sorts the rows using sortExpression as the sorting criteria +// /// +// /// +// /// Rows to be inserted into the table +// /// +// /// +// /// Sorting criteria +// /// +// public SortedTable (IBlueRowCollection rows, SortExpression sortExpression) +// { +// mySortCols = sortExpression.columns; +// myConstructorHelper (rows); +// } + +// public SortedTable (IBlueRowCollection rows, IEnumerable sortCols) +// { +// mySortCols = sortCols; +// myConstructorHelper (rows); +// } + +// public SortedTable (IBlueRowCollection rows, IComparer comparer) +// { +// myComparer = comparer; +// myConstructorHelper (rows); +// } + +// private void myConstructorHelper (IBlueRowCollection rows) +// { +// mySchema = rows.schema; +// foreach (IBlueRow row in rows) +// { +// SortRow newRow = +// myComparer == null ? +// new SortRow (row, mySortCols) : new SortRow (row, myComparer); +// myRows.Add (newRow); +// myIndex[row["id"]] = newRow; +// } +// myRows.Sort (); +// } + +// #endregion + +// #region IBlueRowCollections members +// public IBlueRow this[int id] +// { +// get +// { +// if (myIndex[id] == null) +// throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); +// return (IBlueRow)myIndex[id]; +// } +// } +// public IBlueTableSchema schema {get{return mySchema;}} + +// public bool containsID (int id) {return myIndex[id] != null;} + +// #endregion + +// #region ICollection members +// public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();} + +// public bool IsSynchronized {get{return false;}} + +// public object SyncRoot {get{return this;}} + +// public void CopyTo (Array array, int index){myRows.CopyTo (array, index);} + +// public int Count{get{return myRows.Count;}} +// #endregion +// } + +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs b/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs new file mode 100644 index 000000000..d1b7c519c --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections; +using System.Reflection; +using BCG.Utilities; + +namespace BCG +{ + [DataStreamUserRespresentation (false)] + public class BlueRowCollectionStaticStream : ViewStream + { + public BlueRowCollectionStaticStream () {} + + public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);} + + public void loadCollection (IBlueRowCollection rows) + { + myFireEvent (new DataPacket (rows, rows, myLastValue != null ? + new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema))); + } + + public void crush_with_single_row(IBlueRow r) + { + BlueTable t = new BlueTable(r.schema); + t.addRow(r); + loadCollection(t); + } + + public void loadPacket (DataPacket dp) {myFireEvent (dp);} + + /// + /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc + /// The other arguments are provided by the user. + /// + /// + /// + /// + /// + public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args) + { + ArrayList totArgs = new ArrayList (args); + BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream (); + totArgs.Insert (0, v1); + ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream; + v1.loadCollection (rows); + return retVal.rows; + } + + } + + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/MutableStream.cs b/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/MutableStream.cs new file mode 100644 index 000000000..4dcd8e6c6 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/MutableStream.cs @@ -0,0 +1,43 @@ +using System; + +namespace BCG +{ + /// + /// A more complete and 'correct' version of BlueRowCollectionStaticStream. Should replace it finally, though + /// at this time, to expedite the process, both exist simultaniously + /// + public class MutableStream : ViewStream + { + public MutableStream () {} + + public MutableStream (IBlueRowCollection rows) + { + loadCollection (rows); + } + + /// + /// Fires a datapacket with newRows set to the argument, and the whole previous state (if existed) + /// set as the deletedRows. + /// + /// + public void loadCollection (IBlueRowCollection rows) + { + myFireEvent (new DataPacket (rows, new BlueTable(rows), + myLastValue != null ? + new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema))); + } + + /// + /// Fires the whole datapacket as one. Its the callers responsibility to make sure that it makes incremental sense + /// + /// + public void loadPacket (DataPacket dp) {myFireEvent (dp);} + + protected override void myOnFirstSubscriber() + { + if (myLastValue != null) + loadPacket (myLastValue as DataPacket); + } + } + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextReaderStream.cs b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextReaderStream.cs new file mode 100644 index 000000000..62b4c9f0b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextReaderStream.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace BCG +{ + public class TextReaderStream : ViewStream + { + #region constructors + public TextReaderStream(string file) + { + myFile = file; + } + #endregion + + #region ViewStream + protected override void myOnFirstSubscriber() + { + StreamReader reader = new StreamReader(myFile); + + #region get schema + IBlueTableSchema schema = null; string schemaStr; + if ((schemaStr = reader.ReadLine()) == null) + { + reader.Close(); + return; + } + + schema = myGetSchema(schemaStr); + #endregion + + #region read rows and fire update + BlueTable adds = new BlueTable(schema); + BlueTable dels = new BlueTable(schema); + + string rowStr; + while ((rowStr = reader.ReadLine()) != null) + { + adds.addRow(myGetRow(schema, rowStr)); + } + + myFireEvent(new DataPacket(adds, adds, dels)); + #endregion + + reader.Close(); + } + #endregion + + #region helpers + private IBlueTableSchema myGetSchema(string schemaStr) + { + List columnInfos = new List(); + + List columnInfoStrs = new List(schemaStr.Split(new string[] { TextWriterStream.DELIM }, StringSplitOptions.None)); + for(int i = 0; i < columnInfoStrs.Count; ++i) + { + string columnInfoStr = columnInfoStrs[i]; + + List columnInfoStrParts = new List(columnInfoStr.Split(new char[] { ':' }, StringSplitOptions.None)); + if (columnInfoStrParts.Count != 2) + { + throw (new Exception(String.Format("Unable to decode schema string '{0}'", schemaStr))); + } + + columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, Type.GetType(columnInfoStrParts[1]))); + } + + return (new BlueTableSchema(columnInfos.ToArray(), "")); + } + + private IBlueRow myGetRow(IBlueTableSchema schema, string rowStr) + { + List valueStrs = new List(rowStr.Split(new string[] { TextWriterStream.DELIM }, StringSplitOptions.None)); + + if (schema.columnInfo.Length != valueStrs.Count) + { + throw (new Exception(String.Format("Unable to decode row string '{0}'", rowStr))); + } + + List values = new List(); + for(int i = 0; i < schema.columnInfo.Length; ++i) + { + values.Add(Convert.ChangeType(valueStrs[i], schema.columnInfo[i].colType)); + } + + return (new BlueRow(schema, values.ToArray())); + } + #endregion + + #region state + private string myFile; + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs new file mode 100644 index 000000000..3b49e5d55 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs @@ -0,0 +1,145 @@ +using System; +using System.Diagnostics; +using System.IO; + +using BCG.Utilities.General; + +namespace BCG +{ + [DataStreamUserRespresentation("textWriter")] + public class TextWriterStream : ViewStream, IWorker + { + #region constructor + public TextWriterStream(ViewStream stream, string fileNamePrefix, int writeFreqSecs) + : this(ViewStreamDatabase.instance, stream, fileNamePrefix, writeFreqSecs) + { + } + + public TextWriterStream(ViewStreamDatabase vsd, ViewStream stream, string fileNamePrefix, int writeFreqSecs) + { + myVsd = vsd; + myStream = stream; + myFileNamePrefix = fileNamePrefix; + myWriteFreqSecs = writeFreqSecs; + myNextWrite = DateTime.MinValue; + myGotUpdate = false; + mySchemaStr = null; + + if (!myFileNamePrefix.EndsWith("_")) + { + myFileNamePrefix += "_"; + } + } + #endregion + + #region ViewStream + protected override void myOnFirstSubscriber() + { + myStream.onChange +=new ChangeDelegate(myStreamChange); + myVsd.addWorker(this); + } + + protected override void myOnLastSubscriber() + { + myVsd.removeWorker(this); + myStream.onChange -= new ChangeDelegate(myStreamChange); + } + #endregion + + #region IWorker + public void doWork() + { + if ((myNextWrite <= DateTime.Now && myGotUpdate)) + { + myGotUpdate = false; + myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs); + myWrite(myStream.rows); + } + } + #endregion + + private void myStreamChange(object o) + { + myGotUpdate = true; + } + + #region helpers + public void myWrite(IBlueRowCollection rows) + { + #region create schema str + if (mySchemaStr == null) + { + myCreateSchemaStr(rows.schema); + } + #endregion + + // create file name + string filename = String.Format("{0}{1}.txt", myFileNamePrefix, myDateTimeStr(DateTime.Now)); + + // open file + StreamWriter writer = new StreamWriter(filename); + + // write schema str + if (mySchemaStr != null) + { + writer.WriteLine(mySchemaStr); + } + + foreach (IBlueRow row in rows) + { + #region write row str + string rowStr = ""; + foreach (string columnName in row.schema.columnNames) + { + if (rowStr != "") + { + rowStr += DELIM; + } + + rowStr += row[columnName].ToString(); + } + + writer.WriteLine(rowStr); + #endregion + } + + // close file + writer.Close(); + } + + public void myCreateSchemaStr(IBlueTableSchema schema) + { + mySchemaStr = ""; + + foreach (ColumnInfo columnInfo in schema.columnInfo) + { + if (mySchemaStr != "") + { + mySchemaStr += DELIM; + } + + mySchemaStr += String.Format("{0}:{1}", columnInfo.colName, columnInfo.colType.ToString()); + } + } + + private string myDateTimeStr(DateTime dateTime) + { + return (String.Format("{0}{1}{2}{3}{4}{5}", + dateTime.Year.ToString("D4"), dateTime.Month.ToString("D2"), dateTime.Day.ToString("D2"), + dateTime.Hour.ToString("D2"), dateTime.Minute.ToString("D2"), dateTime.Second.ToString("D2"))); + } + #endregion + + public static readonly string DELIM = ", "; + + #region state + private ViewStreamDatabase myVsd; + private ViewStream myStream; + private string myFileNamePrefix; + private int myWriteFreqSecs; + private DateTime myNextWrite; + private bool myGotUpdate; + private string mySchemaStr; + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/ThreadSafe/ThreadSafeStream.cs b/src/DataStreamSuite/DataStream/Primitives/ThreadSafe/ThreadSafeStream.cs new file mode 100644 index 000000000..9102987f7 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/ThreadSafe/ThreadSafeStream.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using BCG.Utilities.General; + +namespace BCG +{ + /* + * The threadsafe stream wraps around an input stream, and amalgamates all the values from that stream + * locally. It only fires packets on the thread where the VSD's doWork function is called. + * + * */ + public class ThreadSafeStream : ViewStream, IWorker + { + + private ViewStream myParentStream; + private ThreadSafeQueue myQueuedUpdates; + private ViewStreamDatabase myVSD; + + public ThreadSafeStream(ViewStream parent) : this(ViewStreamDatabase.instance, parent) { } + public ThreadSafeStream(ViewStreamDatabase vsd, ViewStream parentStream) + { + myParentStream = parentStream; + myVSD = vsd; + myQueuedUpdates = new ThreadSafeQueue(); + } + + protected override void myOnFirstSubscriber() + { + myParentStream.onChange += new ChangeDelegate(myParentStreamChangeHandler); + myVSD.addWorker(this); + } + + + + protected override void myOnLastSubscriber() + { + myParentStream.onChange -= new ChangeDelegate(myParentStreamChangeHandler); + myVSD.removeWorker(this); + } + + void myParentStreamChangeHandler(object newValue) + { + myQueuedUpdates.enqueue(newValue as DataPacket); + } + + public void doWork() + { + foreach (DataPacket dp in myQueuedUpdates.dequeueAll()) + { + myFireEvent(dp); + } + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/ThreadSafe/VSDBMarshallerStream.cs b/src/DataStreamSuite/DataStream/Primitives/ThreadSafe/VSDBMarshallerStream.cs new file mode 100644 index 000000000..887c07ac3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/ThreadSafe/VSDBMarshallerStream.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; + +using BCG.Utilities.General; +using BCG.Utilities; + +namespace BCG +{ + // Marshal updates from one VSDB to another, the assumption being + // that the two vsdb's are driven by different threads + // constructor is called from the main thread + public class VSDBMarshallerStream : ViewStream, IDisposable, IWorker + { + public class TargetDoWorkRelay : IWorker + { + public TargetDoWorkRelay(VSDBMarshallerStream owner) + { + _owner = owner; + } + public void doWork() + { + _owner.onTargetDoWork(); + } + VSDBMarshallerStream _owner; + } + // source_stream is assumed to be driven by + // source_db + public VSDBMarshallerStream ( + ViewStreamDatabase source_db, + ViewStream source_stream, + ViewStreamDatabase target_db) + { + _source_thread = Thread.CurrentThread.ManagedThreadId; + _source_db = source_db; + _source_stream = source_stream; + _target_db = target_db; + _queued_actions = new ThreadSafeQueue(); + _queued_updates = new ThreadSafeQueue(); + _source_db.addWorker(this); + } + + // source doWork + public void doWork() + { + // BlueEventLog2.instance().assert(_source_thread == Thread.CurrentThread.ManagedThreadId, "doWork can only be called from source thread"); + + var r = _queued_actions.dequeue(); + if (r != null) + { + if (r.is_subscription) + { + _current_source_subscription = r.subscription_id; + _source_stream.onChange += new ChangeDelegate(_source_stream_onChange); + } + else + { + _source_stream.onChange -= new ChangeDelegate(_source_stream_onChange); + } + } + } + + public void Dispose() + { + // BlueEventLog2.instance().assert(_source_thread == Thread.CurrentThread.ManagedThreadId, "Dispose can only be called from source thread"); + doWork(); + _source_db.removeWorker(this); + } + + protected override void myOnFirstSubscriber() + { + _target_thread = Thread.CurrentThread.ManagedThreadId; + // BlueEventLog2.instance().assert(_source_thread != _target_thread, "Subscriptions should only occur in the target thread"); + + + _current_target_subscription = new Guid(); + _queued_actions.enqueue(new Request { is_subscription = true, subscription_id = _current_target_subscription}); + _do_work_relay = new TargetDoWorkRelay(this); + _target_db.addWorker(_do_work_relay); + } + + protected override void myOnLastSubscriber() + { + // BlueEventLog2.instance().assert(Thread.CurrentThread.ManagedThreadId == _target_thread, + // "Unsubscriptions should only occur in the target thread"); + _target_db.removeWorker(_do_work_relay); + _queued_actions.enqueue(new Request { is_subscription = false}); + } + + void _source_stream_onChange(object newValue) + { + // BlueEventLog2.instance().assert(_source_thread == Thread.CurrentThread.ManagedThreadId, "Updates should come from source thread"); + _queued_updates.enqueue(new Update { dp = (DataPacket)newValue, subscription = _current_target_subscription }); + } + + // target doWork + private void onTargetDoWork() + { + // BlueEventLog2.instance().assert(Thread.CurrentThread.ManagedThreadId == _target_thread, + // "onTargetDoWork should only be called from target thread"); + var updates = _queued_updates.dequeueAll(); + foreach (var u in updates) { + if (u.subscription == _current_target_subscription) + myFireEvent(u.dp); + } + } + + ViewStreamDatabase _source_db; + ViewStreamDatabase _target_db; + ViewStream _source_stream; + class Request + { + public bool is_subscription; + public Guid subscription_id; + } + private ThreadSafeQueue _queued_actions; + + class Update + { + public DataPacket dp; + public Guid subscription; + } + private ThreadSafeQueue _queued_updates; + TargetDoWorkRelay _do_work_relay; + Guid _current_source_subscription; + Guid _current_target_subscription; + int _source_thread; + int _target_thread; + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/DayStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/DayStream.cs new file mode 100644 index 000000000..34591314d --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Time/DayStream.cs @@ -0,0 +1,88 @@ +using System; +using BCG.Utilities.General; + +namespace BCG +{ + public class DayStream : ViewStream, IWorker + { + public DayStream () + : + this (0) + {} + + public DayStream (int offsetInDays) + : + this (offsetInDays,new TimeSpan(0)) + {} + + public DayStream (int offsetInDays, TimeSpan zeroH) + : + this (offsetInDays, zeroH, ViewStreamDatabase.instance) + {} + + public DayStream (int offsetInDays, int h, int m, int s) + : + this (offsetInDays, new TimeSpan(0,h,m,s,0), ViewStreamDatabase.instance) + {} + + public DayStream (int offsetInDays, TimeSpan zeroHour, ViewStreamDatabase db) + { + myOffset = offsetInDays; + myZeroHour = zeroHour; + myDB = db; + } + + protected override void myOnFirstSubscriber() + { + myDB.addWorker(this); + myInitialize(); + } + + protected override void myOnLastSubscriber() + {myDB.removeWorker(this);} + + public void doWork() + {myInitialize();} + + private void myInitialize() + { + DateTime now = DateTime.Now; + + DateTime zero; + if ((now - now.Date) > myZeroHour) + zero = now.Date; + else + zero = (now.Date - new TimeSpan(TimeSpan.TicksPerDay)); + + DateTime v = zero + myZeroHour + new TimeSpan(myOffset * TimeSpan.TicksPerDay); + + if (myLastValue == null || !v.Equals(myGetLastDay())) + { + BlueTableSchema s = + new BlueTableSchema("day", + new ColumnInfo("id",0,typeof(Int32)), + new ColumnInfo("dateTime",1,typeof(DateTime))); + + BlueTable newRows = new BlueTable(s); + newRows.addRow(new BlueRow(s,1,v)); + + IBlueRowCollection deletedRows = new BlueTable(s); + if (myLastValue != null) + deletedRows = ((DataPacket) myLastValue).newRows; + + myFireEvent(new DataPacket(newRows,newRows,deletedRows)); + } + } + + private object myGetLastDay() + { + return + BlueDataUtility.extractRow(((DataPacket) myLastValue).newRows,0)["dateTime"]; + } + + private int myOffset; + private TimeSpan myZeroHour; + private ViewStreamDatabase myDB; + } + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs new file mode 100644 index 000000000..226408212 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Collections; + +namespace BCG +{ + public class LatencyStream : ViewStream, IStreamMapper + { + private ViewStream _src; + private string _time_stamp_field; + private IBlueTableSchema _schema; + private string _latency_field; + + public LatencyStream(ViewStream input, string time_stamp_field, string latency_field) + { + _src = input; + _time_stamp_field = time_stamp_field; + _latency_field = latency_field; + mySetRealSource(new MapStream(input, this)); + } + + #region IStreamMapper Members + public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) + { + List cols = new List(inputSchema.columnInfo); + cols.Add(new ColumnInfo(_latency_field, cols.Count, typeof(double))); + _schema = new BlueTableSchema(cols.ToArray(), "latency"); + return _schema; + } + + #endregion + + #region IDynamicStreamMapper Members + + public IBlueRow map(IBlueRow row) + { + var ht = new Hashtable(row); + if (row[_time_stamp_field].Equals(DBNull.Value)) + ht[_latency_field] = DBNull.Value; + else + ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds; + return new BlueRow(_schema, ht); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs new file mode 100644 index 000000000..1dcb3d2e3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs @@ -0,0 +1,47 @@ +using System; + +namespace BCG +{ + public class MSecondStream : TimeUnitStream + { + public MSecondStream (int intervalInms, int size) + : base (DateTime.Now, new TimeSpan (0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) + {} + + protected override DateTime myRoundTime (DateTime dt) {return dt;} + } + + public class DisplayableMillisecondStream : ViewStream, IStreamMapper + { + private IBlueTableSchema mySchema = new BlueTableSchema ("ms", + new ColumnInfo ("id", 0, typeof(int)), + new ColumnInfo ("time", 1, typeof(string))); + public DisplayableMillisecondStream (int intervalInMs, int sz) + { + mySetRealSource (new MapStream (new MSecondStream (intervalInMs, sz), this)); + } + #region IStreamMapper Members + + public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) + { + return mySchema; + } + + #endregion + + #region IDynamicStreamMapper Members + + public IBlueRow map (IBlueRow row) + { + return new BlueRow (mySchema, row["id"], myToString ((DateTime)row["time"])); + } + + #endregion + + private string myToString (DateTime dt) + { + string sep = ":"; + return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond; + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/SecondStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/SecondStream.cs new file mode 100644 index 000000000..74881651e --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Time/SecondStream.cs @@ -0,0 +1,40 @@ +using System; + +namespace BCG +{ + public class SecondStream : TimeUnitStream + { + public SecondStream (DateTime zeroTime, int intervalInSeconds, int sizeOfStream, ViewStreamDatabase db) : + base (zeroTime, new TimeSpan (0,0,0,intervalInSeconds,0), sizeOfStream, db) + {} + + public SecondStream (DateTime zeroTime, int intervalInSeconds, int sizeOfStream) : + this (zeroTime, intervalInSeconds, sizeOfStream, ViewStreamDatabase.instance) + {} + + public SecondStream (int intervalInSeconds, int size) + : this (DateTime.Now, intervalInSeconds, size) + {} + + public SecondStream (DateTime zeroTime, int sizeOfStream, ViewStreamDatabase db) : + this (zeroTime, 1, sizeOfStream, db) + {} + + public SecondStream (DateTime zeroTime, int sizeOfStream) : + this (zeroTime, 1, sizeOfStream) + {} + + public SecondStream (int sizeOfStream) : + this (DateTime.Now, 1, sizeOfStream) + {} + + + + protected override DateTime myRoundTime (DateTime dt) + { + //return dt.Subtract (new TimeSpan (0,0,0,0,dt.Millisecond)); + return new DateTime (dt.Year, dt.Month, dt.Day, dt.Hour, dt.Minute, dt.Second); + } + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/TickStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/TickStream.cs new file mode 100644 index 000000000..d16f4d799 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Time/TickStream.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Text; + +using BCG.Utilities.General; + +namespace BCG +{ + [DataStreamUserRespresentation("tick")] + public class TickStream : ViewStream, IWorker + { + private DateTime myLastFireTime = DateTime.Now; + private ViewStreamDatabase myVsd; + private TimeSpan myPeriod; + private BlueTable myTable = new BlueTable(new BlueTableSchema("Ticks", + new ColumnInfo("id", 0, typeof(int)), + new ColumnInfo("ticks", 1, typeof(int)))); + + public TickStream() + : this(ViewStreamDatabase.instance, 0) + { } + public TickStream(int periodMs) + : this(ViewStreamDatabase.instance, periodMs) + { } + + public TickStream(ViewStreamDatabase vsd, int periodMs) + { + myVsd = vsd; + myPeriod = new TimeSpan(0, 0, 0, 0, periodMs); + + } + + protected override void myOnFirstSubscriber() + { + myVsd.addWorker(this); + } + + protected override void myOnLastSubscriber() + { + myVsd.removeWorker(this); + } + + #region IWorker Members + + public void doWork() + { + if ( DateTime.Now - myLastFireTime > myPeriod) + { + if (myLastValue == null) + { + + myTable.addRow(new BlueRow(myTable.schema, 0, System.Environment.TickCount)); + myFireEvent(new DataPacket(myTable, myTable, new BlueTable(myTable.schema))); + } + else + { + + BlueTable addedRows = new BlueTable(myTable.schema); + addedRows.addRow(new BlueRow(myTable.schema, 0, System.Environment.TickCount)); + + BlueTable deletedRows = myTable; + + myTable = addedRows; + myFireEvent(new DataPacket(myTable, myTable, deletedRows)); + + } + myLastFireTime = DateTime.Now; + } + } + + #endregion + + + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs new file mode 100644 index 000000000..25d37df0d --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs @@ -0,0 +1,99 @@ +using System; +using BCG.Utilities.General; + +namespace BCG +{ + public abstract class TimeUnitStream : ViewStream, IWorker + { + protected TimeSpan myOffset; + protected DateTime myConstructionTime; + protected int mySizeOfStream; + protected DateTime myZeroTime; + protected CircularRowCollection myCollection; + protected ViewStreamDatabase myDb; + protected int myId = 0; + + public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) + { + myConstructionTime = DateTime.Now; + myZeroTime = zeroTime; + myOffset = offSet; + mySizeOfStream = sizeOfStream; + myDb = db; + myCollection = new CircularRowCollection ( + new BlueTableSchema ("Time", + new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))), + mySizeOfStream); + } + + #region ViewStream overriden methods + + protected override void myOnFirstSubscriber() + { + myDb.addWorker (this); + myInit (); + } + + protected override void myOnLastSubscriber() + { + myDb.removeWorker (this); + } + + #endregion + + protected virtual void myInit () + { + int periods = (int) + (DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds; + + + DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) : + myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods)); + + + if (myGetLatestRow() == null || + !myGetLatestRow()["time"].Equals(dt)) + { + IBlueRow ar = new BlueRow (myCollection.schema, + myId++, dt); + IBlueRow dr = myCollection.addRowWithReturn (ar); + myFireEvent (new DataPacket (myCollection, + myConvertToCollection (ar), + myConvertToCollection (dr))); + } + } + + #region IWorker methods + + public void doWork () + { + myInit (); + } + + #endregion + + protected abstract DateTime myRoundTime (DateTime dt); + + + #region Private Helper methods + + private IBlueRow myGetLatestRow () + { + return myCollection.Count == 0 ? + null : + BlueDataUtility.extractRow (myCollection, myCollection.Count - 1); + } + + private IBlueRowCollection myConvertToCollection (IBlueRow r) + { + BlueTable retVal = new BlueTable (myCollection.schema); + if (r != null) + retVal.addRow (r); + return retVal; + } + + #endregion + + } + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs b/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs new file mode 100644 index 000000000..27ef3313e --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class TimeAverageStream : ViewStream + { + + private IDictionary> myKeyToTimeToValue; + private ViewStream myToAverage; + private string myCol_Key, myCol_Time, myCol_Value; + private double myAveragePeriodSeconds; + public TimeAverageStream(ViewStream toAverage, string key, string timeColumn, string valueColumn, double averagePeriodSeconds) + { + myCol_Key = key; + myCol_Time = timeColumn; + myCol_Value = valueColumn; + myAveragePeriodSeconds = averagePeriodSeconds; + myToAverage = toAverage; + + } + protected override void myOnFirstSubscriber() + { + myKeyToTimeToValue = new Dictionary>(); + myToAverage.onChange += new ChangeDelegate(myToAverage_onChange); + } + + + protected override void myOnLastSubscriber() + { + myToAverage.onChange -= myToAverage_onChange; + } + + private void myToAverage_onChange(object newValue) + { + DataPacket dp = (DataPacket) (newValue); + + int valueCol = dp.schema[myCol_Value].colNo; + + BlueTable result = new BlueTable(dp.schema); + foreach (IBlueRow r in dp.newRows) + { + string key = r[myCol_Key].ToString(); + DateTime dt = (DateTime) r[myCol_Time]; + double value = Convert.ToDouble(r[myCol_Value]); + + object[] rowValues = new object[r.Count]; + r.CopyTo(rowValues, 0); + rowValues[valueCol] = myUpdateValue(key, dt, value); + + result.addRow(new BlueRow(dp.schema, rowValues)); + } + + if (myLastValue == null) + myFireEvent(new DataPacket(result, result, new BlueTable(dp.schema))); + else + { + IBlueRowCollection dels, adds; + + BlueDataUtility.computeDifferences(result, ((DataPacket)myLastValue).newRows, out dels, out adds); + + myFireEvent(new DataPacket(result, adds, dels)); + } + } + + private double myUpdateValue(string key, DateTime time, double value) + { + + SortedList seriesValues; + if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues)) + { + seriesValues = new SortedList(); + myKeyToTimeToValue[key] = seriesValues; + } + + + seriesValues[time] = value; + + + while (seriesValues.Count != 0 && (seriesValues.Last().Key - seriesValues.First().Key).TotalSeconds > myAveragePeriodSeconds) + seriesValues.RemoveAt(0); + + return seriesValues.Sum((kvp) => { return kvp.Value; }) / seriesValues.Count; + + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs b/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs new file mode 100644 index 000000000..10b68a4ad --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs @@ -0,0 +1,49 @@ +// using System; + +// namespace BCG +// { +// public class TopStream : MethodDrivenStream +// { +// private int myTopCount; + +// public TopStream (ViewStream stream, int topCount) : base (stream) {myTopCount = topCount;} + +// public TopStream (ViewStream stream, +// int topCount, +// string sortExpression) +// : +// this (sortExpression == null? +// stream : +// new SortStream (stream, sortExpression), topCount) +// {} + +// protected override void myOnTableClear () +// { +// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; +// myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema); +// myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema); +// } + +// protected override void myHandleFirstCall () +// { +// myLastFiredPacket = new DataPacket (); +// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount); +// myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows); +// myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema); +// } + +// protected override void myHandleIncrementalAdd () {} + +// protected override void myHandleIncrementalDelete() +// { +// IBlueRowCollection oldRows = myLastFiredPacket.newRows; +// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount); +// BlueDataUtility.computeDifferences (myLastFiredPacket.newRows, +// oldRows, +// myLastFiredPacket.schema.columnNames, +// out myLastFiredPacket.deletedRows, +// out myLastFiredPacket.addedRows); +// myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true; +// } +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs new file mode 100644 index 000000000..032d89ebf --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs @@ -0,0 +1,45 @@ + +using System; +using System.Collections; + +namespace BCG { + + public class AlphabetizeColumnsStream : ViewStream { + private ViewStream mySrc; + private IBlueTableSchema mySchema = null; + + public AlphabetizeColumnsStream (ViewStream src) { + mySrc = src; + } + protected override void myOnFirstSubscriber() + { + mySrc.onChange += new ChangeDelegate(mySrc_onChange); + } + protected override void myOnLastSubscriber () + { + mySrc.onChange -= new ChangeDelegate (mySrc_onChange); + } + private void mySrc_onChange (object newValue) { + DataPacket dp = (DataPacket)newValue; + if (mySchema == null) { + SortedList alphabetical = new SortedList (); + foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo) + alphabetical.Add (cinfo.colName,cinfo); + ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count]; + alphabetical.Values.CopyTo (cinfos,0); + for (int i = 0; i < cinfos.Length; ++i) + cinfos[i] = new ColumnInfo (cinfos[i].colName,i,cinfos[i].colType,cinfos[i].colValue); + mySchema = new BlueTableSchema (cinfos,dp.schema.tableName); + } + BlueTable + deleted = new BlueTable (mySchema), + added = new BlueTable (mySchema), + newrows = new BlueTable (mySchema); + deleted.addRows (dp.deletedRows); + added.addRows (dp.addedRows); + newrows.addRows (dp.newRows); + + myFireEvent (new DataPacket (newrows,added,deleted)); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs new file mode 100644 index 000000000..86e782c7a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Represents the AppenedTable. Has 2 hashtables, for ids of the left and + /// right tables pointing to the result rows. + /// + internal class AppendCollection : IBlueRowCollection + { + private int myId = 0; + private IDictionary myLeftIndex = new Hashtable (), + myRightIndex = new Hashtable (); + private IBlueTableSchema mySchema; + + public AppendCollection () {} + + public DataPacket mergeRows (DataPacket dp, bool isLeft) + { + mySchema = mySchema == null ? dp.schema : mySchema; + BlueTable dRows = new BlueTable (mySchema), + aRows = new BlueTable (mySchema); + IDictionary index = isLeft ? myLeftIndex : myRightIndex; + foreach (IBlueRow r in dp.deletedRows) + { + IBlueRow dr = index[r["id"]] as IBlueRow; + dRows.addRow (dr); + index.Remove (r["id"]); + } + foreach (IBlueRow r in dp.addedRows) + { + IBlueRow ar = new AppendRow (r, myId++, mySchema); + aRows.addRow (ar); + index.Add (r["id"], ar); + } + return new DataPacket (this, aRows, dRows); + } + + #region IBlueRowCollection Members + + public IBlueRow this[int id] {get {throw new NotImplementedException ("need not be implemented");}} + + public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");} + + public IBlueTableSchema schema {get {return mySchema;}} + + #endregion + + #region ICollection Members + + public bool IsSynchronized {get {return false;}} + + public int Count {get {return myLeftIndex.Count + myRightIndex.Count;}} + + public void CopyTo (Array array, int index) + { + int i = index; + foreach (IBlueRow r in this) + array.SetValue (r, i++); + } + + public object SyncRoot {get {return null;}} + + #endregion + + #region IEnumerable Members + + public System.Collections.IEnumerator GetEnumerator() + { + return new MyEnumerator (myLeftIndex, myRightIndex); + } + + private class MyEnumerator : IEnumerator + { + private bool myIsLeft = true; + private IEnumerator myEnum; + private IDictionary myLeft, myRight; + #region IEnumerator Members + + public MyEnumerator (IDictionary left, IDictionary right) + { + myLeft = left; + myRight = right; + myEnum = myLeft.Values.GetEnumerator (); + } + + public void Reset() {myEnum = myLeft.GetEnumerator ();} + + public object Current {get {return myEnum.Current;}} + + public bool MoveNext() + { + bool retVal = myEnum.MoveNext (); + if (!retVal && myIsLeft) + { + myIsLeft = false; + retVal = (myEnum = myRight.Values.GetEnumerator()).MoveNext (); + } + return retVal; + } + #endregion + } + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendRow.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendRow.cs new file mode 100644 index 000000000..c7322bfb1 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendRow.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Serves as the row for append operations. Has 2 pointers and one integer. + /// A pointer to the schema is mantained, as the 2 sides of unions can have + /// different columns ordering. + /// + internal class AppendRow : IBlueRow + { + private IBlueRow myRow; + private int myId; + private IBlueTableSchema mySchema; + + public AppendRow (IBlueRow row, int id, IBlueTableSchema schema) + { + myRow = row; + myId = id; + mySchema = schema; + } + + #region IBlueRow Members + + public object this [string colName] + { + get {return colName.Equals("id") ? myId : myRow[colName];} + } + + public object this [int index] + { + get {return this[schema[index].colName];} + } + + public IBlueTableSchema schema {get {return mySchema;}} + + #endregion + + #region IDictionary Members + + public bool IsReadOnly {get {return true;}} + + public System.Collections.IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator (this);} + + object System.Collections.IDictionary.this[object key] + { + get {return key is string ? this[key as string] : this [(int)key];} + set {throw new ApplicationException ("IBlueRow is readonly");} + } + + public void Remove (object key) {throw new ApplicationException ("IBlueRow is readonly");} + + public bool Contains (object key) {return mySchema [key as string] != null;} + + public void Clear () {throw new ApplicationException ("IBlueRow is readonly");} + + public System.Collections.ICollection Values + { + get + { + object[] retVal = new object[Count]; + CopyTo (retVal, 0); + return retVal; + } + } + + public void Add (object key, object value) {throw new ApplicationException ("IBlueRow is readonly");} + + public System.Collections.ICollection Keys {get {return schema.columnNames;}} + + public bool IsFixedSize {get {return true;}} + + #endregion + + #region ICollection Members + + public bool IsSynchronized {get {return myRow.IsSynchronized;}} + + public int Count {get {return myRow.Count;}} + + public void CopyTo (Array array, int index) + { + int i = index; + foreach (object o in this as IEnumerable) + array.SetValue (o, i++); + } + + public object SyncRoot {get {return myRow.SyncRoot;}} + + #endregion + + #region IEnumerable Members + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + {return new BlueRowEnumerator (this);} + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs new file mode 100644 index 000000000..50a2e7339 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs @@ -0,0 +1,118 @@ +// using System; +// using System.Data; +// using System.Collections; +// using BCG.Utilities; +// using BCG.Utilities.General; +// namespace BCG +// { +// /// +// /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id. +// /// The column ordering the 2 tables need not be the same +// /// +// public class AppendStream : UnionStream +// { +// private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend; +// public AppendStream(ViewStream v1, ViewStream v2) +// { +// if (myUseNewAppend) +// mySetRealSource(new MutableAppendStream(v1, v2)); +// else +// { +// myLeftVs = v1; +// myRightVs = v2; +// } +// } + +// public AppendStream (params ViewStream[] vss) +// { + +// if (myUseNewAppend) +// mySetRealSource(new MutableAppendStream(vss)); +// else +// { +// if (vss.Length == 1) +// mySetRealSource(vss[0]); +// else if (vss.Length > 1) +// { +// AppendStream rv = null; +// foreach (ViewStream s in vss) +// rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s); +// mySetRealSource(rv); +// } +// else +// //don't do anything if they didnt' supply any viewstreams +// mySetRealSource(null); +// } + +// } + +// public override void add (IBlueRowCollection rows, bool isLeft) +// { +// IDictionary index = isLeft ? myLeftIndex : myRightIndex; +// int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap; +// foreach (IBlueRow row in rows) +// { +// UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, +// myIdCounter ++, myIdIndex, indexToIndexMap); +// ((BlueTable)myLastFiredPacket.newRows).addRow (newRow); +// index[row["id"]] = newRow; +// ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow); +// } +// } + +// public override void delete (IBlueRowCollection rows, bool isLeft) +// { +// IDictionary index = isLeft ? myLeftIndex : myRightIndex; +// foreach (IBlueRow axedRow in rows) +// { +// int rowId = (int)axedRow["id"]; +// IBlueRow row = (IBlueRow)index[rowId]; +// index.Remove (rowId); +// ((BlueTable)myLastFiredPacket.newRows).deleteRow (row); +// ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row); +// } +// } + +// public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft) +// { +// IDictionary temp = new Hashtable (); +// IDictionary index = isLeft ? myLeftIndex : myRightIndex; +// int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap; +// IBlueRow corrRow; +// foreach (IBlueRow dRow in dels) +// { +// corrRow = index[dRow["id"]] as IBlueRow; +// (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow); +// (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow); +// index.Remove (dRow["id"]); +// if (adds.containsID ((int)dRow["id"])) +// { +// IBlueRow r = adds[(int)dRow["id"]]; +// corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap); +// (myLastFiredPacket.newRows as BlueTable).addRow (corrRow); +// (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow); +// index.Add (r["id"], corrRow); +// temp.Add (r["id"], true); +// } +// } +// foreach (IBlueRow aRow in adds) +// { +// if (!temp.Contains (aRow["id"])) +// { +// corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap); +// (myLastFiredPacket.newRows as BlueTable).addRow (corrRow); +// (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow); +// index.Add (aRow["id"], corrRow); +// } +// } +// } +// } + +// public class MergeStream : ViewStream +// { +// public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols) +// { +// mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols)); +// } +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs new file mode 100644 index 000000000..c10b8a32a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs @@ -0,0 +1,60 @@ +using System; + +namespace BCG +{ + public class AppendStreamModified : ViewStream + { + private AppendCollection myColl = new AppendCollection (); + private ViewStream myLeftStream, myRightstream; + + public AppendStreamModified (ViewStream leftStream, ViewStream rightStream) + { + myLeftStream = leftStream; + myRightstream = rightStream; + } + + public AppendStreamModified (ViewStream[] vss) + { + if (vss.Length == 1) + mySetRealSource (vss[0]); + else + { + AppendStreamModified rv = null; + foreach (ViewStream s in vss) + rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s); + mySetRealSource (rv); + } + } + + protected override void myOnFirstSubscriber() + { + myLeftStream.onChange += new ChangeDelegate(myHandleLeftChanges); + myRightstream.onChange += new ChangeDelegate(myHandleRightChanges); + } + + protected override void myOnLastSubscriber() + { + myLeftStream.onChange -= new ChangeDelegate(myHandleLeftChanges); + myRightstream.onChange -= new ChangeDelegate(myHandleRightChanges); + } + + private void myHandleLeftChanges (object newValue) + { + myFireEvent (myGetDataPacket (newValue as DataPacket, true)); + } + + private void myHandleRightChanges (object newValue) + { + myFireEvent (myGetDataPacket (newValue as DataPacket, false)); + } + + private DataPacket myGetDataPacket (DataPacket dp, bool isLeft) + { + DataPacket retVal = myColl.mergeRows (dp, isLeft); + return new DataPacket (new ImmutableBlueTable (retVal.newRows), + retVal.addedRows, + retVal.deletedRows); + + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/IUnionTable.cs b/src/DataStreamSuite/DataStream/Primitives/Union/IUnionTable.cs new file mode 100644 index 000000000..7c063848a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/IUnionTable.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Interface to hide the 2 implementaions of UnionTables, with and without + /// repititions + /// + public interface IUnionTable : IBlueRowCollection + { + void addRows (IBlueRowCollection rows, bool isLeft, DataPacket dp); + void deleteRows (IBlueRowCollection rows, bool isLeft, DataPacket dp); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs new file mode 100644 index 000000000..a9c679588 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs @@ -0,0 +1,293 @@ +// using System; +// using System.Collections.Generic; +// using System.Text; +// using mbts; + +// namespace BCG +// { +// internal class MutableAppendTable : IBlueRowCollection +// { +// private class MyRow : IBlueRow +// { +// private IBlueRow myRow; +// private int myID; + +// public MyRow(IBlueRow r, int id) +// { +// myRow = r; +// myID = id; +// } + +// #region IBlueRow Members + +// public IBlueTableSchema schema { get { return myRow.schema; } } + +// public object this[int index] +// { +// get +// { +// if (myRow.schema[index].colName == "id") +// return myID; +// else +// return myRow[index]; +// } +// } + +// public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } } + +// #endregion + +// #region IDictionary Members + +// public void Add(object key, object value) { throw new Exception("Row is read only."); } + +// public void Clear() { throw new Exception("Row is read only."); } + +// public bool Contains(object key) { return myRow.Contains(key); } + +// public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + +// public bool IsFixedSize { get { return true; } } + +// public bool IsReadOnly { get { return true; } } + +// public System.Collections.ICollection Keys { get { return myRow.Keys; } } + +// public void Remove(object key) {throw new Exception("The method or operation is not implemented.");} + +// public System.Collections.ICollection Values +// { +// get +// { +// object[] vals = new object[myRow.Count]; +// CopyTo(vals, 0); +// return vals; +// } +// } + +// public object this[object key] +// { +// get { return key is string ? this[key as string] : this[(int)key]; } +// set { throw new Exception("The method or operation is not implemented."); } +// } + +// #endregion + +// #region ICollection Members + +// public void CopyTo(Array array, int index) +// { +// myRow.CopyTo(array, index); +// array.SetValue(myID, index + myRow.schema["id"].colNo); +// } + +// public int Count { get { return myRow.Count; } } + +// public bool IsSynchronized { get { return myRow.IsSynchronized; } } + +// public object SyncRoot { get { return myRow.SyncRoot; } } + +// #endregion + +// #region IEnumerable Members + +// System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() +// { return new BlueRowEnumerator(this); } + +// #endregion +// } + +// private BlueTable myRows; +// private int myID = 0; +// /// +// /// maps the table no to a dictionary containing the +// /// local IDs to a row in the final big table +// /// +// private Dictionary> myIDMap = new Dictionary>(); + +// public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); } + +// public IBlueRow addRow(IBlueRow r, int tableNo) +// { +// if (!myIDMap.ContainsKey(tableNo)) +// myIDMap.Add(tableNo, new Dictionary()); +// IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id"); +// myRows.addRow(nr); +// myIDMap[tableNo].Add((int)r["id"], nr); +// return nr; +// } + +// public IBlueRow removeRow(IBlueRow r, int tableNo) +// { +// IBlueRow dr = myIDMap[tableNo][(int)r["id"]]; +// myRows.deleteRow(dr); +// myIDMap[tableNo].Remove((int)r["id"]); +// return dr; +// } + +// public IBlueRowCollection removeTable(int tableNo) +// { +// if (!myIDMap.ContainsKey(tableNo)) +// return null; +// else +// { +// BlueTable rv = new BlueTable(myRows.schema); +// foreach (IBlueRow r in myIDMap[tableNo].Values) +// rv.addRow(r); +// myRows.deleteRows(rv); +// myIDMap.Remove(tableNo); +// return rv; +// } +// } + +// public void clear() +// { +// myRows.clear(); +// myIDMap.Clear(); +// } + +// #region IBlueRowCollection Members + +// public IBlueRow this[int id] { get { return myRows[id]; } } + +// public IBlueTableSchema schema { get { return myRows.schema; } } + +// public bool containsID(int id) {return myRows.containsID(id);} + +// #endregion + +// #region ICollection Members + +// public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); } + +// public int Count { get { return myRows.Count; } } + +// public bool IsSynchronized { get { return myRows.IsSynchronized; } } + +// public object SyncRoot { get { return myRows.SyncRoot; } } + +// #endregion + +// #region IEnumerable Members + +// public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();} + +// #endregion +// } +// /// +// /// MutableStream can append multiple streams +// /// as well as give the user the flexibility +// /// to append (or remove) more streams at +// /// runtime +// /// +// public class MutableAppendStream : ViewStream +// { +// HighPerformanceTimer _timer = new HighPerformanceTimer(); +// private bool mySubscriptionActive = false; +// private MutableAppendTable myTable; +// private int myMovingIndex; +// private Dictionary> myStreamIndex = new Dictionary>(); +// private bool _allign_col_orders = true; + +// public MutableAppendStream(params ViewStream[] streams) +// : this(new List(streams)) +// { } + +// public MutableAppendStream(bool allign_col_order, params ViewStream[] streams) +// : this(allign_col_order, new List(streams)) +// { } + +// public MutableAppendStream(ICollection streams) +// : this(true, streams) +// { } + +// public MutableAppendStream(bool allign_col_orders, ICollection streams) +// { +// _allign_col_orders = allign_col_orders; +// foreach (ViewStream vs in streams) +// { +// myMovingIndex++; +// ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString()); +// if (_allign_col_orders) +// pj = new ProjectStream(pj, "!"); +// myStreamIndex.Add(myMovingIndex, new Pair(vs, pj)); +// } +// } + +// public void addStream(ViewStream vs) +// { +// myMovingIndex++; +// ///the change name is done to identify the callbacks, and the project to +// ///make all the schemas col orders the same +// ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString()); +// if (_allign_col_orders) +// pj = new ProjectStream(pj, "!"); +// myStreamIndex.Add(myMovingIndex, new Pair(vs, pj)); +// if (mySubscriptionActive) +// pj.onChange += new ChangeDelegate(myHandleStreamChanges); +// } + +// public void removeStream(ViewStream vs) +// { +// int index = -1; +// foreach (KeyValuePair> p in myStreamIndex) +// { +// if (p.Value.first == vs) +// { +// index = p.Key; +// if (mySubscriptionActive) +// { +// p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges); +// if (myTable != null) +// { +// IBlueRowCollection delRows = myTable.removeTable(p.Key); +// if (delRows != null) +// myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows)); +// } +// } +// break; +// } +// if (index > 0) +// myStreamIndex.Remove(index); +// } +// } + +// protected override void myOnFirstSubscriber() +// { +// mySubscriptionActive = true; +// foreach (KeyValuePair> p in myStreamIndex) +// p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges); +// } + +// protected override void myOnLastSubscriber() +// { +// mySubscriptionActive = false; +// foreach (KeyValuePair> p in myStreamIndex) +// p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges); +// if (myTable != null) +// myTable.clear(); +// } + +// private void myHandleStreamChanges(object newValue) +// { +// DataPacket dp = newValue as DataPacket; +// _timer.start(); +// myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo)); +// BlueTable adds = new BlueTable(myTable.schema), +// dels = new BlueTable(myTable.schema); +// int index = Int32.Parse(dp.schema.tableName); +// foreach (IBlueRow r in dp.deletedRows) +// { +// IBlueRow dr = myTable.removeRow(r, index); +// dels.addRow(dr); +// } +// foreach (IBlueRow r in dp.addedRows) +// { +// IBlueRow nr = myTable.addRow(r, index); +// adds.addRow(nr); +// } +// _timer.stop(); +// myFireEvent(new DataPacket(myTable, adds, dels)); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/RenameStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/RenameStream.cs new file mode 100644 index 000000000..cbf5e51a1 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/RenameStream.cs @@ -0,0 +1,38 @@ + +using System; + +namespace BCG { + + public class RenameStream : ViewStream { + private ViewStream mySrc; + private string myTableName; + private IBlueTableSchema mySchema = null; + + public RenameStream (ViewStream src,string tablename) { + mySrc = src; + myTableName = tablename; + } + protected override void myOnFirstSubscriber() + { + mySrc.onChange += new ChangeDelegate(mySrc_onChange); + } + protected override void myOnLastSubscriber () + { + mySrc.onChange -= new ChangeDelegate (mySrc_onChange); + } + private void mySrc_onChange (object newValue) { + DataPacket dp = (DataPacket)newValue; + if (mySchema == null) + mySchema = new BlueTableSchema (dp.newRows.schema.columnInfo,myTableName); + BlueTable + deleted = new BlueTable (mySchema), + added = new BlueTable (mySchema), + newrows = new BlueTable (mySchema); + deleted.addRows (dp.deletedRows); + added.addRows (dp.addedRows); + newrows.addRows (dp.newRows); + + myFireEvent (new DataPacket (newrows,added,deleted)); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/SuperUnionStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/SuperUnionStream.cs new file mode 100644 index 000000000..fc001f7ae --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/SuperUnionStream.cs @@ -0,0 +1,170 @@ +// using System; +// using System.Diagnostics; +// using System.Collections; +// using BCG.Utilities; +// using BCG.Utilities.General; +// using BCG.Evaluator.AddColumnsExpression; + +// namespace BCG +// { +// public class SuperUnionStream : ViewStream +// { +// private bool myFirstCalled, myLastCalled; +// private Hashtable myViewStreams = new Hashtable(); +// private Hashtable myIDMap = new Hashtable(); +// private BlueTable myData; +// private string myName; +// private int myNextID = 0; +// private ICollection myOrigStreams; +// public SuperUnionStream(string endTableName, string maxSchemaColumnDefinition, +// params ViewStream[] streams) +// { +// myName = endTableName; +// syntax parser = new syntax(); +// AddColumnsExpression ace = +// (AddColumnsExpression) parser.Parse(maxSchemaColumnDefinition); + +// BlueEventLog.logAssert(ace != null,"Invalid columnn expression: " + maxSchemaColumnDefinition); +// IDictionary cols = ace.columns; +// string[] orderedCols = myGetOrderedColumns(maxSchemaColumnDefinition); +// ColumnInfo[] info = new ColumnInfo[cols.Count]; +// for(int i = 0; i < orderedCols.Length; i++) +// { +// AddColumnsExpression.ConstantInfo ci = (AddColumnsExpression.ConstantInfo)cols[orderedCols[i]]; +// if(ci.v == DBNull.Value || ci.v == null) +// info[i] = new ColumnInfo(orderedCols[i], i, ci.t); +// else +// info[i] = new ColumnInfo(orderedCols[i], i, ci.t, ci.v); +// } +// myData = new BlueTable(new BlueTableSchema(endTableName, info)); +// myUniquifyStreams(streams); +// } +// private void myUniquifyStreams(ViewStream[] streams) +// { +// Hashtable ht = new Hashtable(); +// int i = 0; +// foreach(ViewStream str in streams) +// { +// string name = str.GetType().Name; +// if(str.rows != null) +// name = str.rows.schema.tableName; +// if(!ht.Contains(name)) +// ht[name] = str; +// else +// { +// string newname = name + "_" + (i++).ToString(); +// ht[newname] = new RenameStream(str, newname); +// } +// } +// myOrigStreams = ht.Values; +// } +// private string[] myGetOrderedColumns(string schema) +// { +// string[] s = schema.Split('/', ' '); +// string[] res = new string[s.Length / 2]; +// for(int i = 0; i < s.Length; i++) +// if(i % 2 == 0) +// res[(int)i/2] = s[i]; +// return res; +// } +// public void add(string key, ViewStream value) +// { +// myViewStreams[key.ToLower()] = value; +// value.onChange += new ChangeDelegate(myStreamChanged); +// } +// public void remove(string key) +// { +// key = key.ToLower(); +// ViewStream vs = (ViewStream)myViewStreams[key]; +// if(vs == null) +// key = "ds_" + key; +// vs = (ViewStream)myViewStreams[key]; +// if(vs != null) +// { +// myViewStreams.Remove(key); +// IBlueRowCollection removed = myRemoveData(vs.rows); +// vs.onChange -= new ChangeDelegate(myStreamChanged); +// if(removed.Count > 0) +// myFireEvent(new DataPacket(myData, new BlueTable(myData.schema), removed)); +// } +// } +// private void myStreamChanged(object newValue) +// { +// DataPacket pack = newValue as DataPacket; +// IBlueRowCollection rows = pack.newRows; +// IBlueRowCollection removed = myRemoveData(pack.deletedRows); +// IBlueRowCollection added = myAddData(pack.addedRows); +// myFireEvent(new DataPacket(myData, added, removed)); +// } +// private string myGetIDMapKey(IBlueTableSchema schema, IBlueRow row) +// { +// string s = schema.tableName + "_" + row["id"].ToString(); +// return s.ToLower(); +// } +// private IBlueRowCollection myAddData(IBlueRowCollection incoming) +// { +// string[] mainColNames = myData.schema.columnNames; +// BlueTable added = new BlueTable(myData.schema); +// foreach(IBlueRow row in incoming) +// { +// object[] values = new object[mainColNames.Length]; +// int newid = myNextID++; +// //map table,origid to newid +// myIDMap[myGetIDMapKey(incoming.schema, row)] = newid; +// int i = 0; +// foreach(string colname in mainColNames) +// { +// object value = DBNull.Value; +// if(colname == "id") +// value = newid; +// else if(row.Contains(colname)) +// value = row[colname]; +// values[i++] = value; +// } +// IBlueRow br = new BlueRow(myData.schema, values); +// added.addRow(br); +// myData.addRow(br); +// } +// return added; +// } +// private IBlueRowCollection myRemoveData(IBlueRowCollection removing) +// { +// BlueTable removed = new BlueTable(myData.schema); +// if(removing != null) +// { +// foreach(IBlueRow row in removing) +// { +// string key = myGetIDMapKey(removing.schema, row); +// object removeid = myIDMap[key]; +// BlueEventLog.logAssert(removeid != null); +// IBlueRow removingrow = (IBlueRow)myData[(int)removeid]; +// removed.addRow(removingrow); +// myData.deleteRow((int)removeid); +// myIDMap.Remove(key); +// } +// } +// return removed; +// } +// protected override void myOnFirstSubscriber() +// { +// BlueEventLog.logAssert(myFirstCalled ? myLastCalled : true, "My myOnFirstSubscriber called twice without one myOnLastSubscriber"); +// if(myOrigStreams != null) +// { +// int i = 0; +// foreach(ViewStream stream in myOrigStreams) +// add("constructorStream_" + (i++).ToString(), stream); +// } +// myFirstCalled = true; +// } +// protected override void myOnLastSubscriber() +// { +// BlueEventLog.logAssert(myLastCalled ? myFirstCalled : true, "My myOnLastSubscriber called without myOnFirstSubscriber ever being called"); + +// string[] keys = new string[myViewStreams.Keys.Count]; +// myViewStreams.Keys.CopyTo(keys, 0); +// foreach(string key in keys) +// remove(key); +// myLastCalled = true; +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs b/src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs new file mode 100644 index 000000000..08c0effb3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs @@ -0,0 +1,63 @@ +using System; + +namespace BCG +{ + /// + /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream + /// on receiving updates + /// + public class UnionOperand + { + private bool myIsLeft; + private DataPacket myLastArrivedPacket; + private UnionStream myMaster; + + public UnionOperand (bool isLeft, UnionStream master) + { + myIsLeft = isLeft; + myMaster = master; + } + + internal void myChangeHandler (object o) + { + if (myLastArrivedPacket != null) + mySubsequentChangeHandler ((DataPacket)o); + else + { + myLastArrivedPacket = (DataPacket)o; + myFirstChangeHandler ((DataPacket)o); + } + } + + private void myFirstChangeHandler(DataPacket dp) + { + myMaster.startSession (); + myMaster.add (dp.newRows, myIsLeft); + myMaster.endSession (); + } + + private void mySubsequentChangeHandler (DataPacket dp) + { + myMaster.startSession (); + //myMaster.delete (dp.deletedRows, myIsLeft); + //myMaster.add (dp.addedRows, myIsLeft); + myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft); + myMaster.endSession (); + } + + public IBlueTableSchema opSchema + { + get + { + if (myLastArrivedPacket == null || myLastArrivedPacket.newRows == null) + return null; + return myLastArrivedPacket.newRows.schema; + } + } + + public void init () + { + myLastArrivedPacket = null; + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/UnionRow.cs b/src/DataStreamSuite/DataStream/Primitives/Union/UnionRow.cs new file mode 100644 index 000000000..22fd96c2d --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/UnionRow.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Encapsulates an IBlueRow. The 2 main functionalities of the UnionRow are generating of ids, and accounting for + /// the different ordering of the 2 operand tables, t + /// + internal class UnionRow : IBlueRow + { + private IBlueRow myRow; + private int myId; + private int myIdIndex; + private IBlueTableSchema mySchema; + private int[] myIndexToIndexMap; + private object[] myCachedValues; + + #region Constructor(s) + /// + /// + /// + /// schema of the parent row + /// parent row + /// generated id + /// column number of the id column + /// mapping from one schema to another + public UnionRow (IBlueTableSchema schema, IBlueRow row, int id, int idIndex, int[] indexToIndexMap) + { + myRow = row; + myId = id; + mySchema = schema; + myIndexToIndexMap = indexToIndexMap; + myIdIndex = idIndex; + myCachedValues = new object[row.Count]; + } + #endregion + + #region IBlueRow members(s) + public IBlueTableSchema schema {get{return mySchema;}} + #endregion + + #region ICollection members + public int Count{get{return myRow.Count;}} + + public bool IsSynchronized {get{return false;}} + + public object SyncRoot{get{return this;}} + + public void CopyTo (Array array, int index) + { + int i = index; + foreach (object o in this as IEnumerable) + array.SetValue (o, i++); + } + #endregion + + #region IDictionary members + public bool IsFixedSize {get{return true;}} + + public bool IsReadOnly {get{return true;}} + + /// + /// The key can either be the string corrsponding to the column name or + /// the order of the column + /// + public object this [object key] + { + get + { + if (key is string) + return this[(string)key]; + return this[(int)key]; + } + set {throw new NotSupportedException ("The IBlueRow is readonly");} + } + + public object this [string colName] {get{return this[mySchema[colName].colNo];}} + + public object this [int index] + { + get + { + if (myCachedValues[index] == null) + myCachedValues[index] = index == myIdIndex ? myId : myRow[myIndexToIndexMap[index]]; + return myCachedValues[index]; + } + } + + public ICollection Keys{get{return mySchema.columnNames;}} + + public ICollection Values + { + get + { + object[] retVal = new object[mySchema.columnNames.Length]; + CopyTo (retVal, 0); + return retVal; + } + } + + public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + + public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} + + public bool Contains (object key){return mySchema[(string)key] != null;} + + /// + /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to + /// iterate through the values, typecast this into an IEnumerable + /// + /// + public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);} + + IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);} + + public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");} + #endregion + } + +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs new file mode 100644 index 000000000..5af486f8f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs @@ -0,0 +1,119 @@ +using System; +using System.Data; +using System.Collections; +using BCG.Utilities; +using BCG.Utilities.General; +namespace BCG +{ + /// + /// Provides a live stream of Union of 2 streams + /// + public abstract class UnionStream : ViewStream + { + protected UnionOperand myLeftOp, myRightOp; + protected ViewStream myLeftVs, myRightVs; + protected DataPacket myLastFiredPacket; + protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable (); + protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap; + protected int myIdCounter, myIdIndex; + + protected override void myOnFirstSubscriber() + { + myLeftOp = new UnionOperand (true, this); + myRightOp = new UnionOperand (false, this); + myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler); + myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler); + } + + protected override void myOnLastSubscriber () + { + myInit (); + myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler); + myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler); + } + + public void startSession () + { + if (myLastFiredPacket == null) + { + myLastFiredPacket = new DataPacket (); + if (myLeftOp.opSchema != null)//ie call back initiated by left operand + { + myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema); + myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema); + myIdIndex = myLeftOp.opSchema["id"].colNo; + } + else + { + myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema); + myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema); + myIdIndex = myRightOp.opSchema["id"].colNo; + } + } + else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null) + myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema); + else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null) + myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema); + myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema); + myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema); + } + + private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2) + { + //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length)); + //foreach (string colName in sch1.columnNames) + // BlueEventLog.logAssert (sch2[colName] != null); + if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length)) + throw new BlueDataException("Union stream: different schemas"); + foreach(string colName in sch1.columnNames) + { + if(sch2[colName] == null) + throw new BlueDataException("Union stream: can't find " + colName); + } + int[] retVal = new int[sch1.columnNames.Length]; + for (int i = 0; i < retVal.Length; i++) + retVal[i] = sch2[sch1[i].colName].colNo; + return retVal; + } + + public virtual void endSession () + { + myFireEvent (myLastFiredPacket.Clone ()); + } + + public abstract void add (IBlueRowCollection rows, bool isLeft); + + public abstract void delete (IBlueRowCollection rows, bool isLeft); + + public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft); + + private void myInit () + { + myLastFiredPacket = null; + myLeftOp.init (); + myRightOp.init (); + myLeftIndex.Clear (); + myRightIndex.Clear (); + myIdCounter = myIdIndex = 0; + myLeftIndexToIndexMap = myRightIndexToIndexMap = null; + } + + /// + /// Used for test purposes only. Users of this class specifically, should ignore it + /// (too bad, the NUnit cant see the private methods) + /// + /// + /// + internal void receiveStream (DataPacket dp, bool isLeft) + { + if (isLeft) + myLeftOp.myChangeHandler (dp); + else + myRightOp.myChangeHandler (dp); + } + public void receiveStreamTest(DataPacket dp, bool isLeft) + { + receiveStream(dp, isLeft); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs b/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs new file mode 100644 index 000000000..a776c6177 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs @@ -0,0 +1,71 @@ +using System; + +namespace BCG +{ + /// + /// Publishes the rows in the form of a circular buffer, ie if the buffer size is n + /// then the n'th row being puclished results in the deletion of the 0th row + /// + public class CircularPublisher + { + private CircularRowCollection myBuffer; + private PubTableBroker myBroker; + private int myBufferSize; + public CircularPublisher (int bufferSize) + { + myBufferSize = bufferSize; + } + + public CircularPublisher (int bufferSize, IBlueRowCollection rows) + : this (bufferSize) + { + publishRows (rows); + } + + public void publishRow (IBlueRow nr) + { + if (myBuffer == null) + { + myBuffer = new CircularRowCollection ( + new BlueTableSchema (nr.schema.tableName, nr.schema.columnInfo), + myBufferSize); + myBroker = new PubTableBroker (); + myBroker.createTable (myBuffer); + } + IBlueRow br = myBuffer.addRowWithReturn (nr); + myBroker.makeBatchChanges (myBuffer, + myReturnCollection (br), + myReturnCollection (nr)); + } + + public void publishRows (IBlueRowCollection nrs) + { + if (myBuffer == null) + { + myBuffer = new CircularRowCollection ( + new BlueTableSchema (BlueDataUtility.extractRow (nrs, 0).schema.tableName, + BlueDataUtility.extractRow (nrs, 0).schema.columnInfo), + myBufferSize); + myBroker = new PubTableBroker (); + myBroker.createTable (myBuffer); + } + BlueTable deletes = new BlueTable (myBuffer.schema); + foreach (IBlueRow nr in nrs) + { + IBlueRow dr = myBuffer.addRowWithReturn (nr); + if (dr != null) + deletes.addRow (dr); + } + myBroker.makeBatchChanges (myBuffer, deletes, nrs); + } + + private IBlueRowCollection myReturnCollection (IBlueRow r) + { + if (r == null) + return new BlueTable (myBuffer.schema); + BlueTable retVal = new BlueTable (r.schema); + retVal.addRow (r); + return retVal; + } + } +} diff --git a/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs b/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs new file mode 100644 index 000000000..2e17ccc25 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections; + +using BCG.Utilities; + +namespace BCG +{ + /// + /// This class will be used by the clients to publish data to the pubsub + /// + public class PubTableBroker + { + protected PublisherCore myPublisherCore; + protected IDictionary myPublisherMap = new Hashtable (); + protected IDictionary mySchemaMap = new Hashtable (); + + protected bool myWaitingForFirstOp = false; + + public PubTableBroker () + : this (new PublisherCore()) + {} + + public PubTableBroker (PublisherCore publisher) + {myPublisherCore = publisher;} + + public PubTableBroker (IBlueRowCollection[] tablesToPublish) + : this (new PublisherCore(), tablesToPublish) + {} + + public PubTableBroker (PublisherCore publisher, IBlueRowCollection[] tablesToPublish) + { + myPublisherCore = publisher; + + myPublisherCore.beginTransaction (); + foreach (IBlueRowCollection table in tablesToPublish) + createTable (table); + myPublisherCore.commitTransaction (); + } + + public void beginTransaction () + { + myWaitingForFirstOp = true; + } + + public void commitTransaction () + { + if (myWaitingForFirstOp) + myWaitingForFirstOp = false; + else + myPublisherCore.commitTransaction (); + } + + public void rollbackTransaction () + { + if (myWaitingForFirstOp) + myWaitingForFirstOp = false; + else + myPublisherCore.abortTransaction (); + } + + public virtual void createTable (IBlueRowCollection newTable) + { + myCheckTransaction(); + if (myPublisherMap [newTable.schema.tableName.ToLower()] == null) + { + myPublisherMap [newTable.schema.tableName.ToLower()] = myGetIPubTablePublisher (); + mySchemaMap [newTable.schema.tableName.ToLower()] = newTable.schema; + } + (myPublisherMap[newTable.schema.tableName.ToLower()] as IPubTablePublisher).createTable (newTable); + } + + /// + /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException + /// will be thrown + /// + /// + public virtual void addRow (IBlueRow r) + { + myCheckTransaction(); + if (myPublisherMap[r.schema.tableName.ToLower()] == null) + throw new ApplicationException ("Row can not be added, as the table that has not been created yet"); + else + (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).addRow (r); + } + + /// + /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException + /// will be thrown + /// + /// + public virtual void deleteRow (IBlueRow r) + { + myCheckTransaction(); + if (myPublisherMap[r.schema.tableName.ToLower()] == null) + throw new ApplicationException ("Row can not be removed, as the table has not been created yet"); + else + (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).deleteRow (r); + } + + /// + /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException + /// will be thrown + /// + /// + public virtual void changeRow (IBlueRow r) + { + myCheckTransaction(); + if (myPublisherMap[r.schema.tableName.ToLower()] == null) + throw new ApplicationException ("Row can not be changed, as the table has not been created yet"); + else + (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).changeRow (r); + } + + /// + /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException + /// will be thrown + /// + /// + public virtual void makeBatchChanges (IBlueRowCollection newRows, + IBlueRowCollection deletedRows, + IBlueRowCollection addedRows) + { + myCheckTransaction(); + if (myPublisherMap[newRows.schema.tableName.ToLower()] == null) + throw new ApplicationException ("Batch changes can not be made, as the table has not been created yet"); + else + (myPublisherMap [newRows.schema.tableName.ToLower()] as IPubTablePublisher).batchRemoveAdd (deletedRows, addedRows); + } + + public virtual void deleteTableRows (string tableName) + { + myCheckTransaction(); + (myPublisherMap [tableName.ToLower()] as IPubTablePublisher).createTable ( + new BlueTable (mySchemaMap[tableName.ToLower()] as IBlueTableSchema)); + } + + public int getNumberOfWaitingMessages() { return myPublisherCore.getNumberOfWaitingMessages(); } + + public void clearTable(string tableName) + {myPublisherCore.clearState(PubStreamPrefixer.prefix(tableName));} + + protected void myCheckTransaction() + { + if (myWaitingForFirstOp) + myPublisherCore.beginTransaction (); + myWaitingForFirstOp = false; + } + + protected virtual IPubTablePublisher myGetIPubTablePublisher () + { + return new PubTablePublisher (myPublisherCore); + } + + } + } diff --git a/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs b/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs new file mode 100644 index 000000000..9477b48b7 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections; +using System.Data; + +using BCG.Utilities; + +namespace BCG +{ + public class ViewPublisher : IDisposable + { + protected ViewPublisher () {} + + /// If an name IsNullOrEmpty then it uses the name + /// in the schema (be careful with that because this + /// could lead to double publishes if the schema name + /// is not uniquely chosen) + public ViewPublisher(string name, PubTableBroker br, ViewStream vs) + { + myName = name; + myView = new TableStream(name); // TODOds String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName); + myBroker = br; + myFirstTime = true; + myView.onChange += new DataStream.ChangeDelegate(myChangeHandler); + } + + public void Dispose() + { + myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler); + if (!String.IsNullOrEmpty(myName)) + myBroker.clearTable(myName); + } + + private void myChangeHandler(object o) + { + DataPacket dp = (DataPacket) o; + if (myFirstTime) + { + myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName; + myLastRowCount = dp.newRows.Count; + myBroker.createTable(dp.newRows); + myFirstTime = false; + } + else + { + if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) + { + myLastRowCount = dp.newRows.Count; + myBroker.makeBatchChanges(dp.newRows,dp.deletedRows, dp.addedRows); + } + } + } + + public int getLastRowPublishedCount() + {return myLastRowCount;} + + public bool hasPublished() + { return !myFirstTime; } + + private int myLastRowCount = -1; + private string myName; + private PubTableBroker myBroker; + protected ViewStream myView; + private bool myFirstTime; + } +} diff --git a/src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs b/src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs new file mode 100644 index 000000000..d38c15491 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs @@ -0,0 +1,92 @@ +using System; +using BCG.Utilities; + +namespace BCG +{ + public class ViewStreamPublisher + { + private IDynamicStreamMapper myMapper; + private PubTableBroker myBroker; + private IBlueRowCollection myOutputTable; + private IBlueRowCollection myInputTable; + + public ViewStreamPublisher (PubTableBroker broker) + { + myBroker = broker; + } + + public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper) + : this (broker) + { + myMapper = mapper; + } + + public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper, ViewStream vs) + : this (broker, mapper) + { + vs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChanges); + } + + private void myHandleChanges (object newValue) + { + DataPacket dp = newValue as DataPacket; + if (myMapper == null) + { + if (myOutputTable == null) + { + myBroker.createTable (dp.newRows); + } + else + { + myBroker.makeBatchChanges (dp.newRows, dp.deletedRows, dp.addedRows); + } + myOutputTable = dp.newRows; + } + else + { + if (myOutputTable != null) + { + BlueTable dRows = new BlueTable (myOutputTable.schema), + aRows = new BlueTable (myOutputTable.schema); + foreach (IBlueRow dRow in dp.deletedRows) + { + (myOutputTable as BlueTable).deleteRow ((int)dRow["id"]); + dRows.addRow (myMapper.map (dRow)); + } + foreach (IBlueRow aRow in dp.addedRows) + { + IBlueRow r = myMapper.map (aRow); + (myOutputTable as BlueTable).addRow (r); + aRows.addRow (r); + } + myBroker.makeBatchChanges (myOutputTable, dRows, aRows); + } + else + { + if (dp.newRows.Count > 0) + { + myOutputTable = new BlueTable ( + myMapper.map (BlueDataUtility.extractRow (dp.newRows, 0)).schema); + myHandleChanges (newValue); + } + } + } + } + + public void addRows (IBlueRowCollection aRows) + { + if (myInputTable != null) + (myInputTable as BlueTable).addRows (aRows); + else + myInputTable = new BlueTable (aRows); + myHandleChanges (new DataPacket(myInputTable, aRows, new BlueTable (aRows.schema))); + } + + public void deleteRows (IBlueRowCollection dRows) + { + // BlueEventLog.logAssert (myInputTable != null); + (myInputTable as BlueTable).deleteRows (dRows); + myHandleChanges (new DataPacket (myInputTable, new BlueTable (myInputTable.schema), dRows)); + } + } +} diff --git a/src/DataStreamSuite/DataStream/TableStream.cs b/src/DataStreamSuite/DataStream/TableStream.cs deleted file mode 100644 index 47098f1f3..000000000 --- a/src/DataStreamSuite/DataStream/TableStream.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace BCG { - public class TableStream : ViewStream { - public TableStream(string name) {} - } -} diff --git a/src/DataStreamSuite/DataStream/ViewStream.cs b/src/DataStreamSuite/DataStream/ViewStream.cs deleted file mode 100644 index 577496a84..000000000 --- a/src/DataStreamSuite/DataStream/ViewStream.cs +++ /dev/null @@ -1,4 +0,0 @@ -namespace BCG { - public class ViewStream { - } -} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataException.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataException.cs new file mode 100644 index 000000000..f76c93c7f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataException.cs @@ -0,0 +1,12 @@ +using System; + +namespace BCG +{ + /// + /// Exception in case of problems with BlueData formulation e.x. the schema not having an id column + /// + public class BlueDataException : ApplicationException + { + public BlueDataException (string message) : base (message) {} + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataPropertyDescriptor.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataPropertyDescriptor.cs new file mode 100644 index 000000000..8e9456c8f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataPropertyDescriptor.cs @@ -0,0 +1,36 @@ +using System; +using System.ComponentModel; + +namespace BCG +{ + /// + /// Used by GridBindableBlueTable to + /// + public class BlueDataPropertyDescriptor : PropertyDescriptor + { + private ColumnInfo myColumnInfo; + + public BlueDataPropertyDescriptor (ColumnInfo ci) + : base (ci.colName, new Attribute[]{}) + { + myColumnInfo = ci; + } + + public override bool IsReadOnly {get{return true;}} + + public override Type PropertyType {get{return myColumnInfo.colType;}} + + public override bool CanResetValue (object component) {return false;} + + public override object GetValue (object component) {return (component as IBlueRow)[myColumnInfo.colName];} + + public override void ResetValue(object component) {} + + public override void SetValue(object component, object value) {} + + public override bool ShouldSerializeValue(object component) {return false;} + + public override Type ComponentType {get {return typeof(IBlueRow);}} + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs new file mode 100644 index 000000000..38008942b --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -0,0 +1,2192 @@ +using System; +using System.Data; +using System.Collections; +using System.Collections.Generic; +using System.Reflection; +using System.IO; +using System.Diagnostics; +using System.Linq; + +//using BCG.Utilities; +using BCG.Utilities.General; +using System.Linq.Expressions; +//using mbts.spark; +using mbts; + +namespace BCG +{ + public static class BlueDataUtility + { + #region Extensions + public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) + { + return BlueDataUtility.get_row_datetime_value(r, col_name, err_value); + } + public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) + { + return BlueDataUtility.get_row_datetime_value(r, col_no, err_value); + } + public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct + { + return BlueDataUtility.get_row_enum_value(r, col_name, err_value); + } + public static T get_enum(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct + { + return BlueDataUtility.get_row_enum_value(r, col_no, err_value); + } + public static T get(this IBlueRow r, string col_name, T err_value = default(T)) + { + return get_row_value(r, col_name, err_value); + } + public static T get(this IBlueRow r, int col_no, T err_value = default(T)) + { + return get_row_value(r, col_no, err_value); + } + + public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) + { + if (rows == null) + return null; + else + { + BlueTable rv = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) + if (where_func(r)) + rv.addRow(r); + return rv; + } + } + + public static IEnumerable to_ienum(this IBlueRowCollection rows) + { + List rv = new List(); + foreach (IBlueRow r in rows) + rv.Add(r); + return rv; + } + + public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple[] cols) + { + List cis = new List(input_schema.columnInfo); + foreach (Tuple c in cols) + cis.Add(new ColumnInfo(c.Item2, cis.Count, c.Item1)); + return new BlueTableSchema(input_schema.tableName, cis.ToArray()); + } + + public static IBlueRowCollection to_collection(this IBlueRow r) + { + if (r == null) + return null; + else + { + BlueTable rv = new BlueTable(r.schema); + rv.addRow(r); + return rv; + } + } + + public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows, + bool handle_datetime_min = false, + bool handle_double_nan = false) + { + if (rows != null) + { + BlueTable rv = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) + { + MutableBlueRow mr = new MutableBlueRow(r); + rv.addRow(mr); + foreach (ColumnInfo ci in rows.schema.columnInfo) + { + if (handle_datetime_min && ci.colType == typeof(DateTime) && DateTime.MinValue.Equals(r[ci.colName])) + mr.setValue(ci.colName, DBNull.Value); + if (handle_double_nan && ci.colType == typeof(double) && Double.IsNaN(r.get(ci.colName))) + mr.setValue(ci.colName, DBNull.Value); + } + } + return rv; + } + else + return null; + } + + #endregion + + private static IReadOnlyDictionary _typecode_to_type = new Dictionary + { + { TypeCode.Boolean, typeof(bool) }, + { TypeCode.Byte, typeof(byte) }, + { TypeCode.Char, typeof(char) }, + { TypeCode.DateTime, typeof(DateTime) }, + { TypeCode.DBNull, typeof(DBNull) }, + { TypeCode.Decimal, typeof(decimal) }, + { TypeCode.Double, typeof(double) }, + { TypeCode.Empty, null }, + { TypeCode.Int16, typeof(short) }, + { TypeCode.Int32, typeof(int) }, + { TypeCode.Int64, typeof(long) }, + { TypeCode.Object, typeof(object) }, + { TypeCode.SByte, typeof(sbyte) }, + { TypeCode.Single, typeof(Single) }, + { TypeCode.String, typeof(string) }, + { TypeCode.UInt16, typeof(UInt16) }, + { TypeCode.UInt32, typeof(UInt32) }, + { TypeCode.UInt64, typeof(UInt64) } + }; + + public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) + { + List cis = new List(); + foreach (Tuple c in cols) + cis.Add(new ColumnInfo(c.Item1, cis.Count, _typecode_to_type[c.Item2])); + if (cis.FirstOrDefault(ci => ci.colName == "id") == null) + cis.Add(new ColumnInfo("id", cis.Count, typeof(int))); + return new BlueTableSchema(table_name, cis.ToArray()); + } + + /// + /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should + /// be alphabetic or not + /// + /// + /// + /// + public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) + { + string retVal = ""; + if (isColAlphabetically) + { + string[] colNames = row.schema.columnNames; + Array.Sort(colNames); + foreach (string col in colNames) + retVal += row[col].ToString(); + } + else + foreach (object o in row) + retVal += o.ToString(); + return retVal.GetHashCode(); + } + + /// + /// Finds the hashcode only inluding the columns (in the order) listed in the column names argument + /// + /// + /// + /// + public static int getRowHashCode(IBlueRow row, params string[] colNames) + { + string retVal = ""; + foreach (string colName in colNames) + retVal += row[colName].ToString(); + return retVal.GetHashCode(); + } + + /// + /// Converts a IBlueRowCollection to a .Net DataTable + /// + /// + /// + public static DataTable toDataTable(IBlueRowCollection rows) + { + DataTable retTable = getTableFromSchema(rows.schema); + retTable.BeginLoadData(); + foreach (IBlueRow row in rows) + retTable.Rows.Add(toDataRow(row, retTable)); + retTable.EndLoadData(); + return retTable; + } + + /// + /// Converts to a DataRow, the preconditioner being that order of the columns should be same + /// in the schema of the row and the table + /// + /// + /// + public static DataRow toDataRow(IBlueRow row, DataTable table) + { + DataRow retRow = table.NewRow(); + foreach (string colName in row.schema.columnNames) + retRow[colName] = row[colName]; + return retRow; + } + + /// + /// Returns the IBlueTableSchema corresponding to the row + /// + /// + /// DataRow whose schmea is desired + /// + /// + public static IBlueTableSchema getSchema(DataRow row) { return getSchema(row.Table); } + + /// + /// Returns the IBlueTableSchema of the DataTable + /// + /// + /// table whos Schema is desired + /// + /// + public static IBlueTableSchema getSchema(DataTable table) + { + ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; + foreach (DataColumn col in table.Columns) + cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); + return new BlueTableSchema(cis, table.TableName); + } + + /// + /// Constructs the schema from an object, using the SchemaMapping attribute. Only those + /// properties are converted into schmea columns that have a get accessor + /// + /// + /// + public static IBlueTableSchema getSchema(object o) + { + return getSchema(o.GetType()); + } + + + /// + /// Builds a new schema based on an existing one + a set of new columns + /// + /// Base schema + /// Columns to add + public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) + { + ArrayList rv = new ArrayList(schema.columnInfo); + rv.AddRange(cis); + return new BlueTableSchema(schema.tableName, rv.ToArray(typeof(ColumnInfo)) as ColumnInfo[]); + } + + /// + /// appends the cols of s2, to s1, subtracting the id col + /// + /// + /// + /// + public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) + { + List cis = new List(s1.columnInfo); + foreach (ColumnInfo c in appendedSchema.columnInfo) + if (c.colName != "id") + cis.Add(new ColumnInfo(c.colName, cis.Count, c.colType)); + return new BlueTableSchema(s1.tableName, cis.ToArray()); + } + + + public static IBlueTableSchema getSchema(Type t) + { + return getSchema(t, false); + } + + + public static IBlueTableSchema getSchema(Type t, bool createIdField) + { + return getSchema(t, t.Name, createIdField); + } + public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) + { + ArrayList colInfos = new ArrayList(); + int i = 0; + foreach (PropertyInfo pi in t.GetProperties()) + if (myIsActiveProperty(pi)) + colInfos.Add(new ColumnInfo(pi.Name, i++, + ColumnInfo.isAllowedType(pi.PropertyType) ? + pi.PropertyType : typeof(String))); + foreach (FieldInfo fi in t.GetFields()) + if (myIsActiveField(fi)) + colInfos.Add(new ColumnInfo(fi.Name, i++, + ColumnInfo.isAllowedType(fi.FieldType) ? fi.FieldType : typeof(String))); + + if (createIdField) + colInfos.Add(new ColumnInfo("id", i++, typeof(int))); + + return new BlueTableSchema(colInfos.ToArray(typeof(ColumnInfo)) as ColumnInfo[], + tableName); + } + + public static IBlueTableSchema getSchema(Type t, string tableName) + { + return new BlueTableSchema(getSchema(t).columnInfo, tableName); + } + + + public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) + { + return createRowFromType(schema, o, null); + } + + //creates a row from the given type, and if idVal is not null, using the given id + public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) + { + object[] rowArray = new object[schema.columnInfo.Length]; + Type t = o.GetType(); + foreach (PropertyInfo pi in t.GetProperties()) + if (myIsActiveProperty(pi)) + rowArray[schema[pi.Name].colNo] = + ColumnInfo.isAllowedType(pi.PropertyType) ? + pi.GetValue(o, null) : pi.GetValue(o, null).ToString(); + foreach (FieldInfo fi in t.GetFields()) + if (myIsActiveField(fi)) + rowArray[schema[fi.Name].colNo] = + ColumnInfo.isAllowedType(fi.FieldType) ? + fi.GetValue(o) : fi.GetValue(o).ToString(); + if (idVal != null) + rowArray[schema["id"].colNo] = idVal; + return new BlueRow(schema, rowArray); + } + + public static MutableBlueRow create_mutable_row(IBlueTableSchema schema, object o, object idVal, + Func type_converter) + { + object[] rowArray = new object[schema.columnInfo.Length]; + Type t = o.GetType(); + foreach (PropertyInfo pi in t.GetProperties()) + if (myIsActiveProperty(pi)) + rowArray[schema[pi.Name].colNo] = + ColumnInfo.isAllowedType(pi.PropertyType) ? + pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null)); + foreach (FieldInfo fi in t.GetFields()) + if (myIsActiveField(fi)) + rowArray[schema[fi.Name].colNo] = + ColumnInfo.isAllowedType(fi.FieldType) ? + fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o)); + if (idVal != null) + rowArray[schema["id"].colNo] = idVal; + return new MutableBlueRow(schema, rowArray); + } + + + public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func type_converter) + { + List colInfos = new List(); + int i = 0; + if (createIdField) + colInfos.Add(new ColumnInfo("id", i++, typeof(int))); + + foreach (PropertyInfo pi in t.GetProperties()) + colInfos.Add(new ColumnInfo(pi.Name, i++, type_converter(pi.PropertyType))); + foreach (FieldInfo fi in t.GetFields()) + colInfos.Add(new ColumnInfo(fi.Name, i++, type_converter(fi.FieldType))); + return new BlueTableSchema(colInfos.ToArray(), tableName); + } + + public static MutableBlueRow create_mutable_row_not_slow(IBlueTableSchema schema, FieldInfo[] fields, PropertyInfo[] props, + object o, object idVal, Func data_converter) + { + object[] rowArray = new object[schema.columnInfo.Length]; + if (props != null) + { + foreach (PropertyInfo pi in props) + rowArray[schema[pi.Name].colNo] = data_converter(pi.PropertyType, pi.GetValue(o, null)); + } + if (fields != null) + { + foreach (FieldInfo fi in fields) + rowArray[schema[fi.Name].colNo] = data_converter(fi.FieldType, fi.GetValue(o)); + } + if (idVal != null) + rowArray[schema["id"].colNo] = idVal; + return new MutableBlueRow(schema, rowArray); + } + + public static T createTypeFromRow(IBlueRow row) + { + return (T)createTypeFromRow(row, typeof(T)); + } + + public static object createTypeFromRow(IBlueRow row, Type t) + { + object outValue = Activator.CreateInstance(t); + + PropertyInfo[] properties = t.GetProperties(); + + foreach (PropertyInfo pi in properties) + { + if (myIsActiveProperty(pi)) + if (row.Contains(pi.Name)) + if (row[pi.Name].Equals(DBNull.Value) && !pi.PropertyType.Equals(typeof(Object))) + pi.SetValue(outValue, null, null); + else + pi.SetValue(outValue, row[pi.Name], null); + } + FieldInfo[] fields = t.GetFields(); + foreach (FieldInfo fi in fields) + { + if (myIsActiveField(fi)) + if (row.Contains(fi.Name)) + if (row[fi.Name].Equals(DBNull.Value) && !fi.FieldType.Equals(typeof(Object))) + fi.SetValue(outValue, null); + else + fi.SetValue(outValue, row[fi.Name]); + } + + return outValue; + + } + + //these functions should originally have been made public + //but changing that would change a bunch of code + public static bool isActiveProperty(PropertyInfo pi) + { + return myIsActiveProperty(pi); + } + + public static bool isActiveField(FieldInfo fi) + { + return myIsActiveField(fi); + } + + private static bool myIsActiveProperty(PropertyInfo pi) + { + if (pi.GetGetMethod() == null) + return false; + foreach (Attribute att in pi.GetCustomAttributes(true)) + if (att is SchemaVisibility && !(att as SchemaVisibility).visible) + return false; + return true; + } + + private static bool myIsActiveField(FieldInfo fi) + { + if (!fi.IsPublic) + return false; + foreach (Attribute att in fi.GetCustomAttributes(true)) + if (att is SchemaVisibility && !(att as SchemaVisibility).visible) + return false; + return true; + } + + private static bool myIsActiveMember(MemberInfo mi) + { + if (!(mi is FieldInfo) || + !(mi is PropertyInfo) || + (mi is PropertyInfo && (mi as PropertyInfo).GetGetMethod() == null) || + (mi is FieldInfo && !(mi as FieldInfo).IsPublic)) + return false; + foreach (Attribute att in mi.GetCustomAttributes(true)) + if (att is SchemaVisibility && !(att as SchemaVisibility).visible) + return false; + return true; + } + + private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) + { + bool rowExists = false; + foreach (IBlueRow br in brc2) + { + bool allElementsSame = true; + for (int i = 0; i < columns.Length; i++) + { + string col = columns[i]; + object o1 = br[col]; + object o2 = oarray[i]; + allElementsSame = (o1.Equals(o2) == false) ? false : allElementsSame; + } + if (allElementsSame == true) + { + rowExists = true; + break; + } + } + return rowExists; + } + + /// + /// Constructs a DataTable from the IBlueTableSchema + /// + /// + /// Schema whose table needs to be created + /// + /// + public static DataTable getTableFromSchema(IBlueTableSchema schema) + { + DataTable retTable = new DataTable(schema.tableName); + foreach (ColumnInfo ci in schema.columnInfo) + retTable.Columns.Add(ci.colName, ci.colType); + return retTable; + } + + /// + /// Constructs a BlueRow from a DataRow. + /// + /// source row + /// + public static IBlueRow toBlueRow(DataRow row) + { + return new BlueRow(getSchema(row), row); + } + + public static IBlueRowCollection toIBlueRowCollection(DataTable dt) + { + BlueTable retVal = new BlueTable(getSchema(dt)); + foreach (DataRow dr in dt.Rows) + retVal.addRow(dr); + return retVal; + } + + public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) + { + BlueTable retVal = null; + foreach (DataRow dr in rows) + { + IBlueRow r = toBlueRow(dr); + if (retVal == null) + retVal = new BlueTable(r.schema); + retVal.addRow(r); + } + return retVal; + } + + /// + /// Returns the nth row of the IBlueRowCollection. This method could be used + /// for any of the nonRandom-Access-IBlueRowCollection-Implementations, as its + /// O(n) + /// + /// + /// + /// + public static IBlueRow extractRow(IBlueRowCollection rows, int index) + { + int i = 0; + foreach (IBlueRow row in rows) + if (i++ == index) + return row; + return null; + } + + /// + /// Computes the equality fo the 2 rows. The rows might have different columns orderings + /// + /// + /// + /// + public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) + { + return computeRowEquality(r1, r2, r1.schema.columnNames); + } + + public static bool computeRowEquality(IBlueRow r1, + IBlueRow r2, + params string[] colNames) + { + return compareRows(r1, r2, colNames) == 0 ? true : false; + } + + /// + /// Computes a string equivelant for the row, using the columns provided + /// in the argument + /// + /// + /// + /// + public static string rowToString(IBlueRow row, params string[] colNames) + { + string retVal = ""; + foreach (string colName in colNames) + retVal += (row[colName] != null ? row[colName].ToString().Trim() : "NULL") + "\n"; + return retVal; + } + + public static string rowToString(IBlueRow row) { return rowToString(row, row.schema.columnNames); } + + public static string rowCollectionToString(IBlueRowCollection rowColl) + { + if (rowColl == null) + return "Null IBlueRowCollection"; + else + { + string retVal = ""; + foreach (IBlueRow r in rowColl) + retVal += retVal == "" ? rowToString(r) : "\n" + rowToString(r); + return retVal; + } + } + + /// + /// Compares 2 rows, using the column names in the order provided, ie the column name with + /// the lower index in the array has higher priority. Also a null is always assumed + /// to be less than a non-null + /// + /// + /// + /// + /// + public static int compareRows(IBlueRow r1, + IBlueRow r2, + params string[] colNames) + { + int retVal = 0; + foreach (string colName in colNames) + { + object val1 = r1[colName]; + object val2 = r2[colName]; + retVal = _compare_row_cols(val1, val2); + if (retVal != 0) + break; + } + return retVal; + } + + private static int _compare_row_cols(object v1, object v2) + { + if (_is_null(v1)) + return _is_null(v2) ? 0 : -1; + else + { + if (_is_null(v2)) + return 1; + else + { + if (v1 is IComparable && v2 is IComparable) + return (v1 as IComparable).CompareTo(v2); + else if (v1 is byte[] && v2 is byte[]) + return _compare_bytes(v1 as byte[], v2 as byte[]) ? 0 : 1; + else + return 1; + } + } + } + + private static bool _compare_bytes(byte[] v1, byte[] v2) + { + if (v1.Length != v2.Length) + return false; + else + for (int i = 0; i < v1.Length; i++) + if (v1[i] != v2[i]) + return false; + return true; + } + + private static bool _is_null(object o) { return o == null || o == DBNull.Value; } + /// + /// Creates a new IBlueRow with a schema matching the schema provided in 'new schema' + /// + /// a row with the values to be converted + /// the desired schema for the output row + /// + public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) + { + object[] outputValues = new object[newSchema.columnInfo.Length]; + + for (int i = 0; i < outputValues.Length; ++i) + outputValues[i] = inRow[newSchema.columnNames[i]]; + + return new BlueRow(newSchema, outputValues); + } + + /// + /// Combines input tables into a single output table, with different ID's + /// this will result in new rows being created, which is kind of expensive + /// + /// + /// + public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) + { + //get some base cases out of the way + if (tables.Length == 0) + return null; + if (tables.Length == 1) + return tables[0]; + + BlueTable result = new BlueTable(tables[0].schema); + int idToUseNext = 0; + int numCols = tables[0].schema.columnInfo.Length; + int idCol = tables[0].schema["id"].colNo; + IBlueRowCollection previousTable = null; + foreach (IBlueRowCollection table in tables) + { + if (previousTable != null) + if (!compareSchemas(previousTable.schema, table.schema)) + throw new ArgumentException("Error, tables must have the same schema to be combined"); + + foreach (IBlueRow r in table) + { + object[] values = new object[numCols]; + r.CopyTo(values, 0); + values[idCol] = idToUseNext++; + result.addRow(new BlueRow(table.schema, values)); + } + + previousTable = table; + + } + + return result; + } + /// + /// Compares two rows (having the same schema) and returns the + /// columns whose values dont match + /// + /// + /// + /// + public static string[] compareRows(IBlueRow r1, IBlueRow r2) + { + ArrayList retVal = new ArrayList(); + foreach (string col in r1.schema.columnNames) + if (!r1[col].Equals(r2[col])) + retVal.Add(col); + return retVal.ToArray(typeof(string)) as string[]; + } + + public static BlueTable cloneRowCollection(IBlueRowCollection rows) + { + BlueTable retVal = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) + retVal.addRow(new BlueRow(r)); + return retVal; + } + + /// + /// Creates a new row with the values of the row passed, except that the newValues are forced into the + /// returned row, in places specified by the indexes (makes any sense ?) + /// + /// + /// + /// + /// + public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) + { + if (!newValues.Length.Equals(indexes.Length)) + throw new Exception("Debug Assert"); + + object[] rowArray = new object[r.Count]; + r.CopyTo(rowArray, 0); + for (int i = 0; i < indexes.Length; i++) + rowArray[indexes[i]] = newValues[i]; + return new BlueRow(r.schema, rowArray); + } + + public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) + { + return getNewReplacedRow(r, new object[] { newValue }, new int[] { index }); + } + + public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) + { + return getNewReplacedRow(r, newValue, r.schema[colName].colNo); + } + + /// + /// Computes the max row, based on the cols provided. If the collection is empty, null is returned + /// + /// + /// + /// + public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) + { + IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; + foreach (IBlueRow r in rows) + retVal = compareRows(r, retVal, cols) > 0 ? r : retVal; + return retVal; + } + + /// + /// Computes the max row, based on the cols provided. If the collection is empty, null is returned + /// + /// + /// + /// + public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) + { + IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; + foreach (IBlueRow r in rows) + retVal = compareRows(r, retVal, cols) < 0 ? r : retVal; + return retVal; + } + + /// + /// Computes the sum of rows, based on the cols provided. If the collection is empty, null is returned + /// + /// + /// + /// + public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) + { + if (rows.Count == 0) return null; + + IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); + + object[] sums = new object[cols.Length]; + int[] idxs = new int[cols.Length]; + for (int i = 0; i < cols.Length; i++) + { + sums[i] = 0.0; + idxs[i] = rows.schema[cols[i]].colNo; + } + + foreach (IBlueRow r in rows) + { + for (int i = 0; i < cols.Length; i++) + { + double dval = BlueDataUtility.get_row_value(r, cols[i], 0.0); + //r[cols[i]] == null || r[cols[i]] is DBNull ? 0.0 : (double)r[cols[i]]; + sums[i] = (double)sums[i] + dval; + } + } + + return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs); + } + + + public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) + { + if (rows.Count == 0) return null; + IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); + if (rows.Count == 1) return retVal; + + object[] sums = new object[cols.Length]; + + int[] idxs = new int[cols.Length]; + for (int i = 0; i < cols.Length; i++) + { + sums[i] = default(T); + idxs[i] = rows.schema[cols[i]].colNo; + } + + foreach (IBlueRow r in rows) + { + for (int i = 0; i < cols.Length; i++) + { + T tval = r[cols[i]] == null || r[cols[i]] is DBNull ? default(T) : (T)r[cols[i]]; + sums[i] = MagicAdder.Add((T)sums[i], tval); + } + } + + return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs); + } + + /// + /// Uses the ID to identify the presence of a row in a collection + /// + /// + /// + /// + public static bool rowPresent(IBlueRowCollection table, IBlueRow row) + { + return table.containsID((int)row["id"]); + } + + public static void computeDifferences(IBlueRowCollection newRows, + IBlueRowCollection oldRows, + out IBlueRowCollection dRows, + out IBlueRowCollection aRows) + { + computeDifferences( + newRows, + oldRows, + newRows.schema.columnNames, + out dRows, + out aRows); + } + + public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) + { + int[] col_indices = new int[new_rows.schema.columnNames.Length]; + for (int i = 0; i < col_indices.Length; i++) + col_indices[i] = i; + compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows); + } + + public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices, + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) + { + Func rows_equal = (r1, r2) => + { + foreach (int col_no in col_indices) + if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value)) + return false; + return true; + }; + + BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema); + // adds + changes + foreach (IBlueRow r in new_rows) + if (!BlueDataUtility.rowPresent(old_rows, r)) + add_rows.addRow(r); + else if (!rows_equal(r, old_rows[(int)r["id"]])) + { + add_rows.addRow(r); + del_rows.addRow(old_rows[(int)r["id"]]); + } + // deletes + foreach (IBlueRow r in old_rows) + if (!BlueDataUtility.rowPresent(new_rows, r)) + del_rows.addRow(r); + out_d_rows = del_rows; + out_a_rows = add_rows; + } + + + public static void computeDifferences(IBlueRowCollection c1, + IBlueRowCollection c2, + string[] columns, + out IBlueRowCollection dRows, + out IBlueRowCollection aRows) + { + dRows = new BlueTable(c1.schema); + aRows = new BlueTable(c1.schema); + + // adds + changes + foreach (IBlueRow nr in c1) + if (!BlueDataUtility.rowPresent(c2, nr)) + (aRows as BlueTable).addRow(nr); + else if (BlueDataUtility.compareRows(nr, c2[(int) + nr["id"]], columns) != 0) + { + (aRows as BlueTable).addRow(nr); + (dRows as BlueTable).addRow(c2[(int)nr["id"]]); + + } + + // deletes + foreach (IBlueRow nr in c2) + if (!BlueDataUtility.rowPresent(c1, nr)) + (dRows as BlueTable).addRow(nr); + } + + //brcDiff = brc1 - brc2 + public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) + { + brcDiff = new BlueTable(brc1.schema); + foreach (IBlueRow br in brc1) + { + object[] oarray = new object[columns.Length]; + for (int i = 0; i < columns.Length; i++) + { + string col = columns[i]; + oarray[i] = br[col]; + } + if (myDoesRowExistsInTable(brc2, columns, oarray) == false) + (brcDiff as BlueTable).addRow(br); + } + } + + /// + /// Abosrbs the deletes and adds into the BlueTable + /// + /// + /// + /// + public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) + { + foreach (IBlueRow r in dels) + if (!t.deleteRow(r)) + throw new Exception(String.Format("{0} does not have row with id {1} to delete", t.schema.tableName, r["id"])); + + foreach (IBlueRow r in adds) + t.addRow(r); + } + + public static IBlueRowCollection toBlueRowCollection(IBlueRow r) + { + BlueTable rv = new BlueTable(r.schema); + rv.addRow(r); + return rv; + } + + /// + /// Converts an ICollection of IBlueRows to IBlueRowCollection. If there are no rows in the + /// collection, a null is returned + /// + /// + /// + public static IBlueRowCollection toBlueRowCollection(ICollection col) + { + BlueTable retVal = null; + foreach (IBlueRow r in col) + { + retVal = retVal ?? new BlueTable(r.schema); + retVal.addRow(r); + } + return retVal; + } + + /// + /// Computes the rows that are truely deleted, added or changed (with respect to relevant columns) + /// + /// all the deleted rows in a certain callback + /// all the added rows in a certain callback + /// the rows that are truely deleted or changed with respect to the relevant columns + /// the rows that are truely added or changed with respect to the relevant columns + /// + public static void getTrueDeletesAndAdds(IBlueRowCollection dels, + IBlueRowCollection adds, + out IBlueRowCollection truelyDels, + out IBlueRowCollection truelyAdds, + string[] cols) + { + truelyDels = new BlueTable(dels); + truelyAdds = new BlueTable(adds); + foreach (IBlueRow r in dels) + { + if (adds.containsID((int)r["id"]) && + BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) + { + (truelyDels as BlueTable).deleteRow((int)r["id"]); + (truelyAdds as BlueTable).deleteRow((int)r["id"]); + } + } + } + + public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) + { + BlueTable retVal = new OrderedBlueTable(coll.schema); + foreach (IBlueRow r in coll) + retVal.addRow(new BlueRow(r)); + return retVal; + } + + /// + /// extracts out a column from a table + /// + /// + /// + /// + public static ArrayList extractColumn(IBlueRowCollection rows, string colName) + { + ArrayList rv = new ArrayList(); + foreach (IBlueRow r in rows) + rv.Add(r[colName]); + return rv; + } + + /// + /// applies the deletedRows and addedRows to the oldState and asserts if it does not + /// match the newRows. for an old state with 10 K rows, and all 10 K rows being deleted and + /// added it takes ~ 35 ms to execute this method. + /// + /// + /// + /// + /// + public static void checkConsistency(IBlueRowCollection oldState, + IBlueRowCollection newRows, + IBlueRowCollection delRows, + IBlueRowCollection addedRows) + { + BlueTable calc = new BlueTable(oldState); + Debug.Assert(oldState != null && newRows != null && delRows != null && addedRows != null); + + foreach (IBlueRow r in delRows) + { + Debug.Assert(calc.containsID((int)r["id"]), "deleted row's id = " + r["id"] + " does not exist"); + calc.deleteRow(r); + } + + foreach (IBlueRow r in addedRows) + { + Debug.Assert(!calc.containsID((int)r["id"]), "added rows's id = " + r["id"].ToString() + " already exists"); + calc.addRow(r); + } + + Debug.Assert(calc.Count == newRows.Count, "row count should be the same. Counts: calc=" + calc.Count + "; newRows=" + newRows.Count); + foreach (IBlueRow r in calc) + Debug.Assert(newRows.containsID((int)r["id"]), r["id"] + " is not found"); + + } + + /// + /// Extracts values from an IBlueRow specified by the columnNames arg + /// + /// + /// + /// + public static object[] getSelectedValues(IBlueRow r, params string[] colNames) + { + object[] retVal = new object[colNames.Length]; + for (int i = 0; i < colNames.Length; i++) + retVal[i] = r[colNames[i]]; + return retVal; + } + + + /// + /// From an IBlueRowCollection, for a set of columns, extracts a row with the same values as + /// provided in the third arg + /// + /// + /// + /// + /// + public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) + { + foreach (IBlueRow r in rows) + { + object[] vals = getSelectedValues(r, keyColumns); + if (CollectionHelpers.equal(vals, colValues)) + return r; + } + return null; + } + + public static IBlueRowCollection deepCopy(IBlueRowCollection rows) + { + BlueTable retVal = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) + retVal.addRow(new BlueRow(r)); + return retVal; + } + + /// + /// Returns the columns, sorted alphabetically. The second arg + /// is for the inclusion of id + /// + /// + /// + /// + public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) + { + ArrayList names = new ArrayList(schema.columnNames), + retVal = new ArrayList(); + names.Sort(); + foreach (string n in names) + if (n != "id" || includeId) + retVal.Add(schema[n].colNo); + return retVal.ToArray(typeof(int)) as int[]; + } + + //this class is used so that we can sort a schema by column name + //that way, prepublished tables will allways get published with schemas in the same order + //regardless of how the underlying streams end up working + private class ColumnInfoComparisonWrapper : IComparable + { + + ColumnInfo myColumn; + + public string ColumnName + { + get { return myColumn.colName; } + //can't set it, these are only for reading columns + } + + public ColumnInfo Column + { + get { return myColumn; } + } + public ColumnInfoComparisonWrapper(ColumnInfo theColumn) + { + myColumn = theColumn; + } + + public int CompareTo(ColumnInfoComparisonWrapper otherGuy) + { + return ColumnName.CompareTo(otherGuy.ColumnName); + } + } + + public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) + { + List sortedColNames = new List(inSchema.columnNames); + sortedColNames.Sort(); + List sortedColInfos = new List(); + int colNo = 0; + foreach (string c in sortedColNames) + { + ColumnInfo ci = inSchema[c]; + sortedColInfos.Add(new ColumnInfo(c, colNo++, ci.colType)); + + } + return new BlueTableSchema(inSchema.tableName, sortedColInfos.ToArray()); + } + + public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) + { + + //for now this function will not do anything + //that is because we don't want to break the C++ deserialization code + int numCols = theSchema.columnNames.Length; + sortedSchemaToOriginalSchemaMapping = new int[numCols]; + for (int col = 0; col < theSchema.columnNames.Length; ++col) + { + //the mapping is just the identity mapping + sortedSchemaToOriginalSchemaMapping[col] = col; + + + } + + idColumnInSortedSchema = theSchema["id"].colNo; + + return theSchema; + + /* + //set this to -1 in case the method doesn't find an id column + idColumnInSortedSchema = 01; + List schemaColumns = new List(); + + foreach (ColumnInfo ci in theSchema.columnInfo) + schemaColumns.Add(new ColumnInfoComparisonWrapper(ci)); + + schemaColumns.Sort(); + + int numCols = theSchema.columnInfo.Length; + ColumnInfo[] outputColumns = new ColumnInfo[numCols]; + sortedSchemaToOriginalSchemaMapping = new int[numCols]; + + for(int i = 0; i < numCols; ++i) + { + string thisColName = schemaColumns[i].ColumnName; + outputColumns[i] = new ColumnInfo(thisColName, i, schemaColumns[i].Column.colType); + sortedSchemaToOriginalSchemaMapping[i] = theSchema[thisColName].colNo; + + //we cache the ID column because it'll be differentin the serialized row than in our schema + if (thisColName == "id") + idColumnInSortedSchema = i; + + } + + + theSchema = new BlueTableSchema(theSchema.tableName,outputColumns); + return theSchema; + */ + + + } + + + + + /// + /// Compares 2 schemas for column-by-column match + /// + /// + /// + /// true if equal + public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) + { + bool result = true; + try + { + if (s1.columnInfo.Length != s2.columnInfo.Length) + result = false; + else + foreach (ColumnInfo ci1 in s1.columnInfo) + { + ColumnInfo ci2 = s2[ci1.colNo]; + if (ci1.colName != ci2.colName || ci1.colType != ci2.colType) + { + result = false; + break; + } + } + } + catch + { + result = false; // the col# wasn't found in s2 + } + return result; + } + + public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) + { + IBlueRowCollection dels = findMatchingRows(table, cols, vals); + table.deleteRows(dels); + return dels; + } + + public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) + { + BlueTable rv = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) + { + bool match = true; + for (int i = 0; i < cols.Length; i++) + match &= vals[i].Equals(r[cols[i]]); + if (match) + rv.addRow(r); + } + return rv; + } + + public static void checkRowTypes(IBlueRowCollection rows) + { + foreach (IBlueRow r in rows) + foreach (string col in r.schema.columnNames) + if (r[col] != null && r[col] != DBNull.Value) + Debug.Assert(r[col].GetType() == rows.schema[col].colType, + String.Format("col {0} in table {1} is of type {2} in schema but is a {3} in the row", col, rows.schema.tableName, r.schema[col].colType.ToString(), r[col].GetType().ToString())); + } + + public static void addRows(ref BlueTable table, ICollection rows) + { + foreach (IBlueRow r in rows) + table.addRow(r); + } + + public static void removeRows(ref BlueTable table, ICollection rows) + { + foreach (IBlueRow r in rows) + table.deleteRow(r); + } + + public static void sortOutPacket(IBlueRowCollection dels, IBlueRowCollection adds, string[] keyCols, + out IBlueRowCollection addedRows, + out IBlueRowCollection deletedRows, + out IBlueRowCollection changedRows) + { + addedRows = new BlueTable(dels.schema); + deletedRows = new BlueTable(dels.schema); + changedRows = new BlueTable(dels.schema); + + KeysBasedTable2 keyedDels = new KeysBasedTable2(dels.schema, keyCols), + keyedAdds = new KeysBasedTable2(dels.schema, keyCols); + + foreach (IBlueRow r in dels) + keyedDels.update(r); + foreach (IBlueRow r in adds) + keyedAdds.update(r); + + foreach (IBlueRow r in keyedDels) + { + object[] key = BlueDataUtility.getSelectedValues(r, keyCols); + if (keyedAdds.getRow(key) != null) + { + ///change + (changedRows as BlueTable).addRow(r); + } + else + { + ///delete + (deletedRows as BlueTable).addRow(r); + } + } + + foreach (IBlueRow r in keyedAdds) + { + object[] key = BlueDataUtility.getSelectedValues(r, keyCols); + if (keyedDels.getRow(key) == null) + { + (addedRows as BlueTable).addRow(r); + } + } + } + + public static int HashExpStrike(DateTime exp, double strike) + { + return ((DateTime)exp).ToShortDateString().GetHashCode() ^ strike.GetHashCode(); + } + + public static IEnumerable collection_to_ienum(IBlueRowCollection rows) + { + List rv = new List(); + foreach (IBlueRow r in rows) + rv.Add(r); + return rv; + } + + public static IBlueRowCollection ienum_to_collection(IEnumerable rows) + { + BlueTable rv = null; + if (rows.Count() > 0) + { + rv = new BlueTable(rows.ElementAt(0).schema); + foreach (IBlueRow r in rows) + rv.addRow(r); + } + return rv; + } + + public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) + { + List cis = new List(input_schema.columnInfo); + cis.Add(new ColumnInfo(new_col_name, cis.Count, new_col_type)); + return new BlueTableSchema(input_schema.tableName, cis.ToArray()); + } + + public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) + { + Dictionary removed_col_names = new Dictionary(); + foreach (string c in col_names) + removed_col_names[c] = true; + List cis = new List(); + foreach (ColumnInfo ci in input_schema.columnInfo) + if (!removed_col_names.ContainsKey(ci.colName)) + cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType)); + return new BlueTableSchema(cis.ToArray(), input_schema.tableName); + } + + public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) + { + List rv = new List(); + foreach (string c in col_names) + rv.Add(schema[c].colNo); + return rv.ToArray(); + } + + public static string get_appended_string_val(IBlueRow r, int[] col_indices) + { + System.Text.StringBuilder sb = new System.Text.StringBuilder(); + for (int i = 0; i < col_indices.Length; i++) + sb.Append(r[col_indices[i]]); + return sb.ToString(); + } + + public delegate string RowValueToString(IBlueRow r, int col_index); + public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) + { + System.Text.StringBuilder sb = new System.Text.StringBuilder(); + for (int i = 0; i < col_indices.Length; i++) + sb.Append(to_string_conv(r, i)); + return sb.ToString(); + } + + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names) + { + BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length); + return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); + } + + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) + { + return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); + } + + public static IDictionary> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) + { + Dictionary> rv = new Dictionary>(); + hasher = hasher ?? new BlueRowHasher(group_by_col_nos.Length); + foreach (IBlueRow r in rows) + { + int row_hash = hasher.get_hash(r, group_by_col_nos); + List this_group_rows = null; + if (!rv.TryGetValue(row_hash, out this_group_rows)) + { + this_group_rows = new List(); + rv.Add(row_hash, this_group_rows); + } + this_group_rows.Add(r); + } + return rv; + } + + public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) + { + foreach (string cn in data_col_names) + output_row.setValue(cn, row[cn]); + } + + /// + /// Copies the column nos specified in data_cols from row, to output_row + /// starting from index start_col_index + /// + public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) + { + foreach (int col_no in data_col_nos) + output_row[start_col_index++] = row[col_no]; + } + + public static object compute_min_max(object v1, object v2, bool is_max) + { + if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) + { + if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) + { + double v1_num = (double)Convert.ChangeType(v1, TypeCode.Double), + v2_num = (double)Convert.ChangeType(v2, TypeCode.Double); + v1_num = Double.IsNaN(v1_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v1_num; + v2_num = Double.IsNaN(v2_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v2_num; + double rv = is_max ? Math.Max(v1_num, v2_num) : Math.Min(v1_num, v2_num); + return Convert.ChangeType(rv, v1.GetType()); + } + else + return v1; + } + else + return TypeHelper.is_dbnull(v1) ? v2 : v1; + } + + #region Sorting + + public static class ParallelSort + { + public class TreeRowEffComparer : IComparer + { + private int _depth_in_tree_col_no; + private int[] _aggregation_tree_col_nos; + private Pair[] _sort_map; + + public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair[] sort_map) + { + _depth_in_tree_col_no = depth_in_tree_col_no; + _aggregation_tree_col_nos = aggregation_tree_col_nos; + _sort_map = sort_map; + } + + public int Compare(IBlueRow x, IBlueRow y) + { + return BlueDataUtility.sort_in_a_tree(x, y, _depth_in_tree_col_no, _aggregation_tree_col_nos, _sort_map); + } + } + + public class TabularRowComparer : IComparer + { + private IEnumerable> _sort_map; + public TabularRowComparer(IEnumerable> sort_map) { _sort_map = sort_map; } + public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.compare_rows(x, y, _sort_map); } + } + + public class TabularRowHashPairComparer : IComparer> + { + private Pair[] _sort_map; + + public TabularRowHashPairComparer(Pair[] sort_map) { _sort_map = sort_map; } + public int Compare(Pair x, Pair y) { return BlueDataUtility.compare_rows(x.first, y.first, _sort_map); } + } + + #region Public Static Methods + + public static void QuicksortSequential(IList arr, IComparer comparer) + { + QuicksortSequential(arr, comparer, 0, arr.Count - 1); + } + + public static void QuicksortParallel(IList arr, IComparer comparer) + { + QuicksortParallel(arr, comparer, 0, arr.Count - 1); + } + + #endregion + + #region Private Static Methods + + private static void QuicksortSequential(IList arr, IComparer comparer, int left, int right) + { + if (right > left) + { + int pivot = Partition(arr, comparer, left, right); + QuicksortSequential(arr, comparer, left, pivot - 1); + QuicksortSequential(arr, comparer, pivot + 1, right); + } + } + + private static void QuicksortParallel(IList arr, IComparer comparer, int left, int right) + { + const int SEQUENTIAL_THRESHOLD = 64; + if (right > left) + { + if (right - left < SEQUENTIAL_THRESHOLD) + { + QuicksortSequential(arr, comparer, left, right); + } + else + { + int pivot = Partition(arr, comparer, left, right); + System.Threading.Tasks.Parallel.Invoke(new Action[] { delegate {QuicksortParallel(arr, comparer, left, pivot - 1);}, + delegate {QuicksortParallel(arr, comparer, pivot + 1, right); } + }); + } + } + } + + private static void Swap(IList arr, int i, int j) + { + T tmp = arr[i]; + arr[i] = arr[j]; + arr[j] = tmp; + } + + private static int Partition(IList arr, IComparer comparer, int low, int high) + { + // Simple partitioning implementation + int pivotPos = (high + low) / 2; + T pivot = arr[pivotPos]; + Swap(arr, low, pivotPos); + + int left = low; + for (int i = low + 1; i <= high; i++) + { + if (comparer.Compare(arr[i], pivot) < 0) + { + left++; + Swap(arr, i, left); + } + } + + Swap(arr, low, left); + return left; + } + + #endregion + } + + public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary sort_map) + { + foreach (KeyValuePair p in sort_map) + { + int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value); + if (comp != 0) + return comp; + } + return 0; + } + + public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable> sort_map) + { + foreach (Pair p in sort_map) + { + int comp = compare_send_dontcare_to_last(r1, p.first, r2, p.first, p.second); + if (comp != 0) + return comp; + } + return 0; + } + + public static int compare_send_dontcare_to_last(IBlueRow r1, int r1_col_no, + IBlueRow r2, int r2_col_no, + bool low_to_high) + { + object o1 = BlueDataUtility.get_row_value(r1, r1_col_no, DBNull.Value), + o2 = BlueDataUtility.get_row_value(r2, r2_col_no, DBNull.Value); + return compare_send_dontcare_to_last(o1, o2, low_to_high); + } + + public static List> get_indexed_sort_map(List> sort_map, IBlueTableSchema schema) + { + List> rv = new List>(); + foreach (Pair p in sort_map) + rv.Add(new Pair(schema[p.first].colNo, p.second)); + return rv; + } + + /// Comapares 2 rows in a way where nulls, nans, infinities, + /// empty strings etc are sent to the last + public static int compare_send_dontcare_to_last(IBlueRow r1, string r1_col_name, + IBlueRow r2, string r2_col_name, + bool low_to_high) + { + object o1 = BlueDataUtility.get_row_value(r1, r1_col_name, DBNull.Value), + o2 = BlueDataUtility.get_row_value(r2, r2_col_name, DBNull.Value); + return compare_send_dontcare_to_last(o1, o2, low_to_high); + } + + public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) + { + if (is_dont_care_val(o1)) + return is_dont_care_val(o2) ? 0 : 1; + else + return is_dont_care_val(o2) ? -1 : (low_to_high ? ((IComparable)o1).CompareTo(o2) + : ((IComparable)o2).CompareTo(o1)); + } + + public static bool is_dont_care_val(object o) + { + return o == null || o == DBNull.Value || + o is string && String.IsNullOrEmpty((string)o) || + o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) || + o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)) || + o is Int32 && (Int32.MaxValue.Equals(o) || Int32.MinValue.Equals(o)); + } + + #endregion + + #region Tree Aggregation + + public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) + { + List rv = new List(); + foreach (string col_name in aggregation_tree_col_names) + rv.Add(schema[col_name].colNo); + return rv.ToArray(); + } + + public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) + { + return BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1) == (aggregation_tree.Count() - 1); + } + + public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) + { + return BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1) == (aggregation_tree.Count() - 1); + } + + public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) + { + int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1); + string rv = String.Empty; + for (int p = 0; p <= aggregation_priority; p++) + rv += (rv == String.Empty ? String.Empty : ":") + String.Format("{0},{1}", p, BlueDataUtility.get_row_value(r, aggregation_tree[p], DBNull.Value)); + return rv; + } + + /// Generates a string based id for a row in a tree + /// by appending the values of the nodes above it + public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) + { + int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1); + string rv = String.Empty; + for (int p = 0; p <= aggregation_priority; p++) + rv += (rv == String.Empty ? String.Empty : ":") + + String.Format("{0},{1}", p, BlueDataUtility.get_row_value(r, aggregation_tree.ElementAt(p), DBNull.Value)); + return rv; + } + /// + /// Given an aggregation hierarchy tree, figures if the first + /// arg is a parent of the second arg. + /// + public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) + { + System.Diagnostics.Debug.Assert(parent_row.Contains(depth_in_tree_col_name) && child_row.Contains(depth_in_tree_col_name)); + int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_name, Int32.MaxValue), + child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_name, Int32.MaxValue); + if (parent_depth < child_depth) + { + for (int p = 0; p <= parent_depth; p++) + { + string col_name = aggregation_tree.ElementAt(p); + object parent_val = BlueDataUtility.get_row_value(parent_row, col_name, DBNull.Value), + child_val = BlueDataUtility.get_row_value(child_row, col_name, DBNull.Value); + if (!parent_val.Equals(child_val)) + return false; + } + return true; + } + else + return false; + } + + public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) + { + int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_no, Int32.MaxValue), + child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_no, Int32.MaxValue); + if (parent_depth < child_depth) + { + for (int p = 0; p <= parent_depth; p++) + { + int col_no = aggregation_tree[p]; + object parent_val = BlueDataUtility.get_row_value(parent_row, col_no, DBNull.Value), + child_val = BlueDataUtility.get_row_value(child_row, col_no, DBNull.Value); + if (!parent_val.Equals(child_val)) + return false; + } + return true; + } + else + return false; + } + + public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary sort_map) + { + if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_name, aggregation_tree)) + return -1; + else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_name, aggregation_tree)) + return 1; + else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) + return 0; + else + { + for (int p = 0; p < aggregation_tree.Count(); p++) + { + /// sort_map is used to leaf nodes + if (p == aggregation_tree.Count() - 1) + { + /// sorting for leaf nodes is special, since they can be + /// sorted by data columns + System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); + int leaf_comp = compare_rows(r1, r2, sort_map); + if (leaf_comp != 0) + return leaf_comp; + } + /// sort_map is used to find sorting order + string aggregation_col_name = aggregation_tree.ElementAt(p); + bool low_to_high = sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true; + int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_name, r2, aggregation_col_name, low_to_high); + if (comp != 0) + return comp; + } + return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree) ? + compare_rows(r1, r2, sort_map) : 0; + } + } + + public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair[] sort_map) + { + if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_no, aggregation_tree)) + return -1; + else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_no, aggregation_tree)) + return 1; + else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) + return 0; + else + { + for (int p = 0; p < aggregation_tree.Count(); p++) + { + /// sort_map is used to leaf nodes + if (p == aggregation_tree.Count() - 1) + { + /// sorting for leaf nodes is special, since they can be + /// sorted by data columns + //System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); + int leaf_comp = compare_rows(r1, r2, sort_map); + if (leaf_comp != 0) + return leaf_comp; + } + /// sort_map is used to find sorting order + int aggregation_col_no = aggregation_tree[p]; + bool low_to_high = true;//sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true; + int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_no, r2, aggregation_col_no, low_to_high); + if (comp != 0) + return comp; + } + return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_no, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_no, aggregation_tree) ? + compare_rows(r1, r2, sort_map) : 0; + } + } + + #endregion + + #region Column accessors + + public static T get_array_value(Array r, int col_no, T error_val) + { + try + { + object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : DBNull.Value; + if (rv == null || rv == DBNull.Value) + return error_val; + else + return (T)Convert.ChangeType(rv, typeof(T)); + } + catch + { + return error_val; + } + } + + public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) + { + try + { + object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : error_val; + return _get_datetime_value(rv, error_val); + } + catch + { + return error_val; + } + } + + public static T get_row_value(IBlueRow r, int col_no, T error_val) + { + try + { + object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; + if (rv == null || rv == DBNull.Value) + return error_val; + else + return (T)Convert.ChangeType(rv, typeof(T)); + } + catch + { + return error_val; + } + } + + public static T get_row_value(IBlueRow r, string col_name, T error_val) + { + try + { + object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value; + if (rv == null || rv == DBNull.Value) + return error_val; + else + return (T)Convert.ChangeType(rv, typeof(T)); + } + catch + { + return error_val; + } + } + + public static T get_row_enum_value(IBlueRow r, string col_name, T error_val) where T : struct + { + try + { + object rv = r.Contains(col_name) ? r[col_name] : DBNull.Value; + return _get_row_enum_value(rv, error_val); + } + catch + { + return error_val; + } + } + + public static T get_row_enum_value(IBlueRow r, int col_no, T error_val) where T : struct + { + try + { + object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; + return _get_row_enum_value(rv, error_val); + } + catch + { + return error_val; + } + } + + private static T _get_row_enum_value(object o, T error_val) where T : struct + { + try + { + if (o == null || o == DBNull.Value) + return error_val; + else if (o is string) + { + T result = error_val; + if (Enum.TryParse((string)o, true, out result)) return result; + return error_val; + } + else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32) + return (T)o; + else if (o is T) + return (T)o; + else + return error_val; + } + catch + { + return error_val; + } + } + + public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) + { + try + { + object rv = (r != null && r.Contains(col_name)) ? r[col_name] : error_val; + return _get_datetime_value(rv, error_val); + } + catch + { + return error_val; + } + } + + public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) + { + try + { + object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : error_val; + return _get_datetime_value(rv, error_val); + } + catch + { + return error_val; + } + } + + private static DateTime _get_datetime_value(object v, DateTime error_val) + { + try + { + if (v == null || v == DBNull.Value) + return error_val; + else + { + if (v is DateTime) + return (DateTime)v; + else if (v is string) + { + DateTime rv_dt = error_val; + DateTime.TryParse((string)v, out rv_dt); + return rv_dt; + } + else + { + double oa_date = (double)Convert.ChangeType(v, typeof(Double)); + return DateTime.FromOADate(oa_date); + } + } + } + catch + { + return error_val; + } + } + + #endregion + + #region CSV methods + /// + /// Reads a schema file name, and return the constructed schema + /// + public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) + { + StreamReader sr = new StreamReader(schemaFileName); + IBlueTableSchema retVal = createSchemaFromCSVSchemaFile(sr); + sr.Close(); + return retVal; + } + + /// + /// Builds a schema from a schema file. Note that the stream is not closed, so + /// the calling code should do so + /// + public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) + { + string tableName = sr.ReadLine(); + ArrayList t = new ArrayList(); + string l = ""; + while ((l = sr.ReadLine()) != null) + t.Add(l.Trim()); + string[] cols = t.ToArray(typeof(string)) as String[]; + ColumnInfo[] cis = new ColumnInfo[cols.Length]; + for (int i = 0; i < cols.Length; i++) + { + string[] colTags = cols[i].Split('/'); + cis[i] = new ColumnInfo(colTags[0], i, TypeInstanceLoader.loadType(colTags[1])); + } + IBlueTableSchema retVal = new BlueTableSchema(cis, tableName); + if (retVal["id"] == null) + { + ArrayList temp = new ArrayList(retVal.columnInfo); + temp.Add(new ColumnInfo("id", temp.Count, typeof(System.Int32))); + retVal = new BlueTableSchema(retVal.tableName, + temp.ToArray(typeof(ColumnInfo)) as ColumnInfo[]); + } + return retVal; + } + + /// + /// Creates an IBlueRow from a CSV data file row; if the string already contains an id, + /// the id argument is ignored + /// + public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) + { + string[] toks = CSVParser.parse(fileRow); + ArrayList rowList = new ArrayList(); + if (toks.Length == schema.columnNames.Length) + { + for (int i = 0; i < toks.Length; i++) + rowList.Add(myConvertTo(schema[i].colType, toks[i])); + } + else + { + int idIndex = schema["id"].colNo; + for (int i = 0; i < toks.Length; i++) + rowList.Add(myConvertTo(schema[i < idIndex ? i : i + 1].colType, toks[i])); + rowList.Insert(idIndex, id); + } + return new BlueRow(schema, rowList.ToArray()); + } + + /// + /// Returns the CSV data string corresponding to the row + /// + public static string toCSVDataString(IBlueRow r) + { + string retVal = ""; + for (int i = 0; i < r.schema.columnNames.Length; i++) + retVal += + r.schema[i].colType.Equals(typeof(string)) ? + "\"" + r[i].ToString().Replace("\"", "\"\"") + "\"" : + r[i].ToString(); + return retVal.Remove(retVal.Length - 1, 1); + } + + public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) + { + string[] retVal = new string[2]; + retVal[0] = schema.tableName; + foreach (ColumnInfo ci in schema.columnInfo) + retVal[1] += ci.colName + "/" + ci.colType.ToString() + " "; + retVal[1] = retVal[1].Remove(retVal[1].Length - 1, 1); + return retVal; + } + + /// + /// Converts a CSV data value to its original type. + /// + private static object myConvertTo(Type t, string val) + { + val = val.Trim(); + if (t.Equals(typeof(System.String))) + return val; + else if (val != "") + { + if (t.Equals(typeof(System.Char))) + return Convert.ToChar(val); + if (t.Equals(typeof(System.Int16))) + return Convert.ToInt16(val); + if (t.Equals(typeof(System.Int32))) + return Convert.ToInt32(val); + if (t.Equals(typeof(System.Int64))) + return Convert.ToInt64(val); + if (t.Equals(typeof(System.Double))) + return Convert.ToDouble(val); + if (t.Equals(typeof(System.Single))) + return Convert.ToSingle(val); + if (t.Equals(typeof(System.DateTime))) + return myToDateTime(val); + } + return DBNull.Value; + } + + private static object myToDateTime(string val) + { + try + { + return Convert.ToDateTime(val); + } + catch (FormatException) //datetime is not stored in one of the string formats + { + return val.IndexOf('.') > -1 ? + DateTime.FromOADate(Convert.ToDouble(val)) : new DateTime(Convert.ToInt64(val)); + } + } + + public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) + { + StreamReader sr = new StreamReader(fileAddress); + string cols = sr.ReadLine(); + string[] columns = cols.Split(','); + ColumnInfo[] ci = new ColumnInfo[columns.Length + 1]; + ci[0] = new ColumnInfo("id", 0, typeof(int)); + for (int i = 1; i < ci.Length; i++) + ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string)); + IBlueTableSchema bts = new BlueTableSchema(ci, "csvTable"); + BlueTable result = new BlueTable(bts); + int id = 0; + while (!sr.EndOfStream) + result.addRow(createRowFromCSVFileRow(sr.ReadLine(), bts, id++)); + sr.Close(); + return result; + } + + #endregion + + #region Magic + /// + /// Uses a cached compiled expression to implement a truly generic add function. Will throw if given types that cannot be added. + /// + internal class MagicAdder + { + static Func add; + static MagicAdder() + { + ParameterExpression left = Expression.Parameter(typeof(T), "a"); + ParameterExpression right = Expression.Parameter(typeof(T), "b"); + add = (Func)Expression.Lambda(Expression.Add(left, right), left, right).Compile(); + } + + public static T Add(T a, T b) + { + return add(a, b); + } + } + #endregion + + #region Misc + + public static void swap(ref T t1, ref T t2) + { + T temp = t1; + t1 = t2; + t2 = temp; + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs new file mode 100644 index 000000000..9425ba613 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs @@ -0,0 +1,45 @@ +using System; + +namespace BCG +{ + /// + /// Allowed types + /// + public class BlueDataValidTypes + { + public static readonly Type BOOL = typeof(bool); + public static readonly Type BYTE = typeof(byte); + public static readonly Type CHAR = typeof(char); + public static readonly Type DECIMAL = typeof(decimal); + public static readonly Type DOUBLE = typeof(double); + public static readonly Type SHORT = typeof(short); + public static readonly Type INT = typeof(int); + public static readonly Type LONG = typeof(long); + public static readonly Type SBYTE = typeof(sbyte); + //public static readonly Type FLOAT = typeof(float); + public static readonly Type STRING = typeof(string); + //public static readonly Type USHORT = typeof(ushort); + public static readonly Type UINT = typeof(uint); + public static readonly Type ULONG = typeof(ulong); + public static readonly Type DATETIME = typeof(DateTime); + public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType); + public static readonly Type SINGLE = typeof(Single); + public static readonly Type BYTEARRAY = typeof(byte[]); + + public static int getValueTypeSize (Type t) + { + if (t == BOOL || t == BYTE || t == SBYTE) + return 1; + else if (t == SHORT || t == CHAR) + return 2; + else if (t == INT || t == UINT || t == SINGLE) + return 4; + else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME) + return 8; + else if (t == DECIMAL) + return 16; + else + throw new ApplicationException("Unsupported Data Type"); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueIntHash.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueIntHash.cs new file mode 100644 index 000000000..3a39f3308 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueIntHash.cs @@ -0,0 +1,70 @@ +using System; +using BCG; +//using BCG.Utilities; + +namespace BCG +{ + [Serializable] + public class BlueIntHash + { + [Serializable] + private struct Element + { + public int k; + public int v; + public int next; + } + + public BlueIntHash(int capacity) + { + myElementPool = new Element[capacity + 1]; + myNumberOfBuckets = capacity * 3 + 1; + myBuckets = new int[myNumberOfBuckets]; + } + + public void add(int k, int v) + { + int index = myGetHash(k); + + myElementPool[myFirstFreeElement].k = k; + myElementPool[myFirstFreeElement].v = v; + myElementPool[myFirstFreeElement].next = myBuckets[index]; + myBuckets[index] = myFirstFreeElement; + myFirstFreeElement++; + } + + [Serializable] + public class ElementNotFound : Exception + { } + + public int find(int k) + { + int b = myFind(k); + if (b == 0) + throw new ElementNotFound(); + else + return myElementPool[b].v; + } + + public bool isPresent(int k) + { return myFind(k) != 0; } + + private int myFind(int k) + { + int b = myBuckets[myGetHash(k)]; + while (b != 0 && myElementPool[b].k != k) + b = myElementPool[b].next; + return b; + } + + + private int myGetHash(int k) + { return k % myNumberOfBuckets; } + + private int myFirstFreeElement = 1; + private Element[] myElementPool; + + private int[] myBuckets; + private int myNumberOfBuckets; + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRow.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRow.cs new file mode 100644 index 000000000..98b89e377 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRow.cs @@ -0,0 +1,318 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data; +using System.Reflection; +using System.Linq; + +//using BCG.Utilities; +using System.Text; + +namespace BCG +{ + /// + /// BlueRow is an analog of DataRow, an immutable one though. It constructs itself either + /// from dotNet DataRow or IBlueRow. One row can belong to any number of tables, 0 or 1 or many. + /// All what the BlueRow requires is a schema to manage its internal information + /// + [Serializable] + public class BlueRow : IBlueRow + { + protected Object[] myValues; + protected IBlueTableSchema mySchema; + + #region Private helper methods + /// + /// Initializes the internal data structures + /// + /// + private void myInit(IBlueTableSchema schema) + { + mySchema = schema; + myValues = new Object[schema.columnNames.Length]; + } + #endregion + + #region Constructors + protected BlueRow() { } + + public BlueRow(IBlueRow r) + { + myInit(r.schema); + foreach (ColumnInfo ci in mySchema.columnInfo) + myValues[ci.colNo] = r[ci.colNo]; + } + /// + /// The BlueRow initializes itself from the row passed. The row is expected + /// to have the same schema (including the col ordering) as the row being created + /// + /// + /// + public BlueRow(IBlueTableSchema schema, IBlueRow row) + { + myInit(schema); + foreach (ColumnInfo ci in mySchema.columnInfo) + myValues[ci.colNo] = row[ci.colNo]; + } + + /// + /// Constructs itself from the schema and the array of values that correspond + /// to the rows + /// + /// + /// Schema of the row + /// + /// + /// Values represent the values in the row, the order should be the same as + /// represented in the schema + /// + public BlueRow(IBlueTableSchema schema, params Object[] values) + { + myValues = values; + mySchema = schema; + } + + /// + /// Initilizes itself from a dotNet DataRow. The row is expected to + /// have the same schema (including the column ordering) as the row being created + /// + /// + /// + public BlueRow(IBlueTableSchema schema, DataRow row) + { + myInit(schema); + foreach (DataColumn col in row.Table.Columns) + myValues[col.Ordinal] = row[col.Ordinal]; + } + + public BlueRow(IBlueTableSchema schema, IDictionary row) + { + myInit(schema); + foreach (DictionaryEntry de in row) + if (mySchema[de.Key as string] != null) + myValues[mySchema[de.Key as string].colNo] = de.Value; + } + + /// + /// Constructs a new BlueRow by extracting the values fields enumerated + /// in schema from the POD (Plain Old Data) object passed. Obviously + /// the object's type must contain at least all the fields defined + /// by the schema and the types of these fields should match their counterparts. + /// See new BlueTableSchema(string tableName, Type). + /// This is a static function rather than a constructor because + /// the type signature of the constructor would not be unique (it would + /// conflict with several of the constructurs above) + /// + /// The schema of the row + /// The id of the row + /// The Plain Old Data object containing the data + static public IBlueRow fromPOD(IBlueTableSchema schema, int id, Object pod) + { return myBlueRowFromPOD(myGetAccessors(schema, pod), schema, id, pod); } + + private static Dictionary myGetAccessors(IBlueTableSchema schema, object o) + { + var accessors = new Dictionary(); + foreach (string fn in schema.columnNames) + if (fn.ToLower() != "id") + { + FieldInfo fi = o.GetType().GetField(fn); + if (fi != null) + accessors[fn] = fi; + } + return accessors; + } + + private static IBlueRow myBlueRowFromPOD(Dictionary accessors, + IBlueTableSchema schema, + int id, + object pod) + { + MutableBlueRow r = new MutableBlueRow(schema); + foreach (var kvp in accessors) + { + object v = kvp.Value.GetValue(pod); + if (v == null) + v = DBNull.Value; + r.setValue(kvp.Key, v); + } + r.setValue("id", id); + return r; + } + + public delegate int IdGenerator(object pod); + static public IBlueRowCollection fromPODCollection(IBlueTableSchema schema, IdGenerator g, IEnumerable pods) + { + Dictionary accessors = null; + BlueTable result = new BlueTable(schema); + foreach (Object o in pods) + { + if (accessors == null) + accessors = myGetAccessors(schema, o); + result.addRow(myBlueRowFromPOD(accessors, schema, g(o), o)); + } + return result; + } + + private static void myPodFromBlueRow(Dictionary accessors, + IBlueRow r, object pod) + { + foreach (var kvp in accessors) + if (!(r[kvp.Key] is DBNull)) + kvp.Value.SetValue(pod, r[kvp.Key]); + } + + /// + /// Copies a blue row to a POD (Plain Old Data) object. + /// + /// + public static T copyToPOD(IBlueRow r) where T : new() + { + T pod = new T(); + myPodFromBlueRow(myGetAccessors(r.schema, pod), r, pod); + return pod; + } + + public static List toPODCollection(IBlueRowCollection tb) where T : new() + { + List result = new List(); + Dictionary accessors = null; + + foreach (IBlueRow r in tb) + { + T pod = new T(); + if (accessors == null) + accessors = myGetAccessors(tb.schema, pod); + myPodFromBlueRow(accessors, r, pod); + result.Add(pod); + } + return result; + } + + // For viewing BlueRow in the debugger more easily + public override string ToString() + { + return this.ToString(true); + } + + public string ToString(bool names) + { + StringBuilder sb = new StringBuilder(); + bool first = true; + for (int x = 0; x < this.myValues.Length; x++) + { + if (!first) sb.Append(", "); + first = false; + if (names) + { + sb.Append(this.mySchema.columnNames[x]); + sb.Append("="); + } + sb.Append((this.myValues[x] ?? "(null)").ToString()); + } + return sb.ToString(); + + } + + /// + /// Constructs the schema and then populates the values, both from the IDictionary. + /// The table name (the only thing that can not be derived from the dictionary) is + /// also provided + /// + /// + /// + public BlueRow(string tableName, IDictionary row) + { + if (row is IBlueRow) + myInit(((IBlueRow)row).schema); + else + { + ColumnInfo[] cis = new ColumnInfo[row.Count]; + int i = 0; + foreach (DictionaryEntry de in row) + cis[i] = new ColumnInfo(de.Key as string, i++, de.Value.GetType()); + myInit(new BlueTableSchema(cis, tableName)); + } + foreach (DictionaryEntry de in row) + myValues[mySchema[de.Key as string].colNo] = de.Value; + } + + /// + /// Constructs the schema and then populates the values, both from the IDictionary. + /// + /// + public BlueRow(IDictionary row) : this("", row) { } + + #endregion + + #region IBlueRow members + public virtual IBlueTableSchema schema { get { return mySchema; } } + public virtual object this[string colName] + { + get + { + ColumnInfo ci = mySchema[colName]; + if (ci != null) + return myValues[ci.colNo]; + else + throw new Exception(colName + " is not a valid column name for " + schema.tableName); + } + } + public virtual object this[int index] { get { return myValues[index]; } } + #endregion + + #region ICollection members + public virtual int Count { get { return myValues.Length; } } + + public virtual bool IsSynchronized { get { return false; } } + + public virtual object SyncRoot { get { return this; } } + + public virtual void CopyTo(Array array, int index) { myValues.CopyTo(array, index); } + #endregion + + #region IDictionary members + public virtual bool IsFixedSize { get { return true; } } + + virtual public bool IsReadOnly { get { return true; } } + + /// + /// The key can either be the string corrsponding to the column name or + /// the order of the column + /// + public virtual object this[object key] + { + get + { + if (key is string) + return this[(string)key]; + else if (key is int) + return this[(int)key]; + throw new BlueDataException("Wrong type while indexing the row"); + } + set { throw new NotSupportedException("The IBlueRow is readonly"); } + } + + public virtual ICollection Keys { get { return mySchema.columnNames; } } + + public virtual ICollection Values { get { return myValues; } } + + public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } + + public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } + + public virtual bool Contains(object key) { return mySchema[(string)key] != null; } + + /// + /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to + /// iterate through the values, typecast this into an IEnumerable + /// + /// + public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + + IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + + public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); } + #endregion + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowComparer.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowComparer.cs new file mode 100644 index 000000000..d76d82b31 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowComparer.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Collections; + +namespace BCG +{ + /// + /// Generic IComparer for IBlueRow. Accepts a list of column names to define the sort order. All included columns must be IComparable. + /// + public class BlueRowComparer : IComparer, IComparer + { + private IEnumerable myCompareColumns; + private bool myIsAscending; + + public BlueRowComparer(bool ascending, params string[] columns) + { + this.myCompareColumns = columns; + this.myIsAscending = ascending; + } + + public BlueRowComparer(params string[] columns) + : this(true, columns) + { + } + + public int Compare(IBlueRow x, IBlueRow y) + { + // Guard against nulls + if (x == y) return 0; + if (x == null) return -1; + if (y == null) return 1; + + int result = 0; + foreach (string compareColumn in this.myCompareColumns) + { + object left = x[compareColumn]; + object right = y[compareColumn]; + if (left == right) result = 0; + else if (left == null || left == DBNull.Value) result = -1; + else if (right == null || right == DBNull.Value) result = 1; + else result = ((IComparable)left).CompareTo(right); + if (result != 0) + break; + } + + return result * (this.myIsAscending ? 1 : -1); + } + + #region IComparer Members + + public int Compare(object x, object y) + { + return this.Compare(x as IBlueRow, y as IBlueRow); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowDictionaryEnumerator.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowDictionaryEnumerator.cs new file mode 100644 index 000000000..03ed1d973 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowDictionaryEnumerator.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// This enumertaor is used to iterate through an IBlueRow when the IBlueRow is + /// used as an IDictionary + /// + public class BlueRowDictionaryEnumerator : BlueRowEnumerator, IDictionaryEnumerator + { + public BlueRowDictionaryEnumerator (IBlueRow row) : base (row) {} + public override object Current {get{return Entry;}} + + #region IDictionaryEnumerator members + public object Key {get{return myRow.schema[myPos].colName;}} + + public object Value {get{return myRow[myPos];}} + + public DictionaryEntry Entry {get{return new DictionaryEntry (Key, Value);}} + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEnumerator.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEnumerator.cs new file mode 100644 index 000000000..c4fababc5 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEnumerator.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Used for enumerating an IEnumerable (ex ICollection). Furthermore, this is the + /// default Enueramtor used when enumerating asn IBlueRow + /// + public class BlueRowEnumerator : IEnumerator + { + protected int myPos = -1; + protected int myRowSize = -1; + protected IBlueRow myRow; + + public BlueRowEnumerator (IBlueRow row) + { + myRow = row; + myRowSize = row.schema.columnNames.Length; + } + + #region IEnumerator members + public void Reset() {myPos = 0;} + + public bool MoveNext () + { + if (myPos == myRowSize - 1) + return false; + myPos ++; + return true; + } + + public virtual object Current {get{return myRow[myPos];}} + #endregion + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowHasher.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowHasher.cs new file mode 100644 index 000000000..c47701f16 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowHasher.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public class BlueRowHasher + { + private int _n_keys; + private int _hash_counter = 0; + private bool _thread_safe = false; + private IDictionary _root_map = new Dictionary(); + + public BlueRowHasher(int n_keys, bool thread_safe = false) + { + _n_keys = n_keys; + _thread_safe = thread_safe; + } + + public string get_string_hash(IBlueRow row, int[] key_cols) + { + return get_hash(row, key_cols).ToString(); + } + + /// + /// gets the hash for a row with given keys, + /// if key does not exist is adds it adds it + /// and returns the hash + /// + public int get_hash(IBlueRow row, int[] key_cols) + { + IDictionary d = _root_map; + for (int i = 0; ; i++) + { + object v = row[key_cols[i]]; + if (i < _n_keys - 1) + { + object inner_d = null; + if (!d.TryGetValue(v, out inner_d)) + { + inner_d = _get_new_dictionary(); + d.Add(v, inner_d); + } + d = (IDictionary)inner_d; + } + else + { + object rv; + if (!d.TryGetValue (v, out rv)) + { + rv = _get_next_hash(); + d.Add(v, rv); + } + return (int)rv; + } + } + } + + public int get_hash(params object[] key_vals) + { + IDictionary d = _root_map; + for (int i = 0; ; i++) + { + object v = key_vals[i]; + if (i < _n_keys - 1) + { + object inner_d = null; + if (!d.TryGetValue(v, out inner_d)) + { + inner_d = _get_new_dictionary(); + d.Add(v, inner_d); + } + d = (IDictionary)inner_d; + } + else + { + object rv; + if (!d.TryGetValue(v, out rv)) + { + rv = _get_next_hash(); + d.Add(v, rv); + } + return (int)rv; + } + } + } + + private int _get_next_hash() + { + return _thread_safe ? System.Threading.Interlocked.Increment(ref _hash_counter) : ++_hash_counter; + } + + private IDictionary _get_new_dictionary() + { + if (_thread_safe) + return new System.Collections.Concurrent.ConcurrentDictionary(); + else + return new Dictionary(); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs new file mode 100644 index 000000000..d368587b6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +/* + * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns. + * if groupNo is zero, then the group is not stord in a lookup table at all. + * + * + * + */ +namespace BCG +{ + public class BlueRowLookupGroup + { + BlueTableSchema myEntrySchema; + public IBlueTableSchema entrySchema + { + get { return myEntrySchema; } + //cant' set this value, have to build it from the row's schema + + } + + //for faster access and such + private int[] myGroupColNos; + public int[] groupColNos + { + get { return myGroupColNos; } + + } + private string[] myGroupColNames; + public string[] groupColNames + { + get { return myGroupColNames; } + } + + private short myGroupNumber; + public short groupNumber + { + get { return myGroupNumber; } + + } + + /* how many bits are the ids into the table? + * In c# everything is stored as integers, but when it goes across the wire + * the data can be represented as a byte, a short, or an int + */ + private short myTableSize; + public short tableSize + { + get { return myTableSize; } + + } + + //if the group number is 0, the group is not stored in tables for serialization + public bool isStoredInTable + { + get { return myGroupNumber != 0; } + + } + public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols) + { + + myGroupNumber = num; + myTableSize = sizeInBits; + myGroupColNames = cols ; + } + + /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing + each of these entries in the lookup table. This function should be called by the LookupTableSchema + during its constructor. + + This function also caches the column numbers for this group, for quicker access*/ + public void buildEntrySchema(LookupTableSchema theSchema) + { + ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1]; + myGroupColNos = new int[myGroupColNames.Length]; + //the first column in our table is always the ID + int idColumn = 0; + columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int)); + //go through each of the columns + for (int col = 1; col < columns.Length; ++col) + { + ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]]; + if (columnInInputTable.isConstant) + columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue); + else + columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType); + + myGroupColNos[col - 1] = columnInInputTable.colNo; + } + + + + myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber); + + } + + /*takes a row as input, and extracts the members of the group from the row */ + public object[] extractGroupFromRow(IBlueRow input) + { + int numCols = myGroupColNames.Length; + object[] results = new object[numCols]; + + for (int col = 0; col < numCols; ++col) + results[col] = input[myGroupColNos[col]]; + + return results; + + } + + + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs new file mode 100644 index 000000000..a66c86654 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs @@ -0,0 +1,239 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data; +//using BCG.Utilities; +using System.Linq; + +namespace BCG +{ + /// + /// Anologue of the dotNet DataTable. + /// + [Serializable] + public class BlueTable : IBlueRowCollection + { + /// + /// Hashtable indexed on "id" to speed up removes + /// + protected Hashtable myIdIndex = new Hashtable(); + /// + /// Schema the table adheres to + /// + protected IBlueTableSchema mySchema; + + #region Constructors + protected BlueTable() { } + /// + /// The BlueTable constructs the schema from the table being passed and copies + /// all the data over + /// + /// + public BlueTable(DataTable table) + { + ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; + int i = 0; + foreach (DataColumn col in table.Columns) + cis[i] = new ColumnInfo(col.ColumnName, i++, col.DataType); + mySchema = new BlueTableSchema(cis, table.TableName); + foreach (DataRow row in table.Rows) + { + IBlueRow newRow = new BlueRow(mySchema, row.ItemArray); + myIdIndex[newRow["id"]] = newRow; + } + } + + public BlueTable(IBlueRowCollection table) + { + mySchema = table.schema; + foreach (IBlueRow row in table) + addRow(row); + } + + public BlueTable(string tableName, + IEnumerable objects, + params CMap[] columnMappers) + { + // extract schema from mappers + ColumnInfo[] cols = new ColumnInfo[columnMappers.Length]; + for (int i = 0; i < columnMappers.Length; i++) + { + cols[i] = new ColumnInfo( + columnMappers[i].getColumnName(), i, + columnMappers[i].getColumnType()); + } + mySchema = new BlueTableSchema(cols, tableName); + + // load rows + int rowNumber = 0; + foreach (object rowObject in objects) + { + object[] values = new object[columnMappers.Length]; + for (int cn = 0; cn < columnMappers.Length; cn++) + values[cn] = columnMappers[cn].getColumnValue(rowNumber, rowObject); + addRow(new BlueRow(mySchema, values)); + rowNumber++; + } + } + + /// + /// Gets the schema, no data is iniliazed at construction time + /// + /// + public BlueTable(IBlueTableSchema schema) { mySchema = schema; } + #endregion + + #region IBlueRowCollection members + /// + /// Returns the row by the index, interface property + /// + public virtual IBlueRow this[int id] + { + get + { + object row = myIdIndex[id]; + if (row == null) + throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); + return (IBlueRow)row; + } + } + + /// + /// Gets the schema + /// + public virtual IBlueTableSchema schema { get { return mySchema; } } + + public virtual bool containsID(int id) { return myIdIndex[id] != null; } + + public virtual bool tryGetValue(int id, out IBlueRow row) + { + row = (IBlueRow)myIdIndex[id]; + return (row != null); + } + + #endregion + + #region BlueTable members + public virtual void clear() + { + myIdIndex.Clear(); + } + + /// + /// If a row with the same id exists, it replaces it + /// + /// + public virtual void forceAddRow(IBlueRow r) + { + myIdIndex[r["id"]] = r; + } + + /// + /// Adds a new row to the table. + /// + /// + public virtual void addRow(IBlueRow row) + { + object val = row["id"]; + + if (val == null || val == DBNull.Value) + throw new BlueDataException("The ID of the row being added is null/DNBull"); + try + { + myIdIndex.Add(val, row); + } + catch (ArgumentException) + { + throw new BlueDataException("ID already exists in the table " + mySchema.tableName + + " , duplication of field ID not allowed"); + } + + } + + public virtual void addRow(DataRow row) + { + object val = row["id"]; + + if (val == null || val == DBNull.Value) + throw new BlueDataException("The ID of the row being added is null/DNBull"); + if (myIdIndex[val] != null) + throw new BlueDataException("ID already exists in the table, duplication of field ID not allowed"); + IBlueRow newRow = new BlueRow(mySchema, row); + myIdIndex[val] = newRow; + } + + public virtual void addRows(IBlueRowCollection rows) + { + foreach (IBlueRow nr in rows) + addRow(nr); + } + + public virtual void addRows(IEnumerable rows) + { + foreach (IBlueRow nr in rows) + addRow(nr); + } + + /// + /// Removes a row from the table. Complexity = O(1) + /// + /// + public virtual bool deleteRow(IBlueRow row) { return deleteRow((int)row["id"]); } + + public virtual bool deleteRow(DataRow row) { return deleteRow((int)row["id"]); } + + /// + /// Removes a row specified by the id. The index helps in making this operation O(1) + /// + /// + public virtual bool deleteRow(int id) + { + if (myIdIndex[id] == null) + return false; + myIdIndex.Remove(id); + return true; + } + + public virtual bool deleteRows(IBlueRowCollection rows) + { + bool retVal = true; + foreach (IBlueRow r in rows) + retVal &= deleteRow(r); + return retVal; + } + + public virtual bool deleteRows(IEnumerable rows) + { + bool retVal = true; + foreach (IBlueRow r in rows) + retVal &= deleteRow(r); + return retVal; + } + + public virtual void changeRow(IBlueRow r) + { + myIdIndex[r["id"]] = r; + } + + public virtual void changeRows(IBlueRowCollection rows) + { + foreach (IBlueRow r in rows) + changeRow(r); + } + + #endregion + + #region ICollection members + public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();} + { return myIdIndex.Values.GetEnumerator(); } + + public virtual bool IsSynchronized { get { return false; } } + + public virtual object SyncRoot { get { return this; } } + + public virtual void CopyTo(Array array, int index) { myIdIndex.Values.CopyTo(array, index); } + + public virtual int Count { get { return myIdIndex.Count; } } + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs new file mode 100644 index 000000000..1a69c2103 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs @@ -0,0 +1,286 @@ +using System; +using System.Data; +using System.Collections; +using System.IO; +using System.Reflection; +using System.Collections.Generic; +using System.Linq; + +using BCG.Utilities.General; + +namespace BCG +{ + /// + /// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created + /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it + /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return + /// the respective data, sorted according to the ColumnOrder. + /// + [Serializable] + public class BlueTableSchema : IBlueTableSchema + { + public class BlueTableSchemaSerializationHelper + { + public static object parse(Type t, string data) + { + if (t == BlueDataValidTypes.BOOL) + return Boolean.Parse(data); + else if (t == BlueDataValidTypes.BYTE) + return Byte.Parse(data); + else if (t == BlueDataValidTypes.CHAR) + return char.Parse(data); + else if (t == BlueDataValidTypes.DATETIME) + return DateTime.Parse(data); + else if (t == BlueDataValidTypes.DOUBLE) + return Double.Parse(data); + else if (t == BlueDataValidTypes.DECIMAL) + return Decimal.Parse(data); + else if (t == BlueDataValidTypes.INT) + return int.Parse(data); + else if (t == BlueDataValidTypes.LONG) + return long.Parse(data); + else if (t == BlueDataValidTypes.SBYTE) + return SByte.Parse(data); + else if (t == BlueDataValidTypes.STRING) + return data; + else if (t == BlueDataValidTypes.UINT) + return uint.Parse(data); + else if (t == BlueDataValidTypes.ULONG) + return ulong.Parse(data); + else if (t == BlueDataValidTypes.SINGLE) + return Single.Parse(data); + else + throw new ApplicationException("Can not be parsed"); + } + public static void serialize(BinaryWriter bw, Type t, object o) + { + if (t == BlueDataValidTypes.BOOL) + bw.Write((bool)o); + else if (t == BlueDataValidTypes.BYTE) + bw.Write((byte)o); + else if (t == BlueDataValidTypes.CHAR) + bw.Write((char)o); + else if (t == BlueDataValidTypes.DATETIME) + bw.Write(((DateTime)o).Ticks); + else if (t == BlueDataValidTypes.DOUBLE) + bw.Write((double)o); + else if (t == BlueDataValidTypes.DECIMAL) + bw.Write((decimal)o); + else if (t == BlueDataValidTypes.INT) + bw.Write((int)o); + else if (t == BlueDataValidTypes.LONG) + bw.Write((long)o); + else if (t == BlueDataValidTypes.SBYTE) + bw.Write((sbyte)o); + else if (t == BlueDataValidTypes.STRING) + bw.Write(o as string); + else if (t == BlueDataValidTypes.SINGLE) + bw.Write((Single)o); + else if (t == BlueDataValidTypes.BYTEARRAY) + { + byte[] bs = o as byte[]; + bw.Write(bs.Length); + bw.Write(bs); + } + else if (t == BlueDataValidTypes.UINT) + bw.Write((uint)o); + else if (t == BlueDataValidTypes.ULONG) + bw.Write((ulong)o); + else + throw new ApplicationException("Unsupported type"); + } + + public static object deserialize(BinaryReader br, Type t) + { + if (t == BlueDataValidTypes.BOOL) + return br.ReadBoolean(); + else if (t == BlueDataValidTypes.BYTE) + return br.ReadByte(); + else if (t == BlueDataValidTypes.CHAR) + return br.ReadChar(); + else if (t == BlueDataValidTypes.DATETIME) + return new DateTime(br.ReadInt64()); + else if (t == BlueDataValidTypes.DOUBLE) + return br.ReadDouble(); + else if (t == BlueDataValidTypes.DECIMAL) + return br.ReadDecimal(); + else if (t == BlueDataValidTypes.INT) + return br.ReadInt32(); + else if (t == BlueDataValidTypes.LONG) + return br.ReadInt64(); + else if (t == BlueDataValidTypes.SBYTE) + return br.ReadSByte(); + else if (t == BlueDataValidTypes.STRING) + return br.ReadString(); + else if (t == BlueDataValidTypes.BYTEARRAY) + return br.ReadBytes(br.ReadInt32()); + else if (t == BlueDataValidTypes.UINT) + return br.ReadUInt32(); + else if (t == BlueDataValidTypes.ULONG) + return br.ReadUInt64(); + else if (t == BlueDataValidTypes.SINGLE) + return br.ReadSingle(); + else + throw new ApplicationException("Unsupported type"); + } + } + + protected Hashtable myIndexByColName = new Hashtable(); + protected ColumnInfo[] myIndexByColOrder; + private string[] myTableColNames; + protected string myTableName; + + public BlueTableSchema() { } + + public BlueTableSchema(DataTable table) + { + myIndexByColOrder = new ColumnInfo[table.Columns.Count]; + myTableName = table.TableName; + foreach (DataColumn col in table.Columns) + { + ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); + myIndexByColName[ci.colName] = ci; + myIndexByColOrder[ci.colNo] = ci; + } + } + + public BlueTableSchema(ColumnInfo[] cis, string tableName) + { + myConstruct(cis, tableName); + } + + private void myConstruct(ColumnInfo[] cis, string tableName) + { + myIndexByColOrder = new ColumnInfo[cis.Length]; + myTableName = tableName; + foreach (ColumnInfo ci in cis) + { + myIndexByColOrder[ci.colNo] = ci; + myIndexByColName[ci.colName] = ci; + } + } + + public BlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { } + + /// + /// Creates a BlueTableSchema that exactly mirrors the public fields + /// present in fromType except that an id field will be added regardless + /// of it being present in fromType or not. + /// + /// the table name + /// the plain old data type whose fields will be mirrored in the schema + public BlueTableSchema(string tableName, Type fromType) + { + List cis = new List(); + int i = 0; + // we sort the field name so that we have a deterministic order given + // the same type. + bool foundId = false; + foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; })) + { + if (fi.Name.ToLower().Equals("id")) + foundId = true; + cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType)); + } + if (!foundId) + cis.Add(new ColumnInfo("id", cis.Count, typeof(Int32))); + + myConstruct(cis.ToArray(), tableName); + } + + public ColumnInfo this[string colName] + { + get + { + object retVal = myIndexByColName[colName]; + return retVal != null ? retVal as ColumnInfo : null; + } + } + + public ColumnInfo this[int colNo] + { + get { return myIndexByColOrder[colNo]; } + } + + /// + /// The returned Array is sorted according to the Column No + /// + public string[] columnNames + { + get + { + if (myTableColNames == null) + { + myTableColNames = new string[myIndexByColName.Count]; + foreach (DictionaryEntry de in myIndexByColName) + { + ColumnInfo temp = (ColumnInfo)de.Value; + myTableColNames[temp.colNo] = temp.colName; + } + } + return myTableColNames; + } + } + + /// + /// The returned Array is sorted according to the Column No + /// + public ColumnInfo[] columnInfo + { + get { return myIndexByColOrder; } + } + + public string tableName { get { return myTableName; } } + + + #region ISerializedType methods + public void serialize(BinaryWriter bw) + { + bw.Write(columnNames.Length); + bw.Write(tableName); + foreach (ColumnInfo ci in columnInfo) + { + bw.Write(ci.colName); + bw.Write(ci.colType.ToString()); + if (ci.isConstant) + { + bw.Write(true); + BlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue); + } + else + { + bw.Write(false); + } + } + } + + public void deserialze(BinaryReader br) + { + ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()]; + string name = br.ReadString(); + for (int i = 0; i < cis.Length; i++) + { + string colName = br.ReadString(); + Type t = TypeInstanceLoader.loadType(br.ReadString(), true); + bool isConstant = br.ReadBoolean(); + if (isConstant) + { + cis[i] = new ColumnInfo(colName, i, t, BlueTableSchemaSerializationHelper.deserialize(br, t)); + } + else + { + cis[i] = new ColumnInfo(colName, i, t); + } + } + myConstruct(cis, name); + } + + public int getSerializedSize() + { + throw new NotImplementedException("Feature for provided constant values to" + + "non-primitive types in schema, not provided as of now"); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/CMap.cs b/src/DataStreamSuite/DataStreamCore/BlueData/CMap.cs new file mode 100644 index 000000000..10908d036 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/CMap.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public abstract class CMap + { + public abstract string getColumnName(); + public abstract Type getColumnType(); + public abstract object getColumnValue(int rowIndex,object rowObject); + } + + public class CMap : CMap + { + public CMap(string columnName, MapF mapperFunction) + { + myColumnName = columnName; + myExtractColumn = mapperFunction; + } + + private string myColumnName; + public override string getColumnName() + { + return myColumnName; + } + public override Type getColumnType() + { + return typeof(T); + } + + public delegate T MapF(int rowIndex, object rowObject); + private MapF myExtractColumn; + public override object getColumnValue(int rowIndex, object rowObject) + { + return myExtractColumn(rowIndex, rowObject); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs new file mode 100644 index 000000000..b4a8d7607 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs @@ -0,0 +1,92 @@ +using System; +using System.Data; +using System.Collections.Generic; +using System.Linq; + +namespace BCG +{ + /// + /// Circular buffer for rows. + /// + public class CircularRowCollection : OrderedBlueTable + { + private int myBufferSize; + public CircularRowCollection (IBlueTableSchema schema, int bufferSize) + : base (schema) + { + myBufferSize = bufferSize ; + } + + public CircularRowCollection (IBlueRowCollection rows, int bufferSize) + : this (rows.schema, bufferSize) + { + foreach (IBlueRow r in rows) + addRow (r); + } + + public int bufferSize + { + get + { + return (myBufferSize); + } + } + + /// + /// Adds a new row to the collection. If the size of the + /// collection becomes greater that + /// buffer size, the first row is returned; + /// + /// + /// + public IBlueRow addRowWithReturn (IBlueRow r) + { + base.addRow (r); + if (myOrderedRows.Count > myBufferSize) + { + IBlueRow retVal = myOrderedRows [0] as IBlueRow; + deleteRow (retVal); + return retVal; + } + return null; + } + + public override void addRow (IBlueRow row) + { + base.addRow (row); + if (myOrderedRows.Count > myBufferSize) + deleteRow (myOrderedRows [0] as IBlueRow); + } + + public override void addRow (DataRow row) + { + base.addRow (row); + if (myOrderedRows.Count > myBufferSize) + deleteRow (myOrderedRows [0] as IBlueRow); + } + + public void add_rows(IEnumerable rows_to_add, out IBlueRowCollection added_rows, out IBlueRowCollection deleted_rows) + { + BlueTable added_rv = new BlueTable(mySchema), deleted_rv = new BlueTable(mySchema); + if (rows_to_add.Count() < myBufferSize) + { + foreach (IBlueRow r in rows_to_add) + { + IBlueRow dr = addRowWithReturn(r); + added_rv.addRow(r); + if (dr != null) + deleted_rv.addRow(dr); + } + } + else + { + deleted_rv = new BlueTable(this); + clear(); + addRows(rows_to_add); + added_rv = new BlueTable(this); + } + added_rows = added_rv; + deleted_rows = deleted_rv; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs new file mode 100644 index 000000000..9337d9420 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs @@ -0,0 +1,95 @@ +using System; + +namespace BCG +{ + /// + /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable + /// + [Serializable] + public class ColumnInfo : IComparable + { + public static bool isAllowedType (Type t) + { + return + t.Equals (typeof (System.Int32)) || + t.Equals (typeof (System.Int64)) || + t.Equals (typeof (System.Double)) || + t.Equals (typeof (System.String)) || + t.Equals (typeof (System.DateTime)) ? true : + false; + } + + //the default column info does not cach values in a lookup table, therefore it is in group 0 + public ColumnInfo(string colName, int colNo, Type colType) + : this(colName, colNo, colType, 0) + { + + + } + + public ColumnInfo(string colName, int colNo, Type colType, short colGroup) + { + myColName = colName; + myColNo = colNo; + myColType = colType; + myLookupTableGroup = colGroup; + myColValue = null; + } + + + public ColumnInfo (string colName, int colNo, object colValue) + : this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;} + + public ColumnInfo (string colName, int colNo, Type colType, object colValue) + : this (colName, colNo, colType) + { + + myColValue = colValue == null ? DBNull.Value : colValue; + + } + + + public int CompareTo (object obj) + { + int otherColNo = ((ColumnInfo)(obj)).colNo; + if (myColNo < otherColNo) + return -1; + else if (myColNo > otherColNo) + return 1; + return 0; + } + + //public bool isCalculated{get{return myExpression != null;}} + + public bool isConstant {get{return myColValue != null;}} + + private int myColNo; + private string myColName; + private Type myColType; + //private string myExpression; + private object myColValue; + + //no table will ever have more than 65,536 columns + //if it does, we are in serious trouble + private short myLookupTableGroup; + + public int colNo{get{return myColNo;}} + public string colName {get{return myColName;}} + public Type colType {get{return myColType;}} + + + //this value corresponds to how the data in the column is serialized. + //if the group number is 0, the values are not stored in a table + //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together + //are stored together in a lookup table + + + public short lookupTableGroup + { + get { return myLookupTableGroup; } + set { myLookupTableGroup = value; } + } + //public string expression {get{return myExpression;}} + public object colValue {get{return myColValue;}} + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnKeyedData.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnKeyedData.cs new file mode 100644 index 000000000..b7ec368fa --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnKeyedData.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public class ColumnKeyedData + { + private int[] _key_col_indices = null; + private string[] _key_col_names; + private BlueRowHasher _row_hasher; + private Dictionary _row_map = new Dictionary(); + + public ColumnKeyedData(params string[] key_col_names) + { + _key_col_names = key_col_names; + _row_hasher = new BlueRowHasher(key_col_names.Length); + } + + public void delete_rows(IBlueRowCollection rows) + { + if (rows != null) + { + _init_schema(rows.schema); + foreach (IBlueRow r in rows) + { + int row_hash = _row_hasher.get_hash(r, _key_col_indices); + if (_row_map.ContainsKey(row_hash)) + _row_map.Remove(row_hash); + } + } + } + + public void add_rows(IBlueRowCollection rows) + { + if (rows != null) + { + _init_schema(rows.schema); + foreach (IBlueRow r in rows) + { + int row_hash = _row_hasher.get_hash(r, _key_col_indices); + _row_map[row_hash] = r; + } + } + } + + public void reset_state (IBlueRowCollection rows) + { + _row_map.Clear(); + add_rows(rows); + } + + public IEnumerable get_all_rows() { return _row_map.Values; } + + private void _init_schema(IBlueTableSchema schema) + { + if (_key_col_indices == null) + _key_col_indices = BlueDataUtility.get_column_indices(schema, _key_col_names); + } + + public IBlueRow get_row(params object[] key_vals) + { + int row_hash = _row_hasher.get_hash(key_vals); + IBlueRow rv = null; + _row_map.TryGetValue(row_hash, out rv); + return rv; + } + + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/FileRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/FileRowCollection.cs new file mode 100644 index 000000000..035b57874 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/FileRowCollection.cs @@ -0,0 +1,54 @@ +using System; +using System.IO; +using System.Collections; + + +namespace BCG +{ + public class FileRowCollection : CircularRowCollection + { + public FileRowCollection(string dataFileName, + string schemaFileName) + : this(dataFileName, schemaFileName, Int32.MaxValue) + { } + + /// + /// + /// file with the rows + /// file with schema (written as columns names in the first line and types in the 2nd + /// if max size is mentioned then a buffer is mantained for only + /// that number of rows, ie the last maxSize number of rows in the file + /// + public FileRowCollection(string dataFileName, + string schemaFileName, + int maxSize) + : this(new StreamReader(dataFileName), + new StreamReader(schemaFileName), + maxSize) + { } + + public FileRowCollection(StreamReader dataReader, + StreamReader schemaReader) + : this(dataReader, schemaReader, Int32.MaxValue) + { } + + public FileRowCollection(StreamReader dataReader, + StreamReader schemaReader, + int maxSize) + : base(BlueDataUtility.createSchemaFromCSVSchemaFile(schemaReader), maxSize) + { + int id = 0; + while (dataReader.Peek() > 0) + { + try + { + addRow(BlueDataUtility.createRowFromCSVFileRow(dataReader.ReadLine(), mySchema, id++)); + } + catch (Exception) + { + //BlueEventLog.logEvent (e.Message, BlueEventLevel.Warning); + } + } + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/GridBindableBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/GridBindableBlueTable.cs new file mode 100644 index 000000000..b041bc171 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/GridBindableBlueTable.cs @@ -0,0 +1,384 @@ +// using System; +// using System.Reflection; +// using System.Reflection.Emit; +// using System.Collections; +// using System.ComponentModel; + +// namespace BCG +// { +// public class GridBindableBlueTable : IBindingList, IBlueRowCollection, ITypedList +// { +// private class MyComparer : IComparer +// { +// private string myColName; +// public MyComparer (string colName) +// { +// myColName = colName; +// } +// #region IComparer Members + +// public int Compare (object x, object y) +// { +// IBlueRow r1 = x as IBlueRow; +// IBlueRow r2 = y as IBlueRow; +// try +// { +// return +// r1[myColName] == DBNull.Value ? +// r2[myColName] == DBNull.Value ? +// 0 +// : +// -1 +// : +// r2[myColName] == DBNull.Value ? +// 1 +// : +// (r1[myColName] as IComparable).CompareTo (r2[myColName]); +// } +// catch (Exception e) +// { +// Console.WriteLine (e); +// return 0; +// } +// } + +// #endregion + +// } + +// private PropertyDescriptor mySortPropertyDescriptor; +// private IBlueTableSchema mySchema; +// private PropertyDescriptorCollection myPDColl; +// protected ArrayList myList = new ArrayList (); +// protected ListSortDirection mySortDirection; +// protected IComparer myComparer; +// public event System.ComponentModel.ListChangedEventHandler ListChanged; + +// public GridBindableBlueTable (IBlueTableSchema schema) +// : this (schema, new ArrayList()) +// {} + +// public GridBindableBlueTable (IBlueTableSchema schema, ICollection data) +// { +// myComparer = new MyComparer ("id"); +// mySchema = schema; +// myInitPropertyDescriptors (); +// foreach (object o in data) +// Add (o); +// } + +// private void myInitPropertyDescriptors () +// { +// ArrayList temp = new ArrayList (); +// foreach (ColumnInfo ci in mySchema.columnInfo) +// temp.Add (new BlueDataPropertyDescriptor (ci)); +// myPDColl = new PropertyDescriptorCollection ( +// temp.ToArray (typeof(PropertyDescriptor)) as PropertyDescriptor[]); +// } + +// #region IList Members + +// public bool IsReadOnly {get {return false;}} + +// public virtual object this [int index] +// { +// get {return myList[index];} +// set {throw new NotSupportedException ("Illegal operation for a Sorted list");} +// } + +// public virtual void RemoveAt(int index) +// { +// myList.RemoveAt (index); +// if (ListChanged != null) +// ListChanged (this, new ListChangedEventArgs (ListChangedType.ItemDeleted, index)); +// } + +// public virtual void Insert (int index, object value) +// { +// throw new NotSupportedException ("Illegal operation for a Sorted list"); +// } + +// public virtual void Remove (object value) +// { +// RemoveAt (myList.IndexOf (value)); +// } + +// public virtual bool Contains (object value) +// { +// return myList.Contains (value); +// } + +// public virtual void Clear() +// { +// myList.Clear (); +// if (ListChanged != null) +// ListChanged (this, new ListChangedEventArgs (ListChangedType.Reset, 0)); +// } + +// public virtual int IndexOf (object value) +// { +// return myList.IndexOf (value); +// } + +// /// +// /// Adds the value to the right index in the sorted list +// /// +// /// +// /// +// public virtual int Add (object value) +// { +// int insertionIndex = -1; +// if (Count > 0) +// { +// for (int i = 0; i < myList.Count; i++) +// { +// if ((insertionIndex = myGetInsertionIndex (value, i)) > -1) +// { +// myList.Insert (insertionIndex, value); +// break; +// } +// insertionIndex = -1; +// } +// } +// else +// { +// insertionIndex = 0; +// myList.Add (value); +// } +// if (ListChanged != null) +// ListChanged (this, new ListChangedEventArgs (ListChangedType.ItemAdded, insertionIndex)); +// return insertionIndex; +// } + +// private int myGetInsertionIndex (object obj, int listIndex) +// { +// if (mySortPropertyDescriptor == null) +// return myList.Count; +// if (mySortDirection.Equals (ListSortDirection.Ascending)) +// { +// if (myComparer.Compare (obj, myList[listIndex]) <= 0) +// return listIndex; +// else if (listIndex == myList.Count-1) +// return myList.Count; +// } +// else +// { +// if (myComparer.Compare (obj, myList[listIndex]) >= 0) +// return listIndex; +// else if (listIndex == myList.Count-1) +// return myList.Count; +// } +// return -1; +// } + +// public bool IsFixedSize {get {return false;}} + +// #endregion + +// #region ICollection Members + +// public bool IsSynchronized {get {return false;}} + +// public int Count {get {return myList.Count;}} + +// public void CopyTo(Array array, int index) {myList.CopyTo (array, index);} + +// public object SyncRoot {get {return null;}} + +// #endregion + +// #region IEnumerable Members + +// public IEnumerator GetEnumerator() {return myList.GetEnumerator ();} + +// #endregion + +// #region IBindingList Members + +// public void AddIndex(PropertyDescriptor property) +// { +// throw new NotSupportedException ("Adding index is not supported"); +// } + +// public bool AllowNew +// { +// get {return false;} +// } + +// public void ApplySort (PropertyDescriptor property, System.ComponentModel.ListSortDirection direction) +// { +// if (mySortPropertyDescriptor != null +// && mySortPropertyDescriptor.Name == property.Name) +// { +// if (mySortDirection == direction) +// return; +// else +// { +// mySortDirection = direction; +// myList.Reverse (); +// return; +// } +// } +// foreach (PropertyDescriptor pd in myPDColl) +// if (pd.Name == property.Name) +// mySortPropertyDescriptor = pd; +// myComparer = new MyComparer (property.Name); +// mySortDirection = direction; +// myList.Sort (myComparer); +// } + +// public PropertyDescriptor SortProperty +// { +// get {return mySortPropertyDescriptor;} +// } + +// public int Find (PropertyDescriptor property, object key) +// { +// throw new NotSupportedException ("Find not supported"); +// } + +// public bool SupportsSorting +// { +// get {return true;} +// } + +// public bool IsSorted {get {return mySortPropertyDescriptor != null;}} + +// public bool AllowRemove {get {return false;}} + +// public bool SupportsSearching {get {return false;}} + +// public System.ComponentModel.ListSortDirection SortDirection {get {return mySortDirection;}} + + + +// public bool SupportsChangeNotification {get {return true;}} + +// public void RemoveSort() +// { +// mySortPropertyDescriptor = null; +// } + +// public object AddNew() +// { +// throw new NotSupportedException ("Find not supported"); +// } + +// public bool AllowEdit +// { +// get {return false;} +// } + +// public void RemoveIndex(PropertyDescriptor property) +// { +// throw new NotSupportedException ("Find not supported"); +// } + +// #endregion + +// #region IBlueRowCollection Members + +// IBlueRow BCG.IBlueRowCollection.this[int id] +// { +// get +// { +// foreach (IBlueRow r in (this as IList)) +// { +// if (r["id"].Equals(id)) +// return r; +// } +// return null; +// } +// } + +// public bool containsID (int id) +// { +// return (this as IBlueRowCollection) [id] != null; +// } + +// public IBlueTableSchema schema {get {return mySchema;}} + +// #endregion + +// #region Dynamic Type Creation + +// private IBlueRow myCreateDynamicRow (IBlueRow r) +// { +// //return Activator.CreateInstance (myRowType, new object[] {r}) as IBlueRow; +// return r; +// } + +// private Type myGenerateType () +// { +// ILGenerator ilg; +// AssemblyName an = new AssemblyName (); +// an.Name = "DynamicRowAssembly"; +// AssemblyBuilder aBuilder = +// AppDomain.CurrentDomain.DefineDynamicAssembly (an, AssemblyBuilderAccess.Run); +// ModuleBuilder mBuilder = +// aBuilder.DefineDynamicModule ("DefaultModule"); +// TypeBuilder tBuilder = +// mBuilder.DefineType ("DynamicBlueRow", TypeAttributes.Public, typeof(BlueRow)); +// ConstructorBuilder cb = +// tBuilder.DefineConstructor ( +// MethodAttributes.Public, +// CallingConventions.Standard, +// new Type[] {typeof(IBlueRow)} +// ); +// ilg = cb.GetILGenerator (); +// ilg.Emit (OpCodes.Ldarg_0); +// ilg.Emit (OpCodes.Ldarg_1); +// ilg.Emit (OpCodes.Call, +// typeof(BlueRow).GetConstructor (new Type[] {typeof(IBlueRow)})); +// ilg.Emit (OpCodes.Ret); + +// foreach (string col in mySchema.columnNames) +// { +// myAddProperty (col, tBuilder); +// } + +// Type dynType = tBuilder.CreateType (); +// return dynType; +// } + +// private void myAddProperty (string colName, TypeBuilder tBuilder) +// { +// MethodBuilder methodBuilder = +// tBuilder.DefineMethod ("get_" + colName, +// MethodAttributes.Public, +// CallingConventions.Standard, +// typeof(object), +// new Type[]{} +// ); +// ILGenerator ilg = +// methodBuilder.GetILGenerator (); +// ilg.Emit (OpCodes.Ldarg_0); +// ilg.Emit (OpCodes.Ldstr, colName); +// ilg.Emit (OpCodes.Call, typeof(BlueRow).GetMethod ("get_Item", new Type[]{typeof(string)})); +// ilg.Emit (OpCodes.Ret); + +// PropertyBuilder propBuilder = +// tBuilder.DefineProperty (colName, System.Reflection.PropertyAttributes.None , typeof(object), new Type[]{}); +// propBuilder.SetGetMethod (methodBuilder); +// } + +// #endregion + +// #region ITypedList Members + +// public PropertyDescriptorCollection GetItemProperties(PropertyDescriptor[] listAccessors) +// { +// return myPDColl; +// } + +// public string GetListName(PropertyDescriptor[] listAccessors) +// { +// return "GridBindableBlueTable"; +// } + +// #endregion +// } + +// } + + diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/HashTableND.cs b/src/DataStreamSuite/DataStreamCore/BlueData/HashTableND.cs new file mode 100644 index 000000000..93bf424b5 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/HashTableND.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// An N-domensional hashtable , providing add and retrieve features + /// for the time being. Does not deal with null keys right now + /// + /// + [Serializable] + public class HashTableND + { + private int myDim, myCount = 0; + private IDictionary myRootDic = new Hashtable (); + + public HashTableND (int dim) {myDim = dim;} + + + public object add (object[] keys, object val) + { + ArrayList t = new ArrayList (keys); + t.Add (val); + return add (t.ToArray()); + } + /// + /// Adds a value to the hashtable. the size of the arg array should be dim+1, with the + /// the first dim entries corresponding to keys and the last to the value to be added + /// + /// + public object add (object[] os) + { + if (os.Length != myDim+1) + throw new ApplicationException ("size of arg array not right"); + else + { + myCount ++; + IDictionary d = myRootDic; + for (int i = 0 ; ; i++) + { + if (i < myDim - 1) + { + if (!d.Contains (os[i])) + d.Add (os[i], new Hashtable ()); + d = d[os[i]] as IDictionary; + } + else + { + d[os[i]] = os[i+1]; + return os [myDim]; + } + } + } + } + + + /// + /// Retrieves values from the hastable, with the arg array specifying the keys. + /// + /// + /// + public object getValue (params object[] keys) + { + IDictionary d = myRootDic; + for (int i = 0; ; i++) + if (i < keys.Length-1) + { + d = d[keys[i]] as IDictionary; + if (d == null) + return null; + } + else + return d[keys[i]]; + } + + + public void setValue (object[] keys, object val) + { + ArrayList t = new ArrayList (keys); + t.Add (val); + setValue (t.ToArray()); + } + + public void setValue (object[] os) + { + if (os.Length != myDim+1) + throw new ApplicationException ("size of arg array not right"); + IDictionary d = myRootDic; + for (int i = 0; i < myDim-1; i++) + d = d[os[i]] as IDictionary; + d[os[myDim-1]] = os[myDim]; + } + + public void remove (params object[] keys) + { + myCount --; + IDictionary d = myRootDic; + for (int i = 0; ; i++) + if (i < keys.Length-1) + d = d[keys[i]] as IDictionary; + else + { + d.Remove (keys[i]); + return; + } + } + + public int count {get {return myCount;}} + + public ICollection topLevelKeys { get {return myRootDic.Keys;}} + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRow.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRow.cs new file mode 100644 index 000000000..d351ef36d --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRow.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Represents a primitive immutable anologue of a dotNet DataRow. The user assumes no info about the row + /// but the number of columns, and the way to access data through the indexer by providing either the + /// ordinal or the name. Also note that the default iteration through the IBlueRow return DictionaryEntry + /// (ie the default iteration is the same as an IDictionary's iteration) + /// + public interface IBlueRow : IDictionary + { + /// + /// Gets the schema of the row, which is same as that of the IBlueRowCollections to which it belongs + /// + IBlueTableSchema schema{get;} + /// + /// Though the IDictionary supports an untyped indexer, having the typed indexer makes retreivals + /// faster, specially in the case of large tables, where one type checking per cell would amount to + /// a sizeable time, eg for a table with 20000 rows and a schema of 20 columns, we will save about + /// half a million type checkings + /// + object this[int index] {get;} + object this[string colName] {get;} + } + namespace Extensions + { + public static class IBlueRowExtensions + { + public static bool isDBNull(this object val) + { + return (val == null || val == DBNull.Value); + } + public static bool isNotDBNull(this object val) + { + return (val != null && val != DBNull.Value); + } + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs new file mode 100644 index 000000000..ad4932bb6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Represents a readonly, very primitive form of dotNet DataTable. All it has has an indexer for + /// getting the rows + /// + public interface IBlueRowCollection : ICollection + { + /// + /// Gets the row corresponding to the id + /// + IBlueRow this[int id]{get;} + IBlueTableSchema schema {get;} + bool containsID (int id); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollectionBuilder.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollectionBuilder.cs new file mode 100644 index 000000000..4c8726f02 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollectionBuilder.cs @@ -0,0 +1,228 @@ +using System; +using System.Diagnostics; +using System.Collections; +//using BCG.Utilities; +//using BCG.Utilities.General; + +namespace BCG +{ + public class IBlueRowCollectionBuilder + { + static public bool isBlueDataType(Type t) + { + bool a = t.Equals(BlueDataValidTypes.BOOL) || t.Equals(BlueDataValidTypes.BYTE) + || t.Equals(BlueDataValidTypes.BYTEARRAY) || t.Equals(BlueDataValidTypes.CHAR) + || t.Equals(BlueDataValidTypes.DATETIME) || t.Equals(BlueDataValidTypes.DECIMAL) + || t.Equals(BlueDataValidTypes.DOUBLE) || t.Equals(BlueDataValidTypes.INT) + || t.Equals(BlueDataValidTypes.ISERIALIZEDTYPE) || t.Equals(BlueDataValidTypes.LONG) + || t.Equals(BlueDataValidTypes.SBYTE) || t.Equals(BlueDataValidTypes.SINGLE) + || t.Equals(BlueDataValidTypes.STRING) || t.Equals(BlueDataValidTypes.UINT) + || t.Equals(BlueDataValidTypes.ULONG); + return a; + } + static public double[] subtract(object o1, object o2) + { + Type t = o1.GetType(); + if (t == typeof(DateTime)) + return new double[] { ((DateTime)o1 - (DateTime)o2).TotalDays }; + else if (t.IsPrimitive && t.IsValueType) + return new double[] { Convert.ToDouble(o1) - Convert.ToDouble(o2) }; + else if (t.IsArray && t.GetElementType().IsValueType && t.GetElementType().IsPrimitive) + { + double[] a1 = (double[])Array.ConvertAll((object[])o1, + delegate (object o) { return Convert.ToDouble(o); }); + double[] a2; + if (o2.GetType().IsArray) + a2 = (double[])Array.ConvertAll((object[])o2, + delegate (object o) { return Convert.ToDouble(o); }); + else + { + a2 = new double[a1.Length]; + for (int i = 0; i < a1.Length; i++) + a2[i] = (double)o2; + } + double[] op = new double[a1.Length]; + for (int i = 0; i < a1.Length; i++) + op[i] = a1[i] - a2[i]; + return op; + } + else + return new double[] { 0.0 }; + } + static public double[] add(object o1, object o2) + { + Type t = o1.GetType(); + if (t.IsPrimitive && t.IsValueType) + return new double[] { Convert.ToDouble(o1) + Convert.ToDouble(o2) }; + else if (t.IsArray && t.GetElementType().IsValueType && t.GetElementType().IsPrimitive) + { + double[] a1 = (double[])Array.ConvertAll((object[])o1, + delegate (object o) { return Convert.ToDouble(o); }); + double[] a2; + if (o2.GetType().IsArray) + a2 = (double[])Array.ConvertAll((object[])o2, + delegate (object o) { return Convert.ToDouble(o); }); + else + { + a2 = new double[a1.Length]; + for (int i = 0; i < a1.Length; i++) + a2[i] = (double)o2; + } + double[] op = new double[a1.Length]; + for (int i = 0; i < a1.Length; i++) + op[i] = a1[i] + a2[i]; + return op; + } + else + return new double[] { 0.0 }; + } + static public double[] mult(object o1, object o2) + { + Type t = o1.GetType(); + if (t.IsPrimitive && t.IsValueType) + return new double[] { Convert.ToDouble(o1) * Convert.ToDouble(o2) }; + else if (t.IsArray && t.GetElementType().IsValueType && t.GetElementType().IsPrimitive) + { + double[] a1 = (double[])Array.ConvertAll((object[])o1, + delegate (object o) { return Convert.ToDouble(o); }); + double[] a2; + if (o2.GetType().IsArray) + a2 = (double[])Array.ConvertAll((object[])o2, + delegate (object o) { return Convert.ToDouble(o); }); + else + { + a2 = new double[a1.Length]; + for (int i = 0; i < a1.Length; i++) + a2[i] = (double)o2; + } + double[] op = new double[a1.Length]; + for (int i = 0; i < a1.Length; i++) + op[i] = a1[i] * a2[i]; + return op; + } + else + return new double[] { 0.0 }; + } + public delegate double[] BinaryDelegate(object o1, object o2); + static public IBlueRowCollection operate(IBlueRowCollection d1, IBlueRowCollection d2, + BinaryDelegate del) + { + int jj = 0; + ColumnInfo[] newinfo = new ColumnInfo[d1.schema.columnInfo.Length]; + foreach (ColumnInfo info in d1.schema.columnInfo) + newinfo[jj++] = new ColumnInfo(info.colName, info.colNo, typeof(double)); + IBlueTableSchema schem = new BlueTableSchema(newinfo, d1.schema.tableName + "_" + + d2.schema.tableName + "_processed"); + BlueTable res = new BlueTable(schem); + for (int i = 0; i < d1.Count; i++) + { + IBlueRow r1 = BlueDataUtility.extractRow(d1, i); + IBlueRow r2 = BlueDataUtility.extractRow(d2, i); + object[] vals = new object[r1.Count]; + int idx = 0; + foreach (ColumnInfo info in res.schema.columnInfo) + vals[idx++] = del(r1[info.colName], r2[info.colName]); + res.addRow(new BlueRow(schem, vals)); + } + return res; + } + static public IBlueRowCollection merge(string mergedName, IBlueRowCollection[] data) + { + SortedList colinfos = new SortedList(); + foreach (IBlueRowCollection col in data) + { + IBlueTableSchema s = col.schema; + foreach (ColumnInfo info in s.columnInfo) + if (info.colName != "id") + colinfos[info.colName] = info; + } + int colidx = 1; + ColumnInfo[] ciary = new ColumnInfo[colinfos.Count + 1]; + ciary[0] = new ColumnInfo("id", 0, typeof(int)); + foreach (ColumnInfo ci in colinfos.Values) + { + ciary[colidx] = new ColumnInfo(ci.colName, colidx, ci.colType); + colidx++; + } + + BlueTableSchema newschem = new BlueTableSchema(mergedName, ciary); + BlueTable tab = new BlueTable(newschem); + int numRows = 0; + foreach (IBlueRowCollection col in data) + if (col.Count > numRows) + numRows = col.Count; + for (int i = 0; i < numRows; i++) + { + Hashtable ht = new Hashtable(); + ht["id"] = i; + foreach (string name in newschem.columnNames) + { + if (name != "id") + { + ht[name] = DBNull.Value; + foreach (IBlueRowCollection col in data) + { + IBlueRow r = BlueDataUtility.extractRow(col, i); + if (r != null && r.Contains(name)) + ht[name] = r[name]; + } + } + } + BlueRow br = new BlueRow(newschem, ht); + tab.addRow(br); + } + return tab; + } + static public IBlueRowCollection build(string tblname, object data) + { + return build(tblname, "value", data); + } + static public IBlueRowCollection build(string tblname, string dataname, object data) + { + BlueTable result = null; + if (data == null || tblname == null || tblname == string.Empty) + return null; + if (data is IBlueRowCollection) + { + IBlueRowCollection rows = data as IBlueRowCollection; + result = new BlueTable(new BlueTableSchema(rows.schema.columnInfo, tblname)); + result.addRows(rows); + } + else if (data is Array) + { + Array da = (Array)data; + Type et = da.GetType().GetElementType(); + if (!isBlueDataType(et)) + et = typeof(string); + ColumnInfo ci = new ColumnInfo(dataname, 1, et); + ColumnInfo cii = new ColumnInfo("id", 0, typeof(int)); + BlueTableSchema schem = new BlueTableSchema(tblname, cii, ci); + result = new BlueTable(schem); + int i = 0; + foreach (object dat in da) + { + object obj = (isBlueDataType(dat.GetType()) ? dat : dat.ToString()); + result.addRow(new BlueRow(schem, i++, obj)); + } + } + else if (isBlueDataType(data.GetType())) + { + ColumnInfo cii = new ColumnInfo("id", 0, typeof(int)); + Type dt = data.GetType(); + ColumnInfo ci = new ColumnInfo(dataname, 1, dt); + BlueTableSchema schem = new BlueTableSchema(tblname, cii, ci); + result = new BlueTable(schem); + result.addRow(new BlueRow(schem, 0, data)); + } + else + { + ColumnInfo cii = new ColumnInfo("id", 0, typeof(int)); + ColumnInfo ci = new ColumnInfo("value", 1, typeof(string)); + BlueTableSchema schem = new BlueTableSchema(tblname, cii, ci); + result = new BlueTable(schem); + result.addRow(new BlueRow(schem, 0, data.ToString())); + } + return result; + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs new file mode 100644 index 000000000..de72a4e2e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs @@ -0,0 +1,252 @@ +using System; +using System.Collections; +//using BCG.Utilities; + +namespace BCG +{ + /// + /// Provides a multiple indexing mechanism, required to increase the peformance of the streams + /// + public class BlueTableIndexException : Exception + { + public BlueTableIndexException(string msg) : base(msg) { } + } + + public interface IBlueTableIndex + { + void addRow(IBlueRow row); + void deleteRowOnPrimaryKey(object pkValue); + void deleteRowOnPrimaryKey(IBlueRow row); + void deleteRowOnBothKeys(object pkValue, object skValue); + void deleteRowOnBothKeys(IBlueRow row); + ArrayList searchRowOnPrimaryKey(object pkValue); + ArrayList searchRowOnPrimaryKey(IBlueRow row); + ArrayList searchRowOnBothKeys(object pkValue, object skValue); + ArrayList searchRowOnBothKeys(IBlueRow row); + + + } + + /* + public class BlueTableDoubleIndex : IBlueTableIndex + { + private IBlueRowCollection myTable; + private string myPrimaryIndex; + private string mySecondaryIndex; + private Hashtable myHt = new Hashtable (); + + /// + /// Builds an index with 2 columns of the table acting as primary index and secondary index + /// respectively + /// + /// + /// table to index + /// + /// + /// primary key to index upon + /// + /// + /// column serving as the secondary key + /// + public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex) + { + myTable = table; + myPrimaryIndex = primaryIndex; + mySecondaryIndex = secondaryIndex; + myInit (); + } + + private void myInit() + { + foreach (IBlueRow row in myTable) + addRow (row); + } + + /// + /// To add a row to the index + /// + /// + /// row to be added + /// + public void addRow (IBlueRow row) + { + if (myHt[row[myPrimaryIndex]] != null) + ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row; + else + { + TimeManager.start (); + Hashtable ht = new Hashtable(); + TimeManager.stop (); + ht[row[mySecondaryIndex]] = row; + myHt[row[myPrimaryIndex]] = ht; + } + } + + /// + /// If index or the primary key has already been deleted appropriate exceptions are thrown + /// + /// + public void deleteRow (IBlueRow row) + { + if (myHt[row[myPrimaryIndex]] == null) + { + throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value"); + } + ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]); + if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0) + myHt.Remove(row[myPrimaryIndex]); + } + + /// + /// Returns an array of DataRow that match the primary key being passed as an argument + /// + /// + /// Primary Key of the table + /// + /// + /// Array of rows matching the criteria + /// + public IBlueRow[] findValueFromPrimaryKey (object o) + { + object oht = myHt[o]; + if (oht == null) + return new IBlueRow[0]; + Hashtable ht = (Hashtable)oht; + IBlueRow[] rows = new IBlueRow[ht.Keys.Count]; + ht.Values.CopyTo(rows, 0); + return rows; + } + + + /// + /// returns as array of the primary keys + /// + /// + /// object[] of primary keys + /// + public Array getPrimaryKeys() + { + object[] retval = new object [myHt.Keys.Count]; + myHt.Keys.CopyTo (retval, 0); + return retval; + } + } + */ + + /// + /// Provides a 1-Level indexing structure. + /// + public class BlueTableSingleIndex : IBlueTableIndex + { + string myPrimaryKey; + string mySecondaryKey; + Hashtable myHt = new Hashtable(); + + public BlueTableSingleIndex(string primaryKey, string secondaryKey) + { + myPrimaryKey = primaryKey; + mySecondaryKey = secondaryKey; + } + + /// + /// The primary key is the indexing key, and the secondary key is used as the key for + /// searching and hence deletion + /// + /// + /// + /// + public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey) + : this(primaryKey, secondaryKey) + { + foreach (IBlueRow row in table) + addRow(row); + } + + + + /// + /// Adds row to the indexing new structure + /// + /// + public void addRow(IBlueRow row) + { + if (myHt[row[myPrimaryKey]] == null) + myHt[row[myPrimaryKey]] = new ArrayList(); + ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row); + } + + /// + /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria + /// + /// + /// + public ArrayList searchRowOnPrimaryKey(IBlueRow row) + { + return searchRowOnPrimaryKey(row[myPrimaryKey]); + } + + public ArrayList searchRowOnPrimaryKey(object pkValue) + { + if (myHt[pkValue] == null) + return new ArrayList(); + return (ArrayList)myHt[pkValue]; + } + + /// + /// Gets rows based on the primary AND the secondary key + /// + /// + /// + public ArrayList searchRowOnBothKeys(IBlueRow row) + { + return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]); + } + + public ArrayList searchRowOnBothKeys(object pkValue, object skValue) + { + if (myHt[pkValue] == null) + return new ArrayList(); + ArrayList matchingList = (ArrayList)myHt[pkValue]; + ArrayList retVal = new ArrayList(); + foreach (IBlueRow r in matchingList) + if (r[mySecondaryKey].Equals(skValue)) + retVal.Add(r); + return retVal; + } + + + /// + /// Deletes rows based on the primary key only + /// + /// + public void deleteRowOnPrimaryKey(IBlueRow row) + { + deleteRowOnPrimaryKey(row[myPrimaryKey]); + } + + public void deleteRowOnPrimaryKey(object pkValue) + { + if (myHt[pkValue] == null) + throw new BlueTableIndexException("Key to be deleted was not found in the index"); + myHt.Remove(pkValue); + } + + public void deleteRowOnBothKeys(IBlueRow row) + { + deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]); + } + + public void deleteRowOnBothKeys(object pkValue, object skValue) + { + if (myHt[pkValue] == null) + throw new BlueTableIndexException("Key to be deleted was not found in the index"); + ArrayList matchingList = (ArrayList)myHt[pkValue]; + ArrayList aux = new ArrayList(); + foreach (IBlueRow r in matchingList) + if (r[mySecondaryKey].Equals(skValue)) + aux.Add(r); + foreach (IBlueRow r in aux) + matchingList.Remove(r); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableSchema.cs new file mode 100644 index 000000000..c56cdc841 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableSchema.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Represents a schema for the table, which can be indexed either with the column name or + /// the index of the column + /// + public interface IBlueTableSchema : ISerializedType + { + /// + /// index w.r.t. the ordinal + /// + ColumnInfo this[int index]{get;} + /// + /// index w.r.t. column name + /// + ColumnInfo this[string colName]{get;} + /// + /// Utility method to get all the column names + /// + string[] columnNames {get;} + /// + /// Utility method to get all the info about all the columns + /// + ColumnInfo[] columnInfo{get;} + /// + /// Name of the table, whose schema it holds + /// + string tableName {get;} + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IImmutableRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IImmutableRowCollection.cs new file mode 100644 index 000000000..e3bf8ad15 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IImmutableRowCollection.cs @@ -0,0 +1,10 @@ +using System; + +namespace BCG +{ + /// + /// Tables that dont need to be cloned by the ViewStream into an ImmutableBlueTable should + /// implement this interface + /// + public interface IImmutableRowCollection : IBlueRowCollection {} +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ISerializedType.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ISerializedType.cs new file mode 100644 index 000000000..bff395d9e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ISerializedType.cs @@ -0,0 +1,31 @@ +using System; +using System.IO; + +namespace BCG +{ + /// + /// Each type that needs to be transported through the PubSub mechanism, should + /// implement this interface, so that PubSub knows how to serialize it and + /// how to contruct an object out of a byte array + /// + public interface ISerializedType + { + /// + /// serializes itself and writes into a BinaryWriter; note that the underlying stream's pointer + /// moves to the end of the written bytes + /// + /// + void serialize (BinaryWriter bw); + /// + /// construct itself from the bytes read off the BinaryReader; note that the underlying stream's pointer + /// moves to the end of the bytes read + /// + /// + void deserialze (BinaryReader br); + /// + /// Should return the size in bytes, when the instance is written to a stream of binary data + /// + /// + int getSerializedSize (); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ITagType.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ITagType.cs new file mode 100644 index 000000000..2b667f9a7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ITagType.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +//using BCG.Utilities.General; + +/* + * An instance of this class represents one particular type of tag. The class encapsulates + * the grammer of the tag, and contains a method for extracting metadata from tags of this type, + * as well as a method for converting metatdata back into a tag + * + * */ + +namespace BCG +{ + public interface ITagType + { + + //parses a given tag to see if it matches this type. If the tag does not match this type, null is returned. + //If the tag DOES match this type, then the metadata about the tag is returned. + IDictionary parse(string tag); + + //takes metadata about the tag and turns it back into the tag + string generateTag(IDictionary metaData); + + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs new file mode 100644 index 000000000..e2bdcd708 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs @@ -0,0 +1,103 @@ +using System; +using BCG; +//using BCG.Utilities; + +namespace BCG +{ + [Serializable] + public class ImmutableBlueTable : IImmutableRowCollection + { + public ImmutableBlueTable(IBlueRowCollection c) + { + mySchema = c.schema; + myHash = new BlueIntHash(c.Count); + myRows = new IBlueRow[c.Count]; + int id_col = c.schema["id"].colNo; + + int n = 0; + foreach (IBlueRow r in c) + { + myRows[n] = r; + var id = (int)r[id_col]; + if (myHash.isPresent(id)) + throw new Exception("Duplicate row"); + + myHash.add(id, n); + n++; + } + } + + + #region IBlueRowCollection Members} + + public IBlueRow this[int id] + { + get + { + try + { + return myRows[myHash.find(id)]; + } + catch (BlueIntHash.ElementNotFound) + { + throw new BlueDataException("Row with id " + id.ToString() + + " was not found in table " + + mySchema.tableName); + } + } + } + + public IBlueTableSchema schema + { + get + { + return mySchema; + } + } + + public bool containsID(int id) + { return myHash.isPresent(id); } + + #endregion + + #region ICollection Members + + public bool IsSynchronized + { + get + { + return false; + } + } + + public int Count + { + get + { return myRows.Length; } + } + + public void CopyTo(Array array, int index) + { + myRows.CopyTo(array, index); + } + + public object SyncRoot + { + get + { return null; } + } + + #endregion + + #region IEnumerable Members + + public System.Collections.IEnumerator GetEnumerator() + { return myRows.GetEnumerator(); } + + #endregion + + private BlueIntHash myHash; + private IBlueRow[] myRows; + private IBlueTableSchema mySchema; + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/InvalidMetaDataException.cs b/src/DataStreamSuite/DataStreamCore/BlueData/InvalidMetaDataException.cs new file mode 100644 index 000000000..5da17e654 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/InvalidMetaDataException.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + + +namespace BCG +{ + public class InvalidMetaDataException : Exception + { + public InvalidMetaDataException(MetaDataField field) : base("Error: Invalid metadata: field " + field.ToString() + " was expected but missing.") { } + + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs b/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs new file mode 100644 index 000000000..181526e3b --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs @@ -0,0 +1,139 @@ +using System; + +namespace BCG +{ + public class KeysBasedTable2 : IBlueRowCollection + { + private int myID = 0; + private HashTableND myHash; + private BlueTable myTable; + private string[] myKeys; + + public KeysBasedTable2 (IBlueTableSchema schema, params string[] keys) + { + myTable = new BlueTable (schema); + myKeys = keys; + myHash = new HashTableND (myKeys.Length); + } + + #region IBlueRowCollection Members + + public IBlueRow this[int id] {get {return myTable[id];}} + + public bool containsID (int id) {return myTable.containsID (id);} + + public IBlueTableSchema schema {get {return myTable.schema;}} + + #endregion + + #region ICollection Members + + public bool IsSynchronized {get {return myTable.IsSynchronized;}} + + public int Count {get {return myTable.Count;}} + + public void CopyTo (Array array, int index) {myTable.CopyTo (array, index);} + + public object SyncRoot {get {return myTable.SyncRoot;}} + + #endregion + + #region IEnumerable Members + + public System.Collections.IEnumerator GetEnumerator() {return myTable.GetEnumerator ();} + + #endregion + + public void update (IBlueRow row, out IBlueRow replacedRow) + { + object[] keyVals = myGetKeyVals (row); + replacedRow = myHash.getValue (keyVals) as IBlueRow; + if (replacedRow == null) + { + myHash.add (keyVals, row); + } + else + { + myTable.deleteRow ((int)replacedRow["id"]); + myHash.setValue (keyVals, row); + } + myTable.addRow (row); + } + + public void updateWithIDGeneration(IBlueRow row, out IBlueRow addedRow, out IBlueRow deletedRow) + { + object[] keyVals = myGetKeyVals(row); + deletedRow = myHash.getValue(keyVals) as IBlueRow; + if (deletedRow == null) + { + addedRow = BlueDataUtility.getNewReplacedRow(row, myID++, "id"); + myHash.add(keyVals, addedRow); + myTable.addRow(addedRow); + } + else + { + addedRow = BlueDataUtility.getNewReplacedRow(row, deletedRow["id"], "id"); + myHash.setValue(keyVals, addedRow); + myTable.deleteRow(deletedRow); + myTable.addRow(addedRow); + } + } + + public void updateMultipleRows(IBlueRowCollection updatesRows, + out IBlueRowCollection removedRows, out IBlueRowCollection addedRows) + { + BlueTable preUpdateState = new BlueTable(myTable); + removedRows = new BlueTable(schema); addedRows = new BlueTable(schema); + foreach (IBlueRow r in updatesRows) + { + IBlueRow rRow, aRow; + updateWithIDGeneration(r, out aRow, out rRow); + if (rRow != null && preUpdateState.containsID ((int)rRow["id"])) + (removedRows as BlueTable).forceAddRow(rRow); + (addedRows as BlueTable).forceAddRow(aRow); + } + } + + public void deleteRow (IBlueRow row, out IBlueRow deletedRow) + { + object[] keys = myGetKeyVals(row); + deletedRow = myHash.getValue(keys) as IBlueRow; + if (deletedRow != null) { + myTable.deleteRow(deletedRow); + myHash.remove(keys); + } + } + + public void deleteRow (IBlueRow row) + { + IBlueRow t1; + deleteRow (row, out t1); + } + + public void deleteMultipleRows(IBlueRowCollection rows) + { + foreach (IBlueRow r in rows) + deleteRow(r); + } + + public void update (IBlueRow row) + { + IBlueRow t1; + update (row, out t1); + } + + public IBlueRow getRow (object[] keys) + { + return myHash.getValue (keys) as IBlueRow; + } + + public IBlueRow getRow(IBlueRow r) + { + return getRow(myGetKeyVals(r)); + } + + public IBlueRow this[params object[] keyVals] {get {return myHash.getValue (keyVals) as IBlueRow;}} + + private object[] myGetKeyVals (IBlueRow r) {return BlueDataUtility.getSelectedValues (r, myKeys);} + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs new file mode 100644 index 000000000..426050353 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs @@ -0,0 +1,324 @@ +using System; +using System.Data; +using System.Collections; +using System.IO; +using System.Reflection; +using System.Collections.Generic; +using System.Linq; + +using BCG.Utilities.General; + +namespace BCG +{ + /// + /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created + /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it + /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return + /// the respective data, sorted according to the ColumnOrder. + /// + [Serializable] + public class LookupTableSchema : IBlueTableSchema + { + public class LookupTableSchemaSerializationHelper + { + public static object parse(Type t, string data) + { + if (t == BlueDataValidTypes.BOOL) + return Boolean.Parse(data); + else if (t == BlueDataValidTypes.BYTE) + return Byte.Parse(data); + else if (t == BlueDataValidTypes.CHAR) + return char.Parse(data); + else if (t == BlueDataValidTypes.DATETIME) + return DateTime.Parse(data); + else if (t == BlueDataValidTypes.DOUBLE) + return Double.Parse(data); + else if (t == BlueDataValidTypes.DECIMAL) + return Decimal.Parse(data); + else if (t == BlueDataValidTypes.INT) + return int.Parse(data); + else if (t == BlueDataValidTypes.LONG) + return long.Parse(data); + else if (t == BlueDataValidTypes.SBYTE) + return SByte.Parse(data); + else if (t == BlueDataValidTypes.STRING) + return data; + else if (t == BlueDataValidTypes.UINT) + return uint.Parse(data); + else if (t == BlueDataValidTypes.ULONG) + return ulong.Parse(data); + else if (t == BlueDataValidTypes.SINGLE) + return Single.Parse(data); + else + throw new ApplicationException("Can not be parsed"); + } + public static void serialize(BinaryWriter bw, Type t, object o) + { + if (t == BlueDataValidTypes.BOOL) + bw.Write((bool)o); + else if (t == BlueDataValidTypes.BYTE) + bw.Write((byte)o); + else if (t == BlueDataValidTypes.CHAR) + bw.Write((char)o); + else if (t == BlueDataValidTypes.DATETIME) + bw.Write(((DateTime)o).Ticks); + else if (t == BlueDataValidTypes.DOUBLE) + bw.Write((double)o); + else if (t == BlueDataValidTypes.DECIMAL) + bw.Write((decimal)o); + else if (t == BlueDataValidTypes.INT) + bw.Write((int)o); + else if (t == BlueDataValidTypes.LONG) + bw.Write((long)o); + else if (t == BlueDataValidTypes.SBYTE) + bw.Write((sbyte)o); + else if (t == BlueDataValidTypes.STRING) + bw.Write(o as string); + else if (t == BlueDataValidTypes.SINGLE) + bw.Write((Single)o); + else if (t == BlueDataValidTypes.BYTEARRAY) + { + byte[] bs = o as byte[]; + bw.Write(bs.Length); + bw.Write(bs); + } + else if (t == BlueDataValidTypes.UINT) + bw.Write((uint)o); + else if (t == BlueDataValidTypes.ULONG) + bw.Write((ulong)o); + else + throw new ApplicationException("Unsupported type"); + } + + public static object deserialize(BinaryReader br, Type t) + { + if (t == BlueDataValidTypes.BOOL) + return br.ReadBoolean(); + else if (t == BlueDataValidTypes.BYTE) + return br.ReadByte(); + else if (t == BlueDataValidTypes.CHAR) + return br.ReadChar(); + else if (t == BlueDataValidTypes.DATETIME) + return new DateTime(br.ReadInt64()); + else if (t == BlueDataValidTypes.DOUBLE) + return br.ReadDouble(); + else if (t == BlueDataValidTypes.DECIMAL) + return br.ReadDecimal(); + else if (t == BlueDataValidTypes.INT) + return br.ReadInt32(); + else if (t == BlueDataValidTypes.LONG) + return br.ReadInt64(); + else if (t == BlueDataValidTypes.SBYTE) + return br.ReadSByte(); + else if (t == BlueDataValidTypes.STRING) + return br.ReadString(); + else if (t == BlueDataValidTypes.BYTEARRAY) + return br.ReadBytes(br.ReadInt32()); + else if (t == BlueDataValidTypes.UINT) + return br.ReadUInt32(); + else if (t == BlueDataValidTypes.ULONG) + return br.ReadUInt64(); + else if (t == BlueDataValidTypes.SINGLE) + return br.ReadSingle(); + else + throw new ApplicationException("Unsupported type"); + } + } + + protected Hashtable myIndexByColName = new Hashtable(); + protected ColumnInfo[] myIndexByColOrder; + private string[] myTableColNames; + private BlueRowLookupGroup[] myLookupGroups; + protected string myTableName; + + public LookupTableSchema() { } + + + public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups) + { + myConstruct(tableName, cis, groups); + } + + private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups) + { + myIndexByColOrder = new ColumnInfo[cis.Length]; + myTableName = tableName; + foreach (ColumnInfo ci in cis) + { + myIndexByColOrder[ci.colNo] = ci; + myIndexByColName[ci.colName] = ci; + } + myLookupGroups = groups; + + foreach (BlueRowLookupGroup group in groups) + { + group.buildEntrySchema(this); + + + } + } + + + + + public ColumnInfo this[string colName] + { + get + { + object retVal = myIndexByColName[colName]; + return retVal != null ? retVal as ColumnInfo : null; + } + } + + public BlueRowLookupGroup[] lookupGroups + { + get { return myLookupGroups; } + + } + public ColumnInfo this[int colNo] + { + get { return myIndexByColOrder[colNo]; } + } + + /// + /// The returned Array is sorted according to the Column No + /// + public string[] columnNames + { + get + { + if (myTableColNames == null) + { + myTableColNames = new string[myIndexByColName.Count]; + foreach (DictionaryEntry de in myIndexByColName) + { + ColumnInfo temp = (ColumnInfo)de.Value; + myTableColNames[temp.colNo] = temp.colName; + } + } + return myTableColNames; + } + } + + /// + /// The returned Array is sorted according to the Column No + /// + public ColumnInfo[] columnInfo + { + get { return myIndexByColOrder; } + } + + public string tableName { get { return myTableName; } } + + + #region ISerializedType methods + public void serialize(BinaryWriter bw) + { + bw.Write(columnNames.Length); + bw.Write(tableName); + + //first, write the column names + foreach (ColumnInfo ci in columnInfo) + { + bw.Write(ci.colName); + bw.Write(ci.colType.ToString()); + if (ci.isConstant) + { + bw.Write(true); + LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue); + } + else + { + bw.Write(false); + bw.Write(ci.lookupTableGroup); + } + + } + + //this is here for error checking + bw.Write(myLookupGroups.Length); + //now, write the size of each lookup table group + foreach (BlueRowLookupGroup group in myLookupGroups) + { + bw.Write(group.groupNumber); + bw.Write(group.tableSize); + bw.Write(group.groupColNames.Length); + foreach (string colName in group.groupColNames) + { + //write the index of this column in the table + bw.Write(this[colName].colNo); + + } + + + } + + + + } + + public void deserialze(BinaryReader br) + { + int numCols = br.ReadInt32(); + //these are the final output columns that will be created + ColumnInfo[] columnInfos = new ColumnInfo[numCols]; + + Dictionary> colLookupGroupMap = new Dictionary>(); + + + string tableName = br.ReadString(); + for (int colNo = 0; colNo < numCols; ++colNo) + { + string colName = br.ReadString(); + Type colType = TypeInstanceLoader.loadType(br.ReadString(), true); + bool colIsConst = br.ReadBoolean(); + if (colIsConst) + { + columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType)); + + } + else + { + short thisColGroup = br.ReadInt16(); + columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup); + } + } + + int numGroups = br.ReadInt32(); + BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups]; + + for (int group = 0; group < numGroups; ++group) + { + short groupNo = br.ReadInt16(); + short groupSizeInBits = br.ReadInt16(); + int numColsInGroup = br.ReadInt32(); + int[] groupCols = new int[numColsInGroup]; + string[] groupColNames = new string[numColsInGroup]; + for (int col = 0; col < numColsInGroup; ++col) + { + groupCols[col] = br.ReadInt32(); + groupColNames[col] = columnInfos[groupCols[col]].colName; + } + + //create the group + theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames); + + + } + + + myConstruct(tableName, columnInfos, theGroups); + + + + } + + public int getSerializedSize() + { + throw new NotImplementedException("Feature for provided constant values to" + + "non-primitive types in schema, not provided as of now"); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs b/src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs new file mode 100644 index 000000000..9b0f80302 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + /* + These fields are first going to be used by BlueRay to help it determine what differnet tags mean, but eventually + * the goal is that they will be used for all services in this system. + * + * not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service + * probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated + */ + public enum MetaDataField { Protocol, //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what + Type, //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever + Symbol, + Exchange, + Platform, + MarketDataType, // BAM (bid, ask, mid), last, etc + RealName }; //does this thing have a better name we could call it +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/Misc/CSVParser.cs b/src/DataStreamSuite/DataStreamCore/BlueData/Misc/CSVParser.cs new file mode 100644 index 000000000..11fba7c0f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/Misc/CSVParser.cs @@ -0,0 +1,141 @@ +using System; +using System.Reflection; +using System.Collections; +using System.Text; +using System.Text.RegularExpressions; + +namespace BCG.Utilities.General +{ + public class CSVParser + { + enum state + { + waitingForToken, + inQuote, + waitingForSeparator, + possibleEndOfQuote, + inToken, + } + + class CSVParseException : Exception + { + public CSVParseException (string msg) : base (msg) + {} + } + + public static string [] parse (string csv) + { + ArrayList tokens = new ArrayList(); + StringBuilder currentToken = null; + state s = state.waitingForToken; + + int n = 0; + foreach (char c in csv) + { + n++; + switch (s) + { + case state.waitingForToken: + if (c == ',') + tokens.Add(""); + else if (c == '"') + { + s = state.inQuote; + currentToken = new StringBuilder(); + } + else if (Char.IsWhiteSpace(c)) + continue; + else + { + s = state.inToken; + currentToken = new StringBuilder(); + currentToken.Append(c); + } + break; + case state.inQuote: + if (c == '"') + { + s = state.possibleEndOfQuote; + } + else + { + currentToken.Append(c); + } + break; + case state.possibleEndOfQuote: + if (c == '"') + { + s = state.inQuote; + currentToken.Append(c); + } + else if (Char.IsWhiteSpace(c)) + { + tokens.Add(currentToken.ToString()); + s = state.waitingForSeparator; + } + else if (c == ',') + { + tokens.Add(currentToken.ToString()); + s = state.waitingForToken; + } + else + throw new Exception("Expected ',' or end of line at column " + n.ToString()); + break; + case state.waitingForSeparator : + if (c == ',') + { + s = state.waitingForToken; + } + else if (Char.IsWhiteSpace(c)) + continue; + else + throw new Exception("Expected ',' or end of line at column " + n.ToString()); + break; + case state.inToken : + if (c == ',') + { + s = state.waitingForToken; + tokens.Add(currentToken.ToString()); + } + else + { + currentToken.Append(c); + } + break; + } + + } + + if (s == state.inToken || s == state.possibleEndOfQuote) + tokens.Add(currentToken.ToString()); + else if (s == state.waitingForToken) + tokens.Add(""); + else if ( s != state.waitingForSeparator) + throw new Exception("Missing column or unterminated string at end of line"); + + string [] result = new string[tokens.Count]; + tokens.CopyTo(result,0); + return result; + } + + + static void test() + { + for (;;) + { + try + { + foreach (string s in parse(Console.ReadLine())) + { + Console.WriteLine(s); + } + } + catch (Exception e) + { + Console.WriteLine(e); + } + } + } + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/Misc/CollectionHelpers.cs b/src/DataStreamSuite/DataStreamCore/BlueData/Misc/CollectionHelpers.cs new file mode 100644 index 000000000..ce41d9b9e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/Misc/CollectionHelpers.cs @@ -0,0 +1,397 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace BCG +{ + [Serializable] + public class CollectionHelpers + { + public delegate bool EqualDelegate(object a, object b); + public static ArrayList difference (ICollection a, ICollection b, + EqualDelegate ed) + { + ArrayList result = new ArrayList(); + + foreach (Object aa in a) + { + bool found = false; + foreach (Object bb in b) + { + if (ed(aa, bb)) + { + found = true; + break; + } + } + + if (!found) + result.Add (aa); + } + return result; + } + + public static ArrayList difference (ICollection a, ICollection b) + { + ArrayList result = new ArrayList(); + + foreach (Object aa in a) + { + bool found = false; + foreach (Object bb in b) + { + if (aa.Equals(bb)) + { + found = true; + break; + } + } + + if (!found) + result.Add (aa); + } + return result; + } + + public static ArrayList sorted_difference (ICollection sa, ICollection sb) { + ArrayList result = new ArrayList (); + + IEnumerator ia = sa.GetEnumerator (); + IEnumerator ib = sb.GetEnumerator (); + IEnumerator iremainder = ia; + bool moreA,moreB; + for (moreA = ia.MoveNext (), moreB = ib.MoveNext (); moreA && moreB;) { + int cmp = ((IComparable)ia.Current).CompareTo (ib.Current); + if (cmp < 0) { + result.Add (ia.Current); + moreA = ia.MoveNext (); + iremainder = ia; + } + else if (cmp > 0) + moreB = ib.MoveNext (); + else { + moreA = ia.MoveNext (); + moreB = ib.MoveNext (); + } + } + for (; moreA; moreA = iremainder.MoveNext ()) + result.Add (iremainder.Current); + + return result; + } + + public static bool typedequal(ICollection lhs, ICollection rhs) + { + if(lhs == null || rhs == null) + { + return false; + } + + T[] l = new T[lhs.Count]; + T[] r = new T[rhs.Count]; + lhs.CopyTo(l, 0); + rhs.CopyTo(r, 0); + return equal((IList)l, (IList)r); + } + public static bool equal(ICollection lhs, ICollection rhs) + { + if ( lhs == null || rhs == null ) + { + return false; + } + + Array l = new object[lhs.Count]; + Array r = new object[rhs.Count]; + lhs.CopyTo(l, 0); + rhs.CopyTo(r,0); + return equal(l,r); + } + + public static bool equal(IList lhs, IList rhs) + { + if ( lhs == null || rhs == null ) + { + return false; + } + + if(lhs.Count != rhs.Count) + return false; + else + { + for(int i = 0; i < lhs.Count; i++) + { + if(!lhs[i].Equals(rhs[i])) + return false; + } + return true; + } + } + + public static Array fromArrayList(ArrayList al, Type t) + { + Array result = Array.CreateInstance(t, al.Count); + al.CopyTo(result,0); + return result; + } + + public static Array append (Array a1, Array a2, Type t) + { + Array result = Array.CreateInstance(t, a1.Length + a2.Length); + a1.CopyTo(result,0); + a2.CopyTo(result,a1.Length); + return result; + } + + public static Array subtract (Array from, params object [] elemens) + { + return fromArrayList( + difference(from, elemens),from.GetType(). + GetElementType()); + + } + + public static bool hasDuplicates (ICollection collection) + { + Hashtable h = new Hashtable(); + foreach (object o in collection) + h[o] = true; + + return h.Keys.Count != collection.Count; + } + + public static ICollection subtract (IDictionary d1, IDictionary d2) + { + ArrayList result = new ArrayList(); + foreach (object d1k in d1.Keys) + if (!d2.Contains(d1k)) + result.Add(d1k); + + return result; + } + + /// + /// Is c1 a subset of c2? + /// + /// + /// + /// + public static bool isSubset (ICollection c1, ICollection c2) + {return difference(c1,c2).Count == 0;} + + public static void orderedUniqueInsert (ArrayList coll,object item) { + int ifnd = coll.BinarySearch (item); + if (ifnd < 0) + coll.Insert (~ifnd,item); + else + coll[ifnd] = item; + } + public static void orderedRemove (ArrayList coll,object item) { + int ifnd = coll.BinarySearch (item); + if (ifnd >= 0) + coll.RemoveAt (ifnd); + } + + public static bool t_equals(T t1, T t2) { return (t1 != null && t2 != null && t1.Equals(t2)) || (t1 == null && t2 == null); } + + public static bool dictionaries_equal(IDictionary new_d, IDictionary old_d, Func comp = null) + { + IDictionary deleted_d, added_d, changed_d; + compare_dictionaries(new_d, old_d, out deleted_d, out added_d, out changed_d, comp); + return deleted_d.Count == 0 && added_d.Count == 0 && changed_d.Count == 0; + } + + public static void compare_dictionaries(IDictionary new_d, IDictionary old_d, + out IDictionary deleted_d, out IDictionary added_d, out IDictionary changed_d, + Func comp = null) + { + new_d = new_d ?? new Dictionary(); + old_d = old_d ?? new Dictionary(); + deleted_d = new Dictionary(); + added_d = new Dictionary(); + changed_d = new Dictionary(); + comp = comp ?? t_equals; + foreach (KeyValuePair p in new_d) + { + V old_d_val = default(V); + if (!old_d.TryGetValue(p.Key, out old_d_val)) + added_d.Add(p); + else if (!comp(p.Value, old_d_val)) + changed_d.Add(p); + } + foreach (KeyValuePair p in old_d) + if (!new_d.ContainsKey(p.Key)) + deleted_d.Add(p); + } + + /// Compares 2 dictionaries with the same key types + /// but different data types. Takes a function to + /// compare the 2 data types + public static void compare_dictionaries(IReadOnlyDictionary new_d, IReadOnlyDictionary old_d, + out IDictionary deleted_d, out IDictionary added_d, out IDictionary> changed_d, + Func comp) + { + new_d = new_d ?? new Dictionary(); + old_d = old_d ?? new Dictionary(); + deleted_d = new Dictionary(); + added_d = new Dictionary(); + changed_d = new Dictionary>(); + + foreach (KeyValuePair p in new_d) + { + O old_val = default(O); + if (!old_d.TryGetValue(p.Key, out old_val)) + added_d.Add(p); + else if (!comp(p.Value, old_val)) + changed_d.Add(p.Key, new Tuple(p.Value, old_val)); + } + foreach (KeyValuePair p in old_d) + if (!new_d.ContainsKey(p.Key)) + deleted_d.Add(p); + } + + + public static void compare_dictionary_keys(IDictionary new_d, IDictionary old_d, + out HashSet deleted_keys, out HashSet added_keys) + { + new_d = new_d ?? new Dictionary(); + old_d = old_d ?? new Dictionary(); + + deleted_keys = new HashSet(); + added_keys = new HashSet(); + + foreach (KeyValuePair p in new_d) + if (!old_d.ContainsKey(p.Key)) + added_keys.Add(p.Key); + + foreach (KeyValuePair p in old_d) + if (!new_d.ContainsKey(p.Key)) + deleted_keys.Add(p.Key); + } + + public static void compare_hashsets(HashSet new_d, HashSet old_d, + out HashSet deleted_d, out HashSet added_d) + { + added_d = new HashSet(); + deleted_d = new HashSet(); + foreach (T t in new_d) + if (!old_d.Contains(t)) + added_d.Add(t); + foreach (T t in old_d) + if (!new_d.Contains(t)) + deleted_d.Add(t); + } + + public static IEnumerable append_arrays(params IEnumerable[] ienum_list) + { + List rv = new List(); + foreach (IEnumerable ienum in ienum_list) + rv.AddRange(ienum); + return rv; + } + + public static bool hashset_equals(HashSet new_d, HashSet old_d) + { + HashSet deleted_d, added_d; + compare_hashsets(new_d, old_d, out deleted_d, out added_d); + return deleted_d.Count == 0 && added_d.Count == 0; + } + + public static void find_neighbors(IEnumerable items, Comparison comp_func, T comp, out int low_index, out int high_index) + { + low_index = high_index = -1; + for (int i = 0; i < items.Count(); i++) + { + T p = items.ElementAt (i); + if (comp_func (comp, p) <= 0) + { + high_index = i; + low_index = i - 1; + if (low_index >= 0) + { + T low_index_val = items.ElementAt (low_index); + if (comp_func(comp, low_index_val) >= 0) + break; + } + else + break; + } + else + { + low_index = i; + high_index = i + 1; + if (high_index <= items.Count() - 1) + { + T high_index_val = items.ElementAt (high_index); + if (comp_func(comp, high_index_val) <= 0) + break; + } + else + { + high_index = -1; + break; + } + } + } + } + + public static void find_neighbors(IEnumerable list, P nearest_to, Func diff_measure, + out T low_neighbor, out T high_neighbor) + { + low_neighbor = default(T); high_neighbor = default(T); + list = list ?? new List(); + double low_min_diff = double.MaxValue, high_min_diff = double.MaxValue; + foreach (T t in list) + { + double this_diff = Math.Abs(diff_measure(t, nearest_to)); + if (this_diff <= 0) + { + this_diff = Math.Abs(this_diff); + if (this_diff < low_min_diff) + { + low_min_diff = this_diff; + low_neighbor = t; + } + } + else + { + if (this_diff < high_min_diff) + { + high_min_diff = this_diff; + high_neighbor = t; + } + } + } + } + + public static T find_nearest(IEnumerable list, P nearest_to, Func diff_measure) + { + list = list ?? new List(); + double min_diff = double.MaxValue; + T rv = default(T); + foreach (T t in list) + { + double this_diff = Math.Abs(diff_measure(t)); + if (this_diff < min_diff) + { + rv = t; + min_diff = this_diff; + } + } + return rv; + } + + public static bool are_maps_equal(IDictionary m1, IDictionary m2, Func equality_comparer = null) + { + m1 = m1 ?? new Dictionary(); + m2 = m2 ?? new Dictionary(); + if (equality_comparer == null) + equality_comparer = (t1, t2) => { return t1.Equals(t2); }; + IDictionary deleted_d, added_d; + IDictionary> changed_d; + compare_dictionaries(new Dictionary(m1), new Dictionary(m2), + out deleted_d, out added_d, out changed_d, equality_comparer); + return deleted_d.Count + added_d.Count + changed_d.Count == 0; + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/Misc/TypeHelper.cs b/src/DataStreamSuite/DataStreamCore/BlueData/Misc/TypeHelper.cs new file mode 100644 index 000000000..b042efb23 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/Misc/TypeHelper.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public class TypeHelper + { + private static HashSet _numeric_type_codes = new HashSet + { + TypeCode.Byte, TypeCode.Decimal, TypeCode.Double, TypeCode.Int16, TypeCode.Int32, + TypeCode.Int64, TypeCode.SByte, TypeCode.Single, TypeCode.UInt16, TypeCode.UInt32, + TypeCode.UInt64 + }; + + public static bool is_dbnull(object o) { return o == null || o == DBNull.Value; } + + public static bool is_numeric_type(object o) + { + return is_dbnull(o) ? false : _numeric_type_codes.Contains(Type.GetTypeCode(o.GetType())); + } + + public static T[] parse_string_to_typed_array(string text, params char[] seperators) + { + if (text == null || text.Length == 0) + return new T[] { }; + else + { + List rv = new List(); + string[] toks = text.Split(seperators); + foreach (string t in toks) + { + string fin_t = t.Trim(); + if (!String.IsNullOrEmpty(fin_t)) + { + try + { + T ct = (T)Convert.ChangeType(fin_t, typeof(T)); + if (ct != null) + rv.Add(ct); + } + catch (Exception) { } + } + } + return rv.ToArray(); + } + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/Misc/TypeInstanceLoader.cs b/src/DataStreamSuite/DataStreamCore/BlueData/Misc/TypeInstanceLoader.cs new file mode 100644 index 000000000..92ffc955a --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/Misc/TypeInstanceLoader.cs @@ -0,0 +1,48 @@ +using System; +using System.Reflection; +using System.Diagnostics; + +namespace BCG.Utilities.General +{ + public class TypeInstanceLoader + { + private TypeInstanceLoader () + { + } + + public static object loadInstance (string typename, params object[] parameters) + { + Type t = null; + try + { + t = loadType (typename); + } + catch (ApplicationException e) + { + Debug.Assert (false, e.Message); + } + + return Activator.CreateInstance (t, parameters); + } + + public static Type loadType (string name) + { + return loadType(name, false); + } + public static Type loadType (string name, bool throwOnError) + { + Type t = null; + Assembly[] ass = AppDomain.CurrentDomain.GetAssemblies(); + foreach (Assembly a in ass) + { + t = a.GetType(name, false, true); + if (t != null) + return t; + } + if(throwOnError) + throw new ApplicationException ("Can't find type '" + name + "' in all assemblies."); + else + return null; + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs new file mode 100644 index 000000000..eae986b4a --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs @@ -0,0 +1,145 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; + +namespace BCG +{ + /// + /// A generalization of TimeWindowTable. Derive a class from this and implement the IsInTable() method to create a table with a rolling window. + /// Note: When the window is rolled forward it will only call IsInTable() until it finds a row that it can keep. Make sure that the sort is ascending and that IsInTable works on a sorted collection. + /// + public abstract class MovingWindowTable : IBlueRowCollection + { + protected IComparer myRowComparer; + protected IBlueTableSchema mySchema; + protected List mySortedRows = new List(); + protected Lazy> myIDToRows; + + #region constructors + + public MovingWindowTable(IBlueTableSchema schema, IComparer comparer) + { + this.myRowComparer = comparer; + this.mySchema = schema; + this.myIDToRows = new Lazy>(InitializeIDToRows); + } + + #endregion + + public bool addRow(IBlueRow r) + { + if (windowContains(r)) + { + int insertIndex = this.mySortedRows.BinarySearch(r, this.myRowComparer); + if (insertIndex < 0) insertIndex = ~insertIndex; + this.mySortedRows.Insert(insertIndex, r); + if (this.myIDToRows.IsValueCreated) + { + this.myIDToRows.Value[(int)r["id"]] = r; + } + return true; + } + return false; + } + + public IBlueRowCollection moveWindow() + { + BlueTable removed = new BlueTable(this.mySchema); + + if (this.mySortedRows.Count == 0) + return removed; + + // we don't necessarily know whether the table is sorted ascending or descending since we let the user pass in a comparer + // so we'll test both ends of the table and start pulling from the end that is outside the window + int index; + if (windowContains(this.mySortedRows[0])) + { + for (index = this.mySortedRows.Count - 1; index >= 0; index--) + { + if (windowContains(this.mySortedRows[index])) break; + removed.addRow(this.mySortedRows[index]); + } + index++; + this.mySortedRows.RemoveRange(index, this.mySortedRows.Count - index); + } + else + { + for (index = 0; index < this.mySortedRows.Count; index++) + { + if (windowContains(this.mySortedRows[index])) break; + removed.addRow(this.mySortedRows[index]); + } + this.mySortedRows.RemoveRange(0, index); + } + + return removed; + } + + protected abstract bool windowContains(IBlueRow row); + + private Dictionary InitializeIDToRows() + { + Dictionary idMap = new Dictionary(this.mySortedRows.Count); + foreach (var row in this.mySortedRows) + { + idMap[(int)row["id"]] = row; + } + return idMap; + } + + #region IBlueRowCollection Members + + public IBlueRow this[int id] + { + get { return this.myIDToRows.Value[id]; } + } + + public IBlueTableSchema schema + { + get { return this.mySchema; } + } + + public bool containsID(int id) + { + return this.myIDToRows.Value.ContainsKey(id); + } + + #endregion + + #region ICollection Members + + public void CopyTo(Array array, int index) + { + ((System.Collections.ICollection)this.mySortedRows).CopyTo(array, index); + } + + public int Count + { + get { return this.mySortedRows.Count; } + } + + public bool IsSynchronized + { + get { return false; } + } + + public object SyncRoot + { + // Throw notimplemented rather than return null which will probably do bad things + get { throw new NotImplementedException(); } + } + + #endregion + + #region IEnumerable Members + + public System.Collections.IEnumerator GetEnumerator() + { + return this.mySortedRows.GetEnumerator(); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/MutableBlueRow.cs b/src/DataStreamSuite/DataStreamCore/BlueData/MutableBlueRow.cs new file mode 100644 index 000000000..a21920bf8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/MutableBlueRow.cs @@ -0,0 +1,56 @@ +using System; +using System.Data; +using System.Collections; + +namespace BCG +{ + [Serializable] + public class MutableBlueRow : BlueRow + { + public MutableBlueRow (IBlueTableSchema schema) + : this (schema, MutableBlueRow.myCreateArray (schema.columnNames.Length, DBNull.Value)) + {} + + public MutableBlueRow (IBlueTableSchema schema, params object[] values) + : base (schema, MutableBlueRow.myCloneArray (schema,values)) + {} + + public MutableBlueRow (string tableName, IDictionary row) + : base (tableName, row) + {} + public MutableBlueRow (IBlueTableSchema schema, IDictionary row) + : base (schema, row) { } + + public MutableBlueRow (IBlueRow r) + : base (r) + {} + + + public void setValue (string columnName, object newValue) + { + setValue (mySchema[columnName].colNo, newValue); + } + + public void setValue (int colNo, object newValue) + { + myValues[colNo] = newValue; + } + + private static object[] myCloneArray (IBlueTableSchema schema,object[] vals) + { + object[] retVal = new object[schema.columnNames.Length]; + for (int i =0; i < vals.Length; i++) + retVal[i] = vals[i]; + return retVal; + } + + private static object[] myCreateArray (int sz, object o) + { + object[] rv = new object[sz]; + for (int i = 0; i < sz; i++) + rv[i] = DBNull.Value; + return rv; + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs new file mode 100644 index 000000000..43ce64e02 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections; +using System.Data; +//using BCG.Utilities; + +namespace BCG +{ + [Serializable] + public class OrderedBlueTable : BlueTable + { + protected ArrayList myOrderedRows = new ArrayList(); + + protected OrderedBlueTable() { } + + public OrderedBlueTable(IBlueTableSchema schema) : base(schema) { } + + public OrderedBlueTable(IBlueRowCollection table) : base(table) { } + + public OrderedBlueTable(IBlueRowCollection table, int count) : this(table, 0, count) { } + + public OrderedBlueTable(IBlueRowCollection table, int startIndex, int count) + { + //BlueEventLog.logAssert (count >= 0 && startIndex + count <= table.Count); + mySchema = table.schema; + int index = 0; + foreach (IBlueRow r in table) + { + if (index++ >= startIndex) + addRow(r); + if (index == startIndex + count) + break; + } + } + + + public override void clear() + { + base.clear(); + myOrderedRows.Clear(); + } + + public override void addRow(IBlueRow row) + { + base.addRow(row); + myOrderedRows.Add(row); + } + + public override void addRow(DataRow row) + { + base.addRow(row); + myOrderedRows.Add(myIdIndex[row["id"]]); + } + + public override bool deleteRow(int id) + { + if (myIdIndex[id] == null) + return false; + myOrderedRows.Remove(myIdIndex[id]); + myIdIndex.Remove(id); + return true; + } + + #region ICollection members + public override IEnumerator GetEnumerator() { return myOrderedRows.GetEnumerator(); } + + public override void CopyTo(Array array, int index) { myOrderedRows.CopyTo(array, index); } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/OrderedInplaceCircularBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/OrderedInplaceCircularBlueTable.cs new file mode 100644 index 000000000..8110a473a --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/OrderedInplaceCircularBlueTable.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public class OrderedInplaceCircularBlueTable + { + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SchemaVisibility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SchemaVisibility.cs new file mode 100644 index 000000000..c104c9f68 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SchemaVisibility.cs @@ -0,0 +1,27 @@ +using System; + +namespace BCG.Utilities.General +{ + /// + /// Properties or public fields of a type can be mapped by this attribute, governing + /// their visiblity in the schema that can be constructed out of the object + /// + public class SchemaVisibility : Attribute + { + private bool myVisible = true; + + /// + /// Tagging visbible as true, results in the property/field to + /// map to itself in the schema. This has the same result as + /// not tagging a field/property + /// + /// + public SchemaVisibility (bool visible) + { + myVisible = visible; + } + + public bool visible{get{return myVisible;}} + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs new file mode 100644 index 000000000..33438d513 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// Optimized for the scneario when id's are sequential starting from 0 (no indexing is necessary etc) + /// + public class SequentialIDBlueTable : IBlueRowCollection + { + private IBlueRow[] myRows; + private IBlueTableSchema mySchema; + + /// + /// The refernce to the rows is kept. So if user wants to reuse the variable, its best + /// to pass in a copied value + /// + /// + /// + public SequentialIDBlueTable (IBlueTableSchema schema, IBlueRow[] rows) + { + myRows = rows; + mySchema = schema; + } + + public SequentialIDBlueTable(IBlueTableSchema schema, System.Collections.Generic.IReadOnlyList rows) + { + mySchema = schema; + myRows = new IBlueRow[rows.Count]; + for (int i = 0; i < myRows.Length; i++) + myRows[i] = new BlueRow(mySchema, rows[i]); + } + + #region IBlueRowCollection Members + + public IBlueRow this[int id] { get { return myRows[id]; } } + + public IBlueTableSchema schema {get {return mySchema;}} + + public bool containsID (int id) + { + return id > -1 && id < myRows.Length; + } + + #endregion + + #region ICollection Members + + public bool IsSynchronized {get {return false;}} + + public int Count {get {return myRows.Length;}} + + public void CopyTo (Array array, int index) {myRows.CopyTo (array, index);} + + public object SyncRoot {get {return null;}} + + #endregion + + #region IEnumerable Members + + public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator ();} + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueRow.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueRow.cs new file mode 100644 index 000000000..811bd3db4 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueRow.cs @@ -0,0 +1,400 @@ +using System; +using System.Collections; +using System.Data; +//using BCG.Utilities; +using System.IO; + + +namespace BCG +{ + /// + /// SerializedBlueRow is an analog of DataRow, an immutable one though. It constructs itself either + /// from dotNet DataRow or IBlueRow. One row can belong to any number of tables, 0 or 1 or many. + /// All what the SerializedBlueRow requires is a schema to manage its internal information + /// + /// + /// Data in serialized rows is stored in two places: a byte array contains all nonstring data, + /// and a string array contains the string data. The strings in this array are all just references + /// to strings stored in the internin module (STring.intern) + /// Additdionally, the ids are cached locally in the row, so that they can be accessed faster. + /// + [Serializable] + public class SerializedBlueRow : IBlueRow + { + protected SerializedBlueTableSchema mySchema; + byte[] myRowData; + string[] myRowStrings; + + + + #region Private helper methods + /// + /// Initializes the internal data structures + /// + /// + private void myInit(IBlueTableSchema schema) + { + if (schema is SerializedBlueTableSchema) + mySchema = schema as SerializedBlueTableSchema; + else + mySchema = new SerializedBlueTableSchema(schema); + + myRowData = null; + } + #endregion + + #region Constructors + protected SerializedBlueRow() { } + + public SerializedBlueRow(IBlueRow r) + { + myInit(r.schema); + Object[] thisRowObjects = new Object[r.schema.columnInfo.Length]; + foreach (ColumnInfo col in r.schema.columnInfo) + thisRowObjects[col.colNo] = r[col.colNo]; + myBuildFromObjects(thisRowObjects); + } + /// + /// The SerializedBlueRow initializes itself from the row passed. The row is expected + /// to have the same schema (including the col ordering) as the row being created + /// + /// + /// + public SerializedBlueRow(IBlueTableSchema schema, IBlueRow row) + { + myInit(schema); + + } + + /// + /// Constructs itself from the schema and the array of values that correspond + /// to the rows + /// + /// + /// Schema of the row + /// + /// + /// Values represent the values in the row, the order should be the same as + /// represented in the schema + /// + public SerializedBlueRow(IBlueTableSchema schema, params Object[] values) + { + myInit(schema); + myBuildFromObjects(values); + } + + /// + /// Initilizes itself from a dotNet DataRow. The row is expected to + /// have the same schema (including the column ordering) as the row being created + /// + /// + /// + public SerializedBlueRow(IBlueTableSchema schema, DataRow row) + { + myInit(schema); + Object[] thisRowObjects = new Object[row.Table.Columns.Count]; + foreach (DataColumn col in row.Table.Columns) + thisRowObjects[col.Ordinal] = row[col.Ordinal]; + + myBuildFromObjects(thisRowObjects); + } + + public SerializedBlueRow(IBlueTableSchema schema, IDictionary row) + { + myInit(schema); + Object[] thisRowObjects = new Object[row.Count]; + foreach (DictionaryEntry de in row) + if (mySchema[de.Key as string] != null) + thisRowObjects[mySchema[de.Key as string].colNo] = de.Value; + myBuildFromObjects(thisRowObjects); + } + + + + /// + /// Constructs the schema and then populates the values, both from the IDictionary. + /// The table name (the only thing that can not be derived from the dictionary) is + /// also provided + /// + /// + /// + public SerializedBlueRow(string tableName, IDictionary row) + { + if (row is IBlueRow) + myInit(((IBlueRow)row).schema); + else + { + ColumnInfo[] cis = new ColumnInfo[row.Count]; + int i = 0; + foreach (DictionaryEntry de in row) + cis[i] = new ColumnInfo(de.Key as string, i++, de.Value.GetType()); + myInit(new SerializedBlueTableSchema(cis, tableName)); + } + Object[] thisRowObjects = new Object[row.Count]; + + foreach (DictionaryEntry de in row) + thisRowObjects[mySchema[de.Key as string].colNo] = de.Value; + myBuildFromObjects(thisRowObjects); + } + + /// + /// Constructs the schema and then populates the values, both from the IDictionary. + /// + /// + public SerializedBlueRow(IDictionary row) : this("", row) { } + + #endregion + + + /// + /// Creates the serialized blue row from the list of objects. Serializes them using the schema. + /// + /// + private void myBuildFromObjects(Object[] theObjects) + { + + + using(BinaryWriter theBinaryWriter = new BinaryWriter(new MemoryStream())) + { + + + //store the id + int idCol = mySchema["id"].colNo; + theBinaryWriter.Write((int)theObjects[idCol]); + + // + //serialize all the data + // + + + int numCols = theObjects.Length; + //first, write the null map + BitArray NullMap = new BitArray(numCols); + + for (int col = 0; col < numCols; ++col) + NullMap.Set(col,(theObjects[col] == null || theObjects[col] == DBNull.Value)); + byte[] NullMapBytes = new Byte[mySchema.getNullMapSize()]; + NullMap.CopyTo(NullMapBytes,0); + theBinaryWriter.Write(NullMapBytes); + + //then, write the non null, non string objects into the byte array + //and copy the string references into the string array + string[] thisColStrings = new string[numCols]; + int numStringCols = 0; + for (int col = 0; col < numCols; ++col) + { + //don't serialize null values into the array. They'll just take up space. + //also don't serialize the id + if(NullMap.Get(col) == false && col != idCol) + { + Type thisColType = mySchema[col].colType; + if (thisColType == typeof(string)) + thisColStrings[numStringCols++] = String.Intern(((string)theObjects[col])); + else + { + long numBytesBefore = theBinaryWriter.BaseStream.Length; + mySchema.serializeObject(theObjects[col], theBinaryWriter, col); + long numBytesAfter = theBinaryWriter.BaseStream.Length; + + + } + } + } + + //now that we knmow how many string columns are in this row, we should store those references + //we do it thsi way to avoid storing null string refernces. + //it complicates string access a tiny but, it the savings is probably worth it + //becuase a lot of strings are null in tables like trade + myRowStrings = new string[numStringCols]; + for (int col = 0; col < numStringCols; ++col) + { + myRowStrings[col] = thisColStrings[col]; + + } + + + //now save the bytes of the memory writer + long numBytes = theBinaryWriter.BaseStream.Length; + myRowData = new Byte[numBytes]; + theBinaryWriter.Seek(0, SeekOrigin.Begin); + for (int i = 0; i < numBytes; ++i) + myRowData[i] = Convert.ToByte(theBinaryWriter.BaseStream.ReadByte()); + + + /*/This code is for verifying that the row construction actually worked. + //it is test code, and should be commented out when actually used + ICollection thisRowValues = Values; + BlueRow thisBlueRow = new BlueRow(mySchema, theObjects); + string[] diffcolumns = BlueDataUtility.compareRows(this, thisBlueRow); + if(diffcolumns.Length != 0) + throw new BlueDataException(string.Format("Row Serialization Failed on column: {0}", diffcolumns[0])); + //*/ + + } + + + + } + + public string Bytes + { + get { return new System.Text.ASCIIEncoding().GetString(myRowData); } + } + + #region IBlueRow members + public virtual IBlueTableSchema schema { get { return mySchema; } } + public virtual object this[string colName] + { + get + { + //see if they're askign for the id. These should be much faster than querying the hashtable + //(this is slow right now, but soon we should swithc to unsafe code and just typecase the byte poitner); + if (colName == "id") + return this[mySchema.IdColumn]; + + + ColumnInfo ci = mySchema[colName]; + if (ci != null) + return this[ci.colNo]; + else + throw new Exception(colName + " is not a valid column name for " + schema.tableName); + } + } + public unsafe virtual object this[int index] + { + get + { + if (index == mySchema.IdColumn) + { + //the id is stored at the start of the row, to speed up access to it + fixed (byte* id = &myRowData[0]) + { + return *((int*)id); + } + + } + + //the id is the first thing stored in the row. WE want to eat this up first. + if (mySchema[index].colType == typeof(string)) + { + BitArray nullMap = mySchema.getNullMap(myRowData); + int indexIntoStringTable = 0; + for (int colNo = 0; colNo < mySchema.columnInfo.Length; ++colNo ) + { + if (nullMap.Get(colNo)) + { + //this is the string they are asking for + if (colNo == index) + return DBNull.Value; //we want to break out of all the loops and such here + + //otherwise, don't increment the index into the string table + } + else + { + if (colNo == index) + return myRowStrings[indexIntoStringTable]; + else if (mySchema[colNo].colType == typeof(string)) + ++indexIntoStringTable; + } + } + + + }//end if this column is a string + else + return mySchema.getValueNonString(myRowData, index); + + + //you should never get to this point. + throw new BlueDataException("Error: Malformed Serialized Blue Row, couldn't pull data item"); + } + + + + } + #endregion + + #region ICollection members + public virtual int Count { get { return mySchema.columnInfo.Length; } } + + public virtual bool IsSynchronized { get { return false; } } + + public virtual object SyncRoot { get { return this; } } + + public virtual void CopyTo(Array array, int index) { + Values.CopyTo(array, index); + + } + #endregion + + #region IDictionary members + public virtual bool IsFixedSize { get { return true; } } + + virtual public bool IsReadOnly { get { return true; } } + + /// + /// The key can either be the string corrsponding to the column name or + /// the order of the column + /// + public virtual object this[object key] + { + get + { + if (key is string) + return this[(string)key]; + else if (key is int) + return this[(int)key]; + throw new BlueDataException("Wrong type while indexing the row"); + } + set { throw new NotSupportedException("The IBlueRow is readonly"); } + } + + public virtual ICollection Keys { get { return mySchema.columnNames; } } + + public virtual ICollection Values + { + get + { + return mySchema.getAllValues(myRowData, myRowStrings); + + } + } + + public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } + + public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } + + public virtual bool Contains(object key) { return mySchema[(string)key] != null; } + + /// + /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to + /// iterate through the values, typecast this into an IEnumerable + /// + /// + public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + + IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + + public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); } + #endregion + + #region IComparable Members + + public int CompareTo(IBlueRow other) + { + int thisID = (int)this["id"]; + int otherID = (int)other["id"]; + + if (thisID > otherID) + return 1; + else if (thisID == otherID) + return 0; + else + return -1; + + + } + + #endregion + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs new file mode 100644 index 000000000..958a6f431 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs @@ -0,0 +1,476 @@ +using System; +using System.Data; +using System.Collections; +using System.IO; +using BCG.Utilities.General; +//using BCG.Utilities; + +namespace BCG +{ + /// + /// The SerializedBlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created + /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it + /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return + /// the respective data, sorted according to the ColumnOrder. + /// + /// It is designed to work in concert with serialized blue rows. The serialized blue rows hold most information inside + /// memory streams, and uses this schema to unpack the values when they are needed. + /// + [Serializable] + public class SerializedBlueTableSchema : IBlueTableSchema + { + public class SerializedBlueTableSchemaSerializationHelper + { + public static object parse(Type t, string data) + { + if (t == BlueDataValidTypes.BOOL) + return Boolean.Parse(data); + else if (t == BlueDataValidTypes.BYTE) + return Byte.Parse(data); + else if (t == BlueDataValidTypes.CHAR) + return char.Parse(data); + else if (t == BlueDataValidTypes.DATETIME) + return DateTime.Parse(data); + else if (t == BlueDataValidTypes.DOUBLE) + return Double.Parse(data); + else if (t == BlueDataValidTypes.DECIMAL) + return Decimal.Parse(data); + else if (t == BlueDataValidTypes.INT) + return int.Parse(data); + else if (t == BlueDataValidTypes.LONG) + return long.Parse(data); + else if (t == BlueDataValidTypes.SBYTE) + return SByte.Parse(data); + else if (t == BlueDataValidTypes.STRING) + return data; + else if (t == BlueDataValidTypes.UINT) + return uint.Parse(data); + else if (t == BlueDataValidTypes.ULONG) + return ulong.Parse(data); + else if (t == BlueDataValidTypes.SINGLE) + return Single.Parse(data); + else + throw new ApplicationException("Can not be parsed"); + } + public static void serialize(BinaryWriter bw, Type t, object o) + { + if (t == BlueDataValidTypes.BOOL) + bw.Write((bool)o); + else if (t == BlueDataValidTypes.BYTE) + bw.Write((byte)o); + else if (t == BlueDataValidTypes.CHAR) + bw.Write((char)o); + else if (t == BlueDataValidTypes.DATETIME) + bw.Write(((DateTime)o).Ticks); + else if (t == BlueDataValidTypes.DOUBLE) + bw.Write((double)o); + else if (t == BlueDataValidTypes.DECIMAL) + bw.Write((decimal)o); + else if (t == BlueDataValidTypes.INT) + bw.Write((int)o); + else if (t == BlueDataValidTypes.LONG) + bw.Write((long)o); + else if (t == BlueDataValidTypes.SBYTE) + bw.Write((sbyte)o); + else if (t == BlueDataValidTypes.STRING) + bw.Write(o as string); + else if (t == BlueDataValidTypes.SINGLE) + bw.Write((Single)o); + else if (t == BlueDataValidTypes.BYTEARRAY) + { + byte[] bs = o as byte[]; + bw.Write(bs.Length); + bw.Write(bs); + } + else if (t == BlueDataValidTypes.UINT) + bw.Write((uint)o); + else if (t == BlueDataValidTypes.ULONG) + bw.Write((ulong)o); + else + throw new ApplicationException("Unsupported type"); + } + + public static object deserialize(BinaryReader br, Type t) + { + if (t == BlueDataValidTypes.BOOL) + return br.ReadBoolean(); + else if (t == BlueDataValidTypes.BYTE) + return br.ReadByte(); + else if (t == BlueDataValidTypes.CHAR) + return br.ReadChar(); + else if (t == BlueDataValidTypes.DATETIME) + return new DateTime(br.ReadInt64()); + else if (t == BlueDataValidTypes.DOUBLE) + return br.ReadDouble(); + else if (t == BlueDataValidTypes.DECIMAL) + return br.ReadDecimal(); + else if (t == BlueDataValidTypes.INT) + return br.ReadInt32(); + else if (t == BlueDataValidTypes.LONG) + return br.ReadInt64(); + else if (t == BlueDataValidTypes.SBYTE) + return br.ReadSByte(); + else if (t == BlueDataValidTypes.UINT) + return br.ReadUInt32(); + else if (t == BlueDataValidTypes.ULONG) + return br.ReadUInt64(); + else if (t == BlueDataValidTypes.SINGLE) + return br.ReadSingle(); + else + throw new ApplicationException("Unsupported type"); + } + } + + protected Hashtable myIndexByColName = new Hashtable(); + protected ColumnInfo[] myIndexByColOrder; + private string[] myTableColNames; + protected string myTableName; + private int[] myColumnSizesTable; + int myIdCol; + + + public SerializedBlueTableSchema() { } + + //copies the schema from an existing schema, building the dispatch table + public SerializedBlueTableSchema(IBlueTableSchema toCopy) + { + int numCols = toCopy.columnInfo.Length; + ColumnInfo[] theColumns = new ColumnInfo[toCopy.columnInfo.Length]; + foreach (ColumnInfo toCopyCI in toCopy.columnInfo) + { + theColumns[toCopyCI.colNo] = toCopyCI; + } + myConstruct(theColumns, toCopy.tableName); + } + + public SerializedBlueTableSchema(DataTable table) + { + ColumnInfo[] theseCols = new ColumnInfo[table.Columns.Count]; + myTableName = table.TableName; + foreach (DataColumn col in table.Columns) + { + theseCols[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); + } + myConstruct(theseCols, myTableName); + } + + public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName) + { + myConstruct(cis, tableName); + } + + public void serializeObject(object data, BinaryWriter bw, int colNo) + { + mySerializeDispatchTable[colNo](data, bw); + + } + + private void myConstruct(ColumnInfo[] cis, string tableName) + { + myIndexByColOrder = new ColumnInfo[cis.Length]; + myTableName = tableName; + foreach (ColumnInfo ci in cis) + { + myIndexByColOrder[ci.colNo] = ci; + myIndexByColName[ci.colName] = ci; + } + myBuildDispatchTable(); + } + + public SerializedBlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { } + + public ColumnInfo this[string colName] + { + get + { + object retVal = myIndexByColName[colName]; + return retVal != null ? retVal as ColumnInfo : null; + } + } + + public ColumnInfo this[int colNo] + { + get { return myIndexByColOrder[colNo]; } + } + + /// + /// The returned Array is sorted according to the Column No + /// + public string[] columnNames + { + get + { + if (myTableColNames == null) + { + myTableColNames = new string[myIndexByColName.Count]; + foreach (DictionaryEntry de in myIndexByColName) + { + ColumnInfo temp = (ColumnInfo)de.Value; + myTableColNames[temp.colNo] = temp.colName; + } + } + return myTableColNames; + } + } + + /// + /// The returned Array is sorted according to the Column No + /// + public ColumnInfo[] columnInfo + { + get { return myIndexByColOrder; } + } + + public string tableName { get { return myTableName; } } + + + public int IdColumn + { + get { return myIdCol; } + } + + + #region ISerializedType methods + public void serialize(BinaryWriter bw) + { + bw.Write(columnNames.Length); + bw.Write(tableName); + foreach (ColumnInfo ci in columnInfo) + { + bw.Write(ci.colName); + bw.Write(ci.colType.ToString()); + if (ci.isConstant) + { + bw.Write(true); + SerializedBlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue); + } + else + { + bw.Write(false); + } + } + } + + public void deserialze(BinaryReader br) + { + ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()]; + string name = br.ReadString(); + for (int i = 0; i < cis.Length; i++) + { + string colName = br.ReadString(); + Type t = TypeInstanceLoader.loadType(br.ReadString(), true); + bool isConstant = br.ReadBoolean(); + if (isConstant) + { + cis[i] = new ColumnInfo(colName, i, t, SerializedBlueTableSchemaSerializationHelper.deserialize(br, t)); + } + else + { + cis[i] = new ColumnInfo(colName, i, t); + } + } + myConstruct(cis, name); + } + + public int getSerializedSize() + { + throw new NotImplementedException("Feature for provided constant values to" + + "non-primitive types in schema, not provided as of now"); + } + + #endregion + + + //builds a dispatch table used to pull values out of the memory streams in the rows. + private void myBuildDispatchTable() + { + int numCols = myIndexByColOrder.Length; + mySerializeDispatchTable = new SerializeDelegate[numCols]; + myPullFromByteBufferDispatchTable = new PullFromByteBufferDelegate[numCols]; + myColumnSizesTable = new int[numCols]; + ColumnInfo idColumn = (ColumnInfo)myIndexByColName["id"]; + + myIdCol = idColumn.colNo; + + for (int col = 0; col < numCols; ++col) + { + Type thisColType = myIndexByColOrder[col].colType; + //we don't store serializers for the id column, or string columns. + if (col == myIdCol || thisColType == typeof(string)) + { + mySerializeDispatchTable[col] = null; + myColumnSizesTable[col] = 0; + myPullFromByteBufferDispatchTable[col] = null; + } + else + { + mySerializeDispatchTable[col] = myGetDispatchMethodSerialize(thisColType); + myPullFromByteBufferDispatchTable[col] = myGetDispatchMethodPullFromBuffer(thisColType); + myColumnSizesTable[col] = BlueDataValidTypes.getValueTypeSize(thisColType); + } + + } + + + } + + + public int getNullMapSize() + { + int numCols = myIndexByColOrder.Length; + return Convert.ToInt32(Math.Ceiling((Convert.ToDouble(numCols) / 8.0))); + } + + //this function reads the null array from inside the byte array. + //it has to copy the bytes over. + public BitArray getNullMap(byte[] theBytes) + { + int nullMapSize = getNullMapSize(); + //the null map starts after the id, so that we have instant access to the id. + int byteCursor = BlueDataValidTypes.getValueTypeSize(typeof(Int32)); + byte[] nullMapArray = new byte[nullMapSize]; + for (int i = 0; i < nullMapSize; ++i) + nullMapArray[i] = theBytes[byteCursor++]; + + return new BitArray(nullMapArray); + } + //gets the value either from the bytes stream but only if the value is not a string. + //doesn't work if you're getting the id. + public unsafe object getValueNonString(byte[] theBytes, int colNo) + { + if (colNo == myIdCol) + throw new BlueDataException("Error: do not use getValueNonString to try to fetch the id of a row"); + BitArray nullMap = getNullMap(theBytes); + if (nullMap.Get(colNo)) + return DBNull.Value; + //the rest of the row starts after the id and the null map + int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize(); + for (int currentColumn = 0; currentColumn < colNo; ++currentColumn) + { + //pull the data from the binary reader, but only if this type is not null or a string + //if the type wer'e looking for is a string, it would have been returned already by the accesor + //in the bluerow. + //if it's the id, it also would have been returned already + if (!nullMap.Get(currentColumn) + && myIndexByColOrder[currentColumn].colType != typeof(string) + && currentColumn != myIdCol) + + thisOffset += myColumnSizesTable[currentColumn]; + } + fixed (byte* thisBytePtr = &theBytes[thisOffset]) + { + return myPullFromByteBufferDispatchTable[colNo](thisBytePtr); + } + + + } + + public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray) + { + int numCols = myColumnSizesTable.Length; + object[] theObjs = new Object[numCols]; + + BitArray nullMap = getNullMap(theBytes); + + int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize(); + int thisStringNo = 0; + for (int currentColumn = 0; currentColumn < numCols; ++currentColumn) + { + //pull the data from the binary reader, but only if this type is not null or a string + //if the type wer'e looking for is a string, it would have been returned already by the accesor + //in the bluerow. + //if it's the id, it also would have been returned already + if (nullMap.Get(currentColumn)) + theObjs[currentColumn] = DBNull.Value; + else if (myIndexByColOrder[currentColumn].colType == typeof(string)) + theObjs[currentColumn] = stringArray[thisStringNo++]; + else if (currentColumn != myIdCol) + { + fixed (byte* thisGuy = &theBytes[thisOffset]) + { + theObjs[currentColumn] = myPullFromByteBufferDispatchTable[currentColumn](thisGuy); + } + thisOffset += myColumnSizesTable[currentColumn]; + } + } + + return theObjs; + + + } + + private delegate void SerializeDelegate(object data, BinaryWriter bw); + private SerializeDelegate[] mySerializeDispatchTable; + + private void mySerializeInt32(object data, BinaryWriter bw) { bw.Write((Int32)data); } + private void mySerializeDouble(object data, BinaryWriter bw) { bw.Write((double)data); } + private void mySerializeChar(object data, BinaryWriter bw) { bw.Write((char)data); } + private void mySerializeLong(object data, BinaryWriter bw) { bw.Write((long)data); } + private void mySerializeSByte(object data, BinaryWriter bw) { bw.Write((sbyte)data); } + private void mySerializeBool(object data, BinaryWriter bw) { bw.Write((bool)data); } + private void mySerializeByte(object data, BinaryWriter bw) { bw.Write((byte)data); } + private void mySerializeDecimal(object data, BinaryWriter bw) { bw.Write((decimal)data); } + private void mySerializeUInt(object data, BinaryWriter bw) { bw.Write((uint)data); } + private void mySerializeString(object data, BinaryWriter bw) { bw.Write((string)data); } + private void mySerializeULong(object data, BinaryWriter bw) { bw.Write((ulong)data); } + private void mySerializeDateTime(object data, BinaryWriter bw) { bw.Write(((DateTime)data).ToOADate()); } + private void mySerializeSingle(object data, BinaryWriter bw) { bw.Write((Single)data); } + + private SerializeDelegate myGetDispatchMethodSerialize(Type t) + { + if (t.Equals(BlueDataValidTypes.INT)) return new SerializeDelegate(mySerializeInt32); + else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new SerializeDelegate(mySerializeDouble); + else if (t.Equals(BlueDataValidTypes.CHAR)) return new SerializeDelegate(mySerializeChar); + else if (t.Equals(BlueDataValidTypes.LONG)) return new SerializeDelegate(mySerializeLong); + else if (t.Equals(BlueDataValidTypes.SBYTE)) return new SerializeDelegate(mySerializeSByte); + else if (t.Equals(BlueDataValidTypes.BOOL)) return new SerializeDelegate(mySerializeBool); + else if (t.Equals(BlueDataValidTypes.BYTE)) return new SerializeDelegate(mySerializeByte); + else if (t.Equals(BlueDataValidTypes.DECIMAL)) return new SerializeDelegate(mySerializeDecimal); + else if (t.Equals(BlueDataValidTypes.UINT)) return new SerializeDelegate(mySerializeUInt); + else if (t.Equals(BlueDataValidTypes.ULONG)) return new SerializeDelegate(mySerializeULong); + else if (t.Equals(BlueDataValidTypes.DATETIME)) return new SerializeDelegate(mySerializeDateTime); + else if (t.Equals(BlueDataValidTypes.SINGLE)) return new SerializeDelegate(mySerializeSingle); + else if (t.Equals(BlueDataValidTypes.STRING)) return new SerializeDelegate(mySerializeString); + else throw new ApplicationException(t.FullName + " is not a supported type"); + } + + private unsafe delegate object PullFromByteBufferDelegate(byte* bytePtr); + private PullFromByteBufferDelegate[] myPullFromByteBufferDispatchTable; + + + private unsafe object myPullInt32(byte* bytePtr) { return *((int*)bytePtr); } + private unsafe object myPullDouble(byte* bytePtr) { return *((double*)bytePtr); } + private unsafe object myPullChar(byte* bytePtr) { return *((char*)bytePtr); } + private unsafe object myPullLong(byte* bytePtr) { return *((long*)bytePtr); } + private unsafe object myPullSByte(byte* bytePtr) { return *((SByte*)bytePtr); } + private unsafe object myPullBool(byte* bytePtr) { return *((bool*)bytePtr); } + private unsafe object myPullByte(byte* bytePtr) { return *(bytePtr); } + private unsafe object myPullDecimal(byte* bytePtr) { return *((decimal*)bytePtr); } + private unsafe object myPullUInt(byte* bytePtr) { return *((UInt32*)bytePtr); } + private unsafe object myPullULong(byte* bytePtr) { return *((UInt64*)bytePtr); } + private unsafe object myPullDateTime(byte* bytePtr) + { + double thisDateTime = *((double*)bytePtr); + return DateTime.FromOADate(thisDateTime); + } + private unsafe object myPullSingle(byte* bytePtr) { return *((Single*)bytePtr); ; } + + + private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t) + { + if (t.Equals(BlueDataValidTypes.INT)) return new PullFromByteBufferDelegate(myPullInt32); + else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new PullFromByteBufferDelegate(myPullDouble); + else if (t.Equals(BlueDataValidTypes.CHAR)) return new PullFromByteBufferDelegate(myPullChar); + else if (t.Equals(BlueDataValidTypes.LONG)) return new PullFromByteBufferDelegate(myPullLong); + else if (t.Equals(BlueDataValidTypes.SBYTE)) return new PullFromByteBufferDelegate(myPullSByte); + else if (t.Equals(BlueDataValidTypes.BOOL)) return new PullFromByteBufferDelegate(myPullBool); + else if (t.Equals(BlueDataValidTypes.BYTE)) return new PullFromByteBufferDelegate(myPullByte); + else if (t.Equals(BlueDataValidTypes.DECIMAL)) return new PullFromByteBufferDelegate(myPullDecimal); + else if (t.Equals(BlueDataValidTypes.UINT)) return new PullFromByteBufferDelegate(myPullUInt); + else if (t.Equals(BlueDataValidTypes.ULONG)) return new PullFromByteBufferDelegate(myPullULong); + else if (t.Equals(BlueDataValidTypes.DATETIME)) return new PullFromByteBufferDelegate(myPullDateTime); + else if (t.Equals(BlueDataValidTypes.SINGLE)) return new PullFromByteBufferDelegate(myPullSingle); + else throw new ApplicationException(t.FullName + " is not a supported type to pull from a buffer"); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs new file mode 100644 index 000000000..09577d2f9 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs @@ -0,0 +1,121 @@ +using System; +using System.ComponentModel; +using System.Collections; + +namespace BCG +{ + public class SimpleBindableCollection : IBlueRowCollection, ITypedList, IList + { + private ArrayList myRows; + private IBlueTableSchema mySchema; + private PropertyDescriptorCollection myPDColl; + + public SimpleBindableCollection (IBlueTableSchema schema, ICollection rows) + { + mySchema = schema; + ArrayList temp = new ArrayList (); + foreach (ColumnInfo ci in mySchema.columnInfo) + temp.Add (new BlueDataPropertyDescriptor (ci)); + myPDColl = new PropertyDescriptorCollection (temp.ToArray (typeof(PropertyDescriptor)) as PropertyDescriptor[]); + myRows = new ArrayList (rows); + } + + public SimpleBindableCollection (IBlueTableSchema schema) + : this (schema, new ArrayList ()) + {} + + #region IBlueRowCollection Members + + public IBlueRow this[int id] + { + get + { + foreach (IBlueRow r in this) + if (r["id"].Equals (id)) + return r; + throw new ApplicationException ("ID + " + id.ToString () + " not found"); + } + } + + public bool containsID(int id) + { + try + { + return this[id] != null; + } + catch (ApplicationException) + { + return false; + } + } + + public IBlueTableSchema schema {get {return mySchema;}} + + #endregion + + #region ICollection Members + + public bool IsSynchronized {get {return false;}} + + public int Count {get {return myRows.Count;}} + + public void CopyTo(Array array, int index) {myRows.CopyTo (array, index);} + + public object SyncRoot { get {return null;}} + + #endregion + + #region IEnumerable Members + + public System.Collections.IEnumerator GetEnumerator() + { + return myRows.GetEnumerator (); + } + + #endregion + + #region ITypedList Members + + public PropertyDescriptorCollection GetItemProperties(PropertyDescriptor[] listAccessors) + { + return myPDColl; + } + + public string GetListName(PropertyDescriptor[] listAccessors) + { + return "SimpleBindableCollection"; + } + + #endregion + + #region IList Members + + public bool IsReadOnly {get {return true;}} + + object System.Collections.IList.this[int index] + { + get {return myRows[index];} + set {throw new ApplicationException ("Collection is read only");} + } + + public void RemoveAt(int index) {throw new ApplicationException ("Collection is read only");} + + public void Insert(int index, object value) {throw new ApplicationException ("Collection is read only");} + + public void Remove(object value) {throw new ApplicationException ("Collection is read only");} + + public bool Contains(object value) {return myRows.Contains (value);} + + public void Clear() {throw new ApplicationException ("Collection is read only");} + + public int IndexOf(object value) {return myRows.IndexOf (value);} + + public int Add(object value) {throw new ApplicationException ("Collection is read only");} + + public bool IsFixedSize {get {return true;}} + + #endregion + + public void reset (ICollection rows) {myRows = new ArrayList(rows);} + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs new file mode 100644 index 000000000..33a3a0ff6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public class SparseVectorBlueTable : IBlueRowCollection + { + private List _rows = new List(50); + + public SparseVectorBlueTable(IBlueTableSchema schema) { + this.schema = schema; + } + + public void reserve(int length) { + if (_rows.Capacity < length) { + _rows.Capacity += 1024; + } + while (_rows.Count <= length) { + _rows.Add(null); + } + } + + public IBlueRow this[int id] { + get { + return _rows[id]; + } + set { + this.reserve(id); + _rows[id] = value; + } + } + public IBlueTableSchema schema { get; private set; } + public bool containsID(int id) { + return (id >= 0 && id < _rows.Count && this[id] != null); + } + + public void CopyTo(Array array, int index) { _rows.CopyTo((IBlueRow[])array, index);} + public int Count { get { return _rows.Count; } } + public bool IsSynchronized {get { return false; }} + public object SyncRoot { get { return this; } } + public System.Collections.IEnumerator GetEnumerator() { return _rows.Where(r => r != null).GetEnumerator(); } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SpreadIndexer.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SpreadIndexer.cs new file mode 100644 index 000000000..ebdea2160 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SpreadIndexer.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class SpreadIndexer + { + public enum IndexerDataType + { + SpreadSpec = 0, + Family, + LongDesc, + ShortDesc, + LegArray + } + + private static Dictionary> _cached_data = new Dictionary>(); + public static object get_value(string spread_def, IndexerDataType data_type) + { + Dictionary type_to_data = null; + if (!_cached_data.TryGetValue(spread_def, out type_to_data)) + { + type_to_data = new Dictionary (); + _cached_data.Add(spread_def, type_to_data); + } + object rv = null; + type_to_data.TryGetValue(data_type, out rv); + return rv; + } + + public static void set_value(string spread_def, IndexerDataType data_type, object val) + { + Dictionary type_to_data = null; + if (!_cached_data.TryGetValue(spread_def, out type_to_data)) + { + type_to_data = new Dictionary(); + _cached_data.Add(spread_def, type_to_data); + } + type_to_data[data_type] = val; + } + + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/TimeWindowTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/TimeWindowTable.cs new file mode 100644 index 000000000..0aab88557 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/TimeWindowTable.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + /// + /// keeps rows that are within n mseconds of now + /// + public class TimeWindowTable : MovingWindowTable + { + private int myWindowSizeMS; + private string myTimeCol; + + public TimeWindowTable(IBlueTableSchema schema, int windowSizeMS, string timeCol) + :base(schema, new BlueRowComparer(timeCol, "id")) + { + this.myTimeCol = timeCol; + this.myWindowSizeMS = windowSizeMS; + } + + public bool RemoveRow(IBlueRow row) + { + int index = this.mySortedRows.BinarySearch(row, this.myRowComparer); + if (index >= 0) + { + this.mySortedRows.RemoveAt(index); + return true; + } + return false; + } + + protected override bool windowContains(IBlueRow row) + { + DateTime rowTime = (DateTime)row[this.myTimeCol]; + return DateTime.Now.Subtract(rowTime).TotalMilliseconds <= this.myWindowSizeMS; + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/WrapperRow.cs b/src/DataStreamSuite/DataStreamCore/BlueData/WrapperRow.cs new file mode 100644 index 000000000..d9d6736c0 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/WrapperRow.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public class WrapperRow : IBlueRow + { + private readonly IBlueTableSchema _schema; + private readonly T _data; + private readonly int? _id = null; + private Func _getter; + private readonly int _id_col_index; + + public WrapperRow(IBlueTableSchema schema, T data, Func getter, int? id = 0) + { + _schema = schema; + _data = data; + _getter = getter; + _id = id; + _id_col_index = schema["id"].colNo; + } + + public IBlueTableSchema schema { get { return _schema; } } + + public object this[int i] { get { return (i == _id_col_index && _id != null) ? _id.Value : _getter(i, _data); } } + + public object this[string c] { get { return this[_schema[c].colNo]; } } + + public void Add(object key, object value) { throw new NotImplementedException(); } + + public void Clear() { throw new NotImplementedException(); } + + public bool Contains(object key) { return key != null && key is string && schema.columnNames.Contains(key as string); } + + public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + + public bool IsFixedSize { get { return true; } } + + public bool IsReadOnly { get { return true; } } + + public System.Collections.ICollection Keys { get { return schema.columnNames; } } + + public void Remove(object key) { throw new NotImplementedException("IBlueRow is read only"); } + + public System.Collections.ICollection Values { get { return _schema.columnInfo.Select(c => this[c.colNo]).ToArray(); } } + + public virtual object this[object key] + { + get + { + if (key is string) + return this[(string)key]; + else if (key is int) + return this[(int)key]; + throw new BlueDataException("Wrong type while indexing the row"); + } + set { throw new NotSupportedException("The IBlueRow is readonly"); } + } + + public void CopyTo(Array array, int index) { Values.CopyTo(array, index); } + + public int Count { get { return schema.columnNames.Length; } } + + public virtual bool IsSynchronized { get { return false; } } + + public virtual object SyncRoot { get { return this; } } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/DataPacket.cs b/src/DataStreamSuite/DataStreamCore/DataPacket.cs new file mode 100644 index 000000000..441d2a06e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DataPacket.cs @@ -0,0 +1,316 @@ +using System; +using System.Collections; +using System.Data; +using BCG.Utilities.General; + +namespace BCG +{ + [Serializable] + public class DataPacket : ICloneable, IDataPacket + { + public class BlueRowIdComparator + { + private string[] myIdColumns = new string[]{"id"}; + + public BlueRowIdComparator(string[] idCols) + { + if ( idCols.Length != 0 ) + { + myIdColumns = idCols; + } + } + + public bool blueRowIdEqualEvaluator(object a, object b) + { + if ( a == null || b == null ) + { + return false; + } + + if ( !(a is IBlueRow) || !(b is IBlueRow) ) + { + return false; + } + + IBlueRow lhs = (IBlueRow)a; + IBlueRow rhs = (IBlueRow)b; + + bool result = true; + foreach(string idColumn in myIdColumns) + { + result = result && lhs[idColumn].Equals(rhs[idColumn]); + if ( !result ) + break; + } + + return result; + } + } + + public IBlueRowCollection newRows, addedRows, deletedRows; + public DataPacket () {} + public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) + { + this.newRows = newRows; + this.addedRows = addedRows; + this.deletedRows = deletedRows; + } + + public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) + { + new_rows = newRows; + added_rows = addedRows; + deleted_rows = deletedRows; + } + + public IBlueRowCollection getTrulyAddedRows(params string[] idColumns) + { + BlueTable truly = new BlueTable(addedRows.schema); + BlueRowIdComparator c = new BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); + + foreach(IBlueRow r in ta) + { + truly.addRow(r); + } + + return truly; + + } + + public virtual IBlueRowCollection getTrulyAddedRows () + { + BlueTable retVal = new BlueTable (schema); + foreach (IBlueRow r in addedRows) + if (!deletedRows.containsID ((int)r["id"])) + retVal.addRow (r); + return retVal; + } + + public virtual IBlueRowCollection getTrulyDeletedRows () + { + BlueTable retVal = new BlueTable (schema); + foreach (IBlueRow r in deletedRows) + if (!addedRows.containsID ((int)r["id"])) + retVal.addRow (r); + return retVal; + } + + public virtual IBlueRowCollection getTrulyChangedRows () + { + BlueTable retVal = new BlueTable (schema); + foreach (IBlueRow r in addedRows) + if (deletedRows.containsID ((int)r["id"])) + retVal.addRow (r); + return retVal; + } + + + public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns) + { + BlueTable truly = new BlueTable(deletedRows.schema); + BlueRowIdComparator c = new BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); + + foreach(IBlueRow r in td) + { + truly.addRow(r); + } + + return truly; + } + + public IBlueRowCollection getTrulyChangedRows(params string[] idColumns) + { + + BlueTable truly = new BlueTable(addedRows.schema); + BlueRowIdComparator c = new BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); + ICollection tc = CollectionHelpers.difference(addedRows, ta, ed); + + foreach(IBlueRow r in tc) + { + truly.addRow(r); + } + + return truly; + } + + public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns) + { + BlueTable truly = new BlueTable(deletedRows.schema); + BlueRowIdComparator c = new BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); + ICollection tco = CollectionHelpers.difference(deletedRows, td, ed); + + foreach(IBlueRow r in tco) + { + truly.addRow(r); + } + + return truly; + } + + public TablePacket toTablePacket () + { + return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows), + BlueDataUtility.toDataTable (deletedRows)); + + } + + public IBlueTableSchema schema + { + get + { + IBlueTableSchema retVal = newRows.schema != null ? newRows.schema : + addedRows.schema != null ? addedRows.schema : deletedRows.schema; + return retVal; + } + } + + /* + status D total A total D this A This D' total A' Total + x 0 0 0 0 0 0 + 0 0 0 1 1 this+ + 0 0 1 0 1this + 0 0 1 1 1 this 1 this+ + x 0 1 0 0 + error 0 1 0 1 + 0 1 1 0 1 total- + 0 1 1 1 1 total-, 1 this + + x 1 0 0 0 + 1 0 0 1 1 this+ + error 1 0 1 0 + error 1 0 1 1 + x 1 1 0 0 + error 1 1 0 1 + 1 1 1 0 1 total -1 keep the old delete and remove the old add + 1 1 1 1 1 total-, 1 this + + + d' = dthis - (dthis intersect a) + d // keep the old delete + a' = a - (dthis intersect a) + athis // keep the new add + * */ + public static DataPacket combineWithIds(ICollection packets, string[] idColumns) + { + if ( packets.Count == 0 ) + { + return new DataPacket( + new BlueTable(new DataTable()), + new BlueTable(new DataTable()), + new BlueTable(new DataTable())); + } + + DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns); + + CollectionHelpers.EqualDelegate ed = + new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); + + ArrayList added = new ArrayList(); + ArrayList deleted = new ArrayList(); + BlueTable dt = null; + BlueTable at = null; + IBlueRowCollection nt = null; + + foreach(DataPacket dp in packets) + { + ArrayList oldDeleted = deleted; + ArrayList oldAdded = added; + + added = new ArrayList(); + deleted = new ArrayList(); + if ( dt == null ) + { + dt = new BlueTable(dp.schema); + at = new BlueTable(dp.schema); + } + + deleted.AddRange(oldDeleted); + deleted.AddRange( CollectionHelpers.difference( + CollectionHelpers.difference(dp.deletedRows, oldAdded, ed), + oldDeleted, ed) ); + + added.AddRange(dp.addedRows); + added.AddRange( CollectionHelpers.difference( + CollectionHelpers.difference(oldAdded, dp.deletedRows, ed), + added, ed) ); + nt = dp.newRows; + } + + foreach ( IBlueRow dr in added ) + { + at.addRow(dr); + } + + foreach ( IBlueRow dr in deleted ) + { + dt.addRow(dr); + } + + return new DataPacket(nt,at,dt); + } + + public DataPacket deepCopy () + { + return new DataPacket (new OrderedBlueTable (newRows), + new OrderedBlueTable (newRows), + new BlueTable (deletedRows)); + } + + public virtual DataPacket createImmutable () + { + return + new DataPacket( + newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows), + addedRows, + deletedRows); + } + #region ICloneable Members + + public object Clone() + { + return MemberwiseClone(); + } + + #endregion + + public string toIncrementalIDString() + { + System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName); + sb.Append ("\ndeleted : "); + foreach (IBlueRow r in deletedRows) + { + sb.Append(r["id"]); + sb.Append(','); + } + sb.Append("\nadded : "); + foreach (IBlueRow r in addedRows) + { + sb.Append(r["id"]); + sb.Append(','); + } + sb.Append("\n"); + return sb.ToString(); + } + + public override string ToString() + { + return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj b/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj index de3c61cba..d696ddc8d 100644 --- a/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj +++ b/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj @@ -1,8 +1,12 @@ - + + + + true + net7.0 - enable + disable diff --git a/src/DataStreamSuite/DataStreamCore/IDataPacket.cs b/src/DataStreamSuite/DataStreamCore/IDataPacket.cs new file mode 100644 index 000000000..8165cb5dd --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/IDataPacket.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public interface IDataPacket + { + //this is an ICollection of IDictionary + void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/IDataPump.cs b/src/DataStreamSuite/DataStreamCore/IDataPump.cs new file mode 100644 index 000000000..02f21c95b --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/IDataPump.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BCG +{ + public interface IDataPump + { + string tag + { + get; + } + void start(); + void stop(); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/AccumulatorFactory.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/AccumulatorFactory.cs new file mode 100644 index 000000000..61c15ad61 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/AccumulatorFactory.cs @@ -0,0 +1,17 @@ +using System; + +namespace BCG +{ + + /// + /// A central place to get all ITableAccumulators. This is helpful, since specific implementations + /// of ITableAccumulators can be instantiated using config. + /// + public class AccumulatorFactory + { + public static ITableAccumulator getAccumulator (string tag, ISubscriberCore subCore) + { + return new TableAccumulator (tag, subCore); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/ITableAccumulator.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/ITableAccumulator.cs new file mode 100644 index 000000000..e712e5aa3 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/ITableAccumulator.cs @@ -0,0 +1,24 @@ +using System; +using System.IO; + +namespace BCG +{ + public delegate void TableUpdateDelegate (IBlueRowCollection newRows, + IBlueRowCollection addedRows, + IBlueRowCollection deletedRows); + + /// + /// This interface is supposed to hook upto pubsub on one end, receive data in bytes, + /// deserialize and call the client (that wil hook upto this interface) and give tables + /// + public interface ITableAccumulator + { + event TableUpdateDelegate onUpdate; + /// + /// return trues if this table acumulator has any subscribers. + /// + /// + bool isActive(); + void update (Stream[] streams); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs new file mode 100644 index 000000000..c2978b55f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs @@ -0,0 +1,349 @@ +using System; +using System.IO; +//using BCG.Utilities; +using mbts; + +namespace BCG +{ + public class TableAccumulator : ITableAccumulator + { + private HighPerformanceTimer _timer = new HighPerformanceTimer(); + private IClientDeserializer myDeserializer = new ClientDeserializationController (); + private BlueTable myStateTable; + private ISubscriberCore mySubscriptionCore; + private TableUpdateDelegate myOnUpdateDelegate; + private string myTag; + private bool myUseIncrementsForMultipleOps = DataStreamGatewaySettings.instance.useIncrementsForMultipleOps; + + public TableAccumulator (string tag, ISubscriberCore subCore) + { + myTag = tag; + mySubscriptionCore = subCore; + } + + #region ITableAccumulator Members + + public bool isActive() + {return myOnUpdateDelegate != null;} + + public event TableUpdateDelegate onUpdate + { + add + { + // BlueEventLog.logEvent ("TableSubscriber being Subscribed for tag :" + myTag, BlueEventLevel.Information); + if (myOnUpdateDelegate == null) + { + //ie first subscription + // BlueEventLog.logEvent ("Subscription made to PubSub for tag : " + myTag, BlueEventLevel.Information); + mySubscriptionCore.subscribe (myTag); + } + myOnUpdateDelegate = + Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate; + if (myStateTable != null) + value (myStateTable, myStateTable, new BlueTable (myStateTable.schema)); + } + remove + { + // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information); + if (myOnUpdateDelegate != null) + { + myOnUpdateDelegate = + Delegate.Remove (myOnUpdateDelegate, value) as TableUpdateDelegate; + if (myOnUpdateDelegate == null) + { + //all subscribers gone + // BlueEventLog.logEvent ("Unubscription made to PubSub for tag : " + myTag, BlueEventLevel.Information); + mySubscriptionCore.unsubscribe (myTag); + myStateTable = null; + } + } + + } + } + + public void update(Stream[] streams) + { + //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one + //data element for that tag in the ViewStreamDatabase + if (myOnUpdateDelegate != null) + { + byte[] ops; + Array rows; + _timer.start(); + myDeserializer.deserialize (streams, out ops, out rows); + _timer.stop(); + // BlueEventLog.logEvent(String.Format("deserializing {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance); + _timer.start(); + myAccumulateAndFire (ops, rows); + _timer.stop(); + // BlueEventLog.logEvent(String.Format("myAccumulateAndFire for {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance); + } + } + + #endregion + + private void myAccumulateAndFire (byte[] ops, Array rows) + { + //if a non create comes before the create, the deserializer returns 0 sized arrays + if (ops.Length == 0 || rows.Length == 0) + { + //ie if one is zero sized so should be the other one + // BlueEventLog.logAssert (ops.Length + rows.Length == 0); + } + else + { + if (ops.Length > 1) + { + int index = 0; + if (myStateTable == null) + { + for(index=0; index 0 || dRows.Count > 0) + { + myStateTable = new BlueTable(newState); + myOnUpdateDelegate(myStateTable, aRows, dRows); + } + } + } + else + { + if (op == StreamOpCode.add) + { + myStateTable.addRows(aRows = tableArray[0]); + dRows = new BlueTable(myStateTable.schema); + } + else if (op == StreamOpCode.remove) + { + myStateTable.deleteRows(dRows = tableArray[0]); + aRows = new BlueTable(myStateTable.schema); + } + else if (op == StreamOpCode.change) + { + IBlueRow cRow = BlueDataUtility.extractRow(tableArray[0], 0); + aRows = new BlueTable(myStateTable.schema); + dRows = new BlueTable(myStateTable.schema); + (aRows as BlueTable).addRow(cRow); + (dRows as BlueTable).addRow(myStateTable[(int)cRow["id"]]); + myStateTable.changeRow(cRow); + } + else if (op == StreamOpCode.multipleChange) + { + if (myStateTable.Count == tableArray[0].Count) + { + //all rows are changed, so we can optimize this + dRows = myStateTable; + aRows = myStateTable = new BlueTable(tableArray[0]); + } + else + { + dRows = new BlueTable(myStateTable.schema); + aRows = tableArray[0]; + foreach (IBlueRow r in tableArray[0]) + { + (dRows as BlueTable).addRow(myStateTable[(int)r["id"]]); + myStateTable.changeRow(r); + } + } + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + BlueDataUtility.absorbChanges(myStateTable, dRows = tableArray[0], aRows = tableArray[1]); + } + else + { + aRows = dRows = null; + // BlueEventLog.logAssert(false, "unknown opcode"); + } + myOnUpdateDelegate(myStateTable, aRows, dRows); + } + } + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs new file mode 100644 index 000000000..d27998c44 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; +using System.Collections; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it. + /// + public class ClientDeserializationController : IClientDeserializer + { + private IClientDeserializer myDeserializer; + + public ClientDeserializationController () {} + + #region IClientDeserializer Members + + /// + /// Deserializes bytes into operation and data. If it gets a non create before a create, both + /// the out arrays get initialized to empty arrays + /// + /// + /// + /// + public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows) + { + if (myDeserializer != null) + { + myDeserializer.deserialize (streams, out theOps, out theRows); + } + else + { + Stream totalStream = UtilityMethods.getAggregatedStream (streams); + long initialPos = totalStream.Position; + BinaryReader br = new BinaryReader (totalStream); + byte op; + try + { + op = br.ReadByte (); + } + catch (EndOfStreamException) + { + op = Byte.MaxValue; + } + if (op == StreamOpCode.create) + { + br.ReadString (); //pubsubclientdeserializer + string deserName = br.ReadString (); + myDeserializer = Activator.CreateInstance (Type.GetType (deserName)) + as IClientDeserializer; + totalStream.Position = initialPos; + myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows); + } + else + { + // BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning); + theOps = new byte[]{}; + theRows = new object[]{}; + } + } + } + + + #endregion + + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializer.cs new file mode 100644 index 000000000..3998a1a95 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializer.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections; +using System.IO; + +using BCG.Utilities; +//using BCG.Utilities.BlueConfiguration; + +namespace BCG +{ + /// + /// Pairs up with the UniversalSerializer to deserialize the corresponding data. Everythign anologous + /// to UniversalSerializer. + /// + public abstract class ClientDeserializer : IClientDeserializer + { + protected IBlueTableSchema mySchema; + protected IBlueTableSchema myFirstSchema; + protected IClientRowDeserializer myRowDeserializer; + + public ClientDeserializer() + { + myFirstSchema = null; + mySchema = null; + + } + + #region IClientDeserializer Members + + void BCG.IClientDeserializer.deserialize(Stream[] streams, out byte[] theOps, out Array theRows) + { + BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams)); + + // BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length); + + // emit # of streams and total length + ArrayList ops = new ArrayList(), rows = new ArrayList(); + IBlueRowCollection[] result; + while (true) + { + byte op; + try + { + int value = br.BaseStream.ReadByte(); + if (value == -1) break; + op = (byte)value; + + // BlueTrace.WriteDeserializeOperation(op); + } + catch (EndOfStreamException) + { + break; + } + if (op == StreamOpCode.create) + { + + br.ReadString(); //pubsubDeserializer + br.ReadString(); //clientDeserializer + mySchema = new BlueTableSchema(); + mySchema.deserialze(br); + //keep track of the first schema you've ever seen, so you know how to reorder columns + if (myFirstSchema == null) + myFirstSchema = mySchema; + + result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) }; + myEncorporateRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else + { + if (mySchema != null) + { + if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) + { + result = new IBlueRowCollection[] { myGetRows(br, op, 1) }; + myEncorporateRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleChange) + { + result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) }; + myEncorporateRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + result = + new IBlueRowCollection[] { + myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()), + myGetRows (br, StreamOpCode.add, br.ReadInt32 ()) + }; + + myEncorporateRows(result[0], StreamOpCode.remove); + myEncorporateRows(result[1], StreamOpCode.add); + ops.Add(op); + rows.Add(result); + } + else + { + // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); + } + } + else + { + // BlueEventLog.logEvent("Received a non create operation even though the schema" + + // "has never been created", BlueEventLevel.Warning); + } + } + } + theOps = ops.ToArray(typeof(byte)) as byte[]; + theRows = rows.ToArray(); + // BlueTrace.WriteDeserializeStop(rows.Count); + } + + #endregion + + protected abstract IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows); + + protected abstract void myEncorporateRows(IBlueRowCollection rows, byte op); + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientReorderingRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientReorderingRowDeserializer.cs new file mode 100644 index 000000000..e9bf2b6b1 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientReorderingRowDeserializer.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace BCG +{ + class ClientReorderingRowDeserializer : IClientRowDeserializer + { + private IBlueTableSchema myOriginalSchema; + private IBlueTableSchema myOutputSchema; + private ClientUniversalRowDeserializer myDeserializer; + public ClientReorderingRowDeserializer(IBlueTableSchema originalSchema, IBlueTableSchema newSchema) + { + myOriginalSchema = originalSchema; + myOutputSchema = newSchema; + myDeserializer = new ClientUniversalRowDeserializer(newSchema); + + } + + #region IClientRowDeserializer Members + + public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) + { + IBlueRow temp = null; + myDeserializer.deserializeRow(currentState, br, op, out temp); + r = BlueDataUtility.changeSchema(temp, myOutputSchema); + + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalDeserializer.cs new file mode 100644 index 000000000..514d00e41 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalDeserializer.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections; +using System.IO; + +namespace BCG +{ + public class ClientUniversalDeserializer : ClientDeserializer + { + public ClientUniversalDeserializer () {} + + protected override IBlueRowCollection myGetRows (BinaryReader br, byte op, int nRows) + { + BlueTable retVal = new BlueTable (mySchema); + + if (myRowDeserializer == null) + myRowDeserializer = new ClientUniversalRowDeserializer(mySchema); + else if (op == StreamOpCode.create) + myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema); + + for (int i = 0; i < nRows; i++) + { + IBlueRow row; + myRowDeserializer.deserializeRow (null, br, op, out row); + retVal.addRow (row); + } + return retVal; + } + + protected override void myEncorporateRows (IBlueRowCollection rows, byte op) + { + //this does not keep state so ignore calls + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs new file mode 100644 index 000000000..6ba3158e7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections; +using System.IO; + +using BCG.Utilities; +//using BCG.Utilities.BlueConfiguration; + +namespace BCG +{ + /// + /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process + /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) + /// + internal class ClientUniversalRowDeserializer : IClientRowDeserializer + { + private IBlueTableSchema mySchema; + private int myNullMapLength = 0; + private PrimitiveDeserializationHelper myDeserializationHelper; + private BitArray myNullMap; + private int[] mySerIndexToSchemaIndexMap; + + public ClientUniversalRowDeserializer (IBlueTableSchema schema) + { + myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema); + myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8))); + myInitReorderSchema (); + + + } + + #region Schema Reordering initialization + private void myInitReorderSchema () + { + int idColIndex = mySchema["id"].colNo; + int[] temp = new int [mySchema.columnInfo.Length]; + foreach (ColumnInfo ci in mySchema.columnInfo) + { + temp[ci.colNo] = + ci.colNo == idColIndex ? 0 : + ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo; + } + mySerIndexToSchemaIndexMap = new int[temp.Length]; + for (int i = 0; i < temp.Length; i ++) + mySerIndexToSchemaIndexMap [temp[i]] = i; + } + #endregion + + #region IClientRowDeserializer Members + + public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) + { + br.ReadInt32 (); //length information for the row not needed + myNullMap = new BitArray(br.ReadBytes (myNullMapLength)); + object[] values = new object [mySchema.columnNames.Length]; + for (int i = 0; i < values.Length; i++) + { + int colNo = mySerIndexToSchemaIndexMap[i]; + values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br); + } + + + r = new BlueRow(mySchema, values); + + } + + #endregion + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs new file mode 100644 index 000000000..6be1a959a --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections; +using System.IO; + +namespace BCG +{ + /// + /// Performs deserialization inside the client app. The client gets either the complete data set or the + /// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point. + /// + public interface IClientDeserializer + { + /// + /// deserializes the data received by the client from the pubsub + /// + /// the data in byte format received from pubsub + /// the operations that the data maps onto + /// the data corresponding to the operations, is an array of IblueRowCollection[] + void deserialize (Stream[] streams, out byte[] theOps, out Array theRows); + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs new file mode 100644 index 000000000..5f35a2423 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs @@ -0,0 +1,14 @@ +using System; +using System.IO; +using System.Collections; + +namespace BCG +{ + /// + /// Deserializes one row. the current state is also provided as some deserialization schemes might use it. + /// + public interface IClientRowDeserializer + { + void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/PrimitiveDeserializationHelper.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/PrimitiveDeserializationHelper.cs new file mode 100644 index 000000000..2d94dc878 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/PrimitiveDeserializationHelper.cs @@ -0,0 +1,79 @@ +using System; +using System.IO; + +namespace BCG +{ + /// + /// Deserializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1) + /// rather than searching it up and making it O(n) + /// + public class PrimitiveDeserializationHelper + { + private delegate object DeserializeDelegate (int colNo, BinaryReader br); + private DeserializeDelegate[] myDispatchTable; + private ColumnInfo[] myCis; + + public PrimitiveDeserializationHelper (IBlueTableSchema schema) + : this (schema.columnInfo) + {} + + public PrimitiveDeserializationHelper (ColumnInfo[] cis) + { + myDispatchTable = new DeserializeDelegate [(myCis = cis).Length]; + foreach (ColumnInfo ci in cis) + myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType); + } + + public object deserializeColumn (int colNo, BinaryReader br) + { + return myDispatchTable[colNo] (colNo, br); + } + + private DeserializeDelegate myGetDispatchMethod (Type t) + { + if (t.Equals (BlueDataValidTypes.INT)) return new DeserializeDelegate (myDeserializeInt32); + else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new DeserializeDelegate (myDeserializeDouble); + else if (t.Equals (BlueDataValidTypes.STRING)) return new DeserializeDelegate (myDeserializeString); + else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new DeserializeDelegate (myDeserializeByteArray); + else if (t.Equals (BlueDataValidTypes.CHAR)) return new DeserializeDelegate (myDeserializeChar); + else if (t.Equals (BlueDataValidTypes.LONG)) return new DeserializeDelegate (myDeserializeLong); + else if (t.Equals (BlueDataValidTypes.SBYTE)) return new DeserializeDelegate (myDeserializeSByte); + else if (t.Equals (BlueDataValidTypes.BOOL)) return new DeserializeDelegate (myDeserializeBool); + else if (t.Equals (BlueDataValidTypes.BYTE)) return new DeserializeDelegate (myDeserializeByte); + else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new DeserializeDelegate (myDeserializeDecimal); + else if (t.Equals (BlueDataValidTypes.UINT)) return new DeserializeDelegate (myDeserializeUInt); + else if (t.Equals (BlueDataValidTypes.ULONG)) return new DeserializeDelegate (myDeserializeULong); + else if (t.Equals (BlueDataValidTypes.DATETIME)) return new DeserializeDelegate (myDeserializeDateTime); + else if (t.Equals (BlueDataValidTypes.SINGLE)) return new DeserializeDelegate (myDeserializeSingle); + else if (SerializationUtilityMethods.isSerializedType (t)) return new DeserializeDelegate (myDeserializeISerializedType); + else throw new ApplicationException (t.FullName + " is not a supported type"); + } + + private object myDeserializeInt32 (int colNo, BinaryReader br) {return br.ReadInt32 ();} + private object myDeserializeISerializedType (int colNo, BinaryReader br) + { + ISerializedType o = Activator.CreateInstance (myCis[colNo].colType) + as ISerializedType; + o.deserialze (br); + return o; + } + private object myDeserializeDouble (int colNo, BinaryReader br) {return br.ReadDouble ();} + private object myDeserializeString (int colNo, BinaryReader br) {return br.ReadString ();} + private object myDeserializeByteArray (int colNo, BinaryReader br) {return br.ReadBytes (br.ReadInt32 ());} + private object myDeserializeChar (int colNo, BinaryReader br) {return br.ReadChar ();} + private object myDeserializeLong (int colNo, BinaryReader br) {return br.ReadInt64 ();} + private object myDeserializeSByte (int colNo, BinaryReader br) {return br.ReadSByte ();} + private object myDeserializeBool (int colNo, BinaryReader br) {return br.ReadBoolean ();} + private object myDeserializeByte (int colNo, BinaryReader br) {return br.ReadByte ();} + private object myDeserializeDecimal (int colNo, BinaryReader br) {return br.ReadDecimal ();} + private object myDeserializeUInt (int colNo, BinaryReader br) {return br.ReadUInt32 ();} + private object myDeserializeDateTime (int colNo, BinaryReader br) + { + return new DateTime(br.ReadInt64()); + //return DateTime.FromOADate (br.ReadDouble ()); + //return new DateTime(br.ReadInt64()); + } + private object myDeserializeULong (int colNo, BinaryReader br) {return br.ReadUInt64 ();} + private object myDeserializeSingle (int colNo, BinaryReader br) {return br.ReadSingle ();} + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Config/DataStreamGatewaySettings.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Config/DataStreamGatewaySettings.cs new file mode 100644 index 000000000..4714d80f6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Config/DataStreamGatewaySettings.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections; +//using BCG.Utilities.BlueConfiguration; +using System.Text.RegularExpressions; + +namespace BCG +{ + public class DataStreamGatewaySettings //: ConfigurationRequest + { + private static object myLockObj = new object(); + private static DataStreamGatewaySettings mySelf; + public DataStreamGatewaySettings () + { + SendBlueCommMonitor = false; + } + + private string[] myNewSerializationSchemeTableList; + private SerializationScheme[] mySerializationScheme; + private bool myEnablePubsubStats; + private int mySingleTagStatsUpdateInterval; + private int myPubsubStatsUpdateInterval; + private bool myUseIncrementsForMultipleOps; + private bool myUseCppDataStreams = false; + private bool myUseSerializedBlueRows = false; + + //public override bool isValid() {return true;} + + public string[] newSerializationSchemeTableList + { + get {return myNewSerializationSchemeTableList;} + set {myNewSerializationSchemeTableList = value;} + } + + public bool enablePubsubStats + { + get { return myEnablePubsubStats; } + set { myEnablePubsubStats = Convert.ToBoolean(value); } + } + + public int singleTagStatsUpdateInterval + { + get { return mySingleTagStatsUpdateInterval; } + set { mySingleTagStatsUpdateInterval = value; } + } + + public int pubsubStatsUpdateInterval + { + get { return myPubsubStatsUpdateInterval; } + set { myPubsubStatsUpdateInterval = value; } + } + + public SerializationScheme[] serializationScheme + { + get {return mySerializationScheme;} + set {mySerializationScheme = value;} + } + + public bool useCppDataStreams + { + get {return myUseCppDataStreams;} + set { myUseCppDataStreams = value; } + } + + + public bool isSerializableUnderNewScheme (string tableName) + { + ArrayList temp = new ArrayList (); + foreach (string s in newSerializationSchemeTableList) + if (new Regex (s).Match (tableName).Success) + return true; + return false; + } + + public string getSubscriber (string tableName) + { + foreach (SerializationScheme ss in mySerializationScheme) + if (new Regex (ss.tableRegExp).Match (tableName).Success) + return ss.subscriber; + return null; + } + + public bool useIncrementsForMultipleOps + { + get { return myUseIncrementsForMultipleOps; } + set { myUseIncrementsForMultipleOps = value; } + } + + public string getSerializer (string tableName) + { + foreach (SerializationScheme ss in mySerializationScheme) + if (new Regex (ss.tableRegExp).Match (tableName).Success) + return ss.serializer; + return null; + } + + public object[] getSerializerArgs(string tableName) + { + foreach (SerializationScheme ss in mySerializationScheme) + if (new Regex(ss.tableRegExp).Match(tableName).Success) + return ss.args; + return null; + } + + public bool useSerializedBlueRows + { + get { return myUseSerializedBlueRows; } + set { myUseSerializedBlueRows = value; } + } + + public bool SendBlueCommMonitor + { + get; + set; + } + + public static DataStreamGatewaySettings instance + { + get + { + lock(myLockObj) + { + if (mySelf == null) + { + // ConfigurationRequest req = ConfigurationRequester.request(typeof(DataStreamGatewaySettings)); + mySelf = new DataStreamGatewaySettings(); + } + } + return mySelf; + + + } + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Config/SerializationScheme.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Config/SerializationScheme.cs new file mode 100644 index 000000000..2af9b19b7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Config/SerializationScheme.cs @@ -0,0 +1,20 @@ +using System; + +namespace BCG +{ + + public class SerializationScheme + { + public string tableRegExp, serializer, subscriber; + public object[] args; + public SerializationScheme () {} + public SerializationScheme (string theTableRegExp, string theSerializer, string theSubscriber, object[] theArgs) + { + tableRegExp = theTableRegExp; + serializer = theSerializer; + subscriber = theSubscriber; + args = theArgs; + + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/DynamicLoading/CorePrimitiveManager.cs b/src/DataStreamSuite/DataStreamCore/Messaging/DynamicLoading/CorePrimitiveManager.cs new file mode 100644 index 000000000..78cb71a2d --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/DynamicLoading/CorePrimitiveManager.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections; +using System.Reflection; +//using BCG.Utilities.BlueConfiguration; +using BCG.Utilities; + +namespace BCG +{ + public class CorePrimtiveManager + { + private IDictionary myTypeLookupTable = new Hashtable (); + private static CorePrimtiveManager mySelf; + private ArrayList myAssemblyNames = new ArrayList (); + + private CorePrimtiveManager() + { + myLoadAllTypes (); + } + + private void myLoadAllTypes () + { + //we only load one type here + loadAssembly ("X:\\bcg\\blue2\\Debug\\CppToCSharpDataStreamBridge.dll"); + } + + private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) + { + foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies()) + if (args.Name.Equals(a.FullName)) + return a; + return null; + } + + /// + /// Loads all the types associated with the assembly + /// + /// + public void loadAssembly (string assemblyName) + { + if (myAssemblyNames.Contains (assemblyName)) + return; + myAssemblyNames.Add (assemblyName); + Assembly loadedAssembly = GetType ().Assembly.FullName.Equals (assemblyName) ? + GetType ().Assembly : + Assembly.LoadFrom (assemblyName); + + foreach (Type type in loadedAssembly.GetTypes()) + if (!type.IsInterface + && !type.IsAbstract + && type.IsPublic) + { + bool isVisible = true; + string userRep = type.Name; + + if (isVisible) + { + // BlueEventLog.logAssert(myTypeLookupTable [userRep] == null); + myTypeLookupTable[userRep] = type; + } + } + } + + + + + public static CorePrimtiveManager instance + { + get {return mySelf = mySelf == null ? new CorePrimtiveManager () : mySelf;} + } + + /// + /// Returns the Type name corresponding to the primitive name + /// + /// + /// + public string find (string friendlyName) + { + if (myTypeLookupTable[friendlyName] == null) + throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types"); + return (myTypeLookupTable[friendlyName] as Type).Name; + } + + /// + /// Constructs and returns an object of type corresponding to the friendly name + /// + /// friendly primitive name + /// args can be null or an empty ICollection if the paramterless constructor + /// is the desired one + /// + public Object build (string friendlyName, ICollection args) + { + object[] constructorArgs = null; + if (args != null && args.Count > 0) + { + constructorArgs = new object [args.Count]; + args.CopyTo (constructorArgs, 0); + } + if (myTypeLookupTable[friendlyName] == null) + throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types"); + return + constructorArgs == null ? + Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type): + Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type, constructorArgs); + } + + /// + /// Returns the name of all the primitives + /// + /// + public string[] getPrimitiveNames () + { + string[] retVal = new string [myTypeLookupTable.Count]; + myTypeLookupTable.Keys.CopyTo (retVal, 0); + return retVal; + } + + /// + /// Return the types of all the streams + /// + /// + public Type[] getStreamTypes () + { + Type[] retVal = new Type [myTypeLookupTable.Count]; + myTypeLookupTable.Values.CopyTo (retVal, 0); + return retVal; + } + + public ICollection getConstructorSigns (string primitiveName) + { + ArrayList retVal = new ArrayList (); + // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null); + Type t = myTypeLookupTable[primitiveName] as Type; + foreach (ConstructorInfo ci in t.GetConstructors()) + { + string sign = ""; + sign = t.Name + " ("; + foreach (ParameterInfo pi in ci.GetParameters ()) + sign += pi.ParameterType.Name + + " " + + pi.Name + ", "; + sign = sign.Length == (t.Name + " (").Length ? sign + ")": sign.Remove (sign.Length - 2, 2) + ")"; + retVal.Add (sign); + } + return retVal; + } + + public string getAssemblyName (string primitiveName) + { + // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null); + return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name; + + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Misc/PubStreamPrefixer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Misc/PubStreamPrefixer.cs new file mode 100644 index 000000000..44733ce15 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Misc/PubStreamPrefixer.cs @@ -0,0 +1,20 @@ +using System; + +namespace BCG +{ + public class PubStreamPrefixer + { + private static readonly string myPrefixString = "DS"; + + public static string prefix (string tableName) + { + return myPrefixString + "_" + tableName.ToLower(); + } + + public static string extractTableNameFromTag (string tag) + { + return tag.Substring ((myPrefixString + "_").Length); + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Misc/SerializationUtilityMethods.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Misc/SerializationUtilityMethods.cs new file mode 100644 index 000000000..8879a23c8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Misc/SerializationUtilityMethods.cs @@ -0,0 +1,339 @@ +using System; +using System.IO; +using System.Collections; + +namespace BCG +{ + public class SerializationUtilityMethods + { + private static IDictionary mySerializationDispatchTable, + myDeserializationDispatchTable; + private delegate void MySerialize (object data, BinaryWriter bw); + private delegate object MyDeserialize (Type t, BinaryReader br); + + static SerializationUtilityMethods () + { + mySerializationDispatchTable = new Hashtable (); + mySerializationDispatchTable [BlueDataValidTypes.BOOL.Name] = + new MySerialize (mySerializeBool); + mySerializationDispatchTable [BlueDataValidTypes.BYTE.Name] = + new MySerialize (mySerializeByte); + mySerializationDispatchTable [BlueDataValidTypes.CHAR.Name] = + new MySerialize (mySerializeChar); + mySerializationDispatchTable [BlueDataValidTypes.DATETIME.Name] = + new MySerialize (mySerializeDateTime); + mySerializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] = + new MySerialize (mySerializeDecimal); + mySerializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] = + new MySerialize (mySerializeDouble); + mySerializationDispatchTable [BlueDataValidTypes.INT.Name] = + new MySerialize (mySerializeInt32); + mySerializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] = + new MySerialize (mySerializeISerializedType); + mySerializationDispatchTable [BlueDataValidTypes.LONG.Name] = + new MySerialize (mySerializeLong); + mySerializationDispatchTable [BlueDataValidTypes.SBYTE.Name] = + new MySerialize (mySerializeSByte); + mySerializationDispatchTable [BlueDataValidTypes.STRING.Name] = + new MySerialize (mySerializeString); + mySerializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] = + new MySerialize (mySerializeByteArray); + mySerializationDispatchTable [BlueDataValidTypes.UINT.Name] = + new MySerialize (mySerializeUInt); + mySerializationDispatchTable [BlueDataValidTypes.ULONG.Name] = + new MySerialize (mySerializeULong); + mySerializationDispatchTable [BlueDataValidTypes.SINGLE.Name] = + new MySerialize (mySerializeSingle); + + myDeserializationDispatchTable = new Hashtable (); + myDeserializationDispatchTable [BlueDataValidTypes.BOOL.Name] = + new MyDeserialize (myDeserializeBool); + myDeserializationDispatchTable [BlueDataValidTypes.BYTE.Name] = + new MyDeserialize (myDeserializeByte); + myDeserializationDispatchTable [BlueDataValidTypes.CHAR.Name] = + new MyDeserialize (myDeserializeChar); + myDeserializationDispatchTable [BlueDataValidTypes.DATETIME.Name] = + new MyDeserialize (myDeserializeDateTime); + myDeserializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] = + new MyDeserialize (myDeserializeDecimal); + myDeserializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] = + new MyDeserialize (myDeserializeDouble); + myDeserializationDispatchTable [BlueDataValidTypes.INT.Name] = + new MyDeserialize (myDeserializeInt32); + myDeserializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] = + new MyDeserialize (myDeserializeISerializedType); + myDeserializationDispatchTable [BlueDataValidTypes.LONG.Name] = + new MyDeserialize (myDeserializeLong); + myDeserializationDispatchTable [BlueDataValidTypes.SBYTE.Name] = + new MyDeserialize (myDeserializeSByte); + myDeserializationDispatchTable [BlueDataValidTypes.STRING.Name] = + new MyDeserialize (myDeserializeString); + myDeserializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] = + new MyDeserialize (myDeserializeByteArray); + myDeserializationDispatchTable [BlueDataValidTypes.UINT.Name] = + new MyDeserialize (myDeserializeUInt); + myDeserializationDispatchTable [BlueDataValidTypes.ULONG.Name] = + new MyDeserialize (myDeserializeULong); + myDeserializationDispatchTable [BlueDataValidTypes.SINGLE.Name] = + new MyDeserialize (myDeserializeSingle); + } + + public static int getObjectSerializedSize (object o) + { + Type t = o.GetType (); + if (t == BlueDataValidTypes.STRING) + { + int l = (o as string).Length; + if (l < 128) + return l + 1; + if (l < 16384) + return l + 2; + if (l < 2097152) + return l + 3; + if (l < Math.Pow (2, 28)) + return l + 4; + throw new ApplicationException ("String too big"); + } + else if (t == BlueDataValidTypes.BYTEARRAY) + { + return getFixedLengthTypeSize (BlueDataValidTypes.INT) + + (o as byte[]).Length; + } + else if (isSerializedType (t)) + { + return (o as ISerializedType).getSerializedSize (); + } + return getFixedLengthTypeSize (t); + } + + public static int getFixedLengthTypeSize (Type t) + { + if (t == BlueDataValidTypes.BOOL || + t == BlueDataValidTypes.BYTE || + t == BlueDataValidTypes.CHAR || + t == BlueDataValidTypes.SBYTE) + return 1; + else if (t == BlueDataValidTypes.INT || + t == BlueDataValidTypes.UINT|| + t == BlueDataValidTypes.SINGLE) + return 4; + else if (t == BlueDataValidTypes.DOUBLE || + t == BlueDataValidTypes.LONG || + t== BlueDataValidTypes.ULONG) + return 8; + else if (t == BlueDataValidTypes.DECIMAL) + return 16; + else if (t == BlueDataValidTypes.DATETIME) + //as objects of type DateTime are stored as doubles + return getFixedLengthTypeSize (typeof(long)); + else + throw new ApplicationException (t.FullName + " is not a supported type"); + + } + + public static bool isFixedLengthType (Type t) + { + return !(t.Equals (BlueDataValidTypes.STRING) || t.Equals (BlueDataValidTypes.BYTEARRAY) || + isSerializedType (t)); + } + + public static bool isSerializedType (Type t) + { + return t.GetInterface ("BCG.ISerializedType") != null; + } + + #region Serialization + public static void serializeValue (object data, BinaryWriter bw) + { + Type t = data.GetType (); + if (isSerializedType (t)) + (mySerializationDispatchTable [typeof(ISerializedType).Name] + as MySerialize)(data, bw); + else + (mySerializationDispatchTable [t.Name] as MySerialize) (data, bw); + } + public static void serializeEnum(object data, BinaryWriter bw) + { + mySerializeInt32(data, bw); + } + private static void mySerializeInt32 (object data, BinaryWriter bw) + { + bw.Write ((Int32)data); + } + + private static void mySerializeDouble (object data, BinaryWriter bw) + { + bw.Write ((double)data); + } + + private static void mySerializeString (object data, BinaryWriter bw) + { + bw.Write (data as string); + } + + private static void mySerializeByteArray (object data, BinaryWriter bw) + { + byte[] bs = data as byte[]; + bw.Write (bs.Length); + bw.Write (bs); + } + + + private static void mySerializeSingle (object data, BinaryWriter bw) + { + bw.Write ((Single)data); + } + + private static void mySerializeChar (object data, BinaryWriter bw) + { + bw.Write ((char)data); + } + + private static void mySerializeLong (object data, BinaryWriter bw) + { + bw.Write ((long)data); + } + + private static void mySerializeSByte (object data, BinaryWriter bw) + { + bw.Write ((sbyte)data); + } + + private static void mySerializeBool (object data, BinaryWriter bw) + { + bw.Write ((bool)data); + } + + private static void mySerializeByte (object data, BinaryWriter bw) + { + bw.Write ((byte)data); + } + + private static void mySerializeDecimal (object data, BinaryWriter bw) + { + bw.Write ((decimal)data); + } + + private static void mySerializeUInt (object data, BinaryWriter bw) + { + bw.Write ((uint)data); + } + + private static void mySerializeULong (object data, BinaryWriter bw) + { + bw.Write ((ulong)data); + } + + private static void mySerializeDateTime (object data, BinaryWriter bw) + { + //DateTime is written is serialiazed as OADATE + bw.Write (((DateTime)data).ToOADate()); + } + + private static void mySerializeISerializedType (object data, BinaryWriter bw) + { + (data as ISerializedType).serialize (bw); + } + #endregion + + #region Deserialization + + public static object deserializeValue(Type t, BinaryReader br) + { + return myDeserialize(t, br); + } + private static object myDeserialize (Type t, BinaryReader br) + { + return isSerializedType (t) ? + (myDeserializationDispatchTable [typeof(ISerializedType).Name] + as MyDeserialize)(t, br) + : + (myDeserializationDispatchTable [t.Name] as MyDeserialize) (t, br); + } + + public static object deserializeEnum(Type t, BinaryReader br) + { + return myDeserializeInt32(t, br); + } + + private static object myDeserializeInt32 (Type t, BinaryReader br) + { + return br.ReadInt32 (); + } + + private static object myDeserializeISerializedType (Type t, BinaryReader br) + { + ISerializedType o = Activator.CreateInstance (t) + as ISerializedType; + o.deserialze (br); + return o; + } + + private static object myDeserializeDouble (Type t, BinaryReader br) + { + return br.ReadDouble (); + } + + private static object myDeserializeString (Type t, BinaryReader br) + { + return br.ReadString (); + } + + private static object myDeserializeByteArray (Type t, BinaryReader br) + { + return br.ReadBytes (br.ReadInt32 ()); + } + + private static object myDeserializeSingle (Type t, BinaryReader br) + { + return br.ReadSingle (); + } + + private static object myDeserializeChar (Type t, BinaryReader br) + { + return br.ReadChar (); + } + + private static object myDeserializeLong (Type t, BinaryReader br) + { + return br.ReadInt64 (); + } + + private static object myDeserializeSByte (Type t, BinaryReader br) + { + return br.ReadSByte (); + } + + private static object myDeserializeBool (Type t, BinaryReader br) + { + return br.ReadBoolean (); + } + + private static object myDeserializeByte (Type t, BinaryReader br) + { + return br.ReadByte (); + } + + private static object myDeserializeDecimal (Type t, BinaryReader br) + { + return br.ReadDecimal (); + } + + private static object myDeserializeUInt (Type t, BinaryReader br) + { + return br.ReadUInt32 (); + } + + private static object myDeserializeDateTime (Type t, BinaryReader br) + { + return DateTime.FromOADate(br.ReadDouble()); + //return new DateTime(br.ReadInt64()); + } + + private static object myDeserializeULong (Type t, BinaryReader br) + { + return br.ReadUInt64 (); + } + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/IPubTablePublisher.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/IPubTablePublisher.cs new file mode 100644 index 000000000..6e9d07725 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/IPubTablePublisher.cs @@ -0,0 +1,44 @@ +using System; + +namespace BCG +{ + /// + /// The table publishing happens here + /// + public interface IPubTablePublisher + { + /// + /// Adds a row to the existing table + /// + /// + void addRow (IBlueRow row); + /// + /// Deletes a row from an existing table + /// + /// + void deleteRow (IBlueRow row); + /// + /// Changes an existing row + /// + /// + void changeRow (IBlueRow row); + /// + /// Creates a table. if already exists overwrites it. + /// + /// + void createTable (IBlueRowCollection rows); + /// + /// Removes and adds more than one row; however the assumption here is that delete is + /// applied first and then add. If that violates the current state, exception is thrown + /// + /// + /// + void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows); + /// + /// Changes more than one row at a time. This is specially helpful, since changes make a huge + /// percentage of the pubsub traffic. + /// + /// + void batchChange (IBlueRowCollection rows); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs new file mode 100644 index 000000000..632f5c541 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs @@ -0,0 +1,175 @@ +using System; +using System.Collections; +using System.IO; + +namespace BCG +{ + /// + /// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer + /// that it initializes according to the table, to optimize serialiazation, hence bw and performance + /// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls, + /// ie the first call will always be create. + /// + public class PubTablePublisher : IPubTablePublisher + { + protected IBlueDataSerializer mySerializer; + protected PublisherCore myPubCore; + + public PubTablePublisher (PublisherCore publisherCore) + { + myPubCore = publisherCore; + mySerializer = new BlueDataSerializationController (); + } + + #region IPubTablePublisher Members + + public virtual void addRow (IBlueRow row) + { + byte[] data; + mySerializer.serialize (StreamOpCode.add, + myGetRowCollectionArray (row), + out data); + myPublish (data, row.schema.tableName); + } + + public virtual void deleteRow (IBlueRow row) + { + byte[] data; + mySerializer.serialize (StreamOpCode.remove, + myGetRowCollectionArray (row), + out data); + myPublish (data, row.schema.tableName); + } + + public virtual void changeRow(IBlueRow row) + { + byte[] data; + mySerializer.serialize (StreamOpCode.change, + myGetRowCollectionArray (row), + out data); + myPublish (data, row.schema.tableName); + } + + public virtual void createTable (IBlueRowCollection rows) + { + byte[] data; + mySerializer.serialize (StreamOpCode.create, + new IBlueRowCollection [] {rows}, + out data); + myPublish (data, rows.schema.tableName); + } + + public virtual void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows) + { + byte[] data; + if (myIsRealChange (delRows, addRows)) + batchChange (addRows); + else + { + mySerializer.serialize (StreamOpCode.multipleRemoveAdd, + new IBlueRowCollection []{delRows, addRows}, + out data); + myPublish (data, delRows.schema.tableName); + } + } + + public virtual void batchChange (IBlueRowCollection rows) + { + byte[] data; + + mySerializer.serialize (StreamOpCode.multipleChange, + new IBlueRowCollection [] {rows}, + out data); + myPublish (data, rows.schema.tableName); + } + + #endregion + + #region private methods + private void myPublish (byte[] data, string tableName) + { + if (data.Length > 0) + myPubCore.publish ("DataStreamTag", + PubStreamPrefixer.prefix (tableName), + new MemoryStream (data, 0, data.Length, false, true)); + } + + private IBlueRowCollection[] myGetRowCollectionArray (IBlueRow r) + { + return new IBlueRowCollection []{ + BlueDataUtility.toBlueRowCollection (new IBlueRow[] {r}) + }; + } + + private bool myIsRealChange (IBlueRowCollection delRows, IBlueRowCollection addRows) + { + if (delRows.Count == addRows.Count) + { + foreach (IBlueRow dRow in delRows) + if (!addRows.containsID ((int)dRow["id"])) + return false; + return true; + } + else + return false; + } + #endregion + } + + public class IndependentTablePublisher : PubTablePublisher + { + bool myIsCreated = false; + + public IndependentTablePublisher (PublisherCore pubCore) : base (pubCore) + {} + + public override void addRow (IBlueRow row) + { + myCheckCreate (row.schema); + base.addRow (row); + } + + public override void batchChange (IBlueRowCollection rows) + { + myCheckCreate (rows.schema); + base.batchChange (rows); + } + + public override void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows) + { + myCheckCreate (delRows.schema); + base.batchRemoveAdd (delRows, addRows); + } + + public override void changeRow (IBlueRow row) + { + myCheckCreate (row.schema); + base.changeRow (row); + } + + public override void deleteRow (IBlueRow row) + { + myCheckCreate (row.schema); + base.deleteRow (row); + } + + public override void createTable(IBlueRowCollection rows) + { + myIsCreated = true; + base.createTable (rows); + } + + + private void myCheckCreate (IBlueTableSchema schema) + { + if (!myIsCreated) + { + //so that the initiazliation work is done on the serializer + byte[] placeHolder; + mySerializer.serialize (StreamOpCode.create, new IBlueRowCollection [] {new BlueTable (schema)}, out placeHolder); + myIsCreated = true; + } + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs new file mode 100644 index 000000000..f5f674e2c --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs @@ -0,0 +1,65 @@ +using System; + +namespace BCG +{ + /// + /// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to + /// the table name, from the config, and wraps around it. The external code that will deal with user calls for + /// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized + /// serialization. + /// + public class BlueDataSerializationController : IBlueDataSerializer + { + private IBlueDataSerializer mySerializer; + + public BlueDataSerializationController () {} + + #region IBlueDataSerializer Members + + /// + /// Serializes an op and data; however if a non create is called before a create, + /// the serializedData is set to an empty byte array. + /// + /// + /// + /// + public void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData) + { + if (op == StreamOpCode.create) + { + if (mySerializer == null) + mySerializer = myGetSerializerForTable(rows[0].schema); + + mySerializer.serialize (op, rows, out serializedData); + } + else + { + if (mySerializer != null) + { + mySerializer.serialize (op, rows, out serializedData); + } + else + { + serializedData = new byte[] {}; + } + } + } + #endregion + + private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema) + { + string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName); + + object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName); + + if(serializerArgs != null) + return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer; + else + return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer; + + + + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializer.cs new file mode 100644 index 000000000..d8e0c8f53 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializer.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections; +using System.IO; + +namespace BCG +{ + /// + /// Abstract base class that can potentially serve all implementations of IBlueDataSerializer. The IBlueRowSerializer + /// is used to serialize individual rows. The implementaions should, in the constructor, contruct the myRowSerializer + /// to do the specific row serializations. Every nonabtract implementation of the this, should have a contructor + /// taking in an IBlueTableSchema. + /// + public abstract class BlueDataSerializer : IBlueDataSerializer + { + protected IBlueRowSerializer myRowSerializer; + protected string myClientDeserializerName, myPubsubDeserializerName; + + public BlueDataSerializer(string pubsubDeserializerName, + string clientDeserializerName) + { + myPubsubDeserializerName = pubsubDeserializerName; + myClientDeserializerName = clientDeserializerName; + } + + + #region IBlueDataSerializer Members + /// + /// Provides the following serialization + /// + /// -add, remove, change: + /// op + /// serializedRow (depending on the implementation of myRowSerializer) + /// + /// -create + /// op + /// pubsubDeserializer name + /// clientDeserializer name + /// schema + /// # of rows + /// serializedRows + /// + /// -multipleAddRemoves + /// op + /// # of deleted rows + /// serializedRows + /// # of added rows + /// serializedRows + /// + /// -multipleChanges + /// op + /// # of changed rows + /// serializedRows + /// + /// + /// + /// + public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) + { + // BlueTrace.WriteSerializeStart(op, rows.Length); + + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change) + { + bw.Write(op); + myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw); + } + else if (op == StreamOpCode.create) + { + IBlueRowCollection createdRows = rows[0]; + bw.Write(op); + bw.Write(myPubsubDeserializerName); + bw.Write(myClientDeserializerName); + int dummyIdCol = 0; + int[] dummyIdList = new int[1]; + //the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order + IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol); + sortedSchema.serialize(bw); + bw.Write(createdRows.Count); + foreach (IBlueRow r in createdRows) + myRowSerializer.serializeRow(r, StreamOpCode.add, bw); + + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + bw.Write(op); + IBlueRowCollection delRows = rows[0], + addRows = rows[1]; + bw.Write(delRows.Count); + foreach (IBlueRow dr in delRows) + myRowSerializer.serializeRow(dr, StreamOpCode.remove, bw); + bw.Write(addRows.Count); + foreach (IBlueRow ar in addRows) + myRowSerializer.serializeRow(ar, StreamOpCode.add, bw); + } + else if (op == StreamOpCode.multipleChange) + { + bw.Write(op); + IBlueRowCollection changedRows = rows[0]; + bw.Write(changedRows.Count); + foreach (IBlueRow cr in changedRows) + myRowSerializer.serializeRow(cr, StreamOpCode.change, bw); + } + bw.BaseStream.Position = 0; + serializedData = (bw.BaseStream as MemoryStream).ToArray(); + + // BlueTrace.WriteSerializeStop(serializedData.Length); + } + #endregion + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs new file mode 100644 index 000000000..3e9bc628a --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections; + +namespace BCG +{ + /// + /// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[]. + /// + public interface IBlueDataSerializer + { + /// + /// f(op, rows) = byte[] + /// + /// Operation that can be add, delete, change etc + /// The data to serialize. The only case when it will be an array of more than 1 element + /// is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd. + /// + /// The data serialized as bytes to be send over the wire + void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueRowSerializer.cs new file mode 100644 index 000000000..a7bf20d22 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueRowSerializer.cs @@ -0,0 +1,23 @@ +using System; +using System.IO; + +namespace BCG +{ + /// + /// Responsible for serilializing rows + /// The way the serialization scheme works is that the header is same + /// for all serialization algorithms, only the way the row is serialized is different. Abstracting it out and we + /// can have each specific serializer have a unique implementation of IBlueRowSerializer. + /// + public interface IBlueRowSerializer + { + /// + /// Serializes a row and writes it into the BinaryWriter. Since its only one row, the ops should be appropriate + /// for one row; the ones allowed are StreamopCode.add, remove and change only. + /// + /// + /// + void serializeRow (IBlueRow r, byte op, BinaryWriter bw); + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/PrimitiveSerializationHelper.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/PrimitiveSerializationHelper.cs new file mode 100644 index 000000000..f55fd8a58 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/PrimitiveSerializationHelper.cs @@ -0,0 +1,96 @@ +using System; +using System.IO; + +namespace BCG +{ + /// + /// Serializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1) + /// rather than searching it up and making it O(n), n = types supported. + /// + public class PrimitiveSerializationHelper + { + private delegate void SerializeDelegate (object data, BinaryWriter bw); + private SerializeDelegate[] myDispatchTable; + + #region constructors + public PrimitiveSerializationHelper (IBlueTableSchema schema) + : this (schema.columnInfo) {} + + public PrimitiveSerializationHelper (ColumnInfo[] cis) + { + myDispatchTable = new SerializeDelegate [cis.Length]; + foreach (ColumnInfo ci in cis) + myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType); + } + #endregion + + #region public methods + /// + /// Serializes the column no in the row and writes it into the BinaryWriter + /// + /// + /// + /// + public void serializeColumn (IBlueRow r, int colNo, BinaryWriter bw) + { + myDispatchTable[colNo] (r[colNo], bw); + } + + /// + /// Writes the data to the BinaryWriter. The ColNo is provided to get the appropriate + /// serializer. + /// + /// + /// + /// + public void serializeColumn (object data, int colNo, BinaryWriter bw) + { + myDispatchTable[colNo] (data, bw); + } + #endregion + + #region private methods + private void mySerializeInt32 (object data, BinaryWriter bw) {bw.Write ((Int32)data);} + private void mySerializeDouble (object data, BinaryWriter bw) {bw.Write ((double)data);} + private void mySerializeString (object data, BinaryWriter bw){bw.Write (data as string);} + private void mySerializeByteArray (object data, BinaryWriter bw) + { + byte[] bs = data as byte[]; + bw.Write (bs.Length); + bw.Write (bs); + } + private void mySerializeChar (object data, BinaryWriter bw) {bw.Write ((char)data);} + private void mySerializeLong (object data, BinaryWriter bw) {bw.Write ((long)data);} + private void mySerializeSByte (object data, BinaryWriter bw) {bw.Write ((sbyte)data);} + private void mySerializeBool (object data, BinaryWriter bw) {bw.Write ((bool)data);} + private void mySerializeByte (object data, BinaryWriter bw) {bw.Write ((byte)data);} + private void mySerializeDecimal (object data, BinaryWriter bw) {bw.Write ((decimal)data);} + private void mySerializeUInt (object data, BinaryWriter bw) {bw.Write ((uint)data);} + private void mySerializeULong (object data, BinaryWriter bw){bw.Write ((ulong)data);} + private void mySerializeDateTime (object data, BinaryWriter bw) {bw.Write (((DateTime)data).Ticks);} + private void mySerializeSingle (object data, BinaryWriter bw) {bw.Write ((Single)data);} + private void mySerializeISerializedType (object data, BinaryWriter bw) {(data as ISerializedType).serialize (bw);} + + private SerializeDelegate myGetDispatchMethod (Type t) + { + if (t.Equals (BlueDataValidTypes.INT)) return new SerializeDelegate (mySerializeInt32); + else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new SerializeDelegate (mySerializeDouble); + else if (t.Equals (BlueDataValidTypes.STRING)) return new SerializeDelegate (mySerializeString); + else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new SerializeDelegate (mySerializeByteArray); + else if (t.Equals (BlueDataValidTypes.CHAR)) return new SerializeDelegate (mySerializeChar); + else if (t.Equals (BlueDataValidTypes.LONG)) return new SerializeDelegate (mySerializeLong); + else if (t.Equals (BlueDataValidTypes.SBYTE)) return new SerializeDelegate (mySerializeSByte); + else if (t.Equals (BlueDataValidTypes.BOOL)) return new SerializeDelegate (mySerializeBool); + else if (t.Equals (BlueDataValidTypes.BYTE)) return new SerializeDelegate (mySerializeByte); + else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new SerializeDelegate (mySerializeDecimal); + else if (t.Equals (BlueDataValidTypes.UINT)) return new SerializeDelegate (mySerializeUInt); + else if (t.Equals (BlueDataValidTypes.ULONG)) return new SerializeDelegate (mySerializeULong); + else if (t.Equals (BlueDataValidTypes.DATETIME)) return new SerializeDelegate (mySerializeDateTime); + else if (t.Equals (BlueDataValidTypes.SINGLE)) return new SerializeDelegate (mySerializeSingle); + else if (SerializationUtilityMethods.isSerializedType (t)) return new SerializeDelegate (mySerializeISerializedType); + else throw new ApplicationException (t.FullName + " is not a supported type"); + } + #endregion + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalRowSerializer.cs new file mode 100644 index 000000000..d0aa73a3b --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalRowSerializer.cs @@ -0,0 +1,120 @@ +using System; +using System.Collections; +using System.IO; +using BCG.Utilities; +using System.Collections.Generic; +namespace BCG +{ + /// + /// Helps the UniversalSerializer by serializing one row. Hence this class is agnostic to the operation + /// which is figured out by UniversalSerializer. + /// + /// it sorts the schema it gets, to make sure that the rows are always published the same way. + /// For now, the sorting operation doesn't do anything. The reason for this is that C++ clients + /// are depending upon the data arriving in a certain order. + /// + /// It performs a mapping : (row)->byte[] + /// + /// Row serialized in the following order + /// row size (nullBitMap size + actual row size) + /// null bit map + /// non null row members (with id as the first col) + /// + /// + /// + public void serializeRow(IBlueRow r, byte op, BinaryWriter bw) + { + myNullMap.SetAll(false); + for (int i = 0; i < mySchema.columnNames.Length; i++) + if (r[mySortedSchemaToOriginalSchemaMapping[i]] == DBNull.Value || r[mySortedSchemaToOriginalSchemaMapping[i]] == null) + myNullMap[i] = true; + myNullMap.CopyTo(myNullMapArray, 0); + long initPos = bw.BaseStream.Position; + bw.Write((int)0); ///place holder for size + bw.Write(myNullMapArray); + bw.Write((int)r[mySortedSchemaToOriginalSchemaMapping[myIdColumnInSortedSchema]]); + foreach (ColumnInfo ci in mySchema.columnInfo) + if (!myNullMap[ci.colNo] && ci.colNo != myIdColumnInSortedSchema) + myPrimitiveSerializationHelper.serializeColumn(r, mySortedSchemaToOriginalSchemaMapping[ci.colNo], bw); + long endPos = bw.BaseStream.Position; + bw.BaseStream.Position = initPos; + bw.Write(Convert.ToInt32(endPos - (initPos+4))); /// size does not include the meta data hence substracting 4 + bw.BaseStream.Position = endPos; + } + + public void serializeSchema(BinaryWriter bw) + { + mySchema.serialize(bw); + } + #endregion + + #region size calculations + /// + /// Sets up the fixed length (size in bytes occupied by each of the primitive type) + /// and an index for the non primitive type + /// + private void myInitSizeCalculations() + { + ArrayList temp = new ArrayList(); + myFixedSize = myNullMapArray.Length; + foreach (ColumnInfo ci in mySchema.columnInfo) + { + if (!SerializationUtilityMethods.isFixedLengthType(ci.colType)) + temp.Add(ci.colNo); + else + myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType); + } + myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[]; + } + + /// + /// Find the size of the serialized row, assumes that nullMap has already + /// been built + /// + /// + /// + /// + private int myGetRowSizeInBytes(int fixedSize, IBlueRow r) + { + foreach (int colNo in myNonFixedColIndexes) + { + if (!myNullMap[colNo]) + fixedSize += //myGetStringSizeInBytes (r[colNo] as string); + SerializationUtilityMethods.getObjectSerializedSize(r[mySortedSchemaToOriginalSchemaMapping[colNo]]); + } + return fixedSize; + } + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs new file mode 100644 index 000000000..cc499367e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs @@ -0,0 +1,16 @@ +using System; + +namespace BCG +{ + /// + /// Uses the UniversalRowSerializer as the IBlueRowSerializer implementation. + /// + public class UniversalSerializer : BlueDataSerializer + { + public UniversalSerializer (IBlueTableSchema schema) + : base ("BCG.PubsubUniversalDeserializer, DataStreamGateway", "BCG.ClientUniversalDeserializer") + { + myRowSerializer = new UniversalRowSerializer (schema); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/TablePublishingException.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/TablePublishingException.cs new file mode 100644 index 000000000..4ceee2552 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/TablePublishingException.cs @@ -0,0 +1,15 @@ +using System; + +namespace BCG +{ + /// + /// Exception thrown if the publishing violates the state of table, ex deleted row + /// does not exist in the table + /// + public class TablePublishingException : ApplicationException + { + public TablePublishingException (string message) + : base (message) + {} + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/IPubsubRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/IPubsubRowDeserializer.cs new file mode 100644 index 000000000..04461d55b --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/IPubsubRowDeserializer.cs @@ -0,0 +1,25 @@ +using System; +using System.IO; +using System.Collections; + +namespace BCG +{ + /// + /// Deserializes a row. Returns the id of the row and its full content + /// + public interface IPubsubRowDeserializer + { + /// + /// Reads the data from the BinaryReader and deserializes it in the form of a row + /// + /// Some deserializers might need the current state, ie a Dictionary of rows + /// keyed off the id + /// BinaryReader starting at the row + /// The operation, as this might drive the deserializations scheme in some cases. This op + /// however is one of the basics operations, add remove or change as all other operations can be resolved into this + /// The id of the deserialized row + /// The actual row contents + void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row); + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs new file mode 100644 index 000000000..e2e0e0b2b --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections; +using System.IO; +using BCG.Utilities; +using System.Text; + +namespace BCG +{ + /// + /// Base class for all deserializers. + /// + public abstract class PubsubDeserializer : IPubsubDeserializer + { + protected IDictionary myState = new Hashtable (); + protected IBlueTableSchema mySchema; + protected string myClientDeserializerName; //for the getState method + protected IPubsubRowDeserializer myDeserializer; + + public PubsubDeserializer () {} + + #region IPubSubDeserializer Members + public virtual void deserialize (Stream[] streams, out byte[] theOps, out Array theRows) + { + BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams)); + ArrayList ops = new ArrayList (), rows = new ArrayList (); + while (true) + { + IDictionary[] result; + byte op; + try + { + int value = br.BaseStream.ReadByte(); + if (value == -1) break; + op = (byte)value; + } + catch (EndOfStreamException) + { + break; + } + if (op == StreamOpCode.create) + { + myState.Clear (); + br.ReadString (); //the pubsub-deserializer name, should be the same as this's name + myClientDeserializerName = br.ReadString (); //the client-deserializer name + (mySchema = new BlueTableSchema()).deserialze (br); + result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())}; + myEncorporatRows (result[0], op); + ops.Add (op); + rows.Add (result); + } + else + { + if (mySchema != null) + { + if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) + { + result = new IDictionary[] {myGetRows (br, op, 1)}; + myEncorporatRows (result[0], op); + ops.Add (op); + rows.Add (result); + } + else if (op == StreamOpCode.multipleChange) + { + result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())}; + myEncorporatRows (result[0], op); + ops.Add (op); + rows.Add (result); + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + ops.Add (op); + result = new IDictionary [] { + myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()), + myGetRows (br, StreamOpCode.add, br.ReadInt32 ()) + }; + myEncorporatRows (result[0], StreamOpCode.remove); + myEncorporatRows (result[1], StreamOpCode.add); + rows.Add (result); + } + else + { + // BlueEventLog.logEvent ("Illegal op-code received", BlueEventLevel.Error); + } + } + else + { + // BlueEventLog.logEvent ("Received a non create operation even though the schema" + + // "has never been created", BlueEventLevel.Warning); + } + } + } + theOps = ops.ToArray (typeof(byte)) as byte[]; + theRows = rows.ToArray (); + } + + /// + /// Sends in a create signal, so that new clients can get a create everytime and start + /// afresh from there + /// + /// + public virtual byte[] getState() + { + if (mySchema == null) + return new byte[] {}; + else + { + BinaryWriter bw = new BinaryWriter (new MemoryStream ()); + bw.Write (StreamOpCode.create); + bw.Write (GetType().AssemblyQualifiedName); + bw.Write (myClientDeserializerName); + mySchema.serialize (bw); + bw.Write (myState.Count); + foreach (byte[] row in myState.Values) + bw.Write (row); + return (bw.BaseStream as MemoryStream).ToArray (); + } + } + #endregion + + /// + /// Every row that has been deserialized passes through this method, so that it can update the state. + /// + /// + /// + protected virtual void myEncorporatRows (IDictionary rows, byte op) + { + if (op == StreamOpCode.add || op == StreamOpCode.create) + foreach (DictionaryEntry de in rows) + { + if (myState.Contains(de.Key)) { + // BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key),BlueEventLog2Level.Fatal); + StringBuilder b = new StringBuilder(); + foreach (string n in mySchema.columnNames) + b.AppendFormat(" {0}", n); + // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal); + } + myState.Add(de.Key, de.Value); + } + else if (op == StreamOpCode.remove) + foreach (DictionaryEntry de in rows) + { + if (!myState.Contains(de.Key)) + { + // BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal); + StringBuilder b = new StringBuilder(); + foreach (string n in mySchema.columnNames) + b.AppendFormat(" {0}", n); + // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()),BlueEventLog2Level.Fatal); + } + myState.Remove(de.Key); + } + else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) + foreach (DictionaryEntry de in rows) + myState[de.Key] = de.Value; + } + + /// + /// Mass deserialization happens here, also since there is no knowledge of the schema + /// + /// + /// + /// + /// + protected abstract IDictionary myGetRows (BinaryReader br, byte op, int noOfRows); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs new file mode 100644 index 000000000..c20947c14 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections; +using System.IO; + +namespace BCG +{ + public class PubsubUniversalDeserializer : PubsubDeserializer + { + public PubsubUniversalDeserializer() {} + + protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows) + { + myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myDeserializer; + Hashtable retVal = new Hashtable (); + for (int i = 0; i < noOfRows; i++) + { + int id; object row; + myDeserializer.deserializeRow (myState, br, op, out id, out row); + retVal.Add (id, row); + } + return retVal; + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalRowDeserializer.cs new file mode 100644 index 000000000..330c91c38 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalRowDeserializer.cs @@ -0,0 +1,34 @@ +using System; +using System.IO; +using System.Collections; + + +namespace BCG +{ + public class PubsubUniversalRowDeserializer : IPubsubRowDeserializer + { + private int myNullMapLength; + private IBlueTableSchema mySchema; + private int myRowOverheadSize = SerializationUtilityMethods.getFixedLengthTypeSize (typeof(int)); + + public PubsubUniversalRowDeserializer (IBlueTableSchema schema) + { + mySchema = schema; + myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8))); + } + + #region IRowDeserializer Members + + public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row) + { + //current state is ignored + long pos = br.BaseStream.Position; + int rowSize = br.ReadInt32 (); + br.BaseStream.Position = pos; + row = br.ReadBytes (rowSize + myRowOverheadSize); + id = BitConverter.ToInt32 (row as byte[], myNullMapLength + myRowOverheadSize); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs new file mode 100644 index 000000000..e046be78e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs @@ -0,0 +1,57 @@ +using System; +using System.IO; +using System.Collections; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Keeps a collection of TagStateBanks to hold the state of each tag + /// + public class PubsubServerTagHandler : IPubSubServerTagHandler + { + private IDictionary myTagStates = new Hashtable (); + private bool myPublishStatistics = //DataStreamGatewaySettings.instance.enablePubsubStats; + false; + + public PubsubServerTagHandler() {} + + #region IPubSubServerTagHandler Members + + public byte[] getContent (string tag) + { + TagStateBank stateBank = myTagStates[tag] as TagStateBank; + if (stateBank != null) + { + return stateBank.getConent (); + } + else + { + // BlueEventLog.logEvent (String.Format ("Information requested for tag {0} which does not exist now", tag), BlueEventLevel.Warning); + return new byte[] {}; + } + } + + public void update (string tag, Stream[] messageBodies) + { + if (myPublishStatistics) + { + StatisticsLogger.instance.log(tag, messageBodies); + } + if (!myTagStates.Contains (tag)) + { + // BlueEventLog.logEvent (String.Format ("Tag {0} received for the first time and its state created", tag), BlueEventLevel.Information); + myTagStates.Add (tag, new TagStateBank ()); + } + (myTagStates[tag] as TagStateBank).update (messageBodies); + } + + public void doWork() {} + + public string[] supportedTagTypes {get {return new string [] {"DataStreamTag"};}} + + public void register(InternalPublishDelegate callback) { } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/StatisticsLogger.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/StatisticsLogger.cs new file mode 100644 index 000000000..f55bd1aa4 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/StatisticsLogger.cs @@ -0,0 +1,102 @@ +using System; +using System.IO; +using System.Collections; +using BCG.Utilities; +using System.Threading; + +namespace BCG +{ + class TagData + { + public DateTime lastUpdateTime; + public long totalSize; + public int lastUpdateSize; + + public TagData(DateTime ut, long ts, int lus) + { + lastUpdateTime = ut; + totalSize = ts; + lastUpdateSize = lus; + } + public override string ToString() + { + return lastUpdateTime.ToString() + "," + totalSize.ToString() + "," + lastUpdateSize.ToString(); + } + } + + class StatisticsLogger + { + private IDictionary myTagData = new Hashtable(); + private static StatisticsLogger mySelf; + private BlockingQueue myQueue = new BlockingQueue(); + private DateTime myLastFireTime = DateTime.MinValue; + private TimeSpan myPubsubStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.pubsubStatsUpdateInterval); + private TimeSpan myTagStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.singleTagStatsUpdateInterval); + + private StatisticsLogger() + { + Thread readThread = new Thread(new ThreadStart(myReadAndPublish)); + readThread.Priority = ThreadPriority.BelowNormal; + readThread.Start(); + } + public static StatisticsLogger instance + { + get + { + if (mySelf == null) + mySelf = new StatisticsLogger(); + return mySelf; + } + } + + public void log(string tag, Stream[] bodies) + { + long nBytes = 0; + foreach (Stream s in bodies) + nBytes += s.Length; + myQueue.enqueue(new object[] { tag, Convert.ToInt32 (nBytes) }); + } + + private void myReadAndPublish() + { + while (true) + { + object qe = myQueue.dequeue(50); + if (qe != null) + { + object[] args = qe as object[]; + string tag = args[0].ToString(); + int dataSize = (int)args[1]; + if (!myTagData.Contains(tag)) + { + myTagData.Add(tag, new TagData(DateTime.Now, Convert.ToInt64(dataSize), dataSize)); + } + else + { + TagData tdToSend = myTagData[tag] as TagData; + tdToSend.lastUpdateSize += dataSize; + tdToSend.totalSize += dataSize; + } + } + myFireMonitors(); + } + } + + private void myFireMonitors() + { + if (DateTime.Now.Subtract(myLastFireTime) >= myPubsubStatsUpdateInterval) + { + foreach (string tag in myTagData.Keys) + { + TagData td = myTagData[tag] as TagData; + if (td.lastUpdateSize > 0 && DateTime.Now.Subtract(td.lastUpdateTime) >= myTagStatsUpdateInterval) + { + td.lastUpdateTime = DateTime.Now; + td.lastUpdateSize = 0; + } + } + myLastFireTime = DateTime.Now; + } + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs new file mode 100644 index 000000000..44ea024e9 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections; +using System.IO; + +namespace BCG +{ + public class TagStateBank + { + //private IDictionary myState = new Hashtable (); + private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController (); + + public TagStateBank () {} + + public byte[] getConent () + { + return myDeserializationController.getState (); + } + + public void update (Stream[] streams) + { + byte[] ops; + Array rowCollection; + myDeserializationController.deserialize (streams, out ops, out rowCollection); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionClientDeserializer.cs new file mode 100644 index 000000000..279594469 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionClientDeserializer.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using BCG.Utilities; + +namespace BCG +{ + public class CompressionClientDeserializer : IClientDeserializer + { + protected IBlueTableSchema mySchema; + protected IBlueTableSchema myFirstSchema; + protected IClientRowDeserializer myRowDeserializer; + + public CompressionClientDeserializer() + { + myFirstSchema = null; + mySchema = null; + } + + #region IClientDeserializer Members + + public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) + { + BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams)); + // BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length); + + // emit # of streams and total length + ArrayList ops = new ArrayList(), rows = new ArrayList(); + IBlueRowCollection[] result; + while (true) + { + byte op; + try + { + int value = br.BaseStream.ReadByte(); + if (value == -1) break; + op = (byte)value; + // BlueTrace.WriteDeserializeOperation(op); + } + catch (EndOfStreamException) + { + break; + } + if (op == StreamOpCode.create) + { + br.ReadString(); //pubsubDeserializer + br.ReadString(); //clientDeserializer + mySchema = new BlueTableSchema(); + mySchema.deserialze(br); + //keep track of the first schema you've ever seen, so you know how to reorder columns + if (myFirstSchema == null) + myFirstSchema = mySchema; + + /// decompress here + int compressedMessageSize = br.ReadInt32(); + byte[] compressedData = br.ReadBytes(compressedMessageSize); + BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData)); + + result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) }; + myEncorporateRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else + { + if (mySchema != null) + { + if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) + { + result = new IBlueRowCollection[] { myGetRows(br, op, 1) }; + myEncorporateRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleChange) + { + int compressedMessageSize = br.ReadInt32(); + byte[] compressedData = br.ReadBytes(compressedMessageSize); + BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData)); + + result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) }; + myEncorporateRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + int compressedMessageSize = br.ReadInt32(); + byte[] compressedData = br.ReadBytes(compressedMessageSize); + BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData)); + + result = + new IBlueRowCollection[] { + myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()), + myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ()) + }; + + myEncorporateRows(result[0], StreamOpCode.remove); + myEncorporateRows(result[1], StreamOpCode.add); + ops.Add(op); + rows.Add(result); + } + else + { + // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); + } + } + else + { + // BlueEventLog.logEvent("Received a non create operation even though the schema" + + // "has never been created", BlueEventLevel.Warning); + } + } + } + theOps = ops.ToArray(typeof(byte)) as byte[]; + theRows = rows.ToArray(); + // BlueTrace.WriteDeserializeStop(rows.Count); + } + + #endregion + + private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows) + { + BlueTable retVal = new BlueTable(mySchema); + + if (myRowDeserializer == null) + myRowDeserializer = new ClientUniversalRowDeserializer(mySchema); + else if (op == StreamOpCode.create) + myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema); + + for (int i = 0; i < nRows; i++) + { + IBlueRow row; + myRowDeserializer.deserializeRow(null, br, op, out row); + retVal.addRow(row); + } + return retVal; + } + + private void myEncorporateRows(IBlueRowCollection rows, byte op) { } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionPubsubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionPubsubDeserializer.cs new file mode 100644 index 000000000..813fca93f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionPubsubDeserializer.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Collections; +using BCG.Utilities; + +namespace BCG +{ + public class CompressionPubsubDeserializer : IPubsubDeserializer + { + protected IDictionary myState = new Hashtable(); + protected IBlueTableSchema mySchema; + protected string myClientDeserializerName; //for the getState method + protected IPubsubRowDeserializer myRowDeserializer; + + public CompressionPubsubDeserializer() { } + + #region IPubSubDeserializer Members + public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) + { + BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams)); + ArrayList ops = new ArrayList(), rows = new ArrayList(); + while (true) + { + IDictionary[] result; + byte op; + try + { + int value = br.BaseStream.ReadByte(); + if (value == -1) break; + op = (byte)value; + } + catch (EndOfStreamException) + { + break; + } + if (op == StreamOpCode.create) + { + myState.Clear(); + br.ReadString(); //the pubsub-deserializer name, should be the same as this's name + myClientDeserializerName = br.ReadString(); //the client-deserializer name + (mySchema = new BlueTableSchema()).deserialze(br); + /// deserialize this part + int compressedMessageSize = br.ReadInt32(); + BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize))); + result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) }; + myEncorporatRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else + { + if (mySchema != null) + { + if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) + { + result = new IDictionary[] { myGetRows(br, op, 1) }; + myEncorporatRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleChange) + { + int compressedMessageSize = br.ReadInt32(); + BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize))); + result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) }; + myEncorporatRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + ops.Add(op); + int compressedMessageSize = br.ReadInt32(); + BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize))); + result = new IDictionary[] { + myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()), + myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ()) + }; + myEncorporatRows(result[0], StreamOpCode.remove); + myEncorporatRows(result[1], StreamOpCode.add); + rows.Add(result); + } + else + { + // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); + } + } + else + { + // BlueEventLog.logEvent("Received a non create operation even though the schema" + + // "has never been created", BlueEventLevel.Warning); + } + } + } + theOps = ops.ToArray(typeof(byte)) as byte[]; + theRows = rows.ToArray(); + } + + /// + /// Sends in a create signal, so that new clients can get a create everytime and start + /// afresh from there + /// + /// + public virtual byte[] getState() + { + if (mySchema == null) + return new byte[] { }; + else + { + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + bw.Write(StreamOpCode.create); + bw.Write(GetType().AssemblyQualifiedName); + bw.Write(myClientDeserializerName); + mySchema.serialize(bw); + /// compress it here + + BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream()); + decompressedBw.Write(myState.Count); + foreach (byte[] row in myState.Values) + decompressedBw.Write(row); + decompressedBw.BaseStream.Position = 0; + byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream); + bw.Write(compressedData.Length); + bw.Write(compressedData); + + return (bw.BaseStream as MemoryStream).ToArray(); + } + } + #endregion + + /// + /// Every row that has been deserialized passes through this method, so that it can update the state. + /// + /// + /// + protected virtual void myEncorporatRows(IDictionary rows, byte op) + { + if (op == StreamOpCode.add || op == StreamOpCode.create) + foreach (DictionaryEntry de in rows) + { + if (myState.Contains(de.Key)) + { + // BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal); + StringBuilder b = new StringBuilder(); + foreach (string n in mySchema.columnNames) + b.AppendFormat(" {0}", n); + // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal); + } + myState.Add(de.Key, de.Value); + } + else if (op == StreamOpCode.remove) + foreach (DictionaryEntry de in rows) + { + if (!myState.Contains(de.Key)) + { + // BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal); + StringBuilder b = new StringBuilder(); + foreach (string n in mySchema.columnNames) + b.AppendFormat(" {0}", n); + // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal); + } + myState.Remove(de.Key); + } + else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) + foreach (DictionaryEntry de in rows) + myState[de.Key] = de.Value; + } + + /// + /// Mass deserialization happens here, also since there is no knowledge of the schema + /// + /// + /// + /// + /// + private IDictionary myGetRows (BinaryReader br, byte op, int noOfRows) + { + myRowDeserializer = myRowDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myRowDeserializer; + Hashtable retVal = new Hashtable (); + for (int i = 0; i < noOfRows; i++) + { + int id; object row; + myRowDeserializer.deserializeRow (myState, br, op, out id, out row); + retVal.Add (id, row); + } + return retVal; + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs new file mode 100644 index 000000000..4b1c51b74 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace BCG +{ + public class CompressionSerializer : IBlueDataSerializer + { + private IBlueRowSerializer myRowSerializer; + private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamGateway", myClientDeserializerName = "BCG.CompressionClientDeserializer"; + + public CompressionSerializer (IBlueTableSchema schema) + { + myRowSerializer = new UniversalRowSerializer (schema); + } + + public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) + { + // BlueTrace.WriteSerializeStart(op, rows.Length); + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change) + { + bw.Write(op); + /// no need to compress the only row + myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw); + } + else if (op == StreamOpCode.create) + { + IBlueRowCollection createdRows = rows[0]; + bw.Write(op); + bw.Write(myPubsubDeserializerName); + bw.Write(myClientDeserializerName); + int dummyIdCol = 0; + int[] dummyIdList = new int[1]; + //the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order + IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol); + sortedSchema.serialize(bw); + /// compress it here + BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream()); + decompressedBw.Write(createdRows.Count); + foreach (IBlueRow r in createdRows) + myRowSerializer.serializeRow(r, StreamOpCode.add, decompressedBw); + decompressedBw.BaseStream.Position = 0; + + byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream); + bw.Write(compressedData.Length); + bw.Write(compressedData); + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + bw.Write(op); + IBlueRowCollection delRows = rows[0], addRows = rows[1]; + /// compress it here + BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream()); + decompressedBw.Write(delRows.Count); + foreach (IBlueRow dr in delRows) + myRowSerializer.serializeRow(dr, StreamOpCode.remove, decompressedBw); + decompressedBw.Write(addRows.Count); + foreach (IBlueRow ar in addRows) + myRowSerializer.serializeRow(ar, StreamOpCode.add, decompressedBw); + decompressedBw.BaseStream.Position = 0; + + byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream); + bw.Write(compressedData.Length); + bw.Write(compressedData); + } + else if (op == StreamOpCode.multipleChange) + { + bw.Write(op); + IBlueRowCollection changedRows = rows[0]; + /// compress it here + BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream()); + decompressedBw.Write(changedRows.Count); + foreach (IBlueRow cr in changedRows) + myRowSerializer.serializeRow(cr, StreamOpCode.change, decompressedBw); + decompressedBw.BaseStream.Position = 0; + + byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream); + bw.Write(compressedData.Length); + bw.Write(compressedData); + } + bw.BaseStream.Position = 0; + serializedData = (bw.BaseStream as MemoryStream).ToArray(); + // BlueTrace.WriteSerializeStop(serializedData.Length); + } + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/BBOOptionDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/BBOOptionDeserializer.cs new file mode 100644 index 000000000..2d91628a7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/BBOOptionDeserializer.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public class BBOOptionDeserializer : ClientFixedDeserializer + { + public BBOOptionDeserializer() { } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedDeserializer.cs new file mode 100644 index 000000000..6631460da --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedDeserializer.cs @@ -0,0 +1,49 @@ +using System; +using BCG.Utilities; + +namespace BCG +{ + public abstract class ClientFixedDeserializer : ClientDeserializer + { + private BlueTable myCurrentState; + + public ClientFixedDeserializer () {} + + protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows) + { + BlueTable retVal = new BlueTable (mySchema); + op = op == StreamOpCode.create ? StreamOpCode.add : + op == StreamOpCode.multipleChange ? StreamOpCode.change : op; + if (myRowDeserializer == null) + { + myRowDeserializer = new ClientFixedRowDeserializer (mySchema); + myCurrentState = new BlueTable (mySchema); + } + for (int i = 0; i < nRows; i++) + { + IBlueRow r; + myRowDeserializer.deserializeRow (myCurrentState, br, op, out r); + retVal.addRow (r); + } + return retVal; + } + + protected override void myEncorporateRows (IBlueRowCollection rows, byte op) + { + if (op == StreamOpCode.create) + myCurrentState = new BlueTable (rows); + else if (op == StreamOpCode.add) + myCurrentState.addRows (rows); + else if (op == StreamOpCode.remove) + myCurrentState.deleteRows (rows); + else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) + myCurrentState.changeRows (rows); + else + { + // BlueEventLog.logAssert (false, "Illegal operation for this context"); + } + } + + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedRowDeserializer.cs new file mode 100644 index 000000000..e673a06f7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedRowDeserializer.cs @@ -0,0 +1,38 @@ +using System; +using System.IO; +using BCG.Utilities; + +namespace BCG +{ + public class ClientFixedRowDeserializer : IClientRowDeserializer + { + private PrimitiveDeserializationHelper myHelper; + private IBlueTableSchema mySchema; + public ClientFixedRowDeserializer (IBlueTableSchema schema) + { + myHelper = new PrimitiveDeserializationHelper (mySchema = schema); + } + + #region IClientRowDeserializer Members + + public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) + { + if (op == StreamOpCode.add || op == StreamOpCode.change) + { + object[] retVal = new object[mySchema.columnInfo.Length]; + foreach (ColumnInfo ci in mySchema.columnInfo) + retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br); + r = new BlueRow (mySchema, retVal); + } + else if (op == StreamOpCode.remove) + r = currentState[br.ReadInt32 ()]; + else + { + // BlueEventLog.logAssert (false, op.ToString() + " not valid in this context"); + r = null; + } + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/MarketOptionTheoClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/MarketOptionTheoClientDeserializer.cs new file mode 100644 index 000000000..700f7b681 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/MarketOptionTheoClientDeserializer.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public class MarketOptionTheoClientDeserializer : ClientFixedDeserializer + { + public MarketOptionTheoClientDeserializer() { } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/BBOOptionSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/BBOOptionSerializer.cs new file mode 100644 index 000000000..c44af6a14 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/BBOOptionSerializer.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public class BBOOptionSerializer : FixedSerializer + { + public BBOOptionSerializer(IBlueTableSchema schema) + : base(schema, "BCG.BBOOptionPubsubDeserializer", "BCG.BBOOptionDeserializer") + { } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedRowSerializer.cs new file mode 100644 index 000000000..bc2a09c5f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedRowSerializer.cs @@ -0,0 +1,35 @@ +using System; +using System.IO; + +namespace BCG +{ + /// + /// Optimized to serialize fixed length rows, ie schemas that have only primitive types and non-nulls + /// + public class FixedRowSerializer : IBlueRowSerializer + { + private PrimitiveSerializationHelper myHelper; + private int myIdColNo; + private IBlueTableSchema mySchema; + public FixedRowSerializer (IBlueTableSchema schema) + { + myIdColNo = schema["id"].colNo; + myHelper = new PrimitiveSerializationHelper (schema); + mySchema = schema; + } + + #region IBlueRowSerializer Members + + public void serializeRow (IBlueRow r, byte op, BinaryWriter bw) + { + if (op == StreamOpCode.remove) + myHelper.serializeColumn (r, myIdColNo, bw); + else + foreach (ColumnInfo ci in r.schema.columnInfo) + myHelper.serializeColumn (r, ci.colNo, bw); + } + + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedSerializer.cs new file mode 100644 index 000000000..064615b0e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedSerializer.cs @@ -0,0 +1,20 @@ +using System; +using System.IO; + +namespace BCG +{ + /// + /// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls) + /// + public abstract class FixedSerializer : BlueDataSerializer + { + public FixedSerializer (IBlueTableSchema schema, + string pubsubDeserializer, + string clientDeserializer) + : base (pubsubDeserializer, clientDeserializer) + { + myRowSerializer = new FixedRowSerializer (schema); + } + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/MarketOptionTheoSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/MarketOptionTheoSerializer.cs new file mode 100644 index 000000000..719cb20e7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/MarketOptionTheoSerializer.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public class MarketOptionTheoSerializer : FixedSerializer + { + public MarketOptionTheoSerializer(IBlueTableSchema schema) + : base(schema, "BCG.MarketOptionTheoPubsubDeserializer", + "BCG.MarketOptionTheoClientDeserializer") + { } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/BBOOptionPubsubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/BBOOptionPubsubDeserializer.cs new file mode 100644 index 000000000..230d3a8e9 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/BBOOptionPubsubDeserializer.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public class BBOOptionPubsubDeserializer : PubsubFixedDeserializer + { + public BBOOptionPubsubDeserializer() { } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/MarketOptionTheoPubsubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/MarketOptionTheoPubsubDeserializer.cs new file mode 100644 index 000000000..0ca4893a5 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/MarketOptionTheoPubsubDeserializer.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public class MarketOptionTheoPubsubDeserializer : PubsubFixedDeserializer + { + public MarketOptionTheoPubsubDeserializer() + { } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedDeserializer.cs new file mode 100644 index 000000000..83e15a2a4 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedDeserializer.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections; +using System.IO; + +namespace BCG +{ + public abstract class PubsubFixedDeserializer : PubsubDeserializer + { + protected BlueTable myCurrentState; + + public PubsubFixedDeserializer () {} + + protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows) + { + if (myDeserializer == null) + myDeserializer = new PubsubFixedRowDeserializer (mySchema); + IDictionary retVal = new Hashtable (); + op = op == StreamOpCode.create ? StreamOpCode.add : + op == StreamOpCode.multipleChange ? StreamOpCode.change : op; + for (int i = 0; i < noOfRows; i++) + { + int id; + object row; + myDeserializer.deserializeRow (myState, br, op, out id, out row); + retVal.Add (id, row); + } + return retVal; + } + + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedRowDeserializer.cs new file mode 100644 index 000000000..f1e21f641 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedRowDeserializer.cs @@ -0,0 +1,37 @@ +using System; +using System.IO; +using System.Collections; + +namespace BCG +{ + public class PubsubFixedRowDeserializer : IPubsubRowDeserializer + { + private int myRowSize, myIdOffset; + + public PubsubFixedRowDeserializer (IBlueTableSchema schema) + { + foreach (ColumnInfo ci in schema.columnInfo) + { + myRowSize += SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType); + myIdOffset += ci.colNo < schema["id"].colNo ? SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType) : 0; + } + } + #region IPubsubRowDeserializer Members + + public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row) + { + if (op == StreamOpCode.add || op == StreamOpCode.change) + { + row = br.ReadBytes (myRowSize); + id = BitConverter.ToInt32 (row as byte[], myIdOffset); + } + else //delete => only the id + { + id = br.ReadInt32 (); + row = currentState[id]; + } + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingDeserializer.cs new file mode 100644 index 000000000..069c27e69 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingDeserializer.cs @@ -0,0 +1,56 @@ +using System; +using System.IO; +using System.Collections; + +using BCG.Utilities; + +namespace BCG +{ + public abstract class ClientFixedChangingDeserializer : ClientDeserializer + { + private string [] myChangingCols; + protected BlueTable myCurrentState; + + public ClientFixedChangingDeserializer (string[] changingCols) + { + myChangingCols = changingCols; + } + + protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows) + { + BlueTable retVal = new BlueTable (mySchema); + op = op == StreamOpCode.create ? StreamOpCode.add : + op == StreamOpCode.multipleChange ? StreamOpCode.change : op; + if (myRowDeserializer == null) + { + myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, myChangingCols); + myCurrentState = new BlueTable (mySchema); + } + for (int i = 0; i < nRows; i++) + { + IBlueRow r; + myRowDeserializer.deserializeRow (myCurrentState, br, op, out r); + retVal.addRow (r); + } + return retVal; + } + + protected override void myEncorporateRows (IBlueRowCollection rows, byte op) + { + if (op == StreamOpCode.create) + myCurrentState = new BlueTable (rows); + else if (op == StreamOpCode.add) + myCurrentState.addRows (rows); + else if (op == StreamOpCode.remove) + myCurrentState.deleteRows (rows); + else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) + myCurrentState.changeRows (rows); + else + { + // BlueEventLog.logAssert (false, "Illegal operation for this context"); + } + } + + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingRowDeserializer.cs new file mode 100644 index 000000000..4e23afd82 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingRowDeserializer.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections; +using System.IO; + +using BCG.Utilities; + +namespace BCG +{ + /// + /// Deserializes a fixed length row with no nulls on the client side. + /// + public class ClientFixedChangingRowDeserializer : IClientRowDeserializer + { + private int[] myChangingColIndices; + protected PrimitiveDeserializationHelper myHelper; + protected IBlueTableSchema mySchema; + + public ClientFixedChangingRowDeserializer (IBlueTableSchema schema, string[] changingCols) + { + myHelper = new PrimitiveDeserializationHelper (mySchema = schema); + myChangingColIndices = new int [changingCols.Length]; + for (int i = 0; i < changingCols.Length; i++) + myChangingColIndices[i] = schema[changingCols[i]].colNo; + Array.Sort (myChangingColIndices); + } + + public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) + { + if (op == StreamOpCode.add) + myDeserializeAdd (currentState, br, out r); + else if (op == StreamOpCode.remove) + myDeserializeRemove (currentState, br, out r); + else if (op == StreamOpCode.change) + myDeserializeChange (currentState, br, out r); + else + { + r = null; + // BlueEventLog.logAssert (false, "Illegal operation for this function, can only deal with single row " + + // "add, delete and change"); + } + } + + protected virtual void myDeserializeAdd (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r) + { + object[] retVal = new object[mySchema.columnInfo.Length]; + foreach (ColumnInfo ci in mySchema.columnInfo) + retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br); + r = new BlueRow (mySchema, retVal); + } + + protected virtual void myDeserializeRemove (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r) + { + r = currentState[br.ReadInt32()]; + } + + + protected virtual void myDeserializeChange (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r) + { + MutableBlueRow rv = new MutableBlueRow (currentState[br.ReadInt32 ()]); + foreach (int colNo in myChangingColIndices) + rv.setValue (colNo, myHelper.deserializeColumn (colNo, br)); + r = rv; + } + + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingRowSerializer.cs new file mode 100644 index 000000000..3ff9b7268 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingRowSerializer.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections; +using System.IO; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Specializes in serializing rows that are sent over as a result of a change; only the changed columns are + /// sent together with the id. + /// + public class FixedChangingRowSerializer : IBlueRowSerializer + { + private int[] myChangingColIndices; + private PrimitiveSerializationHelper myHelper; + private IBlueTableSchema mySchema; + + public FixedChangingRowSerializer (IBlueTableSchema schema, params string[] changingColumns) + { + myHelper = new PrimitiveSerializationHelper (schema); + myChangingColIndices = new int[changingColumns.Length]; + for (int i = 0; i < changingColumns.Length; i++) + myChangingColIndices[i] = schema[changingColumns[i]].colNo; + Array.Sort (myChangingColIndices); + mySchema = schema; + } + + #region IBlueRowSerializer Members + + public void serializeRow (IBlueRow r, byte op, BinaryWriter bw) + { + if (op == StreamOpCode.add) + { + mySerializeAdd (r, bw); + } + else if (op == StreamOpCode.remove) + { + mySerializeRemove (r, bw); + } + else if (op == StreamOpCode.change) + { + mySerializeChange (r, bw); + } + else + { + // BlueEventLog.logAssert (false, String.Format ("{0} is not a valid op in this context", op)); + } + } + + #endregion + + /// + /// The whole row is written, one column after the other with the order defined by the schema. There + /// are no columns headers + /// + /// + /// + protected virtual void mySerializeAdd (IBlueRow r, BinaryWriter bw) + { + foreach (ColumnInfo ci in r.schema.columnInfo) + myHelper.serializeColumn (r, ci.colNo, bw); + } + + /// + /// Only the id is written + /// + /// + /// + protected virtual void mySerializeRemove (IBlueRow r, BinaryWriter bw) + { + myHelper.serializeColumn (r, r.schema["id"].colNo, bw); + } + + /// + /// Serializes only the changed columns, alongside the id. So the format is + /// {id, changedColumns*}. The order of the changedColumns are sorted according + /// to their index in the schema. + /// + /// + /// + protected virtual void mySerializeChange (IBlueRow r, BinaryWriter bw) + { + myHelper.serializeColumn (r, r.schema["id"].colNo, bw); + foreach (int colNo in myChangingColIndices) + myHelper.serializeColumn (r, colNo, bw); + } + + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingSerializer.cs new file mode 100644 index 000000000..7f61008c4 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingSerializer.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections; +using System.IO; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls) + /// and are constantly changing + /// + public abstract class FixedChangingSerializer : BlueDataSerializer + { + + public FixedChangingSerializer (IBlueTableSchema schema, + string[] changingCols, + string pubsubDeserializer, + string clientDeserializer) + : base (pubsubDeserializer, clientDeserializer) + { + myRowSerializer = new FixedChangingRowSerializer (schema, changingCols); + } + } + + #region BidAsk + + public class BidAskSerializer : FixedChangingSerializer + { + public BidAskSerializer (IBlueTableSchema schema) + : base (schema, + new string[] {"bidAskZone"}, + "BCG.BidAskPubsubDeserializer", + "BCG.BidAskClientDeserializer" + ) + {} + } + + public class BidAskPubsubDeserializer : PubsubFixedChangingDeserializer + { + public BidAskPubsubDeserializer () : base (new string[] {"bidAskZone"}) + {} + } + + public class BidAskClientDeserializer : ClientFixedChangingDeserializer + { + public BidAskClientDeserializer () : base (new string[] {"bidAskZone"}) + {} + } + + #endregion + + #region HHMarketPrice + + public class HHMarketPriceSerializer : FixedChangingSerializer + { + public HHMarketPriceSerializer(IBlueTableSchema schema) + : base(schema, + new string[] { "MarketPrice" }, + "BCG.HHMarketPricePubsubDeserializer", + "BCG.HHMarketPriceClientDeserializer" + ) + { } + } + + public class HHMarketPricePubsubDeserializer : PubsubFixedChangingDeserializer + { + public HHMarketPricePubsubDeserializer() + : base(new string[] { "MarketPrice" }) + { } + } + + public class HHMarketPriceClientDeserializer : ClientFixedChangingDeserializer + { + public HHMarketPriceClientDeserializer() + : base(new string[] { "MarketPrice" }) + { } + } + + #endregion + + #region ThrottledMarketPrice + + public class ThrottledPriceSerializer : FixedChangingSerializer + { + public ThrottledPriceSerializer(IBlueTableSchema schema) + : base(schema, + new string[] { "PRICE" }, + "BCG.ThrottledPricePubsubDeserializer", + "BCG.ThrottledPriceClientDeserializer" + ) + { } + } + + public class ThrottledPricePubsubDeserializer : PubsubFixedChangingDeserializer + { + public ThrottledPricePubsubDeserializer() + : base(new string[] { "PRICE" }) + { } + } + + public class ThrottledPriceClientDeserializer : ClientFixedChangingDeserializer + { + public ThrottledPriceClientDeserializer() + : base(new string[] { "PRICE" }) + { } + } + + #endregion + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingDeserializer.cs new file mode 100644 index 000000000..532cd8912 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingDeserializer.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; +using System.Collections; + +using BCG.Utilities; + +namespace BCG +{ + /// + /// Pairs up with FixedSerializer to deserialize messages in the pubsub. + /// + public abstract class PubsubFixedChangingDeserializer : PubsubDeserializer + { + private string[] myChangingCols; + protected BlueTable myCurrentState; + + public PubsubFixedChangingDeserializer (string[] changingColumns) + { + myChangingCols = changingColumns; + } + + protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows) + { + if (myDeserializer == null) + myDeserializer = new PubsubFixedChangingRowDeserializer (mySchema, myChangingCols); + IDictionary retVal = new Hashtable (); + op = op == StreamOpCode.create ? StreamOpCode.add : + op == StreamOpCode.multipleChange ? StreamOpCode.change : op; + for (int i = 0; i < noOfRows; i++) + { + int id; + object row; + myDeserializer.deserializeRow (myState, br, op, out id, out row); + retVal.Add (id, row); + } + return retVal; + } + + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingRowDeserializer.cs new file mode 100644 index 000000000..cfce2c24b --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingRowDeserializer.cs @@ -0,0 +1,84 @@ +using System; +using System.IO; +using System.Collections; + +using BCG.Utilities; + +namespace BCG +{ + + /// + /// Pairs up with FixedSerializer to deserialize messages in the pubsub. + /// + public class PubsubFixedChangingRowDeserializer : IPubsubRowDeserializer + { + private int myFullRowSize; + private PrimitiveDeserializationHelper myDeserHelper; + private PrimitiveSerializationHelper mySerHelper; + private IBlueTableSchema mySchema; + private int[] myOffsets, myChangedColIndices; + private int myIdIndex; + + public PubsubFixedChangingRowDeserializer (IBlueTableSchema schema, params string[] changingColNames) + { + myDeserHelper = new PrimitiveDeserializationHelper (mySchema = schema); + mySerHelper = new PrimitiveSerializationHelper (mySchema); + myIdIndex = mySchema["id"].colNo; + myOffsets = new int[schema.columnInfo.Length]; + //size calulcations + + foreach (ColumnInfo ci in mySchema.columnInfo) + { + myOffsets[ci.colNo] += myFullRowSize; + myFullRowSize += SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType); + } + myChangedColIndices = new int[changingColNames.Length]; + for (int i = 0; i < changingColNames.Length; i++) + myChangedColIndices[i] = mySchema[changingColNames[i]].colNo; + Array.Sort (myChangedColIndices); + } + + #region IPubsubRowDeserializer Members + public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row) + { + if (op == StreamOpCode.add) + myDeserializeAdd (currentState, br, out id, out row); + else if (op == StreamOpCode.remove) + myDeserializeRemove (currentState, br, out id, out row); + else if (op == StreamOpCode.change) + myDeserializeChange (currentState, br, out id, out row); + else + { + id = int.MinValue; + row = null; + // BlueEventLog.logAssert (false, "Illegal operation for this context"); + } + } + #endregion + + protected virtual void myDeserializeAdd (IDictionary currentState, BinaryReader br, out int id, out object row) + { + row = br.ReadBytes (myFullRowSize); + id = BitConverter.ToInt32 (row as byte[], myOffsets[myIdIndex]); + } + + protected virtual void myDeserializeRemove (IDictionary currentState, BinaryReader br, out int id, out object row) + { + id = br.ReadInt32 (); + row = currentState[id]; + } + + protected virtual void myDeserializeChange (IDictionary currentState, BinaryReader br, out int id, out object row) + { + id = br.ReadInt32 (); + BinaryWriter bw = new BinaryWriter (new MemoryStream (currentState[id] as byte[])); + foreach (int colIndex in myChangedColIndices) + { + object newValue = myDeserHelper.deserializeColumn (colIndex, br); + bw.BaseStream.Position = myOffsets[colIndex]; + mySerHelper.serializeColumn (newValue, colIndex, bw); + } + row = (bw.BaseStream as MemoryStream).ToArray (); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs new file mode 100644 index 000000000..61f6c2014 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections; +using System.IO; +using System.IO.Compression; +using BCG.Utilities; +//using BCG.Utilities.BlueConfiguration; + +namespace BCG +{ + public class LookupTableClientDeserializer : IClientDeserializer + { + public LookupTableClientDeserializer() { + + mySchema = new LookupTableSchema(); + myLookupTables = null; + } + + private IClientRowDeserializer myRowDeserializer; + private LookupTableSchema mySchema; + private LookupTable[] myLookupTables; + + public BinaryReader myGetReaderForRestOfStream(Stream theStream) + { + //teh rest of the stream is only compressed data + long compressedBytesLength = theStream.Length - theStream.Position; + byte[] compressedBytes = new byte[compressedBytesLength]; + theStream.Read(compressedBytes, 0, (int)compressedBytesLength); + + + //get a stream to read the uncompressed data + SharpZipCompressor deKompressor = new SharpZipCompressor(); + + MemoryStream decompressed = new MemoryStream(); + deKompressor.decompress(new MemoryStream(compressedBytes), decompressed); + decompressed.Position = 0; + + return new BinaryReader(decompressed); + } + #region IClientDeserializer methods + + public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) + { + ArrayList ops = new ArrayList(); + ArrayList rows = new ArrayList(); + IBlueRowCollection[] result; + foreach (Stream theStream in streams) + { + byte op; + + BinaryReader br = new BinaryReader(theStream); + + op = br.ReadByte(); + + if (op == StreamOpCode.create) + { + br.ReadString(); //pubsub deserializer + br.ReadString(); //client deserializer + + + br = myGetReaderForRestOfStream(theStream); + + result = new IBlueRowCollection[] { myHandleCreate(br) }; + ops.Add(op); + rows.Add(result); + + } + else + { + //the schema and lookup tables should already be created. We just need to update everything. + //figure out how long the rest of the compressed data is + br = myGetReaderForRestOfStream(theStream); + + if (mySchema != null) + { + if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) + { + myDeserializeLookupTableUpdates(br); + result = new IBlueRowCollection[] { myGetRows(br, op, 1) }; + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleChange) + { + myDeserializeLookupTableUpdates(br); + int numRows = br.ReadInt32(); + result = new IBlueRowCollection[] { myGetRows(br, op, numRows) }; + ops.Add(op); + rows.Add(result); + + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + myDeserializeLookupTableUpdates(br); + result = new IBlueRowCollection[2]; + int numDeletedRows = br.ReadInt32(); + result[0] = myGetRows(br, StreamOpCode.remove, numDeletedRows); + int numAddedRows = br.ReadInt32(); + result[1] = myGetRows(br, StreamOpCode.add, numAddedRows); + ops.Add(op); + rows.Add(result); + } + else{} + // BlueEventLog.logEvent("Illegal op-code recieved", BlueEventLevel.Error); + + + } + else + { + // BlueEventLog.logEvent("Received a non create operation even though the schema" + + // "has never been created", BlueEventLevel.Error); + } + + + } + + } + + theOps = ops.ToArray(typeof(byte)) as byte[]; + theRows = rows.ToArray(); + + } + + + #endregion + + IBlueRowCollection myHandleCreate(BinaryReader br) + { + + mySchema.deserialze(br); + + //build the lookup tables now + // There is one less than the length of the lookupTableGroups because the first one + //is always the default group which isn't stored in a lookup table. + myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; + + //start with 1 because we don't store the 0th lookup table + for (int i = 0; i < myLookupTables.Length; ++i) + myLookupTables[i] = new LookupTable(mySchema, i+1); + + myDeserializeLookupTableUpdates(br); + //now read the rows + int numRows = br.ReadInt32(); + return myGetRows(br, StreamOpCode.create, numRows); + + + } + + + private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows) + { + BlueTable retVal = new BlueTable(mySchema); + myRowDeserializer = myRowDeserializer == null ? new LookupTableClientRowDeserializer(mySchema,myLookupTables) : myRowDeserializer; + for (int i = 0; i < nRows; i++) + { + IBlueRow row; + myRowDeserializer.deserializeRow(null, br, op, out row); + retVal.addRow(row); + } + return retVal; + } + + + private void myDeserializeLookupTableUpdates(BinaryReader br) + { + for (int i = 0; i < myLookupTables.Length; ++i) + myLookupTables[i].deserializeUpdates(br); + } + + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs new file mode 100644 index 000000000..d8f794a13 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections; +using System.IO; + +using BCG.Utilities; +//using BCG.Utilities.BlueConfiguration; + +namespace BCG +{ + /// + /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process + /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) + /// + internal class LookupTableClientRowDeserializer : IClientRowDeserializer + { + private LookupTableSchema mySchema; + private LookupTable[] myTables; + private int myNullMapLength = 0; + private PrimitiveDeserializationHelper myDeserializationHelper; + private BitArray myNullMap; + private int myIdColumn; + public LookupTableClientRowDeserializer (LookupTableSchema schema, LookupTable[] theTables) + { + mySchema = schema; + myTables = theTables; + + myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema); + myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8))); + myIdColumn = schema["id"].colNo; + + } + + + + #region IClientRowDeserializer Members + + public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) + { + int rowLength = br.ReadInt32 (); //length information for the row not needed + //rad the null map + myNullMap = new BitArray(br.ReadBytes (myNullMapLength)); + object[] values = new object [mySchema.columnNames.Length]; + //now read the id + values[myIdColumn] = br.ReadInt32(); + //now read the lookup groups + for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo) + { + BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo]; + LookupTable thisTable = myTables[groupNo - 1]; + //read the id in, depending upon the size + int idRead = -1; + switch (thisGroup.tableSize) + { + case 8: + idRead = br.ReadByte(); + break; + case 16: + idRead = br.ReadInt16(); + break; + case 32: + idRead = br.ReadInt32(); + break; + default: + throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specific for lookup group."); + } + + object[] thisGroupValues = thisTable.getValuesForIndex(idRead); + + if (thisGroupValues == null) + throw new BlueDataException("Error: Couldnt' find values for index " + idRead.ToString()); + if (thisGroupValues.Length != thisGroup.groupColNos.Length) + throw new BlueDataException("Error: Values extracted for index " + idRead.ToString() + " had length " + thisGroupValues.Length.ToString() + " but length " + thisGroup.groupColNos.Length.ToString() + " was expected."); + + + //insert those values into our object array + + for (int groupCol = 0; groupCol < thisGroup.groupColNos.Length; ++groupCol) + { + int thisColInOutputRow = thisGroup.groupColNos[groupCol]; + values[thisColInOutputRow] = thisGroupValues[groupCol]; + } + } + for (int colNo = 0; colNo < values.Length; colNo++) + { + + if (myNullMap[colNo]) + { + values[colNo] = DBNull.Value; + } + //if it's non null, not the id column, and not part of a lookup group, get it here. + else if (mySchema[colNo].lookupTableGroup == 0 && colNo != myIdColumn) + { + + values[colNo] = myDeserializationHelper.deserializeColumn(colNo, br); + + } + } + + + r = new BlueRow(mySchema, values); + + } + + #endregion + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs new file mode 100644 index 000000000..08a755f52 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.Collections; +using System.IO; +using System.Linq; +using System.Text; + + +namespace BCG +{ + + /* + * + * LookupTable + * + * This class represents a lookup table used to store values inside data streams. This allows indexes into the table + * to be passed over the wire, instead of actual values of the rows. + * + * Each table can have multiple lookup table groups. This class represents just one of those groups. This means that all the entries + * contained in the table will have the same schema. + * + * + * + * */ + public class LookupTable + { + //each of the entries has the same schema - that is, each of the entries contains the same number of values, with the same + IBlueTableSchema myEntrySchema; + UniversalRowSerializer mySerializer; + ClientUniversalRowDeserializer myDeserializer; + private const int myIdColumn = 0; + + //the table is only allowed to reach a maximum size based upon the number of bits you give to it. + //it throws an error if you try to put too much data into it. + private long myMaxSize; + private int mySizeInBits; + + public int sizeInBits + { + get { return mySizeInBits; } + } + + KeysBasedTable2 myValuesToEntryMap; + + //keep track of the last time somebody published you, so you know what values to publish. + int myIdOfLastPublish; + + int myIdToUse; + + /* builds the local schema, serializer, and desrializer + * initializes to an empty table + */ + public LookupTable(LookupTableSchema theSchema, int lookupGroupNumber) + { + BlueRowLookupGroup group = theSchema.lookupGroups[lookupGroupNumber]; + + myEntrySchema = group.entrySchema; + + //compute how much room you have in the table + mySizeInBits = group.tableSize; + myMaxSize = (long)(Math.Pow(2, group.tableSize) - 1); + + myDeserializer = new ClientUniversalRowDeserializer(myEntrySchema); + mySerializer = new UniversalRowSerializer(myEntrySchema); + + + + //create the map + myValuesToEntryMap = new KeysBasedTable2(myEntrySchema, group.groupColNames); + myIdOfLastPublish = 0; + + + + } + + /*Checks to see if the given values are in the table. IF they are in there, it returns the index to those values. + If they are not present, it adds them to teh table and returns the new index. */ + public int getIndexForValues(params object[] values) + { + + int index = 0; + //see if the key is present + IBlueRow thisRowEntry = myValuesToEntryMap.getRow(values); + + if (thisRowEntry == null) + { + //check to see if we even have room in the table + if (myIdToUse > myMaxSize) + throw new BlueDataException("Error: LookupTable has reached maximum size. Make a bigger table."); + //we need to entrer this new value into the table. Create a blue row using the next ID + //and the values they supplied to us + object[] thisRowValues = new object[myEntrySchema.columnInfo.Length]; + index = myIdToUse; + thisRowValues[myIdColumn] = index; + myIdToUse ++; + //now add in the values they supplied + for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++colNo) + { + thisRowValues[colNo] = values[colNo - 1]; + + } + IBlueRow newEntryRow = new BlueRow(myEntrySchema, thisRowValues); + + myValuesToEntryMap.update(newEntryRow); + } + else + index = (int) thisRowEntry["id"]; + + return index; + + } + + //queries the table to get the values that were stored on a given index. + //returns null if the values are not found + public object[] getValuesForIndex(int index) + { + object[] results = null; + + //get the row from the table + if (myValuesToEntryMap.containsID(index)) + { + IBlueRow thisRow = myValuesToEntryMap[index]; + results = new object[myEntrySchema.columnInfo.Length - 1]; + + //copy the values from the row into the array + for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++ colNo) + { + results[colNo - 1] = thisRow[colNo]; + + } + + } + + //if the index isn't in the tablem, can just return the null value + + + return results; + + } + + public void deserializeUpdates(BinaryReader br) + { + + //get the number of updates that are being published. + int numUpdates = br.ReadInt32(); + + for (int i = 0; i < numUpdates; ++i) + { + + IBlueRow theNewUpdate = null; + myDeserializer.deserializeRow(null, br, StreamOpCode.create, out theNewUpdate); + myValuesToEntryMap.update(theNewUpdate); + + int thisID = (int) theNewUpdate[myIdColumn]; + if (thisID != myIdToUse) + throw new BlueDataException("Error: bad serialized update given to lookup table. Expected id " + myIdToUse.ToString() + ", found " + thisID.ToString()); + ++myIdToUse; + + } + + + } + + public void serializeUpdates(BinaryWriter bw) + { + int numUpdates = myIdToUse - myIdOfLastPublish; + + //serialize the number of updates + bw.Write(numUpdates); + + //serialize each update + for (int i = myIdOfLastPublish; i < myIdToUse; ++i) + mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw); + + //store the last id you published so that next time we don't serialize the same updates + myIdOfLastPublish = myIdToUse; + } + + /*this function serializes EVERYTHING in the table into the writer. This is + used by the pubsub deserializer, which needs to be able to supply the new state + to any clients connection. */ + public void serializeWholeTable(BinaryWriter bw) + { + //serialize the number of updates + bw.Write(myIdToUse); + + //serialize each update + for (int i = 0; i < myIdToUse; ++i) + mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw); + + + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs new file mode 100644 index 000000000..ab8adc94c --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs @@ -0,0 +1,226 @@ +using System; +using System.Collections; +using System.IO; +using BCG.Utilities; +using System.IO.Compression; + +namespace BCG +{ + + public class LookupTablePubSubDeserializer : IPubsubDeserializer + { + protected IDictionary myState = new Hashtable(); + protected LookupTableSchema mySchema; + protected string myClientDeserializerName; //for the getState method + protected IPubsubRowDeserializer myDeserializer; + LookupTable[] myLookupTables; + + public LookupTablePubSubDeserializer() { + + mySchema = new LookupTableSchema(); + + } + + + public BinaryReader myGetReaderForCompressedPortion(Stream theStream) + { + //teh rest of the stream is only compressed data + long compressedBytesLength = theStream.Length - theStream.Position; + byte[] compressedBytes = new byte[compressedBytesLength]; + theStream.Read(compressedBytes, 0, (int)compressedBytesLength); + + //get a stream to read the uncompressed data + SharpZipCompressor deKompressor = new SharpZipCompressor(); + + MemoryStream decompressed = new MemoryStream(); + deKompressor.decompress(new MemoryStream(compressedBytes), decompressed); + decompressed.Position = 0; + + return new BinaryReader(decompressed); + } + + #region IPubsubDeserializer Members + public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) + { + ArrayList ops = new ArrayList(), rows = new ArrayList(); + foreach (Stream theStream in streams) + { + BinaryReader br = new BinaryReader(theStream); + IDictionary[] result; + byte op; + op = br.ReadByte(); + + + + if (op == StreamOpCode.create) + { + myState.Clear(); + br.ReadString(); //the pubsub-deserializer name, should be the same as this's name + myClientDeserializerName = br.ReadString(); //the client-deserializer name + + //at this point, the rest of the bytes in the stream are from a GZIP stream + //so we need to read into a GZipStream and decompress + //figure out how long the rest of the compressed data is + br = myGetReaderForCompressedPortion(theStream); + + + mySchema.deserialze(br); + + //now build the lookup tables from the knowledge gleaned at the schema + myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; + + //start with 1 because we don't store the 0th lookup table + for (int i = 0; i < myLookupTables.Length; ++i) + myLookupTables[i] = new LookupTable(mySchema, i + 1); + + myDeserializeLookupTableUpdates(br); + + int numRows = br.ReadInt32(); + result = new IDictionary[] { myGetRows(br, op, numRows) }; + myEncorporatRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else + { + //figure out how long the rest of the compressed data is + br = myGetReaderForCompressedPortion(theStream); + if (mySchema != null) + { + if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) + { + myDeserializeLookupTableUpdates(br); + result = new IDictionary[] { myGetRows(br, op, 1) }; + myEncorporatRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleChange) + { + myDeserializeLookupTableUpdates(br); + int numRows = br.ReadInt32(); + result = new IDictionary[] { myGetRows(br, op, numRows) }; + myEncorporatRows(result[0], op); + ops.Add(op); + rows.Add(result); + } + else if (op == StreamOpCode.multipleRemoveAdd) + { + myDeserializeLookupTableUpdates(br); + + int numRemoved = br.ReadInt32(); + result = new IDictionary[2]; + + result[0] = myGetRows(br, StreamOpCode.remove, numRemoved); + int numAdded = br.ReadInt32(); + result[1] = myGetRows (br, StreamOpCode.add, numAdded); + + myEncorporatRows(result[0], StreamOpCode.remove); + myEncorporatRows(result[1], StreamOpCode.add); + ops.Add(op); + rows.Add(result); + } + else + { + // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); + } + } + else + { + // BlueEventLog.logEvent("Received a non create operation even though the schema" + + // "has never been created", BlueEventLevel.Warning); + } + } + } + theOps = ops.ToArray(typeof(byte)) as byte[]; + theRows = rows.ToArray(); + } + + /// + /// Sends in a create signal, so that new clients can get a create everytime and start + /// afresh from there + /// + /// + public virtual byte[] getState() + { + if (mySchema == null) + return new byte[] { }; + else + { + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + bw.Write(StreamOpCode.create); + bw.Write(GetType().FullName); + bw.Write(myClientDeserializerName); + + //the rest of this data has to be serialized. + BinaryWriter compressedBw = new BinaryWriter(new MemoryStream()); + mySchema.serialize(compressedBw); + //serialize all the tables + for (int i = 0; i < myLookupTables.Length; ++i) + myLookupTables[i].serializeWholeTable(compressedBw); + compressedBw.Write(myState.Count); + foreach (byte[] row in myState.Values) + compressedBw.Write(row); + + //now compress the bytes + MemoryStream compressedStream = new MemoryStream(); + SharpZipCompressor Kompressor = new SharpZipCompressor(); + compressedBw.BaseStream.Position = 0; + + Kompressor.compress(compressedBw.BaseStream, compressedStream); + + //append the two streams together + byte[] output = new byte[bw.BaseStream.Length + compressedStream.Length]; + bw.BaseStream.Position = 0; + compressedStream.Position = 0; + + + bw.BaseStream.Read(output, 0, (int) bw.BaseStream.Length); + compressedStream.Read(output, (int) bw.BaseStream.Length, (int) compressedStream.Length); + + + + return output; + } + } + #endregion + + private void myDeserializeLookupTableUpdates(BinaryReader br) + { + for (int i = 0; i < myLookupTables.Length; ++i) + myLookupTables[i].deserializeUpdates(br); + } + /// + /// Every row that has been deserialized passes through this method, so that it can update the state. + /// + /// + /// + protected virtual void myEncorporatRows(IDictionary rows, byte op) + { + if (op == StreamOpCode.add || op == StreamOpCode.create) + foreach (DictionaryEntry de in rows) + myState.Add(de.Key, de.Value); + else if (op == StreamOpCode.remove) + foreach (DictionaryEntry de in rows) + myState.Remove(de.Key); + else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) + foreach (DictionaryEntry de in rows) + myState[de.Key] = de.Value; + } + + private IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) + { + //the row deserializer doesn't need to know anything about the lookup tables + myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer; + Hashtable retVal = new Hashtable(); + for (int i = 0; i < noOfRows; i++) + { + int id; object row; + myDeserializer.deserializeRow(myState, br, op, out id, out row); + retVal.Add(id, row); + } + return retVal; + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs new file mode 100644 index 000000000..34ce1f207 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs @@ -0,0 +1,211 @@ +using System; +using System.Collections; +using System.IO; +using BCG.Utilities; +using System.Collections.Generic; +namespace BCG +{ + /// + /// Helps the LookupTableRowSerializer by serializing one row. + /// + /// It does NOT sort the schema, even though the sorted schema doesn't do anything. + /// That is because the schema sorting utility is agnostic of schema type, and so it would destory the grouping information + /// which is VITAL + /// + /// It performs a mapping : (row)->byte[] + /// + /// Row serialized in the following order + /// row size (nullBitMap size + actual row size) + /// null bit map + /// row id + /// row lookup group ids + /// rest of row values + /// + /// + /// + public void serializeRow(IBlueRow r, byte op, BinaryWriter bw) + { + myNullMap.SetAll(false); + int sz = myFixedSize; + //first, build the null map + for (int i = 0; i < mySchema.columnNames.Length; i++) + { + if (r[i] == DBNull.Value || r[i] == null) + { + myNullMap[i] = true; + + //if this thing isn't in a lookup group, it's no longer going to be accounted for in the fixed size of the table + //therefore we need to decrement the size of this row by the size of the type + if (mySchema[i].lookupTableGroup == 0) + { + if (SerializationUtilityMethods.isFixedLengthType(mySchema[i].colType)) + sz -= SerializationUtilityMethods.getFixedLengthTypeSize(mySchema[i].colType); + } + } + + } + //now serialize the null map + myNullMap.CopyTo(myNullMapArray, 0); + int thisRowSize = myGetRowSizeInBytes(sz, r); + bw.Write(thisRowSize); + //tehcnically the size calculatinos are all off by 4 becuase tehy don't account for integer telling how big the row is + long startPos = bw.BaseStream.Position; + bw.Write(myNullMapArray); + //write the ID first, for easy access + bw.Write((int)r[myIdColumn]); + //now write all the value stored in lookup table groups + //we don't do this with the first lookup group becuase it's the default one that isn't stored in lookup tables + for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo) + { + BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo]; + LookupTable thisTable = myTables[groupNo - 1]; + int thisLookupID = thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r)); + + //now, depending on the group size, write this value as either a byte, a short, or an int. + + switch (thisGroup.tableSize) + { + case 8: + bw.Write((byte)thisLookupID); + break; + case 16: + bw.Write((short)thisLookupID); + break; + case 32: + bw.Write(thisLookupID); + break; + default: + throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specified for lookup group."); + + } + + } + //now write the rest of the individual columns + foreach (ColumnInfo ci in mySchema.columnInfo) + //we have already serialized the id column, as well as all the column groups, and we don't serialize any columns that are null + if ( (!myNullMap[ci.colNo]) + && ci.colNo != myIdColumn + && ci.lookupTableGroup == 0) + { + + + myPrimitiveSerializationHelper.serializeColumn(r, ci.colNo, bw); + + } + + long stopPos = bw.BaseStream.Position; + + if (stopPos- startPos != thisRowSize) + throw new BlueDataException("Error: Bad Size Calculation in LookupTableRowSerializer. Expected size was "+thisRowSize.ToString()+", actual size was "+(stopPos-startPos).ToString()); + } + + public void serializeSchema(BinaryWriter bw) + { + mySchema.serialize(bw); + + } + #endregion + + + + + + #region size calculations + /// + /// Sets up the fixed length (size in bytes occupied by each of the primitive type) + /// and an index for the non primitive type + /// + private void myInitSizeCalculations() + { + ArrayList temp = new ArrayList(); + myFixedSize = myNullMapArray.Length; + //go through all the columns that aren't in lookup groups, and add up their sizes + foreach (ColumnInfo ci in mySchema.columnInfo) + { + //if you're in a lookup table group, your size is always fixed. + if (ci.lookupTableGroup == 0) + { + if (SerializationUtilityMethods.isFixedLengthType(ci.colType)) + myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType); + else + temp.Add(ci.colNo); + } + } + //now add on the size of the lookup groups + foreach (BlueRowLookupGroup group in mySchema.lookupGroups) + myFixedSize += (group.tableSize / 8); + + myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[]; + } + + /// + /// Find the size of the serialized row, assumes that nullMap has already + /// been built + /// + /// + /// + /// + private int myGetRowSizeInBytes(int fixedSize, IBlueRow r) + { + foreach (int colNo in myNonFixedColIndexes) + { + if (!myNullMap[colNo]) + fixedSize += //myGetStringSizeInBytes (r[colNo] as string); + SerializationUtilityMethods.getObjectSerializedSize(r[colNo]); + } + return fixedSize; + } + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs new file mode 100644 index 000000000..bc5e130d2 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs @@ -0,0 +1,279 @@ +using System; +using System.IO; +using BCG.Utilities; +using System.Collections.Generic; +using System.IO.Compression; + + +namespace BCG +{ + /// + /// Serializes data stored in all IBlueRowCollections that use a LookupTableSchema. + /// + public class LookupTableSerializer : IBlueDataSerializer + { + private const string myPubSubDeser = "BCG.LookupTablePubSubDeserializer", + myClientDeser = "BCG.LookupTableClientDeserializer"; + + + private LookupTable[] myLookupTables; + private LookupTableSchema mySchema; + private LookupTableRowSerializer myRowSerializer; + + //the header writer writes into a regular stream + private BinaryWriter myHeaderWriter; + //the body writer writes into a compressed binary stream + //that way, we shrink the size of the data that we send over the wire + private BinaryWriter myBodyWriter; + public LookupTableSerializer (IBlueTableSchema inSchema, object[] groups) + { + //get the column infos from the input schema + ColumnInfo[] inputColumns = inSchema.columnInfo; + + //build the lookup table groups + //there is one more group than input string because we have the 'default' group, which isn't stored in a table + int numGroups = groups.Length + 1; + BlueRowLookupGroup[] lookupGroups = new BlueRowLookupGroup[numGroups]; + //we start the groups with group no 1, becuase groups with 0 are not stored in lookup tables + short groupNo = 1; + Dictionary colNameToGroupNo = new Dictionary(); + + foreach (string group in groups) + { + string[] sizeAndColumns = group.Split(';'); + + if (sizeAndColumns.Length != 2) + throw new ArgumentException("Error, bad Lookup Table Group found: " + group); + + short tableSize = short.Parse(sizeAndColumns[0]); + + string[] columns = sizeAndColumns[1].Split(','); + + BlueRowLookupGroup thisGroup = new BlueRowLookupGroup(groupNo, tableSize, columns); + lookupGroups[groupNo] = thisGroup; + foreach (string column in columns) + colNameToGroupNo[column] = groupNo; + + ++groupNo; + + } + + //make the lookup group correpsonding to columns that aren't stored in the lookup table + List nonLookupTableColumns = new List(); + foreach (ColumnInfo ci in inputColumns) + { + if (!colNameToGroupNo.ContainsKey(ci.colName)) + { + nonLookupTableColumns.Add(ci.colName); + } + } + + lookupGroups[0] = new BlueRowLookupGroup(0, 0, nonLookupTableColumns.ToArray()); + + + + //now build new column infos to replace the old ones + int numCols = inputColumns.Length; + ColumnInfo[] newColumns = new ColumnInfo[numCols]; + + for (int col = 0; col < numCols; ++col) + { + ColumnInfo inCol = inputColumns[col]; + if (inCol.isConstant) + newColumns[col] = new ColumnInfo(inCol.colName, col, inCol.colType, inCol.colValue); + else + { + string colName = inCol.colName; + if (!colNameToGroupNo.ContainsKey(inCol.colName)) + newColumns[col] = new ColumnInfo(colName, col, inCol.colType); + else + newColumns[col] = new ColumnInfo(colName, col, inCol.colType, colNameToGroupNo[colName]); + } + + } + + //now build the schema + mySchema = new LookupTableSchema(inSchema.tableName, newColumns, lookupGroups); + + + //build the lookup tables. There is one less than the length of the lookupTableGroups because the first one + //is always the default group which isn't stored in a lookup table. + myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; + + //start with 1 because we don't store the 0th lookup table + for (int i = 0; i < myLookupTables.Length; ++i) + { + myLookupTables[i] = new LookupTable(mySchema, i+1); + } + + //now create the row serializer + myRowSerializer = new LookupTableRowSerializer(mySchema, myLookupTables); + + } + + #region IBlueDataSerializer Members + /// + /// Serializes the data related to a single op. + /// Supports All Ops + /// + /// First, a LookupTableUpdate, which is contained in each op, looks like this: + /// + /// + /// -LookupTableUpdate: + /// group 1 values size \ + /// group 1 value |- all serialized by a single LookupTableObject + /// group 1 value / + /// .... + /// group 2 values size + /// group 2 value key + /// .... + /// + /// + /// Next, the stream formats of the different op codes + /// + /// + /// -create : + /// op + /// clientDeser name + /// pubsubsDeser name + /// schema (which is a LookupTableSchema) + /// LookupTableUpdate + /// num rows + /// row 1 + /// row 2 + /// .... + /// + /// + /// - add, remove, change: + /// op + /// LookupTableUpdate + /// the row + /// + /// -multipleRemoveAdd : + /// op + /// LookupTableUpdate + /// num deleted rows + /// deleted row 1 + /// deleted row 2 + /// .... + /// num added rows + /// added row 1 + /// added row 2 + /// ... + /// + /// -multipleChange + /// op + /// LookupTableUpdate + /// num changed rows + /// changd row 1 + /// changed row 2 + /// ... + /// changd row n + /// + /// + /// + /// + /// + public void serialize(byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData) + { + myHeaderWriter = new BinaryWriter(new MemoryStream()); + myBodyWriter = new BinaryWriter(new MemoryStream()); + //everything start with an op + myHeaderWriter.Write(op); + + /*go through each of the rows we're about the serialize, and populate the lookup table + * that way, when we serialize the lookup tables, well have something to report*/ + foreach (IBlueRowCollection collection in rows) + foreach (IBlueRow r in collection) + myRowSerializer.updateLookupTables(r); + + if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change) + mySerializeSingleRow(op, BlueDataUtility.extractRow(rows[0], 0)); + else if (op == StreamOpCode.create) + mySerializeCreate(rows[0]); + else if (op == StreamOpCode.multipleChange) + mySerializeMultipleChange(rows[0]); + else if (op == StreamOpCode.multipleRemoveAdd) + mySerializeMultipleRemoveAdd(rows[0], rows[1]); + else + { + serializedData = null; + // BlueEventLog.logAssert(false, String.Format("{0} is not a valid op", op)); + } + + + + //copy the data from the header and the body of the message + myHeaderWriter.BaseStream.Position = 0; + myBodyWriter.BaseStream.Position = 0; + int headerLength = (int) myHeaderWriter.BaseStream.Length; + + + + MemoryStream zippedMemStream = new MemoryStream(); + SharpZipCompressor Kompressor = new SharpZipCompressor(); + + Kompressor.compress(myBodyWriter.BaseStream,zippedMemStream); + byte[] compressedBytes = zippedMemStream.ToArray(); + + + serializedData = new byte[headerLength + zippedMemStream.Length]; + + Console.WriteLine(String.Format("Lookuptable Serializer: Header is {0}. Body compresesd from {1} to {2} bytes.", headerLength , myBodyWriter.BaseStream.Length, zippedMemStream.Length)); + + + myHeaderWriter.BaseStream.Read(serializedData, 0, headerLength); + BinaryWriter sizeWriter = new BinaryWriter(new MemoryStream(serializedData)); + zippedMemStream.Position = 0; + zippedMemStream.Read(serializedData, headerLength, (int) zippedMemStream.Length); + + + } + #endregion + + void mySerializeSingleRow(byte op, IBlueRow row) + { + mySerializeLookupTableUpdates(myBodyWriter); + myRowSerializer.serializeRow(row, op, myBodyWriter); + } + + private void mySerializeCreate (IBlueRowCollection rows) + { + myHeaderWriter.Write(myPubSubDeser); + myHeaderWriter.Write(myClientDeser); + mySchema.serialize(myBodyWriter); + mySerializeLookupTableUpdates(myBodyWriter); + myBodyWriter.Write(rows.Count); + foreach (IBlueRow r in rows) + myRowSerializer.serializeRow(r, StreamOpCode.create, myBodyWriter); + } + + private void mySerializeMultipleChange(IBlueRowCollection rows) + { + mySerializeLookupTableUpdates(myBodyWriter); + myBodyWriter.Write(rows.Count); + foreach (IBlueRow r in rows) + myRowSerializer.serializeRow(r,StreamOpCode.change,myBodyWriter); + + } + + private void mySerializeMultipleRemoveAdd(IBlueRowCollection deletedRows, IBlueRowCollection addedRows) + { + mySerializeLookupTableUpdates(myBodyWriter); + myBodyWriter.Write(deletedRows.Count); + foreach (IBlueRow r in deletedRows) + myRowSerializer.serializeRow(r, StreamOpCode.change, myBodyWriter); + myBodyWriter.Write(addedRows.Count); + foreach (IBlueRow r in addedRows) + myRowSerializer.serializeRow(r, StreamOpCode.add, myBodyWriter); + + } + + private void mySerializeLookupTableUpdates(BinaryWriter bw) + { + for (int i = 0; i < myLookupTables.Length; ++i) + myLookupTables[i].serializeUpdates(bw); + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/ClientDeserialization/ClientTheoDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/ClientDeserialization/ClientTheoDeserializer.cs new file mode 100644 index 000000000..2d46651b7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/ClientDeserialization/ClientTheoDeserializer.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; + +namespace BCG +{ + public class ClientTheoDeserializer : IClientDeserializer + { + private IBlueTableSchema mySchema = null; + + public ClientTheoDeserializer() {} + + #region IClientDeserializer Members + + /// + /// Deserialized the streams of bytes, as one operation and the resulting rows. The op + /// is either a create or multipleChange. + /// + /// + /// + /// + public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows) + { + bool isCreate = false; + BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams)); + byte op = Byte.MaxValue; + IBlueRowCollection rows = null; + while (true) + { + try + { + op = br.ReadByte (); + } + catch (EndOfStreamException) + { + break; + } + if (op == StreamOpCode.create) + { + isCreate = true; //we crush all ops (that may be part of this call) but if theres a create we want that to be the + //only op + //ie, we dont want creates to be crushed, as this might lead to clients + //not detecting creates (not sure if its really worth it) + br.ReadString (); //pubsubDeserializer + br.ReadString (); //clientDeserializer + (mySchema = new BlueTableSchema ()) .deserialze (br); + } + rows = myGetRows (br, br.ReadInt32 ()); + } + theOps = new byte[] {isCreate ? StreamOpCode.create : op}; + theRows = new IBlueRowCollection[] {rows}; + } + + private IBlueRowCollection myGetRows (BinaryReader br, int n) + { + IBlueRow[] rows = new IBlueRow[n]; + for (int i = 0; i < n; i++) + rows[i] = new BlueRow (mySchema, new object[]{ + i, + br.ReadByte (), + br.ReadByte (), + new DateTime (br.ReadInt64 ()), + br.ReadDouble () + }); + return new SequentialIDBlueTable (mySchema, rows); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/PubsubDeserialization/PubsubTheoDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/PubsubDeserialization/PubsubTheoDeserializer.cs new file mode 100644 index 000000000..558205f48 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/PubsubDeserialization/PubsubTheoDeserializer.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; +using System.Collections; + +namespace BCG +{ + public class PubsubTheoDeserializer : IPubsubDeserializer + { + private string myClientDeserializerName; + private int myNRows; + private byte[] myState; + private int mySizeOfRow ; + private IBlueTableSchema mySchema; + + public PubsubTheoDeserializer() + { + mySizeOfRow = BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.LONG) + + BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.BYTE) + + BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.DOUBLE) + + BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.BYTE); + } + #region IPubsubDeserializer Members + + public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows) + { + byte op = Byte.MaxValue; + BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams)); + while (true) + { + try + { + op = br.ReadByte (); + } + catch (EndOfStreamException) + { + break; + } + if (op == StreamOpCode.create) + { + br.ReadString (); //pubsubDeser name + myClientDeserializerName = br.ReadString (); + (mySchema = new BlueTableSchema ()).deserialze (br); + } + myState = br.ReadBytes ((myNRows = br.ReadInt32()) * mySizeOfRow); + } + theOps = new byte[] {op}; + theRows = new object[] {myState}; + } + + public byte[] getState() + { + if (myState == null) + return new byte[] {}; + else + { + BinaryWriter bw = new BinaryWriter (new MemoryStream ()); + bw.Write (StreamOpCode.create); + bw.Write (GetType().FullName); + bw.Write (myClientDeserializerName); + mySchema.serialize (bw); + bw.Write (myNRows); + bw.Write (myState); + return (bw.BaseStream as MemoryStream).ToArray (); + } + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/Serialization/TheoSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/Serialization/TheoSerializer.cs new file mode 100644 index 000000000..5bc1a7f1d --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/Serialization/TheoSerializer.cs @@ -0,0 +1,98 @@ +using System; +using System.IO; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Serializes the theos, as one table. This is independent of the operation; we transmit + /// the whole state everytime theres a change of any kind. However the operation is also + /// serialized as part of the data; its eaither a create of batchChange + /// + public class TheoSerializer : IBlueDataSerializer + { + private const string myPubSubDeser = "BCG.PubsubTheoDeserializer", + myClientDeser = "BCG.ClientTheoDeserializer"; + + public TheoSerializer (IBlueTableSchema schema) {} + + public TheoSerializer () {} + + #region IBlueDataSerializer Members + /// + /// Serializes the data related to a single op. The only ops written are either a create + /// or multipleChange + /// -create + /// op + /// clientDeser name + /// pubsubsDeser name + /// schema + /// nRows + /// rows + /// -multipleChange + /// op + /// nRows + /// rows + /// This method extracts the rows accprding to the op, massages the op either to be a create + /// or multipleChange and passes it on to mySerialize + /// + /// + /// + /// + public void serialize (byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData) + { + IBlueRowCollection totalRows = null; + if (op == StreamOpCode.create || op == StreamOpCode.multipleChange) + totalRows = rows[0]; + else if (op == StreamOpCode.multipleRemoveAdd) + totalRows = rows[1]; + else + // BlueEventLog.logAssert (false, String.Format ("{0} is not a valid op", op)); + op = op == StreamOpCode.multipleRemoveAdd ? StreamOpCode.multipleChange : op; + serializedData = mySerialize (op, totalRows); + } + + /// + /// expects a create or multipleChange. + /// + /// + /// + /// + private byte[] mySerialize (byte op, IBlueRowCollection rows) + { + BinaryWriter bw = new BinaryWriter (new MemoryStream ()); + bw.Write (op); + if (op == StreamOpCode.create) + { + bw.Write (myPubSubDeser); + bw.Write (myClientDeser); + myReorderSchema (rows.schema).serialize (bw); + } + bw.Write (rows.Count); + foreach (IBlueRow r in rows) + { + bw.Write ((byte)r["arrayIndex"]); + bw.Write ((byte)r["cell"]); + bw.Write ((long)((DateTime)r["expiration"]).Ticks); + bw.Write ((double)r["value"]); + } + bw.BaseStream.Position = 0; + return (bw.BaseStream as MemoryStream).ToArray (); + } + + #endregion + + public static IBlueTableSchema getSchema() + {return myReorderSchema(new BlueTableSchema("serializedTheos"));} + + private static IBlueTableSchema myReorderSchema (IBlueTableSchema schema) + { + return new BlueTableSchema (schema.tableName, + new ColumnInfo ("id", 0, typeof(int)), + new ColumnInfo ("arrayIndex", 1, typeof(byte)), + new ColumnInfo ("cell", 2, typeof(byte)), + new ColumnInfo ("expiration", 3, typeof(DateTime)), + new ColumnInfo ("value", 4, typeof(double))); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Misc/IPubSubRouter.cs b/src/DataStreamSuite/DataStreamCore/Misc/IPubSubRouter.cs new file mode 100644 index 000000000..13c3e217c --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Misc/IPubSubRouter.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public delegate void PubSubTagURLChangeDelegate(List tagsMoving, string newURL); + + public interface IPubSubRouter + { + + event PubSubTagURLChangeDelegate onSubURLTagRouteChange; + event PubSubTagURLChangeDelegate onPubURLTagRouteChange; + List getAllPubSubSubURLs(); + List getAllPubSubPubURLs(); + string getSubURLForTag(string tag); + string getPubURLForTag(string tag); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Misc/Pair.cs b/src/DataStreamSuite/DataStreamCore/Misc/Pair.cs new file mode 100644 index 000000000..3fb2e52ec --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Misc/Pair.cs @@ -0,0 +1,90 @@ +// //using ProtoBuf; + +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; +// using System.Threading.Tasks; +// //using System.Windows; +// //using System.Windows.Media; + +// namespace mbts +// { +// // TODO move it out or replace it with Tuple +// [Serializable] +// public class Pair +// { +// public F first; +// public S second; + +// public Pair() { } +// public Pair(F f, S s) +// { +// first = f; +// second = s; +// } + +// public Pair(KeyValuePair kvp) +// : this(kvp.Key, kvp.Value) { } + +// public override string ToString() +// { +// return String.Format("{0},{1}", first != null ? first.ToString() : "NULL", +// second != null ? second.ToString() : "NULL"); +// } + +// public override int GetHashCode() +// { +// return ToString().GetHashCode(); +// } + +// public override bool Equals(object obj) +// { +// return obj != null && obj is Pair && ToString() == obj.ToString(); +// } +// } + +// [Serializable] +// public class Triplet +// { +// public F first; +// public S second; +// public T third; + +// public Triplet() { } + +// public Triplet(Tuple tp) +// { +// first = tp.Item1; +// second = tp.Item2; +// third = tp.Item3; +// } + +// public Triplet(F f, S s, T t) +// { +// first = f; +// second = s; +// third = t; +// } + +// public static Triplet from_tuple(Tuple t) { return new Triplet(t.Item1, t.Item2, t.Item3); } + +// public override string ToString() +// { +// return String.Format("{0},{1},{2}", first != null ? first.ToString() : "NULL", +// second != null ? second.ToString() : "NULL", +// third != null ? third.ToString() : "NULL"); +// } + +// public override bool Equals(object obj) +// { +// return obj != null && obj is Triplet && ToString() == obj.ToString(); +// } + +// public override int GetHashCode() +// { +// return ToString().GetHashCode(); +// } +// } + +// } diff --git a/src/DataStreamSuite/DataStreamCore/Misc/SerializationUtilityMethods.cs b/src/DataStreamSuite/DataStreamCore/Misc/SerializationUtilityMethods.cs new file mode 100644 index 000000000..4c498ba5d --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Misc/SerializationUtilityMethods.cs @@ -0,0 +1,339 @@ +// using System; +// using System.IO; +// using System.Collections; + +// namespace BCG +// { +// public class SerializationUtilityMethods +// { +// private static IDictionary mySerializationDispatchTable, +// myDeserializationDispatchTable; +// private delegate void MySerialize (object data, BinaryWriter bw); +// private delegate object MyDeserialize (Type t, BinaryReader br); + +// static SerializationUtilityMethods () +// { +// mySerializationDispatchTable = new Hashtable (); +// mySerializationDispatchTable [BlueDataValidTypes.BOOL.Name] = +// new MySerialize (mySerializeBool); +// mySerializationDispatchTable [BlueDataValidTypes.BYTE.Name] = +// new MySerialize (mySerializeByte); +// mySerializationDispatchTable [BlueDataValidTypes.CHAR.Name] = +// new MySerialize (mySerializeChar); +// mySerializationDispatchTable [BlueDataValidTypes.DATETIME.Name] = +// new MySerialize (mySerializeDateTime); +// mySerializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] = +// new MySerialize (mySerializeDecimal); +// mySerializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] = +// new MySerialize (mySerializeDouble); +// mySerializationDispatchTable [BlueDataValidTypes.INT.Name] = +// new MySerialize (mySerializeInt32); +// mySerializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] = +// new MySerialize (mySerializeISerializedType); +// mySerializationDispatchTable [BlueDataValidTypes.LONG.Name] = +// new MySerialize (mySerializeLong); +// mySerializationDispatchTable [BlueDataValidTypes.SBYTE.Name] = +// new MySerialize (mySerializeSByte); +// mySerializationDispatchTable [BlueDataValidTypes.STRING.Name] = +// new MySerialize (mySerializeString); +// mySerializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] = +// new MySerialize (mySerializeByteArray); +// mySerializationDispatchTable [BlueDataValidTypes.UINT.Name] = +// new MySerialize (mySerializeUInt); +// mySerializationDispatchTable [BlueDataValidTypes.ULONG.Name] = +// new MySerialize (mySerializeULong); +// mySerializationDispatchTable [BlueDataValidTypes.SINGLE.Name] = +// new MySerialize (mySerializeSingle); + +// myDeserializationDispatchTable = new Hashtable (); +// myDeserializationDispatchTable [BlueDataValidTypes.BOOL.Name] = +// new MyDeserialize (myDeserializeBool); +// myDeserializationDispatchTable [BlueDataValidTypes.BYTE.Name] = +// new MyDeserialize (myDeserializeByte); +// myDeserializationDispatchTable [BlueDataValidTypes.CHAR.Name] = +// new MyDeserialize (myDeserializeChar); +// myDeserializationDispatchTable [BlueDataValidTypes.DATETIME.Name] = +// new MyDeserialize (myDeserializeDateTime); +// myDeserializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] = +// new MyDeserialize (myDeserializeDecimal); +// myDeserializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] = +// new MyDeserialize (myDeserializeDouble); +// myDeserializationDispatchTable [BlueDataValidTypes.INT.Name] = +// new MyDeserialize (myDeserializeInt32); +// myDeserializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] = +// new MyDeserialize (myDeserializeISerializedType); +// myDeserializationDispatchTable [BlueDataValidTypes.LONG.Name] = +// new MyDeserialize (myDeserializeLong); +// myDeserializationDispatchTable [BlueDataValidTypes.SBYTE.Name] = +// new MyDeserialize (myDeserializeSByte); +// myDeserializationDispatchTable [BlueDataValidTypes.STRING.Name] = +// new MyDeserialize (myDeserializeString); +// myDeserializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] = +// new MyDeserialize (myDeserializeByteArray); +// myDeserializationDispatchTable [BlueDataValidTypes.UINT.Name] = +// new MyDeserialize (myDeserializeUInt); +// myDeserializationDispatchTable [BlueDataValidTypes.ULONG.Name] = +// new MyDeserialize (myDeserializeULong); +// myDeserializationDispatchTable [BlueDataValidTypes.SINGLE.Name] = +// new MyDeserialize (myDeserializeSingle); +// } + +// public static int getObjectSerializedSize (object o) +// { +// Type t = o.GetType (); +// if (t == BlueDataValidTypes.STRING) +// { +// int l = (o as string).Length; +// if (l < 128) +// return l + 1; +// if (l < 16384) +// return l + 2; +// if (l < 2097152) +// return l + 3; +// if (l < Math.Pow (2, 28)) +// return l + 4; +// throw new ApplicationException ("String too big"); +// } +// else if (t == BlueDataValidTypes.BYTEARRAY) +// { +// return getFixedLengthTypeSize (BlueDataValidTypes.INT) + +// (o as byte[]).Length; +// } +// else if (isSerializedType (t)) +// { +// return (o as ISerializedType).getSerializedSize (); +// } +// return getFixedLengthTypeSize (t); +// } + +// public static int getFixedLengthTypeSize (Type t) +// { +// if (t == BlueDataValidTypes.BOOL || +// t == BlueDataValidTypes.BYTE || +// t == BlueDataValidTypes.CHAR || +// t == BlueDataValidTypes.SBYTE) +// return 1; +// else if (t == BlueDataValidTypes.INT || +// t == BlueDataValidTypes.UINT|| +// t == BlueDataValidTypes.SINGLE) +// return 4; +// else if (t == BlueDataValidTypes.DOUBLE || +// t == BlueDataValidTypes.LONG || +// t== BlueDataValidTypes.ULONG) +// return 8; +// else if (t == BlueDataValidTypes.DECIMAL) +// return 16; +// else if (t == BlueDataValidTypes.DATETIME) +// //as objects of type DateTime are stored as doubles +// return getFixedLengthTypeSize (typeof(long)); +// else +// throw new ApplicationException (t.FullName + " is not a supported type"); + +// } + +// public static bool isFixedLengthType (Type t) +// { +// return !(t.Equals (BlueDataValidTypes.STRING) || t.Equals (BlueDataValidTypes.BYTEARRAY) || +// isSerializedType (t)); +// } + +// public static bool isSerializedType (Type t) +// { +// return t.GetInterface ("BCG.ISerializedType") != null; +// } + +// #region Serialization +// public static void serializeValue (object data, BinaryWriter bw) +// { +// Type t = data.GetType (); +// if (isSerializedType (t)) +// (mySerializationDispatchTable [typeof(ISerializedType).Name] +// as MySerialize)(data, bw); +// else +// (mySerializationDispatchTable [t.Name] as MySerialize) (data, bw); +// } +// public static void serializeEnum(object data, BinaryWriter bw) +// { +// mySerializeInt32(data, bw); +// } +// private static void mySerializeInt32 (object data, BinaryWriter bw) +// { +// bw.Write ((Int32)data); +// } + +// private static void mySerializeDouble (object data, BinaryWriter bw) +// { +// bw.Write ((double)data); +// } + +// private static void mySerializeString (object data, BinaryWriter bw) +// { +// bw.Write (data as string); +// } + +// private static void mySerializeByteArray (object data, BinaryWriter bw) +// { +// byte[] bs = data as byte[]; +// bw.Write (bs.Length); +// bw.Write (bs); +// } + + +// private static void mySerializeSingle (object data, BinaryWriter bw) +// { +// bw.Write ((Single)data); +// } + +// private static void mySerializeChar (object data, BinaryWriter bw) +// { +// bw.Write ((char)data); +// } + +// private static void mySerializeLong (object data, BinaryWriter bw) +// { +// bw.Write ((long)data); +// } + +// private static void mySerializeSByte (object data, BinaryWriter bw) +// { +// bw.Write ((sbyte)data); +// } + +// private static void mySerializeBool (object data, BinaryWriter bw) +// { +// bw.Write ((bool)data); +// } + +// private static void mySerializeByte (object data, BinaryWriter bw) +// { +// bw.Write ((byte)data); +// } + +// private static void mySerializeDecimal (object data, BinaryWriter bw) +// { +// bw.Write ((decimal)data); +// } + +// private static void mySerializeUInt (object data, BinaryWriter bw) +// { +// bw.Write ((uint)data); +// } + +// private static void mySerializeULong (object data, BinaryWriter bw) +// { +// bw.Write ((ulong)data); +// } + +// private static void mySerializeDateTime (object data, BinaryWriter bw) +// { +// //DateTime is written is serialiazed as OADATE +// bw.Write (((DateTime)data).ToOADate()); +// } + +// private static void mySerializeISerializedType (object data, BinaryWriter bw) +// { +// (data as ISerializedType).serialize (bw); +// } +// #endregion + +// #region Deserialization + +// public static object deserializeValue(Type t, BinaryReader br) +// { +// return myDeserialize(t, br); +// } +// private static object myDeserialize (Type t, BinaryReader br) +// { +// return isSerializedType (t) ? +// (myDeserializationDispatchTable [typeof(ISerializedType).Name] +// as MyDeserialize)(t, br) +// : +// (myDeserializationDispatchTable [t.Name] as MyDeserialize) (t, br); +// } + +// public static object deserializeEnum(Type t, BinaryReader br) +// { +// return myDeserializeInt32(t, br); +// } + +// private static object myDeserializeInt32 (Type t, BinaryReader br) +// { +// return br.ReadInt32 (); +// } + +// private static object myDeserializeISerializedType (Type t, BinaryReader br) +// { +// ISerializedType o = Activator.CreateInstance (t) +// as ISerializedType; +// o.deserialze (br); +// return o; +// } + +// private static object myDeserializeDouble (Type t, BinaryReader br) +// { +// return br.ReadDouble (); +// } + +// private static object myDeserializeString (Type t, BinaryReader br) +// { +// return br.ReadString (); +// } + +// private static object myDeserializeByteArray (Type t, BinaryReader br) +// { +// return br.ReadBytes (br.ReadInt32 ()); +// } + +// private static object myDeserializeSingle (Type t, BinaryReader br) +// { +// return br.ReadSingle (); +// } + +// private static object myDeserializeChar (Type t, BinaryReader br) +// { +// return br.ReadChar (); +// } + +// private static object myDeserializeLong (Type t, BinaryReader br) +// { +// return br.ReadInt64 (); +// } + +// private static object myDeserializeSByte (Type t, BinaryReader br) +// { +// return br.ReadSByte (); +// } + +// private static object myDeserializeBool (Type t, BinaryReader br) +// { +// return br.ReadBoolean (); +// } + +// private static object myDeserializeByte (Type t, BinaryReader br) +// { +// return br.ReadByte (); +// } + +// private static object myDeserializeDecimal (Type t, BinaryReader br) +// { +// return br.ReadDecimal (); +// } + +// private static object myDeserializeUInt (Type t, BinaryReader br) +// { +// return br.ReadUInt32 (); +// } + +// private static object myDeserializeDateTime (Type t, BinaryReader br) +// { +// return DateTime.FromOADate(br.ReadDouble()); +// //return new DateTime(br.ReadInt64()); +// } + +// private static object myDeserializeULong (Type t, BinaryReader br) +// { +// return br.ReadUInt64 (); +// } +// #endregion +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/TablePacket.cs b/src/DataStreamSuite/DataStreamCore/TablePacket.cs new file mode 100644 index 000000000..89a0e75d3 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/TablePacket.cs @@ -0,0 +1,26 @@ +using System; +using System.Data; +using System.Collections; + +namespace BCG +{ + /// + /// Defines the type that will be passed as the callback from the TableStream class + /// + public class TablePacket + { + public DataTable newTable = new DataTable(); + public DataTable addedRows = new DataTable(); + public DataTable deletedRows = new DataTable(); + + public TablePacket () {} + + public TablePacket (DataTable nt, DataTable ar, DataTable dr) + { + newTable = nt; + addedRows = ar; + deletedRows = dr; + } + + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 4d41d25ec..9f8ae1e02 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -3,9 +3,18 @@ using mbts.config; using System; +using bts; +using mbts; + +string[] args_ = new string[] { "", "", }; +bts.utils.BTSNativeMethods.provide_command_line(args_, 0); +Configuration.initialize(); BTSNativeMethods.provide_command_line(args, args.Length); // Must initialize config before using log system Console.WriteLine("Initializing config..."); Configuration.initialize(); -bts.Log.Gen.info("Starting DataStreamPublisher..."); +for (int i = 0; i < 10; i++) +{ + bts.Log.Gen.info($"Starting DataStreamPublisher...log: {i}"); +} diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj b/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj index 00076cc97..b7bcc383f 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj @@ -1,5 +1,11 @@ + + + + + + Exe net7.0-windows diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs index 636486e70..6cef82dcf 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs @@ -1,3 +1,79 @@ using System; +using BCG; +using System.IO; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using bts; +using mbts; +using mbts.config; +using bts.utils; +// BTSNativeMethods.provide_command_line(); +// Configuration.initialize(); + +bts.Log.Gen.info($"Starting DSQA"); Console.WriteLine("Starting DataStreamQueryAnalyzer..."); +TestPubliser.publish(); +//TestPubliser.subscribe(); + +class TestPubItem : IFormatter, IPubItem +{ + public DateTime send_time = DateTime.Now; + public object Deserialize(Stream str) + { + BinaryReader br = new BinaryReader(str); + var d = br.ReadDouble(); + TestPubItem rv = new TestPubItem(); + rv.send_time = DateTime.FromOADate(d); + return rv; + } + public void Serialize(Stream str, Object obj) + { + TestPubItem tpi = obj as TestPubItem; + BinaryWriter bw = new BinaryWriter(str); + bw.Write(tpi.send_time.ToOADate()); + } + public System.Runtime.Serialization.SerializationBinder? Binder { get; set; } + + public System.Runtime.Serialization.StreamingContext Context { get; set; } + + public System.Runtime.Serialization.ISurrogateSelector? SurrogateSelector { get; set; } + + public string getTag() + { + return "Test_Tag"; + } +} + +class TestPubliser +{ + public static void publish() + { + int n_pubs = 1000; + Publisher publisher = new Publisher(); + for (int i = 0; i < n_pubs; i++) + { + TestPubItem pi = new TestPubItem(); + pi.send_time = DateTime.Now; + bts.Log.Gen.info($"DSQA publishing tag: {pi.getTag()}"); + publisher.publish(pi); + System.Threading.Thread.Sleep(100); + MasterWorker.getInstance(false).doWork(); + } + } + public static void subscribe() + { + ManagedSubscriber ms = new ManagedSubscriber(); + ms.subscribe("Test_Tag", new Handler(on_data)); + while (true) + { + ms.doWork(); + } + } + public static void on_data(IPubItem pi) + { + Console.WriteLine("Received data"); + } +} + + diff --git a/src/DataStreamSuite/PubSub/AllPublishedTagState.cs b/src/DataStreamSuite/PubSub/AllPublishedTagState.cs new file mode 100644 index 000000000..e54eafe9d --- /dev/null +++ b/src/DataStreamSuite/PubSub/AllPublishedTagState.cs @@ -0,0 +1,100 @@ +using System; +using System.IO; +using System.Collections; +using System.Collections.Generic; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Holds the state for a single tag. If the tagType has an + /// IPubSubServerTagHandler, it is updated and the state stored + /// there. Otherwise, this class saves the last published data. + /// + internal class SingleTagHandler + { + private IPubSubServerTagHandler myHandler = null; + private string myTagType, myTag; + /// + /// this will only play a role when theres no myHandler + /// + private byte[] myData = null; + + public SingleTagHandler(string tag, string tagType) + { + + myTagType = tagType; + myTag = tag; + myHandler = TagType2HandlerMap.getHandler(tagType, false); + } + + public string tagType { get { return myTagType; } } + + public void publish(Stream data) + { + long pos = data.Position; + if (myHandler != null) + { + myHandler.update(myTag, new Stream[] { data }); + } + else + { + myData = new byte[data.Length]; + data.Read(myData, 0, myData.Length); + } + data.Position = pos; + } + + public byte[] getContent() + { + return myHandler == null ? myData : myHandler.getContent(myTag); + } + } + + /// + /// Keeps the state of all the tags published. + /// + public class AllPublishedTagState + { + private Dictionary myTagHandler = new Dictionary(); + + public AllPublishedTagState() { } + + public void publish(string tag, string tagType, Stream data) + { + if (!myTagHandler.ContainsKey(tag)) + myTagHandler.Add(tag, new SingleTagHandler(tag, tagType)); + + myTagHandler[tag].publish(data); + } + + public void publish(string tag, string tagType, byte[] data) + { + publish(tag, tagType, new MemoryStream(data)); + } + + public string[] publishedTags + { get { return new ArrayList(myTagHandler.Keys).ToArray(typeof(string)) as string[]; } } + + public byte[] getContent(string tag, out string tagType) + { + tagType = myTagHandler[tag].tagType; + return myTagHandler[tag].getContent(); + } + + public bool contains(string tag) + { + return myTagHandler.ContainsKey(tag); + } + public void clear(string tag) + { + if (myTagHandler.ContainsKey(tag)) + { + myTagHandler.Remove(tag); + } + else{} + // BlueEventLog.logEvent("AllPublishTagState told to clear nonexisting tag " + tag, BlueEventLog2Level.Warning); + + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/BlutoSubAddress.cs b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/BlutoSubAddress.cs new file mode 100644 index 000000000..71b0ca803 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/BlutoSubAddress.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Text; +//using BCG.Utilities; +using System.Net; + +namespace BCG +{ + public class BlutoSubAddress : SubAddress + { + private string myIp; + private int myPort; + + public static string prefix = "bluto"; + + public BlutoSubAddress(string ip, int port) + { + myIp = ip; + myPort = port; + } + + public BlutoSubAddress(string serverAddress) + { + string[] toks = serverAddress.Split(':'); + // TODO + // BlueEventLog.logAssert(toks.Length == 3); + // BlueEventLog.logAssert(toks[0].ToLower().StartsWith(prefix)); + + string machineNameOrIP = toks[1].Remove(0, 2); + + if (machineNameOrIP.Split('.').Length == 4) + myIp = toks[1].Remove(0, 2); + else + { + //get the IP address from DNS + IPAddress[] theAddressList = Dns.GetHostEntry(machineNameOrIP).AddressList; + //BlueEventLog.logAssert(theAddressList.Length == 1, "Error: machine has multiple ip addresses."); + //if we get multiple addresses, we want the one that is of the corret dotted notation + bool oneWorked = false; + foreach (IPAddress addr in theAddressList) + { + if (addr.AddressFamily != System.Net.Sockets.AddressFamily.InterNetworkV6) + { + myIp = addr.ToString(); + oneWorked = true; + + } + } + + if (!oneWorked) + { + throw new ArgumentException("Couldn't form server address from string " + serverAddress); + } + + + } + myPort = Convert.ToInt32(toks[2]); + } + + + + public string ip + { + get { return myIp; } + } + + public int port + { + get { return myPort; } + } + + public override string ToString() + { + return prefix + "://" + ip + ":" + port; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/ClientToServerConnectionWrapper.cs b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/ClientToServerConnectionWrapper.cs new file mode 100644 index 000000000..2aaa3b59f --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/ClientToServerConnectionWrapper.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Runtime.InteropServices; + +namespace BCG +{ + public class ClientToServerConnectionWrapper + { + [DllImport("BlutoInterop.dll")] + public unsafe static extern void init(); + + [DllImport("BlutoInterop.dll")] + public unsafe static extern void initServerConnection(byte[] ip, int port); + + [DllImport("BlutoInterop.dll")] + public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size); + + [DllImport("BlutoInterop.dll")] + public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts); + + public static byte[] stringToBytes(string s) + { + byte[] rv = new byte[s.Length + 1]; + for (int i = 0; i < s.Length; i++) + rv[i] = Convert.ToByte(s[i]); + rv[s.Length] = Convert.ToByte ('\0'); + return rv; + } + + public static string intPtrToString(IntPtr iptr) + { + return Marshal.PtrToStringAnsi(iptr); + } + + public static byte[] intPtrToByteArray(IntPtr iptr, int sz) + { + byte[] data = new byte[sz]; + Marshal.Copy(iptr, data, 0, data.Length); + return data; + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/PubSubServerUnmanagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/PubSubServerUnmanagedTCPConnection.cs new file mode 100644 index 000000000..41248b79f --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/PubSubServerUnmanagedTCPConnection.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Runtime.InteropServices; +using mbts; +//using BCG.Utilities; + +namespace BCG +{ + public class PubSubServerUnmanagedTCPConnection : IPubSubServerConnection + { + private BlutoSubAddress myServerAddress; + private SmartConnectionManagedWrapper myWrapper; + private ConnectionStatusChange myConnCallback; + private SyncQueue> myAllMsgsQueue = new SyncQueue>(); + + public PubSubServerUnmanagedTCPConnection(string serverURL, SmartConnectionManagedWrapper wrapper) + { + myServerAddress = new BlutoSubAddress(serverURL); + myWrapper = wrapper; + myWrapper.initConnection(new BlutoSubAddress(serverURL)); + myWrapper.resgisterDataCallback(myServerAddress, new SmartConnectionManagedWrapper.MessageDelegate(myHandleServerMessages)); + } + + #region IPubSubServerConnection Members + + public void sendWithConfirmation(byte[] data, ref string confId) + { + //BlueEventLog.logAssert(false, "This function should not be used on this class"); + } + + public void send(byte[] data) + { + myWrapper.send(myServerAddress, data); + } + + public void send(System.IO.Stream str) + { + send (StreamUtilityMethods.getBytesFromStream(str)); + } + + public IList getUpdates(TimeSpan timeout, int maxElements) + { + List> msgs = myAllMsgsQueue.dequeueMultipleElements(Convert.ToInt32 (timeout.TotalMilliseconds), maxElements); + List rv = new List(); + bool connSet = false; + foreach (Pair p in msgs) + { + int msgType = p.first; + byte[] msg = p.second; + if (msgType == 1) + { + rv.Add(msg); + } + else + { + connSet = true; + string connMsg = "connection to " + myServerAddress.ToString() + (msgType == 2 ? " broken" : " established"); + Console.WriteLine(connMsg); + myConnCallback(myServerAddress.ToString(), msgType == 3); + } + } + return connSet ? new List() : rv; + } + + public string serverURL + { + get { return myServerAddress.ToString(); } + } + + public string clientURL + { + get { return myWrapper.getClientAsSeenByServer (serverURL); } + } + + public void registerCallback(ConnectionStatusChange del) + { + myConnCallback = del; + } + + public void registerDataCallback(DataDelegate del) + { + } + + public void registerSendConfirmationCallback(SendConfirmationDelegate del) + { + } + + #endregion + + #region IDisposable Members + + public void Dispose() + { + + } + + #endregion + + #region IWorker Members + public void doWork() + { + } + #endregion + + /// + /// this is the only method that it executed on + /// a different thread + /// + /// + /// + private void myHandleServerMessages(int msgType, byte[] msg) + { + myAllMsgsQueue.enqueue(new Pair(msgType, msg)); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/SmartConnectionManagedWrapper.cs b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/SmartConnectionManagedWrapper.cs new file mode 100644 index 000000000..ce9c62280 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/SmartConnectionManagedWrapper.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Runtime.InteropServices; + +namespace BCG +{ + public class SmartConnectionManagedWrapper + { + public delegate void MessageDelegate(int msgType, byte[] msg); + private Dictionary myServerToDel = new Dictionary(); + + public SmartConnectionManagedWrapper () + { + ClientToServerConnectionWrapper.init(); + Thread readThr = new Thread(new ThreadStart(myReceiveData)); + readThr.Start(); + } + + public void initConnection(BlutoSubAddress subAddress) + { + ClientToServerConnectionWrapper.initServerConnection(ClientToServerConnectionWrapper.stringToBytes(subAddress.ip), + subAddress.port); + } + + public void send(BlutoSubAddress address, byte[] data) + { + byte[] ip = ClientToServerConnectionWrapper.stringToBytes (address.ip); + ClientToServerConnectionWrapper.sendDataToServer(ip, address.port, data, data.Length); + } + + + public void resgisterDataCallback(BlutoSubAddress address, MessageDelegate del) + { + lock (this) + { + myServerToDel.Add(address.ToString(), del); + } + } + + public void unregisterDataCallback(BlutoSubAddress address, MessageDelegate del) + { + lock (this) + { + myServerToDel.Remove(address.ip); + } + } + + public string getClientAsSeenByServer(string serverURL) + { + return ""; + } + + private unsafe void myReceiveData() + { + while (true) + { + BlutoTCPClientMessage** psClientPackets = null; + int npts = 0; + ///returns as soon as it gets the first element + ClientToServerConnectionWrapper.getServerData(1000, 1, &psClientPackets, out npts); + for (int i = 0; i < npts; i++) + { + string ip = Marshal.PtrToStringAnsi(psClientPackets[i]->ip); + int port = psClientPackets[i]->port; + string serverURL = new BlutoSubAddress(ip, port).ToString(); + + byte[] msg = null; + if (psClientPackets[i]->msgType == 1) + { + msg = new byte[psClientPackets[i]->size]; + Marshal.Copy(psClientPackets[i]->msg, msg, 0, msg.Length); + } + + MessageDelegate cb = null; + lock (this) + { + myServerToDel.TryGetValue(serverURL, out cb); + } + if (cb != null) + cb(psClientPackets[i]->msgType, msg); + } + } + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs new file mode 100644 index 000000000..3a24e85e9 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using BCG.Utilities.General; + +namespace BCG +{ + /// + /// This is the interface that both the PublisherCore and the + /// SubscriberCore use to communicate with the PubsubServer. + /// This manages the exchange of information to one + /// pubsub server. + /// Based on the URL, its constructed by the + /// PubsubServerConnectionFactory. All PubCores share + /// common IPubsubServerConnection for each URL, while + /// every instance of the SubscriberCore guarantees a new + /// connection to the Pubsub. + /// Its supposed to be used from a single thread. If + /// multiple threads need to communicate to the PubSub, they + /// should create their own Subscribers. + /// + public interface IPubSubServerConnection : IDisposable, IWorker + { + /// + /// The delegate is fired signalling if the data send was + /// successful or not + /// + void sendWithConfirmation(byte[] data, ref string confId); + /// + /// send is a non waiting call with no guarantee + /// of data being delivered + /// + void send (byte[] data); + /// + /// similiar to above + /// + void send(Stream str); + /// + /// Returns with 0 or more elements, iff the timeout + /// expires or maxElements have been reached (selective polling), + /// whatever comes first. + /// For example, a timeout of TimeSpan.max and maxElements of 1, + /// would wait until it gets the first element and returns + /// with it. + /// A timeout of 0 and maxElements + /// of Int32.Max would return all the messages it has + /// received between last getUpdates and this call. + /// getUpdates(TimeSpan timeout, int maxElements); + /// + /// server URL to which the connection is made + /// + string serverURL { get;} + /// + /// client URL as seen by the server. if a connection has not + /// been established (as can happen in TCP) an empty string + /// is returned + /// + string clientURL { get;} + + /// + /// registers the callback to receive + /// connection notifications + /// + void registerCallback(ConnectionStatusChange del); + + /// + /// registers callbacks to reveive data + /// + /// + void registerDataCallback(DataDelegate del); + + void registerSendConfirmationCallback(SendConfirmationDelegate del); + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQSubAddress.cs b/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQSubAddress.cs new file mode 100644 index 000000000..9d271bbdc --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQSubAddress.cs @@ -0,0 +1,52 @@ +using System; +//using System.Messaging; + +using BCG; +//using BCG.Utilities; + +namespace BCG +{ + public class MSMQSubAddress : SubAddress + { + private string myIP, myQ; + public static readonly string prefix = "msmq"; + + public MSMQSubAddress (string adr) + { + try + { + // throw away prefix + string s = adr; + s = s.Substring(prefix.Length + "://".Length); + // grab IP + myIP = s.Substring(0,s.IndexOf("/")); + // throw away ip + s = s.Substring(myIP.Length + 1); + myQ = s; + + } + catch (Exception) + { + throw (new InvalidSubAddress(adr + " is not a valid queue address")); + } + } + + public MSMQSubAddress (string ip, string simpleQName) + { + myIP = ip; + myQ = simpleQName; + } + + public string ip {get {return myIP;}} + + public string simpleQueueName {get {return myQ;}} + + public override string ToString() { return prefix + "://" + myIP + "/" + myQ; } + + public string formatName + { + get{return "FormatName:DIRECT=TCP:" + myIP + "\\private$\\" + myQ;} + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQWrapper.cs b/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQWrapper.cs new file mode 100644 index 000000000..4d9fc275c --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQWrapper.cs @@ -0,0 +1,300 @@ +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.Messaging; +// using System.IO; +// using System.Collections; +// //using BCG.Utilities; + +// namespace BCG +// { +// internal class MSMQWrapperMessage +// { +// public string msgType; +// public string bindingId; +// public int segsRemaining; +// public byte[] data; + +// public MSMQWrapperMessage() +// : this(new byte[] { }) +// { } + +// public MSMQWrapperMessage(byte[] data) +// : this("", 0, data) +// { } + +// public MSMQWrapperMessage(string theBindingId, int theSegsRemaining, byte[] theData) +// : this("", theBindingId, theSegsRemaining, theData) +// { } + +// public MSMQWrapperMessage(string theMsgType, string theBindingId, int theSegsRemaining, byte[] theData) +// { +// bindingId = theBindingId; +// segsRemaining = theSegsRemaining; +// data = theData; +// msgType = theMsgType; +// } + +// /// +// /// writes the object to the stream passed +// /// +// /// +// public void writeTo(Stream s) +// { +// BinaryWriter bw = new BinaryWriter(s); +// bw.Write(msgType); +// bw.Write(bindingId); +// bw.Write(segsRemaining); +// bw.Write(data.Length); +// bw.Write(data); +// } + +// /// +// /// reads from the stream and initializes itself +// /// +// /// +// private void myReadFrom(Stream s) +// { +// BinaryReader br = new BinaryReader(s); +// msgType = br.ReadString(); +// bindingId = br.ReadString(); +// segsRemaining = br.ReadInt32(); +// BlueEventLog.logAssert(segsRemaining >= 0 && segsRemaining <= 10); +// data = br.ReadBytes(br.ReadInt32()); +// } + +// public static MSMQWrapperMessage constructFromStream(Stream s) +// { +// MSMQWrapperMessage retVal = new MSMQWrapperMessage(); +// retVal.myReadFrom(s); +// return retVal; +// } + +// } +// /// +// /// This is a wrapper around the message queue, as a work around +// /// for some of the problems with it. One of the major problems +// /// it solves is the 4 MB size limit of an MSMQ message. All MSMQ +// /// users use this MSMQWrapper rather than dealing with the message +// /// queue itself. +// /// +// public class MSMQWrapper : IDisposable +// { +// private MessageQueue myQueue; +// private int myDataLimit = Convert.ToInt32(Math.Pow(2, 21)); +// private Dictionary> mySegs = new Dictionary>(); + +// public MSMQWrapper(string qName) +// : this(MessageQueue.Exists(qName) ? new MessageQueue(qName) : +// MessageQueue.Create(qName)) +// {} + +// public MSMQWrapper(MessageQueue q) +// : this (q, Convert.ToInt32(Math.Pow(2, 21))) +// {} + +// public MSMQWrapper(MessageQueue q, int dataLimit) +// { +// myQueue = q; +// myDataLimit = dataLimit; +// } + +// public byte[] receive() +// { +// string msgType; +// return receive(out msgType); +// } + +// public byte[] receive(out string msgType) +// { +// double d = 1000000000; +// while (true) +// { +// byte[] rv = receive(d, out msgType); +// if (rv != null) +// { +// return rv; +// } +// } +// } + +// public byte[] receive(double milliseconds) +// { +// string msgType; +// return receive(milliseconds, out msgType); +// } + +// /// +// /// receives the messages from the queue. if it received a +// /// partial message (ie message that was originally greater +// /// than the limit), it caches it until it receives it in full. +// /// +// /// +// /// +// public byte[] receive(double milliseconds, out string msgType) +// { +// Message msg; +// msgType = null; +// try +// { +// msg = myQueue.Receive(TimeSpan.FromMilliseconds(milliseconds)); +// } +// catch (MessageQueueException e) +// { +// if (e.MessageQueueErrorCode != MessageQueueErrorCode.IOTimeout) +// BlueEventLog.logAssert(false, e.ToString()); +// return null; +// } +// MSMQWrapperMessage recMsg; ; +// try +// { +// recMsg = MSMQWrapperMessage.constructFromStream (msg.BodyStream); +// } +// catch (EndOfStreamException) +// { +// string errMsg = "MSMQ wrapper encountered an error while parsing the message"; +// BlueEventLog.logEvent(errMsg, BlueEventLevel.Error); +// Console.WriteLine(errMsg); +// recMsg = null; +// } +// if (recMsg != null) +// { +// if (recMsg.bindingId == "") +// { +// BlueEventLog.logAssert(recMsg.segsRemaining == 0); +// msgType = recMsg.msgType; +// return recMsg.data; +// } +// else +// { +// if (recMsg.segsRemaining == 0) +// { +// BlueEventLog.logAssert(mySegs.ContainsKey(recMsg.bindingId)); +// List chops = mySegs[recMsg.bindingId]; +// chops.Add(recMsg.data); +// int totalSize = 0, copied = 0; +// foreach (byte[] c in chops) +// totalSize += c.Length; +// byte[] rv = new byte[totalSize]; +// foreach (byte[] c in chops) +// { +// Array.Copy(c, 0, rv, copied, c.Length); +// copied += c.Length; +// } +// msgType = recMsg.msgType; +// return rv; +// } +// else +// { +// if (!mySegs.ContainsKey(recMsg.bindingId)) +// mySegs.Add(recMsg.bindingId, new List()); +// mySegs[recMsg.bindingId].Add(recMsg.data); +// return null; +// } +// } +// } +// else +// return null; +// } + +// public void send(byte[] data) +// { +// send(data, ""); +// } + +// /// +// /// if data > limit, chops the data up and sends them one by one +// /// +// /// +// public void send(byte[] data, string msgType) +// { +// byte[][] choppedDataItems = myChopData(data); +// if (choppedDataItems.Length == 1) +// { +// MSMQWrapperMessage wrapperMsg = new MSMQWrapperMessage(msgType, "", 0, data); +// mySend(wrapperMsg); +// } +// else +// { +// string bindingId = Guid.NewGuid().ToString(); +// for (int i = 0; i < choppedDataItems.Length; i++) +// { +// MSMQWrapperMessage wrapperMsg = new MSMQWrapperMessage(msgType, bindingId, choppedDataItems.Length - (i + 1), +// choppedDataItems[i]); +// mySend(wrapperMsg); +// } +// } +// } + +// public void send(Stream stream) +// { +// send(stream, ""); +// } + + +// public void send(Stream stream, string msgType) +// { +// long initPos = stream.Position; +// BinaryReader br = new BinaryReader(stream); +// byte[] data = br.ReadBytes( Convert.ToInt32 (stream.Length - stream.Position)); +// stream.Position = initPos; +// send(data, msgType); +// } + +// private void mySend(MSMQWrapperMessage wrapperMsg) +// { +// Message msg = new Message(); +// long initPos = msg.BodyStream.Position; +// wrapperMsg.writeTo(msg.BodyStream); +// msg.BodyStream.Position = initPos; +// msg.AttachSenderId = true; +// myQueue.Send(msg); +// } + +// /// +// /// chops the array into multiple arrays s.t. that +// /// the size of each array is less than the limit. +// /// accessor is internal for testing purposes +// /// +// /// +// /// +// internal byte[][] myChopData(byte[] data) +// { +// if (data.Length <= myDataLimit) +// return new byte[][] { data }; +// else +// { +// int remaining = data.Length; +// int nChops = Convert.ToInt32(Math.Ceiling( (double)((double)data.Length / myDataLimit) )); +// byte[][] rv = new byte[nChops][]; +// for (int i = 0; i < nChops; i++) +// { +// rv[i] = remaining >= myDataLimit ? new byte[myDataLimit] : new byte[remaining]; +// Array.Copy(data, data.Length - remaining, rv[i], 0, rv[i].Length); +// remaining -= rv[i].Length; +// } +// return rv; +// } +// } + +// public string queueFormatName +// { +// get { return myQueue.FormatName; } +// } + +// public string queuePath +// { +// get { return myQueue.Path; } +// } + +// #region IDisposable Members + +// public void Dispose() +// { +// myQueue.Close(); +// myQueue.Dispose(); +// } + +// #endregion +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/PubSubServerMSMQConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/PubSubServerMSMQConnection.cs new file mode 100644 index 000000000..156ac8bc1 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/PubSubServerMSMQConnection.cs @@ -0,0 +1,165 @@ +// using System; +// using System.Collections.Generic; +// using System.Text; +// //using System.Messaging; +// using System.IO; +// //using BCG.Utilities.General; +// using System.Threading; +// //using BCG.Utilities; + +// namespace BCG +// { +// internal class PubSubServerMSMQConnection : IPubSubServerConnection +// { +// private ThreadSafeQueue myReceivingQueue; +// private bool myWillReceive = false; +// private byte[] myHeartbeatMessage; +// private string myServerURL; +// private string myClientURL; +// private MSMQWrapper mySendWrapper, myGetWrapper; +// private ConnectionStatusChange myConnCallback; +// private DataDelegate myDataCallback; +// private TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval); +// private DateTime myLastHBTime = DateTime.Now; + +// public PubSubServerMSMQConnection(string serverURL, bool willReceive) +// { +// BlueEventLog.logAssert(!PubSub3Config.instance.PS3_DisableMSMQ, +// "MSMQ is disabled, hence it can not be specified as a transport in the routing table"); +// myWillReceive = willReceive; +// myServerURL = serverURL; +// MSMQSubAddress serverAddress = new MSMQSubAddress(serverURL); +// mySendWrapper = new MSMQWrapper(new MessageQueue(serverAddress.formatName)); +// ///instantiates the receiving queue only if indicated +// ///in the constructor +// if (willReceive) +// { +// myReceivingQueue = new ThreadSafeQueue(); +// string receivingQueueName = "pubsub2_{" + Guid.NewGuid().ToString() + "}"; +// ///if the ip is mapped (ex NAT on merc) +// string ip = (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty) ? +// BCG.Utilities.General.SysInfoManager.getIP() : PubSub3Config.instance.PS3_IPTranslation; +// MSMQSubAddress clientAddress = new MSMQSubAddress(ip, receivingQueueName); +// myClientURL = clientAddress.ToString(); +// myGetWrapper = new MSMQWrapper(MessageQueue.Create(".\\private$\\" + receivingQueueName)); +// BinaryWriter bw = new BinaryWriter (new MemoryStream ()); +// bw.Write (myClientURL); +// myHeartbeatMessage = (bw.BaseStream as MemoryStream).ToArray (); +// System.Threading.Thread receivingThread = new Thread(new ThreadStart(myGetMessages)); +// receivingThread.IsBackground = true; +// receivingThread.Start(); +// } +// } + +// #region IPubSubServerConnection Members + +// public void sendWithConfirmation(byte[] data, ref string confId) +// { +// BlueEventLog.logAssert (false, "This function should not be used on this class"); +// } + +// public void send(byte[] data) +// { +// mySendWrapper.send(data); +// } + +// public void send(Stream str) +// { +// mySendWrapper.send(str); +// } + +// /// +// /// this user call is abstracted out from any tranport +// /// operation and would read the data from an in mem +// /// queue data structure. the read thread fills in the +// /// queue, while the user thread empties it out +// /// +// /// +// /// +// /// +// public IList getUpdates(TimeSpan timeout, int maxElements) +// { +// return myReceivingQueue.dequeueLimited(maxElements); +// } + +// public string serverURL +// { +// get { return myServerURL; } +// } + +// public string clientURL +// { +// get { return myClientURL ; } +// } + +// public void registerCallback(ConnectionStatusChange del) +// { +// myConnCallback = del; +// } + +// public void registerDataCallback(DataDelegate del) +// { +// myDataCallback = del; +// } + +// public void registerSendConfirmationCallback(SendConfirmationDelegate del) { } + + +// #endregion + +// #region IDisposable + +// public void Dispose() +// { +// } + +// #endregion + +// #region IWorker Members + +// public void doWork() +// { +// ///no need to send hbs for publishers +// if (myWillReceive && DateTime.Now - myLastHBTime > myHBTimeSpan) +// { +// myLastHBTime = DateTime.Now; +// mySendWrapper.send(myHeartbeatMessage, "s"); +// } +// } + +// #endregion + +// /// +// /// The message queue read operation is abstracted out +// /// from the user's thread. The read operation is being +// /// done on this thread, enqueued to a queue, and the +// /// user operation reads it from the queue +// /// +// private void myGetMessages() +// { +// while (true) +// { +// try +// { +// while (true) +// { +// byte[] msg = myGetWrapper.receive(); +// if (msg != null) +// { +// if (myDataCallback != null) +// myDataCallback(serverURL, msg); +// else +// myReceivingQueue.enqueue(msg); +// } +// } +// } +// catch (Exception ex) +// { +// BlueEventLog.logEvent(ex); +// if (ex is OutOfMemoryException) +// throw ex; +// } +// } +// } +// } +// } diff --git a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs new file mode 100644 index 000000000..8117a0ae1 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs @@ -0,0 +1,270 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Net; +using System.Net.Sockets; +using System.IO; +//using BCG.Utilities; +//using BCG.Utilities.General; +using System.Threading; +using mbts; + +namespace BCG +{ + /// + /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats + /// from the server, and in case of missing hbs, informs its clients, and tries to + /// restablish the connection. + /// Threads : + /// its assumed that IPubSubServerConnection functions (including doWork) will + /// be called from the same thread. ITCPMessageHandler.onData and + /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread + /// + /// + public class PubSubServerManagedTCPConnection : IPubSubServerConnection, + ITCPMessageHandler, + ITCPConnectionHandler + { + private ITCPConnection myConnection, myTempNewConnection; + private BlutoSubAddress myServerAddress; + private ThreadSafeQueue> myConfs = new ThreadSafeQueue>(); + private SyncQueue myDataQueue = new SyncQueue(); + private DateTime myLastHB = DateTime.Now; + private ConnectionStatusChange myConnCallback; + private DataDelegate myDataCallback; + private SendConfirmationDelegate myOnSendConfirmCallback; + private TimeSpan myConnectionDropThreshold = + new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC); + + public PubSubServerManagedTCPConnection(string serverURL) + : this(new BlutoSubAddress(serverURL)) { } + + public PubSubServerManagedTCPConnection(BlutoSubAddress serverAddress) + { + myServerAddress = serverAddress; + ClientToServerManagedTCPConnector.connectTo(serverAddress, this); + } + + #region IPubSubServerConnection Members + + public void sendWithConfirmation(byte[] data, ref string confId) + { + if (myConnection != null) + { + myConnection.sendWithConfirmation(data, ref confId, new SendConfirmationDelegate(myOnSendConfirm)); + } + else + { + confId = Guid.NewGuid().ToString(); + myConfs.enqueue(new Pair(confId, false)); + } + } + + public void send(byte[] data) + { + Console.WriteLine($"PubSubServerManagedTCPConnection.send server: {myServerAddress} data: {data.Length} myConnection : {myConnection != null}"); + if (myConnection != null) + { + Console.WriteLine("PubSubServerManagedTCPConnection.send server: " + myServerAddress + "data: " + data.Length); + myConnection.send(data); + } + } + + public void send(Stream str) + { + BinaryReader br = new BinaryReader(str); + send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position))); + } + + public IList getUpdates(TimeSpan timeout, int maxElements) + { + return myDataQueue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements); + } + + public string serverURL + { + get { return myServerAddress.ToString(); } + } + + public string clientURL + { + get + { + string rv = ""; + lock (this) + { + if (myConnection == null) + { + rv = ""; + } + else + { + if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty) + { + string ip; + int port; + myConnection.getSelfAddress(out ip, out port); + rv = new BlutoSubAddress(ip, port).ToString(); + } + else + { + rv = ""; + } + } + } + return rv; + } + } + + public void registerCallback(ConnectionStatusChange del) + { + myConnCallback = del; + } + + public void registerDataCallback(DataDelegate del) + { + myDataCallback = del; + } + + public void registerSendConfirmationCallback(SendConfirmationDelegate del) + { + myOnSendConfirmCallback = del; + } + + #endregion + + #region IDisposable Members + + private bool myIsDisposed = false; + public void Dispose() + { + Console.WriteLine("PubSubServerManagedTCPConnection.Dispose"); + //throw new Exception("The method or operation is not implemented."); + if (myConnection != null) + { + (myConnection as AsyncTCPConnection).Dispose(); + + } + myIsDisposed = true; + } + public bool isDisposed + { + get + { + return myIsDisposed; + } + + } + + #endregion + + #region ITCPMessageHandler Members + + /// + /// This method is executed on an arbitrary CLR thread. To avoid blocking it + /// data is written to the queue and method returns. myLastHB being a primitive + /// (long) is safe for multi threaded operations + /// + /// + public void onData(RemoteMessage message) + { + Console.WriteLine($"PubSubServerManagedTCPConnection.onData msg:type: {message.messageType}"); + if (message.messageType == RemoteMessageType.data) + { + if (myDataCallback != null) + myDataCallback(message.remoteConnectionInfo.ToString(), message.message); + else + myDataQueue.enqueue(message.message); + } + else + { + myLastHB = DateTime.Now; + } + } + + #endregion + + #region ITCPConnectionHandler Members + + /// + /// This function is called on the main thread (as all other functions + /// except onData) + /// + /// + public void onConnection(ITCPConnection connection) + { + string ip; + int port; + connection.getPeerAddress(out ip, out port); + BlutoSubAddress add = new BlutoSubAddress(ip, port); + Console.WriteLine($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()}"); + if (add.ToString() == myServerAddress.ToString()) + { + myLastHB = DateTime.Now; + myTempNewConnection = connection; + } + } + + #endregion + + #region IWorker Members + + /// + /// should be called on the same thread as the other user functions. + /// the design only allows for onConnection to be on a different + /// thread. + /// + public void doWork() + { + ///check for heartbeats and let clients know in + ///case of missing hb (connection breaking) + TimeSpan curr_hb_interval = DateTime.Now - myLastHB; + bool hb_missed = curr_hb_interval > myConnectionDropThreshold; + bool conn_disposed = myConnection != null && myConnection.isDisposed; + if (conn_disposed || hb_missed) + { + if (hb_missed) + { + // since conn breakage was diagnosed logically, lets physically close the connection + myConnection.Dispose(); + string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}"; + Console.WriteLine(dispose_msg); + } + myConnection = null; // this is used as a marker for a dead connection + if (myConnCallback != null) + myConnCallback(myServerAddress.ToString(), false); + string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " + + $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {curr_hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}"; + Console.WriteLine(reconn_msg); + ClientToServerManagedTCPConnector.connectTo(myServerAddress, this); + } + if (myTempNewConnection != null) + { + if (myConnection != null) + { + // TODOds + // log error and close it + // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed " + // + " before a new connection is established"); + } + Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection"); + myConnection = myTempNewConnection; + myTempNewConnection = null; + myConnection.register(this); + myConfs.dequeueAll(); + if (myConnCallback != null) + myConnCallback(myServerAddress.ToString(), true); + } + foreach (Pair p in myConfs.dequeueAll()) + if (myOnSendConfirmCallback != null) + myOnSendConfirmCallback(p.first, p.second); + } + + #endregion + + private void myOnSendConfirm(string messageId, bool confirmed) + { + myConfs.enqueue(new Pair(messageId, confirmed)); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs b/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs new file mode 100644 index 000000000..313683bb4 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Text; +//using BCG.Utilities; + +namespace BCG +{ + /// + /// Every instance of PublisherCore and SubscriberCore will have its + /// factory. Rather than have once singelton factory, this approach + /// is preffered since multiple pub/sub core can connect to the same + /// connection and make it difficultt o manage connections. + /// + public class PubSubServerConnectionFactory : IDisposable + { + private bool myIsSub; + private Dictionary myConnections = new Dictionary(); + + internal PubSubServerConnectionFactory (bool isSub) + { + myIsSub = isSub; + } + + public IPubSubServerConnection getConnection(string url) + { + if (!myConnections.ContainsKey(url)) + myConnections.Add(url, myBuildConnection(url)); + return myConnections[url]; + } + + private IPubSubServerConnection myBuildConnection(string url) + { + if (url.StartsWith(MSMQSubAddress.prefix)) + { + // TODO, no more MSMQ + //Diagnostics.assert(false); + return null; + // MSMQSubAddress serverAdd = new MSMQSubAddress(url); + // return new PubSubServerMSMQConnection(serverAdd.ToString(), myIsSub); + } + else if (url.StartsWith(BlutoSubAddress.prefix)) + { + return new PubSubServerManagedTCPConnection(url); + } + else + { + // BlueEventLog.logAssert(false, "Unknown transport, only msmq and blutoTcp allowed"); + return null; + } + } + + #region IDisposable Members + + public void Dispose() {} + + #endregion + } +} diff --git a/src/DataStreamSuite/PubSub/Client/IPubSubServerTagHandler.cs b/src/DataStreamSuite/PubSub/Client/IPubSubServerTagHandler.cs new file mode 100644 index 000000000..b8125d286 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/IPubSubServerTagHandler.cs @@ -0,0 +1,43 @@ +using System; +using System.IO; +using BCG.Utilities.General; +using System.Collections.Generic; +using mbts; + +namespace BCG +{ + public delegate void InternalPublishDelegate (ICollection> pubData); + /// + /// The Pubsub holds an instance of this for every tag + /// type. this interface holds the state of each tag + /// in the pubsub. + /// + public interface IPubSubServerTagHandler : IWorker + { + /// + /// update is called when there is new + /// data published for a tag + /// + /// + /// + void update(string tag, Stream[] messageBodies); + /// + /// gets the entire content of the tag + /// + /// + /// + byte[] getContent(string tag); + /// + /// list of tagTypes this interface supports + /// + string[] supportedTagTypes { get; } + /// + /// registering a callback gives this interface + /// the extra flexibility to publish its own + /// data back to the pubsub. this function will + /// be ignored executed outside the pubsub server + /// + /// + void register(InternalPublishDelegate callback); + } +} diff --git a/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs b/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs new file mode 100644 index 000000000..640f63278 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +//using BCG.Utilities; +using mbts; + +namespace BCG +{ + /// + /// The list of publishers and subscribers for + /// the tag is contained herein. Its used for internal + /// purposes, most noticeably TagInfoStream + /// + public class TagPubSubInfo + { + public string tag; + public IDictionary subscribers, + publishers; + public ICollection> deadPublishers; + public string server; + + public TagPubSubInfo(Stream s) + { + myReadFrom(s); + + //if (tag == "System_AllSubscriptions" && publishers.Count == 0) + //{ + // Console.WriteLine("No publisher , stream cons ?? wtf ??"); + // System.Windows.Forms.MessageBox.Show("No publisher , stream cons ?? wtf ??"); + //} + } + + public TagPubSubInfo(string theTag, IDictionary theSubscribers, + IDictionary theActivePublishers, + ICollection> theDeadPublishers, + string theServer) + { + tag = theTag; + server = theServer; + subscribers = theSubscribers; + publishers = theActivePublishers; + deadPublishers = theDeadPublishers; + + //if (tag == "System_AllSubscriptions" && publishers.Count == 0) + //{ + // Console.WriteLine("No publisher, direct cons ?? wtf ??"); + // System.Windows.Forms.MessageBox.Show("No publisher, direct cons ?? wtf ??"); + //} + } + + /// + /// writes itself to the stream passed on as + /// the arg + /// {tag,subCount,subs,conns,pub,pubConn,server} + /// + /// + public void writeTo(Stream s) + { + BinaryWriter bw = new BinaryWriter(s); + bw.Write(tag); + bw.Write(subscribers.Count); + foreach (KeyValuePair p in subscribers) + { + bw.Write(p.Key); + bw.Write(p.Value); + } + bw.Write(publishers.Count); + foreach (KeyValuePair p in publishers) + { + bw.Write(p.Key); + bw.Write(p.Value); + } + bw.Write(deadPublishers.Count); + foreach (Pair p in deadPublishers) + { + bw.Write(p.first); + bw.Write(p.second); + } + bw.Write(server); + } + + /// + /// constructs itself from the stream + /// + /// + private void myReadFrom(Stream s) + { + BinaryReader br = new BinaryReader(s); + tag = br.ReadString(); + int nSubs = br.ReadInt32(); + subscribers = new Dictionary(); ; + for (int i = 0; i < nSubs; i++) + { + subscribers.Add(br.ReadString(), br.ReadString()); + } + publishers = new Dictionary(); + int nPubs = br.ReadInt32(); + for (int i = 0; i < nPubs; i++) + { + publishers.Add(br.ReadString(), br.ReadString()); + } + deadPublishers = new List>(); + int nDeadPubs = br.ReadInt32(); + for (int i = 0; i < nDeadPubs; i++) + { + (deadPublishers as List>).Add(new Pair(br.ReadString(), br.ReadString())); + } + server = br.ReadString(); + } + + /// + /// If a tag ceases to have a publisher and a subscriber it becomes + /// orphaned. This will happen if all the subscribers of an unpublished + /// tags unsubscribe to it, leaving the tag with no subscribers as + /// well (it had no publisher to begin with). + /// However if a tag is ever published it will never become an orphan + /// + public bool orphaned + { + get + { + return publishers.Count == 0 && subscribers.Count == 0; + } + } + + public string subscribersInString + { + get + { + string subString = ""; + foreach (KeyValuePair p in subscribers) + { + subString += (subString == "" ? "" : ",") + p.Value + ';' + p.Key; + } + return subString; + } + } + + + public string publishersInString + { + get + { + string pubString = ""; + foreach (KeyValuePair p in publishers) + { + pubString += (pubString == "" ? "" : ",") + p.Value + ';' + p.Key; + } + return pubString; + } + } + + public string deadPublishersInString + { + get + { + string deadPubString = ""; + foreach (Pair p in deadPublishers) + { + if (p != null) + deadPubString += (deadPubString == "" ? "" : ",") + p.second + ';' + p.first; + else{} + //BlueEventLog.logEvent("DeadPublishers in deadPublishersInString is null", "Error"); + } + return deadPubString; + } + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs b/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs new file mode 100644 index 000000000..6b79123d5 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; + +namespace BCG +{ + public class TagStatsInfo + { + public string server; + public string tag; + public int minSize = Int32.MaxValue, maxSize = Int32.MinValue, lastUpdateSize; + public double averageSize; + public int totalUpdates; + public long totalBytes; + public long lastUpdateTime; + + public TagStatsInfo(string theTag) + { + server = ClientInfo.thisClient.machineName; + tag = theTag; + } + + public TagStatsInfo(Stream str) + { + BinaryReader br = new BinaryReader(str); + server = br.ReadString(); + tag = br.ReadString(); + minSize = br.ReadInt32(); + maxSize = br.ReadInt32(); + lastUpdateSize = br.ReadInt32(); + averageSize = br.ReadDouble(); + totalUpdates = br.ReadInt32(); + totalBytes = br.ReadInt64(); + lastUpdateTime = br.ReadInt64(); + } + + public void update(int size) + { + totalUpdates++; + totalBytes += size; + averageSize = totalBytes / totalUpdates; + minSize = size < minSize ? size : minSize; + maxSize = size > maxSize ? size : maxSize; + lastUpdateSize = size; + lastUpdateTime = DateTime.Now.Ticks; + } + + public void writeTo(Stream str) + { + BinaryWriter bw = new BinaryWriter(str); + bw.Write(server); + bw.Write(tag); + bw.Write(minSize); + bw.Write(maxSize); + bw.Write(lastUpdateSize); + bw.Write(averageSize); + bw.Write(totalUpdates); + bw.Write(totalBytes); + bw.Write(lastUpdateTime); + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs new file mode 100644 index 000000000..8ee7ff24a --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs @@ -0,0 +1,178 @@ +using System; +using System.Runtime.Serialization; +using System.Collections; +using System.Reflection; +using System.IO; +using System.Text; + + +using BCG; +//using BCG.Utilities; + +namespace BCG +{ + public class MetaData : ICloneable + { + private Hashtable myHash = new Hashtable(); + int mySize = -1; + + public MetaData() + {} + + public MetaData(object o) + {copyFrom(o);} + + public MetaData(Stream s) + { + long initPos = s.Position; + load(s); + mySize = Convert.ToInt32(s.Position - initPos); + } + + public string this[string name] + { + get + {return (string) myHash[name];} + set + { + if (value != null) + myHash[name] = value; + } + } + + public void add(string name, string [] values) + { + StringBuilder sb = new StringBuilder(); + foreach (string s in values) + { + if (sb.Length != 0) + sb.Append(" "); + sb.Append(s); + } + myHash[name] = sb.ToString(); + } + + public int getAsInt(string name) + {return Int32.Parse(this[name]);} + + public string[] getAsStringArray (string name) + { + if (this[name] != "" && this[name] != null) + return this[name].Split(' '); + else + return new string[0]; + } + + public int size() + { + if (mySize < 0) + { + MemoryStream ms = new MemoryStream(); + this.save(ms); + return mySize = (int)ms.Position; + } + else + return mySize; + } + + public bool isEqual (MetaData md) + { + if (md.myHash.Keys.Count != myHash.Keys.Count) + return false; + + foreach (string k in myHash.Keys) + if (!myHash[k].Equals(md.myHash[k])) + return false; + return true; + } + + public void load(Stream s) + { + myHash = new Hashtable(); + string [] state = + new BinaryReader(s).ReadString().Split('\n'); + + for (int i = 0; i < state.Length-1; i+= 2) + myHash[state[i]] = state[i+1]; + } + + public void save(Stream s) + { + new BinaryWriter(s).Write(this.ToString()); + } + + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + foreach (string key in myHash.Keys) + sb.AppendFormat("{0}\n{1}\n",key,myHash[key]); + return sb.ToString(); + } + + public void copyFrom(object o) + { + this["op-code"] = o.GetType().Name; + + + + foreach (FieldInfo fi in o.GetType().GetFields()) + { + + if (fi.GetValue(o) == null) + continue; + + else if (fi.GetValue(o) is String[]) + add(fi.Name, (string[])fi.GetValue(o)); + + else + myHash[fi.Name] = fi.GetValue(o).ToString(); + + } + } + + public void copyTo(object o) + { + + + foreach (FieldInfo fi in o.GetType().GetFields()) + { + + string value = this[fi.Name]; + //this should only be here temporarily, while the new decompression changes are waiting to go into prod. + if (value == null) + { + //BlueEventLog.logEvent("Error: Metadata missing field " + fi.Name, BlueEventLog2Level.MissingMetadata); + } + else + { + if (fi.FieldType.Equals(typeof(string))) + fi.SetValue(o, value); + else if (fi.FieldType.Equals(typeof(int))) + fi.SetValue(o, Int32.Parse(value)); + else if (fi.FieldType.Equals(typeof(long))) + fi.SetValue(o, long.Parse(value)); + else if (fi.FieldType.Equals(typeof(double))) + fi.SetValue(o, double.Parse(value)); + else if (fi.FieldType.Equals(typeof(DateTime))) + fi.SetValue(o, DateTime.Parse(value)); + else if (fi.FieldType.Equals(typeof(Boolean))) + fi.SetValue(o, Boolean.Parse(value)); + else if (fi.FieldType.Equals(typeof(TimeSpan))) + fi.SetValue(o, TimeSpan.Parse(value)); + else if (fi.FieldType.Equals(typeof(String[]))) + fi.SetValue(o, value.Split(' ')); + else + throw new Exception("Field type not supported"); + } + } + } + + public object Clone() + { + MemoryStream ms = new MemoryStream(); + this.save(ms); + ms.Seek(0,SeekOrigin.Begin); + return new MetaData(ms); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/PubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/PubMetaData.cs new file mode 100644 index 000000000..13ab4bbdf --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/MetaData/PubMetaData.cs @@ -0,0 +1,91 @@ +using System; +//using BCG.Utilities.General; +//using BCG.Utilities.BlueConfiguration; +//using BCG.Utilities; + +namespace BCG +{ + + [Serializable] + public class PubMetaData + { + /// + /// tag to be subscribed + /// + public string tag; + /// + /// type of tag + /// + public string type; + /// + /// size of the data published + /// + public int size; + + /// + /// How big is the data when it's decompressed + /// + public int decompressedSize = 0; + + /// + /// is the data compressed? + /// + public bool isCompressed = false; + /// + /// identification of the publisher and the connection to the server to send the data + /// machine; ip; pid; serverURL + /// + public string connection; + + public string publishingApp; + + public long publishTimeTicks; + + public PubMetaData() {} + + public PubMetaData(string theTagType, string theTag, int theSize, string theConnection, string thePublishingApp) + : this(theTagType,theTag,theSize,theSize,theConnection,thePublishingApp) + { + isCompressed = false; + } + + public PubMetaData(string theTagType, string theTag, int theSize, int theDecompressedSize, string theConnection, string thePublishingApp) + { + type = theTagType; + tag = theTag; + size = theSize; + connection = theConnection; + publishingApp = thePublishingApp; + decompressedSize = theDecompressedSize; + isCompressed = true; + + publishTimeTicks = DateTime.Now.ToUniversalTime().Ticks; + myValidateTag(); + } + + //make sure the tag doesn't contain any newlines + private void myValidateTag() + { + if (tag.Contains("\n")) + throw new ArgumentException("Error: tag " + tag + " is not valid; tags cannot contain newlines."); + + } + + /// + /// Returns true if the ticks on this message are greater than the latency report threshold in config + /// + /// + public bool warnIfLatencyExceedsThreshold() + { + DateTime publishTime = new DateTime(publishTimeTicks); + double elapsedSeconds = (DateTime.Now.ToUniversalTime() - publishTime).TotalMilliseconds; + if (elapsedSeconds > PubSub3Config.instance.PS3_LatencyReportThresholdMS) + { + //BlueEventLog.logEvent(string.Concat("Recieved pub message on tag ", tag, " ", elapsedSeconds, " milliseconds after the publisher ",publishingApp," sent it."), PS3LogLevels.latency); + return true; + } + return false; + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/RenewMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/RenewMetaData.cs new file mode 100644 index 000000000..d83c76918 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/MetaData/RenewMetaData.cs @@ -0,0 +1,25 @@ +using System; + +namespace BCG +{ + [Serializable] + public class RenewMetaData + { + public string subscriber; + public int lease; + public bool c; + + public SubAddress getSubAddress() + {return SubAddress.decode(subscriber);} + + public RenewMetaData() + {} + + public RenewMetaData(string s, + int lease) + { + this.subscriber = s; + this.lease = lease; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs new file mode 100644 index 000000000..3b59c69c6 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs @@ -0,0 +1,32 @@ +using System; + +namespace BCG +{ + [Serializable] + public class SubMetaData + { + /// + /// connection URL on which the connection will receive data + /// + public string connection; + /// + /// identification of the client and the connection on which to receive data + /// machine; ip; port; clientURL + /// + public string subscribingApp; + /// + /// tags that are to be subscribed + /// + public string[] tags; + + public SubMetaData() + { } + + public SubMetaData(string theConnection, params string[] theTags) + { + connection = theConnection; + tags = theTags; + subscribingApp = ClientInfo.thisClient.processString; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/TransactionMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/TransactionMetaData.cs new file mode 100644 index 000000000..722c6dfa0 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/MetaData/TransactionMetaData.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + /// + /// when a publisher sends transactional data to the pubsub server, + /// this is the first meta data, followed by pub messages. this indicated + /// the number of messages that are part of the transaction + /// + [Serializable] + public class TransactionMetaData + { + public int nMessages; + + public TransactionMetaData() { } + + public TransactionMetaData(int theNMessages) + { + nMessages = theNMessages; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs new file mode 100644 index 000000000..2bc7540fe --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs @@ -0,0 +1,21 @@ +using System; + +namespace BCG +{ + [Serializable] + public class UnSubMetaData + { + public string connection; + public string[] tags; + public string unsubscribingApp = ClientInfo.thisClient.processString; + + public UnSubMetaData() + {} + + public UnSubMetaData(string theConnection, params string[] theTags) + { + connection = theConnection; + tags = theTags; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs b/src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs new file mode 100644 index 000000000..3f9e1a775 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs @@ -0,0 +1,19 @@ +using System; +using System.Runtime.Serialization; + +namespace BCG +{ + public interface IPublisher + { + void publish( IPubItem pi ); + void publish( IPubItem pi, IFormatter f ); + void publish( IPubItem[] pi ); + void publish( IPubItem[] pi, IFormatter f ); + } + + public interface IPublisher where T : IFormatter + { + void publish (IPubItem pi); + void publish (IPubItem[] pi); + } +} diff --git a/src/DataStreamSuite/PubSub/Client/PublishingUtilities/Publisher.cs b/src/DataStreamSuite/PubSub/Client/PublishingUtilities/Publisher.cs new file mode 100644 index 000000000..c80730484 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/PublishingUtilities/Publisher.cs @@ -0,0 +1,75 @@ +using System; +using System.IO; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; + +namespace BCG +{ + public class Publisher : IPublisher + { + private PublisherCore myPubCore = new PublisherCore (); + private BinaryFormatter myBinFormatter = new BinaryFormatter (); + private const string myTagType = "DefaultTag"; + + public Publisher() {} + + #region IPublisher Members + + public void publish (IPubItem pi) + { publish( pi, myBinFormatter ); } + + public void publish (IPubItem[] pis) + { publish( pis, myBinFormatter ); } + + public void publish( IPubItem pi, IFormatter f ) + { + MemoryStream ms = new MemoryStream(); + #pragma warning disable + f.Serialize( ms, pi ); + #pragma warning restore SYSLIB0011 + ms.Position = 0; + myPubCore.publish( myTagType, pi.getTag(), ms ); + } + + public void publish( IPubItem[] pis, IFormatter f ) + { + myPubCore.beginTransaction(); + foreach ( IPubItem pi in pis ) + publish( pi, f ); + myPubCore.commitTransaction(); + } + + #endregion + } + + public class Publisher : IPublisher where T : IFormatter, new () + { + private PublisherCore myPubCore = new PublisherCore (); + private IFormatter myFormatter = new T (); + private const string myTagType = "DefaultTag"; + + public Publisher () { } + + #region IPublisher Members + + public void publish (IPubItem pi) + { + MemoryStream ms = new MemoryStream (); + #pragma warning disable + myFormatter.Serialize (ms, pi); + #pragma warning restore SYSLIB0011 + ms.Position = 0; + myPubCore.publish (myTagType, pi.getTag (), ms); + } + + public void publish (IPubItem[] pis) + { + myPubCore.beginTransaction (); + foreach (IPubItem pi in pis) + publish (pi); + myPubCore.commitTransaction (); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Routing/IPubSubRouter.cs b/src/DataStreamSuite/PubSub/Client/Routing/IPubSubRouter.cs new file mode 100644 index 000000000..13c3e217c --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Routing/IPubSubRouter.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public delegate void PubSubTagURLChangeDelegate(List tagsMoving, string newURL); + + public interface IPubSubRouter + { + + event PubSubTagURLChangeDelegate onSubURLTagRouteChange; + event PubSubTagURLChangeDelegate onPubURLTagRouteChange; + List getAllPubSubSubURLs(); + List getAllPubSubPubURLs(); + string getSubURLForTag(string tag); + string getPubURLForTag(string tag); + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs new file mode 100644 index 000000000..9bff38d2d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs @@ -0,0 +1,68 @@ +using System; +using BCG; +//using BCG.Utilities; +using System.Text.RegularExpressions; + +namespace BCG +{ + public class PubSubRoute : IComparable + { + public string serverAddress; + public string tagRegExp; + //route names are used in intermediary code (i.e. while substations are being deployed) + //so that MC Hammer can advertise them. + public string name; + private Regex myRegex = null; + + public PubSubRoute() { } + + public PubSubRoute(string theServerAddress, string theTagRegExp) + { + serverAddress = theServerAddress; + tagRegExp = theTagRegExp; + } + + // public PubSubRoute() + // { + // // serverAddress = string.Format("bluto://{0}:{1}", cfg.serverAddress, cfg.serverPort); + // // tagRegExp = cfg.tagsRegex; + // } + + public string[] getAllPubSubs() + { + return serverAddress.Split(';'); + } + + public Regex getRegex() + { + return myRegex ?? new Regex(tagRegExp); + } + + #region IComparable Members + + public int CompareTo(PubSubRoute other) + { + return serverAddress == other.serverAddress ? tagRegExp.CompareTo(other.tagRegExp) + : serverAddress.CompareTo(other.serverAddress); + } + + #endregion + + public override bool Equals(object obj) + { + if (!(obj is PubSubRoute) || obj == null) + return false; + else + { + PubSubRoute other = obj as PubSubRoute; + return serverAddress.Equals(other.serverAddress) && + tagRegExp.Equals(other.tagRegExp); + } + } + + public override int GetHashCode() + { + return base.GetHashCode(); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouteNotFound.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouteNotFound.cs new file mode 100644 index 000000000..c0db1e36d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouteNotFound.cs @@ -0,0 +1,12 @@ +using System; +using BCG; +//using BCG.Utilities; + +namespace BCG +{ + public class PubSubRouteNotFound : Exception + { + public PubSubRouteNotFound(string msg): base(msg) + {} + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs new file mode 100644 index 000000000..cf0fff187 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs @@ -0,0 +1,114 @@ +using System; +using System.Text.RegularExpressions; +using System.Collections; +using System.Collections.Generic; + +using BCG; +//using BCG.Utilities; +//using bts; + +namespace BCG +{ + public class PubSubRouter : IPubSubRouter + { + private Regex[] myPubRouteRegExps; + private Regex[] mySubRouteRegExps; + private PubSubRoute[] myPubRoutes; + private PubSubRoute[] mySubRoutes; + + + public PubSubRouter() : this( + PubSub3Config.instance.getSubRouting(), + PubSub3Config.instance.getPubRouting()) + { } + + public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) + { } + + public PubSubRouter(PubSubRoute[] subRoutes, PubSubRoute[] pubRoutes) + { + mySubRouteRegExps = new Regex[subRoutes.Length]; + myPubRouteRegExps = new Regex[pubRoutes.Length]; + + mySubRoutes = subRoutes; + myPubRoutes = pubRoutes; + + for (int i = 0; i < subRoutes.Length; ++i) + mySubRouteRegExps[i] = new Regex(subRoutes[i].tagRegExp); + for (int i = 0; i < pubRoutes.Length; ++i) + myPubRouteRegExps[i] = new Regex(pubRoutes[i].tagRegExp); + } + + private int myGetRouteIndex(string forTag, PubSubRoute[] routes, Regex[] regexes) + { + for (int i = 0; i < routes.Length; i++) + if (regexes[i].IsMatch(forTag)) + return i; + + throw new PubSubRouteNotFound(String.Format("No route found for {0}", forTag)); + } + + #region IPubSubRouter methods + + private string myPickAServer(string tag, string[] urls) + { + if (urls.Length == 1) + return urls[0]; + else + { + uint hc = (uint)tag.GetHashCode(); + return urls[hc % urls.Length]; + } + } + + public string getPubURLForTag(string tag) + { + var rv = "bluto://127.0.0.1:1976"; //myPickAServer(tag, myPubRoutes[myGetRouteIndex(tag, myPubRoutes, myPubRouteRegExps)].getAllPubSubs()); + bts.Log.Gen.info($"PubSubRouter.getPubURLForTag tag: {tag} route: {rv}"); + return rv; + } + + public string getSubURLForTag(string tag) + { + var rv = "bluto://127.0.0.1:1976"; // myPickAServer(tag, mySubRoutes[myGetRouteIndex(tag, mySubRoutes, mySubRouteRegExps)].getAllPubSubs()); + bts.Log.Gen.info($"PubSubRouter.getSubURLForTag tag: {tag} route: {rv}"); + return rv; + } + + public List getAllPubSubSubURLs() + { + List theURLs = new List(); + + foreach (PubSubRoute subRoute in mySubRoutes) + theURLs.AddRange(subRoute.getAllPubSubs()); + return theURLs; + + } + + public List getAllPubSubPubURLs() + { + List theURLs = new List(); + + foreach (PubSubRoute pubRoute in myPubRoutes) + theURLs.AddRange(pubRoute.getAllPubSubs()); + return theURLs; + + } + public event PubSubTagURLChangeDelegate onSubURLTagRouteChange; + public event PubSubTagURLChangeDelegate onPubURLTagRouteChange; + + #endregion + + //this function is put here so that the events are used, in order to prevent build errors. + private void myNeverCallThisFunction() + { + if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) + { + int aVar = 2; + aVar = aVar + 2; + } + } + + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/SubAddress.cs b/src/DataStreamSuite/PubSub/Client/SubAddress.cs new file mode 100644 index 000000000..7f83d1a0f --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/SubAddress.cs @@ -0,0 +1,31 @@ +using System; +using BCG; +//using BCG.Utilities; + +namespace BCG +{ + public class InvalidSubAddress : Exception + { + public InvalidSubAddress(string msg) : base (msg){} + } + + public abstract class SubAddress + { + public static SubAddress decode (string s) + { + //if (s.StartsWith(TcpSubAddress.prefix)) + // return new TcpSubAddress(s); + //else if (s.StartsWith(SuperSocketAddress.prefix)) + // return new SuperSocketAddress(s); + if (s.StartsWith(MSMQSubAddress.prefix)) + return new MSMQSubAddress(s); + else if (s.StartsWith(BlutoSubAddress.prefix)) + return new BlutoSubAddress(s); + else + { + //BlueEventLog.logAssert(false, "Unknown pubsub transport for : " + s); + return null; + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/IPubItem.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/IPubItem.cs new file mode 100644 index 000000000..3f39f03af --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/IPubItem.cs @@ -0,0 +1,9 @@ +using System; + +namespace BCG +{ + public interface IPubItem + { + string getTag(); + } +} diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/ManagedSubscriber.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/ManagedSubscriber.cs new file mode 100644 index 000000000..70bdf0258 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/ManagedSubscriber.cs @@ -0,0 +1,131 @@ +using System; +using System.Collections; +using System.IO; +using System.Runtime.Serialization.Formatters.Binary; + +using BCG.Utilities; +using BCG.Utilities.General; + + +namespace BCG +{ + public delegate void Handler (IPubItem pi); + public delegate void RawDataHandler (string tag, Stream s); + + /// + /// Provides a callback based interface to the SubscriptionCore (this class is mainly here to allow backward compatibility + /// with PubSub, as some clients are still using ManagedSubcribers) + /// + public class ManagedSubscriber : IWorker, IDisposable + { + private static ManagedSubscriber mySelf; + private SubscriberCore mySubsCore; + private IDictionary myTypedDispatchTable = new Hashtable (), + myRawDispatchTable = new Hashtable (); + + private int myWaitTime = 0; + + public ManagedSubscriber() : this (new SubscriberCore ()) {} + + public ManagedSubscriber (SubscriberCore subsCore) {mySubsCore = subsCore;} + + public void doWork () + { + IDictionary temp = new Hashtable (); + ArrayList updates = mySubsCore.getUpdates (myWaitTime); + foreach (PubSubUpdate psu in updates) + { + temp [psu.tag] = temp [psu.tag] == null ? new ArrayList () : temp [psu.tag]; + (temp [psu.tag] as ArrayList).Add (psu.body); + } + myFireEvents (temp); + } + + public void subscribe (string tag, Handler handler) + { + myTypedDispatchTable[tag] = !myTypedDispatchTable.Contains(tag) ? + new TypedPubItemHandler(tag, mySubsCore) : myTypedDispatchTable[tag]; + (myTypedDispatchTable[tag] as TypedPubItemHandler).addSubscription(handler); + } + + public void subscribe (string tag, Handler handler, IGenericFormatter formatter) + { + myTypedDispatchTable[tag] = !myTypedDispatchTable.Contains (tag) ? + new TypedPubItemHandler (tag, mySubsCore, formatter) : myTypedDispatchTable[tag]; + (myTypedDispatchTable[tag] as TypedPubItemHandler).addSubscription (handler); + } + + public void subscribe (string tag, RawDataHandler handler) + { + myRawDispatchTable [tag] = !myRawDispatchTable.Contains (tag) ? + new RawPubItemHandler (tag, mySubsCore) : myRawDispatchTable [tag]; + (myRawDispatchTable [tag] as RawPubItemHandler).addSubscription (handler); + } + + public void unsubscribe (string tag, Handler handler) + { + if (myTypedDispatchTable.Contains(tag)) + { + (myTypedDispatchTable[tag] as TypedPubItemHandler).removeSubscription(handler); + if ((myTypedDispatchTable[tag] as SingleSubHandler).nSubscribers == 0) + myTypedDispatchTable.Remove(tag); + } + else{} + // BlueEventLog.logEvent("Trying to unsubscribe a PubItem tag that was never subsribed", BlueEventLevel.Error); + } + + public bool alreadySubscribed(string tag) + { + return myTypedDispatchTable.Contains(tag) || myRawDispatchTable.Contains(tag); + } + + public void unsubscribe (string tag, RawDataHandler handler) + { + if (myRawDispatchTable.Contains (tag)) + { + (myRawDispatchTable [tag] as RawPubItemHandler).removeSubscription (handler); + if ((myRawDispatchTable[tag] as SingleSubHandler).nSubscribers == 0) + myRawDispatchTable.Remove(tag); + } + else {} + // BlueEventLog.logEvent ("Trying to unsubscribe a Raw tag that was never subsribed", BlueEventLevel.Warning); + } + + private void myFireEvents (IDictionary updates) + { + foreach (DictionaryEntry de in updates) + { + Stream s = StreamUtilityMethods.getAggregatedStream ((de.Value as ArrayList).ToArray (typeof (Stream)) as Stream[]); + if (myTypedDispatchTable.Contains (de.Key)) + (myTypedDispatchTable [de.Key] as TypedPubItemHandler).update (de.Key.ToString(), s); + else if (myRawDispatchTable.Contains (de.Key)) + (myRawDispatchTable [de.Key] as RawPubItemHandler).update (de.Key.ToString(), s); + else{} + // BlueEventLog.logEvent ("Received update for an unsubscribed tag", BlueEventLevel.Warning); + } + } + + public IPubSubRouter router + { + get + { + return mySubsCore.getRouter(); + } + } + + public static ManagedSubscriber instance + { + get {return mySelf = mySelf == null ? new ManagedSubscriber () : mySelf;} + } + + + #region IDisposable Members + + public void Dispose() + { + mySubsCore.Dispose(); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/RawPubItemHandler.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/RawPubItemHandler.cs new file mode 100644 index 000000000..e8e7a1fe3 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/RawPubItemHandler.cs @@ -0,0 +1,23 @@ +using System; +using System.IO; + +namespace BCG +{ + internal class RawPubItemHandler : SingleSubHandler + { + public RawPubItemHandler (string tag, SubscriberCore subCore) : + base (tag, subCore) + {} + + protected override void myFire (Delegate del) + { + (del as RawDataHandler)(myTag, myData as Stream); + } + + public override void update (string tag, Stream s) + { + myData = s; + myFire (myDelegate); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/SingleSubHandler.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/SingleSubHandler.cs new file mode 100644 index 000000000..5cb23505f --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/SingleSubHandler.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; + +namespace BCG +{ + abstract internal class SingleSubHandler + { + protected SubscriberCore mySubCore; + protected Delegate myDelegate; + protected string myTag; + protected object myData; + + public SingleSubHandler (string tag, SubscriberCore subCore) + { + myTag = tag; + mySubCore = subCore; + } + + public virtual void addSubscription (Delegate del) + { + if (myDelegate == null) + mySubCore.subscribe (myTag); + myDelegate = Delegate.Combine (myDelegate, del); + if (myData != null) + myFire (del); + } + + public virtual void removeSubscription (Delegate del) + { + if (myDelegate != null) + { + myDelegate = Delegate.Remove (myDelegate, del); + if (myDelegate == null) + { + mySubCore.unsubscribe (myTag); + myData = null; + } + } + } + + public int nSubscribers + { + get { return myDelegate == null ? 0 : myDelegate.GetInvocationList().Length; } + } + + public abstract void update (string tag, Stream s); + + protected abstract void myFire (Delegate del); + + } + +} diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/TypedPubItemHandler.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/TypedPubItemHandler.cs new file mode 100644 index 000000000..2a3167686 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/TypedPubItemHandler.cs @@ -0,0 +1,37 @@ +using System; +using System.IO; +using System.Runtime.Serialization.Formatters.Binary; + + +namespace BCG +{ + internal class TypedPubItemHandler : SingleSubHandler + { + private IGenericFormatter myFormatter; + + public TypedPubItemHandler (string tag, SubscriberCore subCore) + : this (tag, subCore, new GenericBinaryFormatter ()) + { + } + + public TypedPubItemHandler (string tag, SubscriberCore subCore, IGenericFormatter formatter) + : base (tag, subCore) + { + myFormatter = formatter; + } + + protected override void myFire (Delegate del) + { + (del as Handler)(myData as IPubItem); + } + + public override void update (string tag, Stream s) + { + while (s.Position < s.Length) + { + myData = myFormatter.Deserialize (s); + myFire (myDelegate); + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/TagSubscriptionState.cs b/src/DataStreamSuite/PubSub/Client/TagSubscriptionState.cs new file mode 100644 index 000000000..fa17f6371 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/TagSubscriptionState.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + /* + This class represents the state of a single tag, as known to the SubscriberCore. + * This was done instead of an enum local to the subscriber core so that we could modify the state of a tag + * while it sits in a hashtable, without modifying the hashtable itself. + */ + class TagSubscriptionState + { + public enum State { watingForSubscription, watingForURL, waitingForUnsubscription, subscribed, unsubscribed }; + + private State myState; + public State state + { + get { return myState; } + set { myState = value; } + + } + public TagSubscriptionState() + { + myState = State.watingForSubscription; + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/TagType2HandlerMap.cs b/src/DataStreamSuite/PubSub/Client/TagType2HandlerMap.cs new file mode 100644 index 000000000..d898016f5 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/TagType2HandlerMap.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections; +using System.Reflection; +using BCG.Utilities; +using System.Collections.Generic; + +namespace BCG +{ + public class TagType2HandlerMap + { + private static TagType2HandlerMap myServerHandlerMap = null, + myPublisherStateMap = null; + private Dictionary myTagTypeToPubHandlerMap = new Dictionary(); + + public TagType2HandlerMap() + { + if (PubSub3Config.instance.PS3_TagTypeToAssemblyMap != null) + { + foreach (TagTypeToAssembly.Entry entry in PubSub3Config.instance.PS3_TagTypeToAssemblyMap.entries) + { + try + { + Assembly assem = Assembly.LoadFrom(entry.assembly); + Type t = assem.GetType(entry.handlerClass, true, false); + object obj = Activator.CreateInstance(t); + IPubSubServerTagHandler tagHandler = obj as IPubSubServerTagHandler; + string[] tagTypes = tagHandler.supportedTagTypes; + foreach (string tagType in tagTypes) + { + myTagTypeToPubHandlerMap[tagType] = tagHandler; + } + } + catch (Exception ex) + { + // BlueEventLog.logEvent(string.Format("error happened while loading type from the assembly ", entry.handlerClass, entry.assembly), BlueEventLevel.Warning); + // BlueEventLog.logEvent(ex); + if (ex is OutOfMemoryException) + throw ex; + } + } + } + } + + private IPubSubServerTagHandler myGetHandlerFromTagType(string tagType) + { + IPubSubServerTagHandler rv; + myTagTypeToPubHandlerMap.TryGetValue(tagType, out rv); + return rv; + } + + public static IPubSubServerTagHandler getHandler(string type, bool isServer) + { + lock (typeof(IPubSubServerTagHandler)) + { + if (isServer) + { + if (myServerHandlerMap == null) + myServerHandlerMap = new TagType2HandlerMap(); + return myServerHandlerMap.myGetHandlerFromTagType(type); + } + else + { + if (myPublisherStateMap == null) + myPublisherStateMap = new TagType2HandlerMap(); + return myPublisherStateMap.myGetHandlerFromTagType(type); + } + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/TagTypeToAssembly.cs b/src/DataStreamSuite/PubSub/Client/TagTypeToAssembly.cs new file mode 100644 index 000000000..3dbbff2fc --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/TagTypeToAssembly.cs @@ -0,0 +1,17 @@ +using System; + +namespace BCG +{ + [Serializable] + public class TagTypeToAssembly + { + //[Serializable] + public class Entry + { + public string handlerClass; + public string assembly; + } + + public Entry[] entries; + } +} diff --git a/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs b/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs new file mode 100644 index 000000000..2a4a2125b --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +// using BCG.Utilities; + using BCG.Utilities.General; +using mbts; + +namespace BCG +{ + /// + /// Workers registering with this class will receive a + /// doWork call in a seperate thread. Since this class + /// manages the thread itself, there is no need to call + /// an external doWork (like in the MasterWorker) + /// + public class ThreadDrivenMasterWorker : MasterWorker + { + internal ThreadDrivenMasterWorker() + : base() + { } + + protected override void myInit() + { + Thread doWorkThread = new Thread(new ThreadStart(myDoWork)); + doWorkThread.IsBackground = true; + doWorkThread.Start(); + } + + public override void addWorker(IWorker worker, TimeSpan period) + { + ///to avoid re-entrance, its added to a temporay dictionary, + ///and will be moved over to the main one, in the doWork + lock (this) + { + myJustAdded.Add(worker, new Pair(period, DateTime.UtcNow)); + } + } + + public override void removeWorker(IWorker worker) + { + ///same as addWorker, added to a temporary store, before moving + ///it to the main one, in the next doWork + lock (this) + { + myJustRemoved.Add(worker, null); + } + } + + private void myDoWork() + { + TimeSpan sleepTime = new TimeSpan(myPeriod.Ticks / 2); + while (true) + { + try + { + lock (this) + { + foreach (IWorker w in myJustRemoved.Keys) + { + // BlueEventLog.logEvent(String.Format("ThreadDrivenMasterWorker : removing worker {0} , worker count before removing this = {1}", + // w.ToString(), myWorkers.Count), PS3LogLevels.connection); + myWorkers.Remove(w); + } + foreach (KeyValuePair> p in myJustAdded) + { + // BlueEventLog.logEvent(String.Format("ThreadDrivenMasterWorker : adding worker {0} , worker count before adding this = {1}", + // p.Key.ToString(), myWorkers.Count), PS3LogLevels.connection); + myWorkers.Add(p.Key, p.Value); + } + myJustRemoved.Clear(); + myJustAdded.Clear(); + } + doWork(); + Thread.Sleep(sleepTime); + } + catch (Exception ex) + { + // BlueEventLog.logEvent(ex); + if (ex is OutOfMemoryException) + throw ex; + } + } + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Config/PubSub3Config.cs b/src/DataStreamSuite/PubSub/Config/PubSub3Config.cs new file mode 100644 index 000000000..7d203e482 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Config/PubSub3Config.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.RegularExpressions; +//using BCG.Utilities.BlueConfiguration; +//using BCG.Utilities; + +namespace BCG +{ + public class PubSub3Config //: ConfigurationRequest + { + private static PubSub3Config myInstance; + public static PubSub3Config instance + { + get + { + if (myInstance == null) + { + myInstance = new PubSub3Config(); + //myInstance = (PubSub3Config) ConfigurationRequester.request(typeof(PubSub3Config)); + } + return myInstance; + } + } + + private int myPS3_millisecondsBetweenSubInfoUpdates = 1000; + private int myPS3_millisecondsBetweenTimeSliceUpdates = 5000; + private int myPS3_retriesOnNoBufferSpace = 1; + private int myPS3_millisecondsBetweenNoBufferSpaceRetries = 100; + + private int myPS3_DoItAllSecondsBetweenLogs = 0; + private int myPS3_HeartBeatInterval = 15000; + private int myPS3_MissingHeartBeatsBeforeGC = 50; + private bool myPS3_UseManagedSockets = true; + private int myPS3_StateUpdatePeriod = 5000; + private int myPS3_StatisticsUpdatePeriod = 5000; + private int myPS3_DisposedConnectionCheckPeriod = 1000; + private int myPS3_MessageProcessingWarningThreshold = 1000; + private int myPS3_ServerPort = 1976; + private int myPS3_MaxPacketSize = 500000000; + private bool myBlueComm_mustDie = true; + + private string[] myPS3_CrushableTags = new string[] { }; + + //moved from PS2 + private TagTypeToAssembly myTagTypeToAssemblyMap; + private string myPS3_ServerQueue; + + //substation stuff + //private PubSubComponentSettings myPS3_Components = null; + private PubSubPortRange myPS3_PortRange = new PubSubPortRange(30400, 31700); + private bool myPS3_BindToRandomPort = false; + + //for debugging stuff on a tag by tag basis + private string myPS3_DebugTags = "^NOTAGS$"; + private Regex myDebugTagsRegex; + + //for reporting the latency of a tag from publisher to reciever + private double myPS3_LatencyReportThresholdMS = 1500f; + + //used to determine what parts to build the pubsub and its clients out of + private bool myPS3_UseSubstations = false; + private bool myPS3_UseDynamicClients = false; + private bool myPS3_IsSubstation = false; + private bool myPS3_InternallyPublishSubInfo = true; + private bool myPS3_UseManagedClientsInNativeCode = false; + + //compression stuff + private string myPS3_CompressedTagsRegex = "^NOTAGS$"; + private string myPS3_TagsToReportCompressionRatio = "^NOTAGS$"; + private Regex myTagsToReportCompressionRatioRegex; + private Regex myCompressedTagsRegex; + + private bool myPS3_DisableMSMQ = false; + private bool myPS3_UsePrepublishedServerState = false; + private string myPS3_IPTranslation; + private long myGCActivationThreshold = 0; + private int myWaitPeriodAfterGC = 1000; + + // recovery + //private ConfigKeys.Recovery_PubSubImports myRecovery_PubSubImports = new ConfigKeys.Recovery_PubSubImports(); + //private ConfigKeys.Recovery_PubSubCopies myRecovery_PubSubCopies = new ConfigKeys.Recovery_PubSubCopies(); + + // bluecomm transition + //public ConfigKeys.BlueComm_PSTransition BlueComm_PSTransition { get; set; } + public PubSub3Config() + { + //BlueComm_PSTransition = new BCG.ConfigKeys.BlueComm_PSTransition(true); + } + + public bool isValid() + { + return true; + } + + public int PS3_retriesOnNoBufferSpace + { + get { return myPS3_retriesOnNoBufferSpace; } + set { myPS3_retriesOnNoBufferSpace = value; } + } + + public int PS3_millisecondsBetweenNoBufferSpaceRetries + { + get { return myPS3_millisecondsBetweenNoBufferSpaceRetries; } + set { myPS3_millisecondsBetweenNoBufferSpaceRetries = value; } + } + + public int PS3_millisecondsBetweenTimeSliceUpdates + { + get { return myPS3_millisecondsBetweenTimeSliceUpdates; } + set { myPS3_millisecondsBetweenTimeSliceUpdates = value; } + } + + + public bool BlueComm_mustDie + { + get { return myBlueComm_mustDie; } + set { myBlueComm_mustDie = value; } + } + + public int PS3_HeartBeatInterval + { + get { return myPS3_HeartBeatInterval; } + set { myPS3_HeartBeatInterval = value; } + } + + public int PS3_DoItAllSecondsBetweenLogs + { + get { return myPS3_DoItAllSecondsBetweenLogs; } + set { myPS3_DoItAllSecondsBetweenLogs = value; } + } + + public int PS3_MissingHeartBeatsForGC + { + get { return myPS3_MissingHeartBeatsBeforeGC; } + set { myPS3_MissingHeartBeatsBeforeGC = value; } + } + + public bool PS3_UseManagedSockets + { + get { return myPS3_UseManagedSockets; } + set { myPS3_UseManagedSockets = value; } + } + + public int PS3_StateUpdatePeriod + { + get { return myPS3_StateUpdatePeriod; } + set { myPS3_StateUpdatePeriod = value; } + } + + public int PS3_StatisticsUpdatePeriod + { + get { return myPS3_StatisticsUpdatePeriod; } + set { myPS3_StatisticsUpdatePeriod = value; } + } + + public int PS3_DisposedConnectionCheckPeriod + { + get { return myPS3_DisposedConnectionCheckPeriod; } + set { myPS3_DisposedConnectionCheckPeriod = value; } + } + + public int PS3_MessageProcessingWarningThreshold + { + get { return myPS3_MessageProcessingWarningThreshold; } + set { myPS3_MessageProcessingWarningThreshold = value; } + } + + public int PS3_millisecondsBetweenSubInfoUpdates + { + get { return myPS3_millisecondsBetweenSubInfoUpdates; } + set { myPS3_millisecondsBetweenSubInfoUpdates = value; } + } + + public double PS3_LatencyReportThresholdMS + { + get { return myPS3_LatencyReportThresholdMS; } + set { myPS3_LatencyReportThresholdMS = value; } + } + + public int PS3_ServerPort + { + get { return myPS3_ServerPort; } + set { myPS3_ServerPort = value; } + } + + public PubSubRoute[] getSubRouting() + { return getRouting(); } + + public PubSubRoute[] getPubRouting() + { return getRouting(); } + + public PubSubRoute[] getRouting() + { + // TODO + return new PubSubRoute[] { }; + // return getRouting(SitesCore.getMySiteName()); + } + + public PubSubRoute[] getRouting(string forSite) + { + // TODO + return new PubSubRoute[] { }; + // if (sites_layout != null) { + // foreach (var s in sites_layout.sites) + // if (s.name == forSite) { + // List result = new List(); + // foreach (var cfgRoute in s.pubSub.routes) + // result.Add(new PubSubRoute(cfgRoute)); + // return result.ToArray(); + // } + // } + //BlueEventLog2.instance().assert(false, "Could not find site {0} in config", forSite); + return null; + } + + public List getAllPubSubs() + { + return new List(); + //return getAllPubSubs(SitesCore.getMySiteName()); + } + + public List getAllPubSubs(string forSite) + { + List result = new List(); + foreach (PubSubRoute r in getRouting(forSite)) + result.AddRange(r.getAllPubSubs()); + return result; + } + + // private ConfigKeys.sites_layout _layout = new ConfigKeys.sites_layout(); + // public ConfigKeys.sites_layout sites_layout { get { return _layout; } } + + // public ConfigKeys.sites_siteName sites_siteName { get; set; } + + public TagTypeToAssembly PS3_TagTypeToAssemblyMap + { + get { return myTagTypeToAssemblyMap; } + set { myTagTypeToAssemblyMap = value; } + } + + //[Description("PUBSUB: The Message queue to look for pub sub messages")] + public string PS3_ServerQueue + { + get + { + if (myPS3_ServerQueue == null || myPS3_ServerQueue == "") + return "PubSubQueue"; + else + return myPS3_ServerQueue; + } + set { myPS3_ServerQueue = value; } + } + + public bool PS3_InternallyPublishSubInfo + { + get { return myPS3_InternallyPublishSubInfo; } + set { myPS3_InternallyPublishSubInfo = value; } + } + + public string[] PS3_CrushableTags + { + get { return myPS3_CrushableTags; } + set { myPS3_CrushableTags = value; } + } + + //lots of logic in this key because it's designed to be transitional, + //to enable mixed use of substations and non-substations. + // public PubSubComponentSettings PS3_Components + // { + // get { + + + // if (myPS3_Components == null) + // { + + // if(myPS3_IsSubstation) + // myPS3_Components = new PubSubComponentSettings(PubSubComponentSettings.ComponentGroup.substationServer); + // else if (ClientInfo.thisClient.app.ToUpper().Contains("PUBSUB2MAIN") && myPS3_InternallyPublishSubInfo) + // myPS3_Components = new PubSubComponentSettings(PubSubComponentSettings.ComponentGroup.interallyPublishingPubSub); + + // else if(myPS3_UseDynamicClients) + // { + // if(myPS3_UseSubstations) + // myPS3_Components = new PubSubComponentSettings(PubSubComponentSettings.ComponentGroup.substationClient); + // else + // myPS3_Components = new PubSubComponentSettings(PubSubComponentSettings.ComponentGroup.dynamicRoutingTableClient); + // } + // else + // myPS3_Components = new PubSubComponentSettings(PubSubComponentSettings.ComponentGroup.staticRoutingTableClient); + + // } + + // return myPS3_Components; + // } + + // //this key cannot be set in config (it would be too messy and only read by the C# clients + // //therefore it's best to use this logic here. + + // } + + public bool PS3_UseSubstations + { + get { return myPS3_UseSubstations; } + set { myPS3_UseSubstations = value; } + } + + public PubSubPortRange PS3_PortRange + { + get { return myPS3_PortRange; } + set { myPS3_PortRange = value; } + } + + public bool PS3_BindToRandomPort + { + get { return myPS3_BindToRandomPort; } + set { myPS3_BindToRandomPort = value; } + + } + + public bool PS3_IsSubstation + { + get { return myPS3_IsSubstation; } + set { myPS3_IsSubstation = value; } + + } + + public bool PS3_UseDynamicClients + { + get { return myPS3_UseDynamicClients; } + set { myPS3_UseDynamicClients = value; } + } + + public bool PS3_UseManagedClientsInNativeCode + { + get { return myPS3_UseManagedClientsInNativeCode; } + set { myPS3_UseManagedClientsInNativeCode = value; } + } + + public string PS3_CompressedTagsRegex + { + get { return myPS3_CompressedTagsRegex; } + + set { myPS3_CompressedTagsRegex = value; } + } + + public string PS3_DebugTags + { + get { return myPS3_DebugTags; } + set { myPS3_DebugTags = value; } + } + + public string PS3_TagsToReportCompressionRatio + { + get { return myPS3_TagsToReportCompressionRatio; } + set { myPS3_TagsToReportCompressionRatio = value; } + } + + public bool PS3_DisableMSMQ + { + get { return myPS3_DisableMSMQ; } + set { myPS3_DisableMSMQ = true; } + } + + public string PS3_IPTranslation + { + get { return myPS3_IPTranslation; } + set { myPS3_IPTranslation = value; } + } + + public long PS3_GCActivationThreshold + { + get { return myGCActivationThreshold; } + set { myGCActivationThreshold = value; } + } + + public int PS3_WaitPeriodAfterGC + { + get { return myWaitPeriodAfterGC; } + set { myWaitPeriodAfterGC = value; } + } + + public bool PS3_UsePrepublishedServerState + { + get { return myPS3_UsePrepublishedServerState; } + set { myPS3_UsePrepublishedServerState = value; } + } + + public int PS3_MaxPacketSize + { + get { return myPS3_MaxPacketSize; } + set { myPS3_MaxPacketSize = value; } + } + + // public ConfigKeys.Recovery_PubSubImports Recovery_PubSubImports + // { + // get { return (myRecovery_PubSubImports); } + // set { myRecovery_PubSubImports = value; } + // } + + // public ConfigKeys.Recovery_PubSubCopies Recovery_PubSubCopies + // { + // get { return (myRecovery_PubSubCopies); } + // set { myRecovery_PubSubCopies = value; } + // } + + public bool isTagDebugTag(string tag) + { + if (myDebugTagsRegex == null) + { + myDebugTagsRegex = new Regex(myPS3_DebugTags, RegexOptions.IgnoreCase); + } + return myDebugTagsRegex.IsMatch(tag); + + } + + public bool isTagCompressed(string tag) + { + + if (myCompressedTagsRegex == null) + { + myCompressedTagsRegex = new Regex(myPS3_CompressedTagsRegex, RegexOptions.IgnoreCase); + + } + + return (myCompressedTagsRegex.IsMatch(tag)); + } + + public bool shouldReportTagsCompressionRatio(string tag) + { + if (myTagsToReportCompressionRatioRegex == null) + { + myTagsToReportCompressionRatioRegex = new Regex(myPS3_TagsToReportCompressionRatio, RegexOptions.IgnoreCase); + } + return (myTagsToReportCompressionRatioRegex.IsMatch(tag)); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Config/PubSubPortRange.cs b/src/DataStreamSuite/PubSub/Config/PubSubPortRange.cs new file mode 100644 index 000000000..8b7cdee9d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Config/PubSubPortRange.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + /* + * PubSubPortRange.cs + * + * This class represents a range of ports that could be used by a server. + * Servers that wish to provide services ask the blue pages for a port they can use, + * and blue pages replies with a port. + * + * */ + public class PubSubPortRange + { + + public PubSubPortRange() + { + + } + + public PubSubPortRange(int min, int max) + { + minPort = min; + maxPort = max; + + } + + public int minPort; + public int maxPort; + + } +} diff --git a/src/DataStreamSuite/PubSub/IGenericFormatter.cs b/src/DataStreamSuite/PubSub/IGenericFormatter.cs new file mode 100644 index 000000000..e5810b0ef --- /dev/null +++ b/src/DataStreamSuite/PubSub/IGenericFormatter.cs @@ -0,0 +1,11 @@ +using System; +using System.IO; + +namespace BCG +{ + public interface IGenericFormatter + { + T Deserialize (Stream serializationStream); + void Serialize (Stream serializationStream, T graph); + } +} diff --git a/src/DataStreamSuite/PubSub/IPublisherCore.cs b/src/DataStreamSuite/PubSub/IPublisherCore.cs new file mode 100644 index 000000000..1d67b00e1 --- /dev/null +++ b/src/DataStreamSuite/PubSub/IPublisherCore.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; + +namespace BCG +{ + public interface IPublisherCore : IDisposable + { + void publish(string tag, Stream content); + void publish(string tagType, string tag, Stream content); + //prevents this tag from being republished if the connection is reset. + void clearState(string tag); + void beginTransaction(); + void commitTransaction(); + void abortTransaction(); + bool inTransaction {get;} + int getNumberOfWaitingMessages(); + } +} diff --git a/src/DataStreamSuite/PubSub/IPubsubDeserializer.cs b/src/DataStreamSuite/PubSub/IPubsubDeserializer.cs new file mode 100644 index 000000000..c80f964fc --- /dev/null +++ b/src/DataStreamSuite/PubSub/IPubsubDeserializer.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace BCG +{ + /// + /// This deserializer resides inside the pubsub. Its deserializes the rows received by the pubsub and hence + /// helps pubsub to mantain its state. Since every row goes through it, its not a bad place to mantain the state + /// as well, hence the getState method. Therefore this Deserializer can not only be used as a deserializer but also + /// as a state holder. + /// + public interface IPubsubDeserializer + { + /// + /// Deserializes the data received by the PubSub + /// + /// Serialized data received by the pubsub from the publisher + /// Array of IDictionary of id vs rows + /// Operations, each pairing up with the correposnding member of theRows + void deserialize(Stream[] streams, out byte[] theOps, out Array theRows); + /// + /// Serializes the entire state + /// + /// + /// + byte[] getState(); + } + +} diff --git a/src/DataStreamSuite/PubSub/ISubscriberCore.cs b/src/DataStreamSuite/PubSub/ISubscriberCore.cs new file mode 100644 index 000000000..e834fadd5 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ISubscriberCore.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public delegate void PubSubConnectionChange(bool isConnected, string connection); + + public interface ISubscriberCore : IDisposable + { + void subscribe(params string[] tags); + void unsubscribe(params string[] tags); + ArrayList getUpdates(); + ArrayList getUpdates(int ms); + ArrayList getUpdates(TimeSpan timeout); + IPubSubRouter getRouter(); + + event PubSubConnectionChange onPubSubConnectionChange; + } +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs new file mode 100644 index 000000000..d43a65b77 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs @@ -0,0 +1,543 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Net; +using System.Net.Sockets; +using System.IO; +//using BCG.Utilities; +using BCG.Utilities.General; +using System.Runtime.InteropServices; + +namespace BCG +{ + /// + /// This class, mantains a connection to a peer socket, using + /// the .Net async TCP library. It sends heartbeats at regular + /// intervals (config). It is in active states when messages + /// are sent and received without an error state. It assumed + /// dead state when either of the following happens + /// BeginReceive returns a SocketError + /// BeginSend returns a SocketError + /// EndReceive returns a SocketError + /// EndSend returns a SocketError + /// As soon as it assumes dead state it stops sending or + /// receiving any messages (including heartbeats). + /// The user of this class can either detect that by + /// missing heartbeats or checking the isConnected property. + /// Missing should be the more general way, since if the + /// peer dies in a 'silent' way, where none of the above + /// methods indicate an error, missing heartbeats is guaranteed + /// to detect it. + /// + public class AsyncTCPConnection : ITCPConnection, IWorker, IDisposable + { + /// + /// This is used as a state for the Async send. Reason being + /// that if onSend indicates that it could not send the full + /// data (ie EndSend does not return the full size), the offset + /// is used to resend the data + /// + public class AsyncMessage + { + public byte[] buffer; + public int offset; + public string messageId; + public SendConfirmationDelegate onSendDel; + + public AsyncMessage(byte[] theBuffer, int theOffset) + : this(theBuffer, theOffset, String.Empty, null) + { } + + public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel) + { + buffer = theBuffer; + offset = theOffset; + messageId = theMessageId; + onSendDel = theOnSendDel; + } + + + public int length + { + get + { + return buffer.Length - offset; + } + } + } + + enum ReadState + { + readingSize, + readingData + }; + + #region work around for .Net socket's LocalEndPoint bug + + [StructLayout(LayoutKind.Sequential)] + private struct in_addr + { + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] + public byte[] sin_addr; + } + + [StructLayout(LayoutKind.Sequential)] + private struct sockaddr + { + public short sin_family; + public ushort sin_port; + public in_addr sin_addr; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] + public byte[] sin_zero; + } + + [DllImport("wsock32.dll")] + private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen); + + [DllImport("wsock32.dll")] + private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen); + + #endregion + + private readonly int myMaxMessageSize = PubSub3Config.instance.PS3_MaxPacketSize; + private RemoteConnectionInfo myPeer, mySelf; + private Socket mySock; + private ITCPMessageHandler myMessageHandler; + + private byte[] myCurrentReadBuffer; + private int myCurrentReadBufferOffset; + private ReadState myCurrentReadState; + + private bool myAlive = true; + + private byte[] myHBMessage; + private TimeSpan myHBTimeSpan; + private MasterWorker myMasterWorker = MasterWorker.getInstance(true); + private bool myHBReceived = false, myDataReceived = false, myHBSent = false; + private DateTime myLastHBReceivedTime = new DateTime(1900, 1, 1); + + public AsyncTCPConnection(Socket sock, TimeSpan heartBeatTimeSpan) + { + myHBTimeSpan = heartBeatTimeSpan; + mySock = sock; + mySock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true); + string ip; + int port; + getPeerAddress(out ip, out port); + myPeer = new RemoteConnectionInfo(ip, port); + getSelfAddress(out ip, out port); + mySelf = new RemoteConnectionInfo(ip, port); + // BlueEventLog.logEvent(String.Format("AsyncTCPConnection created : self = {0} , peer = {1}", + // mySelf.ToString(), myPeer.ToString()), PS3LogLevels.connection); + } + + public override string ToString() + { + return String.Format("peer = {0}, self = {1}", myPeer.ToString(), mySelf.ToString()); + } + + #region ITCPConnection Members + + public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del) + { + confId = Guid.NewGuid().ToString(); + byte[] msg = myBuildDataMessage(data); + myBeginOrResumeSending(new AsyncMessage(msg, 0, confId, del)); + } + + public void send(byte[] data) + { + byte[] msg = myBuildDataMessage(data); + myBeginOrResumeSending(new AsyncMessage(msg, 0)); + } + + public void register(ITCPMessageHandler messageHandler) + { + // BlueEventLog.logAssert(myMessageHandler == null, "Message Handler is already assigned"); + // BlueEventLog.logEvent(String.Format("AsyncTCPConnection for peer = {0} being registered", myPeer.ToString()), + // PS3LogLevels.connection); + myMessageHandler = messageHandler; + myMasterWorker.addWorker(this, myHBTimeSpan); + myReadSize(); + } + + public void getPeerAddress(out string ip, out int port) + { + ip = ""; + port = 0; + if (myAlive) + { + IPEndPoint ipEp = mySock.RemoteEndPoint as IPEndPoint; + ip = ipEp.Address.ToString(); + port = ipEp.Port; + } + ///.Net has some problems getting the port, so using + ///native code for that + //sockaddr addr = new sockaddr(); + //int namelen = 16; + //getsockname(mySock.Handle, ref addr, ref namelen); + //port = addr.sin_port; + } + + public void getSelfAddress(out string ip, out int port) + { + ip = ""; + port = 0; + if (myAlive) + { + IPEndPoint ipEp = mySock.LocalEndPoint as IPEndPoint; + ip = ipEp.Address.ToString(); + port = ipEp.Port; + } + ///due to this bug in .Net socket, we have to resort to + ///native code + //sockaddr addr = new sockaddr(); + //int namelen = 16; + //getsockname(mySock.Handle, ref addr, ref namelen); + //port = addr.sin_port; + } + + public bool isDisposed + { + get { return !myAlive; } + } + + #endregion + + #region IWorker Members + + bool _last_connected = false; + public void doWork() + { + if (mySock != null) + //if (mySock.Connected != _last_connected) + { + Console.WriteLine($"AsyncTCPConnection.doWork connected: {mySock.Connected}"); + _last_connected = mySock.Connected; + } + if (!myHBSent) + { + // BlueEventLog.logEvent(String.Format("Sending heartbeat to peer {0} for the first time", myPeer.ToString()), + // PS3LogLevels.connection); + myHBSent = true; + } + myBeginOrResumeSending(new AsyncMessage(myBuildHeartbeatMessage(), 0)); + } + + #endregion + + #region sending data + + private void myBeginOrResumeSending(AsyncMessage message) + { + if (myAlive) + { + try + { + //Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} connected: {mySock.Connected}"); + SocketError beginSendErr; + mySock.BeginSend(message.buffer, message.offset, message.length, SocketFlags.None, + out beginSendErr, new AsyncCallback(myOnSend), message); + Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} socker_error: {beginSendErr} connected: {mySock.Connected}"); + if (beginSendErr != SocketError.Success) + myHandleError(beginSendErr); + } + catch (Exception ex) + { + myHandleError(ex); + } + } + } + + private void myOnSend(IAsyncResult res) + { + if (myAlive) + { + try + { + SocketError writeErr; + int bytesSent = mySock.EndSend(res, out writeErr); + Console.WriteLine($"AsyncTCPConnection.myOnSend bytes: {bytesSent} writeErr: {writeErr} connected: {mySock.Connected}"); + AsyncMessage message = res.AsyncState as AsyncMessage; + message.offset += bytesSent; + if (writeErr == SocketError.Success) + { + if (message.offset == message.buffer.Length) + { + // if confirmation needs to be sent + if (String.Empty != message.messageId) + { + message.onSendDel(message.messageId, true); + } + } + else + { + ///resume writing rest of the message + // BlueEventLog.logEvent("could not send the whole message all at once , messageSize = " + + // message.buffer.Length + " offset = " + message.offset, PS3LogLevels.info); + myBeginOrResumeSending(message); + } + } + else + { + myHandleError(writeErr); + } + } + catch (Exception ex) + { + myHandleError(ex); + } + } + } + + /// + /// Assembles the heartbeat message, and uses it for + /// future references + /// + /// + private byte[] myBuildHeartbeatMessage() + { + if (myHBMessage == null) + { + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + int sz = 4; + int msgType = 1; + bw.Write(sz); + bw.Write(msgType); + myHBMessage = (bw.BaseStream as MemoryStream).ToArray(); + } + return myHBMessage; + } + + /// + /// Assembles the data message. This is necessary + /// for the other side to understand the message. This can also + /// be achieved by not forming the message and use the + /// BeginSend (List<>) function rather than BeginSend (byte[]) + /// + /// + /// + private byte[] myBuildDataMessage(byte[] data) + { + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + bw.Write(data.Length + 8); + bw.Write(0); + bw.Write(0); ///mid for backward compatibility + bw.Write(data); + return (bw.BaseStream as MemoryStream).ToArray(); + } + + #endregion + + #region receiving data + + private void myReadSize() + { + myCurrentReadBuffer = new byte[4]; + myCurrentReadBufferOffset = 0; + myCurrentReadState = ReadState.readingSize; + myBeginOrResumeReading(); + } + + private void myReadData(int dataSize) + { + myCurrentReadBuffer = new byte[dataSize]; + myCurrentReadBufferOffset = 0; + myCurrentReadState = ReadState.readingData; + myBeginOrResumeReading(); + } + + private void myBeginOrResumeReading() + { + if (myAlive) + { + try + { + int beginReceiveAttempts = 0; + while (true) + { + beginReceiveAttempts++; + SocketError readSockErr; + mySock.BeginReceive(myCurrentReadBuffer, myCurrentReadBufferOffset, + myCurrentReadBuffer.Length - myCurrentReadBufferOffset, + SocketFlags.None, out readSockErr, + new AsyncCallback(myOnRead), + myCurrentReadBuffer); + Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeReading socket_error: {readSockErr} connected: {mySock.Connected}"); + if (readSockErr == SocketError.NoBufferSpaceAvailable) + { + string msg = String.Format("Not enough buffer space to receive {0} bytes. Attempt: {1} of {2}.", + myCurrentReadBuffer.Length - myCurrentReadBufferOffset, + beginReceiveAttempts, PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1); + //BlueEventLog.logEvent(msg, PS3LogLevels.disconnection); + + if (beginReceiveAttempts < (PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1)) + System.Threading.Thread.Sleep(PubSub3Config.instance.PS3_millisecondsBetweenNoBufferSpaceRetries); + else + { + //BlueEventLog.logEvent("Could not recover from NoBufferSpace condition. Breaking connection.", PS3LogLevels.disconnection); + myHandleError(readSockErr); + } + } + else if (readSockErr == SocketError.Success) + { + if (beginReceiveAttempts > 1) + // BlueEventLog.logEvent(String.Format("Recovered from NoBufferSpace after {0} tries",beginReceiveAttempts-1) , + // PS3LogLevels.disconnection); + break; + } + else + myHandleError(readSockErr); + } + } + catch (Exception ex) + { + myHandleError(ex); + } + } + } + + private void myOnRead(IAsyncResult res) + { + if (myAlive) + { + try + { + SocketError readErr; + Console.WriteLine($"AsyncTCPConnection.myOnRead pre connected: {mySock.Connected}"); + int bytesRead = mySock.EndReceive(res, out readErr); + Console.WriteLine($"AsyncTCPConnection.myOnRead bytes: {bytesRead} readErr: {readErr} connected: {mySock.Connected}"); + myCurrentReadBufferOffset += bytesRead; + if (readErr == SocketError.Success) + { + if (myCurrentReadBufferOffset == myCurrentReadBuffer.Length) + { + ///read the whole of the buffer + if (myCurrentReadState == ReadState.readingSize) + { + ///the size has already been read, now convert + ///it into an int, and start reading the data + int dataSize = new BinaryReader(new MemoryStream(myCurrentReadBuffer)).ReadInt32(); + if (dataSize > myMaxMessageSize) + { + string errMessage = "Encountered malformed message, size indicated = " + dataSize; + //BlueEventLog.logEvent(errMessage, "Error"); + myHandleError(new ApplicationException(errMessage)); + } + else + myReadData(dataSize); + } + else + { + ///the whole data has been finally read, time to + ///tell the subscriber about it + BinaryReader br = new BinaryReader(new MemoryStream(myCurrentReadBuffer)); + int msgType = br.ReadInt32(); + if (msgType == 0) + { + ///skips the message ID + br.ReadInt32(); + byte[] data = br.ReadBytes(myCurrentReadBuffer.Length - 8); + ///callbacks can be made through thread pool to avoid this + ///thread from being blocked, by this callback + if (!myDataReceived) + { + myDataReceived = true; + Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}"); + } + myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.data, myPeer)); + + } + else if (msgType == 1) + { + if (!myHBReceived) + { + myHBReceived = true; + Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}"); + } + myLastHBReceivedTime = DateTime.Now; + myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.heartbeat, myPeer)); + } + myReadSize(); + } + } + else + { + ///still part of the buffer left to read + ///continue reading + myBeginOrResumeReading(); + } + } + else + { + myHandleError(readErr); + } + } + catch (Exception ex) + { + myHandleError(ex); + } + } + } + + #endregion + + #region IDisposable Members + + public void Dispose() + { + Console.WriteLine("AsyncTCPConnection.Dispose()"); + lock (this) + { + if (myAlive) + { + myMasterWorker.removeWorker(this); + myAlive = false; + // BlueEventLog.logEvent(String.Format("AsycnTCPConnection with peer {0} being disposed, last HB was received at {1}" , + // myPeer.ToString(), myLastHBReceivedTime.ToLongTimeString()), + // PS3LogLevels.disconnection); + try + { + mySock.Close(); + } + catch (Exception ex) + { + String.Format("Exception of type {0} thrown while trying to close socket connected to {1} , with" + + " the following exception msg {2}", ex.GetType().ToString(), myPeer.ToString(), ex.Message); + //BlueEventLog.logEvent(ex); + if (ex is OutOfMemoryException) + throw ex; + } + } + } + } + + #endregion + + private void myHandleError(object err) + { + lock (this) + { + string errMsg = ""; + if (err is Exception) + { + errMsg = String.Format("AsyncTCPConnection.Dispose Encountered an exception of type '{0}' on the socket connected to ip = {1} , port = {2} ; " + + "Exception message = {3} , if the socket has not been disposed, it will be disposed now", err.GetType(), myPeer.ip, myPeer.port, (err as Exception).Message); + if (err is OutOfMemoryException) + throw (err as OutOfMemoryException); + } + else if (err is SocketError) + { + errMsg = String.Format("AsyncTCPConnection.Dispose Encountered a SocketError of type '{0}' on the socket connected to ip = {1} , port = {2} ; " + + "if the socket has not been disposed, it will be disposed now", err.ToString(), myPeer.ip, myPeer.port); + Console.WriteLine($"AsyncTCPConnection.myHandleError errMsg: {errMsg}"); + } + //BlueEventLog.logEvent(errMsg, PS3LogLevels.disconnection); + if (myAlive) + { + Dispose(); + } + } + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs new file mode 100644 index 000000000..c40017c27 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Net; +using System.Net.Sockets; +using System.Threading; +//using BCG.Utilities; +//using BCG.Utilities.General; + +namespace BCG +{ + internal class SocketConnectionArgs + { + public EndPoint ep; + public Socket sock; + public ITCPConnectionHandler handler; + public string serverURL; + + internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler, + string theServerURL) + { + ep = theEp; + sock = theSock; + handler = theHandler; + serverURL = theServerURL; + } + } + + public class ClientToServerManagedTCPConnector + { + private static Random myRnd = new Random(DateTime.Now.Millisecond); + private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval); + private static int myWaitTimeBeforeRetryingUpperLimit = 5000, + myWaitTimeBeforeRetryingLowerLimit = 1000; + + public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) + { + // TODOds + // BlueEventLog.logEvent(String.Format("Connection being attempted to server {0}", serverAddress.ToString()), + // PS3LogLevels.connection); + Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); + Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port); + sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString())); + } + + private static void myOnConnect(IAsyncResult res) + { + Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString()); + SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs; + try + { + args.sock.EndConnect(res); + } + catch (SocketException sex) + { + string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString()); + //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection); + //Console.WriteLine(errMsg); + if (args.handler.isDisposed) + { + string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString()); + //Console.WriteLine(doneMsg); + // BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection); + return; + } + else + { + string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString()); + //Console.WriteLine(retryMsg); + //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection); + + int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit); + Thread.Sleep(waitTime); + args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args); + return; + } + } + //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection); + args.handler.onConnection(new AsyncTCPConnection(args.sock, myHBTimeSpan)); + } + + } +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs new file mode 100644 index 000000000..5ab5d377a --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + public delegate void SendConfirmationDelegate(string messageId, bool confirmed); + /// + /// This is the very basic connection that sits right + /// on top of the sockets (unmanaged, managed whatever). + /// + public interface ITCPConnection : IDisposable + { + /// + /// The delegate is fired signalling if the data send was + /// successful or not + /// + /// + /// + /// + void sendWithConfirmation (byte[] data, ref string confId, SendConfirmationDelegate del); + /// + /// sends data to the other side. theres no guaranatee that + /// data is delievered. + /// + /// + void send(byte[] data); + /// + /// registers a ITCPMessageHandler, to receive messages + /// from the other side. + /// + /// + void register(ITCPMessageHandler messageHandler); + + /// + /// Gets the ip and port of the peer to which this + /// connection is connected. + /// + /// + /// + void getPeerAddress(out string ip, out int port); + + /// + /// Gets its own ip and port, as seen by the peer. So if this + /// is the client, it will returns its ip and the port, as the + /// server sees it. For server, this is the ip and wellknown + /// port of the server + /// + /// + /// + void getSelfAddress(out string ip, out int port); + /// + /// if the connection is disconnected, it disposes itself, + /// hence this is an indicator of whether the connection is + /// alive or dead + /// + bool isDisposed { get;} + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnectionHandler.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnectionHandler.cs new file mode 100644 index 000000000..ba18730c9 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnectionHandler.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + /// + /// This interface is used to get connections, whenever + /// one is established. + /// This function will be called in an arbitrary thread + /// + public interface ITCPConnectionHandler : IDisposable + { + void onConnection(ITCPConnection connection); + bool isDisposed { get; } + } +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ITCPMessageHandler.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPMessageHandler.cs new file mode 100644 index 000000000..afe664cd5 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPMessageHandler.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + /// + /// This interface is used to get messages back from the + /// ITCPConnection, which is the very low level connection + /// that encapsultes socket send and receive. + /// This function will be called on an arbitrary thread. + /// + public interface ITCPMessageHandler + { + void onData(RemoteMessage message); + } +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs b/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs new file mode 100644 index 000000000..ede494b3f --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Net; + +namespace BCG +{ + public class RemoteConnectionInfo + { + private string myIp; + private int myPort; + private IPEndPoint myEndPoint; + + public RemoteConnectionInfo(string ip, int port) + { + myIp = ip; + myPort = port; + } + + public RemoteConnectionInfo(string ip, short port, IPEndPoint endPoint) + : this (ip, port) + { + myEndPoint = endPoint; + } + + public string ip + { + get { return myIp; } + } + + public int port + { + get { return myPort; } + } + + public IPEndPoint endPoint + { + get { return myEndPoint; } + } + + public override string ToString() + { + return ip + ":" + port; + } + + } +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/RemoteMessage.cs b/src/DataStreamSuite/PubSub/ManagedTCP/RemoteMessage.cs new file mode 100644 index 000000000..4c3ce1fe7 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/RemoteMessage.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + /// + /// Type of messages that can be sent over the transport. + /// + public enum RemoteMessageType + { + heartbeat, data, connection, disconnection + } + + /// + /// Every message from a remote peer, is represented as a RemoteMessage. + /// This can be a transport generated message (heartbeat) or user generated + /// message (data) + /// + public class RemoteMessage + { + private byte[] myMsg; + private RemoteMessageType myMsgType; + private RemoteConnectionInfo myConnInfo; + + public RemoteMessage(byte[] message, RemoteMessageType messageType, RemoteConnectionInfo connInfo) + { + myMsg = message; + myMsgType = messageType; + myConnInfo = connInfo; + } + + public byte[] message + { + get { return myMsg; } + } + + public RemoteMessageType messageType + { + get { return myMsgType; } + } + + public RemoteConnectionInfo remoteConnectionInfo + { + get { return myConnInfo; } + } + + } +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs new file mode 100644 index 000000000..319fa1df5 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +//using BCG.Utilities.General; +//using BCG.Utilities; + +namespace BCG +{ + public class ServerToClientManagedTCPConnectionAcceptor + { + + private static Dictionary myPortToSingleServerAcceptorMap = new Dictionary(); + private static object mySyncObject = new object(); + + public static int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) + { + SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor(); + int portNo = theAcceptor.activateWithRandomPort(handler, clientHBTimeSpan); + + lock (mySyncObject) + { + myPortToSingleServerAcceptorMap[portNo] = theAcceptor; + + } + return portNo; + } + + public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) + { + lock (mySyncObject) + { + if (myPortToSingleServerAcceptorMap.ContainsKey(port)) + { + string msg = string.Format("Error: Application is already accepting clients on port {0}", port); + // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error); + return port; + } + else + { + SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor(); + int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan); + myPortToSingleServerAcceptorMap[assignedport] = theAcceptor; + return assignedport; + } + } //end lock + } //end activate code + + + } +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs b/src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs new file mode 100644 index 000000000..9623e9423 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Net; +using System.Net.Sockets; + +//using BCG.Utilities; + +namespace BCG +{ + public class SingleServerToClientManagedTCPConnectionAcceptor + { + private TimeSpan myHBTimeSpan, + myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250); + private ITCPConnectionHandler myHandler; + private Socket myAcceptSock; + private int myPort; + + + public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) + { + myHBTimeSpan = clientHBTimeSpan; + myHandler = handler; + myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + int port = 0; + bool madeItThrough = false; +// BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection); + Random randomizer = new Random(); + PubSubPortRange theRange = PubSub3Config.instance.PS3_PortRange; + + + //if the range of ports is 1000, then trying 5,000 times means + //your odds of missing an open port are less than 1% + + int maxAttempts = (theRange.maxPort - theRange.minPort) * 5; + int totalAttempts = 0; + while (!madeItThrough) + { + port = randomizer.Next(); + port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort)); + + try + { + myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port)); + madeItThrough = true; + } + catch (Exception) + { + totalAttempts++; + if (totalAttempts < maxAttempts){ + //BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection); + } + else + { + // BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details."); + } + } + } + myPort = port; + myAcceptSock.Listen(250); + myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); + + return myPort; + } + + public int activate (ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) + { + myHBTimeSpan = clientHBTimeSpan; + myHandler = handler; + myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + IPEndPoint endpoint = new IPEndPoint (IPAddress.Any, port); + myAcceptSock.Bind(endpoint); + myAcceptSock.Listen(250); + myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); + myPort = endpoint.Port; + return myPort; // not sure why we need this class variable myPort, but I am just keeping it up with the existing code + } + + private void myOnAccept(IAsyncResult res) + { + Socket newSock; + try + { + newSock = myAcceptSock.EndAccept(res); + } + catch (Exception ex) + { + // BlueEventLog.logEvent(ex); + Thread.Sleep(myWaitTimeBeforeRetrying); + myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); + if (ex is OutOfMemoryException) + throw ex; + return; + } + newSock.LingerState = new LingerOption(false, 0); + newSock.Blocking = false; + ITCPConnection newConn = new AsyncTCPConnection(newSock, myHBTimeSpan); + myHandler.onConnection(newConn); + myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); + string ip;int port; + newConn.getPeerAddress (out ip, out port); + string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort); +// BlueEventLog.logEvent(connMsg, PS3LogLevels.connection); + //Console.WriteLine(connMsg); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/BlockingQueue.cs b/src/DataStreamSuite/PubSub/Misc/BlockingQueue.cs new file mode 100644 index 000000000..0b54f3375 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/BlockingQueue.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections; +using System.Threading; + +using BCG; +using BCG.Utilities; + +namespace BCG +{ + [Serializable] + public class BlockingQueueTerminated : Exception + { + public BlockingQueueTerminated () + :base() + {} + } + + /// + /// Use this class for inter thread communication. + /// + public class BlockingQueue + { + private Queue myQueue = new Queue(); + private bool myTerminated = false; + + public void enqueue (object o) + { + lock (this) + { + if (myTerminated) + throw new BlockingQueueTerminated(); + myQueue.Enqueue(o); + + if (myQueue.Count == 1) + Monitor.Pulse(this); + } + } + + public object dequeue() + { + return dequeue(Timeout.Infinite); + } + + public object dequeue (int millisecondsToWait) + { + for (;;) + lock (this) + { + if (myTerminated) + throw new BlockingQueueTerminated(); + if (myQueue.Count == 0) + { + ///releases the lock and blocks for millisecondsToWait or until Pulse is called by some other thread + ///(which basically means that it will block here until someone calls an enqueue) + if (!Monitor.Wait(this,millisecondsToWait)) + return null; + } + else + return myQueue.Dequeue(); + } + } + + public object[] dequeueMultipleElements(int millisecondsToWait, int maxElements) + { + ArrayList rv = new ArrayList(); + int n = getCount(); + for (int i = 0; i < n; i++) + { + object o = dequeue(0); + if (o != null) + { + rv.Add(o); + if (rv.Count == maxElements) + return rv.ToArray(); + } + } + DateTime startTime = DateTime.Now; + int timeRemaining = millisecondsToWait; + while (true) + { + object o = dequeue(timeRemaining); + if (o != null) + rv.Add(o); + timeRemaining = millisecondsToWait - Convert.ToInt32 (((TimeSpan)(DateTime.Now - startTime)).TotalMilliseconds); + if (timeRemaining < 0 || rv.Count == maxElements) + return rv.ToArray(); + } + } + + public object[] dequeueMultipleElements(TimeSpan ts, int maxElements) + { + return dequeueMultipleElements(Convert.ToInt32 (ts.TotalMilliseconds), maxElements); + } + + public int getCount() + { + lock (this) + { + return myQueue.Count; + } + } + + + public void terminate() + { + lock (this) + { + myTerminated = true; + Monitor.PulseAll(this); + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/ClientInfo.cs b/src/DataStreamSuite/PubSub/Misc/ClientInfo.cs new file mode 100644 index 000000000..27f8f2034 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/ClientInfo.cs @@ -0,0 +1,138 @@ +using System; +using System.Text; +using BCG.Utilities.General; + + + +namespace BCG +{ + [Serializable()] + public class ClientInfo + { + private string myMachineName; + public string machineName + { + get { return myMachineName; } + } + + private int myPid; + public int pid + { + get { return myPid; } + set { myPid = value; } + } + + private string myApp; + public string app + { + get { return myApp; } + set { myApp = value; } + + } + private string myKeyQualifier; + public string keyQualifier + { + get { return myKeyQualifier; } + set { myKeyQualifier = value; } + } + + private DateTime myStartTime; + public DateTime startTime + { + get { return myStartTime; } + set { myStartTime = value; } + + } + private string myProcessString; + public string processString + { + get + { + if (myProcessString == null) + myProcessString = getProcessString(";"); + return myProcessString; + } + + } + + + public ClientInfo(string theMachineName, int thePid, string theProcessName, string theKeyQualifier, DateTime theStartTime) + { + myMachineName = theMachineName; + myPid = thePid; + myApp = theProcessName; + myKeyQualifier = theKeyQualifier; + myProcessString = null; + myStartTime = theStartTime; + } + + public ClientInfo() + { + myMachineName = Environment.MachineName; + myPid = System.Diagnostics.Process.GetCurrentProcess().Id; + myApp = System.Diagnostics.Process.GetCurrentProcess().ProcessName.ToUpper(); + string[] args = System.Environment.GetCommandLineArgs(); + + //the key qualifier is the last argument given. If they didn't give any arguments + //but the program itself, then the KQ is empty. + if (args.Length > 1) + myKeyQualifier = args[args.Length - 1]; + else + myKeyQualifier = ""; + myProcessString = null; + + myStartTime = System.Diagnostics.Process.GetCurrentProcess().StartTime; + } + + private static ClientInfo myLocalClient; + private static object myLock = new object(); + + public static ClientInfo thisClient + { + get + { + if (myLocalClient == null) + { + lock(myLock) + { + if(myLocalClient == null) + myLocalClient = new ClientInfo(); + } + } + return myLocalClient; + } + } + + public static ClientInfo fromProcessString(string procString) + { + + ClientInfo outputGuy = new ClientInfo(); + + string[] info = procString.Split(';'); + + outputGuy.myMachineName = info[0]; + outputGuy.myApp = info[1]; + outputGuy.myKeyQualifier = info[2]; + outputGuy.myPid = Int32.Parse(info[3]); + + + return outputGuy; + + } + + private string getProcessString(string seperator) + { + StringBuilder builder = new StringBuilder(); + + builder.Append(myMachineName); + builder.Append(seperator); + builder.Append(myApp); + builder.Append(seperator); + builder.Append(myKeyQualifier); + builder.Append(seperator); + builder.Append(myPid); + + return builder.ToString(); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/BlueCompressUtilities/IBlueCompressor.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/BlueCompressUtilities/IBlueCompressor.cs new file mode 100644 index 000000000..3d5cec253 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/BlueCompressUtilities/IBlueCompressor.cs @@ -0,0 +1,11 @@ +using System; +using System.IO; + +namespace BCG +{ + public interface IBlueCompressor + { + void compress(Stream uncompressedStream, Stream compressedStream); + void decompress(Stream compressedStream, Stream uncompressedStream); + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/BlueCompressUtilities/SharpZipCompressor.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/BlueCompressUtilities/SharpZipCompressor.cs new file mode 100644 index 000000000..58172cb2d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/BlueCompressUtilities/SharpZipCompressor.cs @@ -0,0 +1,50 @@ +using System; +using System.IO; + +using BCG.Utilities.SharpZipLib.Zip.Compression; +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; + +namespace BCG +{ + public class SharpZipCompressor : IBlueCompressor + { + public SharpZipCompressor(){} + + + public void compress(Stream unCompressedStream, Stream compressedStream) + { + Deflater deflator = new Deflater(Deflater.BEST_COMPRESSION); + + DeflaterOutputStream dfs = new DeflaterOutputStream(compressedStream, deflator); + + while ( true ) + { + byte[] b = new byte[unCompressedStream.Length + 1]; + int numRead = unCompressedStream.Read(b, 0, b.Length); + dfs.Write(b, 0, numRead); + if ( numRead < b.Length ) + break; + } + + dfs.Flush(); + dfs.Finish(); + } + + public void decompress(Stream compressedStream, Stream uncompressedStream) + { + InflaterInputStream infstr = new InflaterInputStream(compressedStream); + + while ( true ) + { + byte[] b = new byte[compressedStream.Length+1]; + int numRead = infstr.Read(b, 0, b.Length); + uncompressedStream.Write(b, 0, numRead); + if ( numRead < b.Length ) + break; + } + + uncompressedStream.Flush(); + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/CompressionUtilities.csproj b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/CompressionUtilities.csproj new file mode 100644 index 000000000..a8b390cd9 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/CompressionUtilities.csproj @@ -0,0 +1,273 @@ + + + + Local + 9.0.30729 + 2.0 + {41DBB1A3-1A4E-457D-BB2C-BFDE486E328B} + Debug + x64 + + + + + CompressionUtilities + ..\..\StrongNameKeys\Builder.snk + JScript + Grid + IE50 + false + Library + CompressionUtilities + OnBuildSuccess + + + false + v4.0 + 0 + 1.0.0.%2a + + + false + 285212672 + false + + + DEBUG;TRACE + + + true + 4096 + false + + + false + false + false + true + 4 + full + prompt + AllRules.ruleset + + + false + 285212672 + false + + + DEBUG;TRACE + + + true + 4096 + false + + + false + false + false + true + 4 + full + prompt + true + AllRules.ruleset + + + true + DEBUG;TRACE + 285212672 + full + x64 + prompt + AllRules.ruleset + + + true + DEBUG;TRACE + 285212672 + 4096 + full + x64 + prompt + AllRules.ruleset + + + true + true + DEBUG;TRACE + 285212672 + 4096 + full + x64 + prompt + AllRules.ruleset + + + true + DEBUG;TRACE + 285212672 + full + x64 + prompt + AllRules.ruleset + + + + System + + + 4.0 + + + System.Data + + + System.XML + + + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + Code + + + + + False + .NET Framework 3.5 SP1 Client Profile + + + False + .NET Framework 3.5 SP1 + + + False + Windows Installer 3.1 + + + + + + + + + + diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2.cs new file mode 100644 index 000000000..5c6cdda7b --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2.cs @@ -0,0 +1,98 @@ +// BZip2.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; + +namespace BCG.Utilities.SharpZipLib.BZip2 { + + /// + /// Does all the compress and decompress pre-operation stuff. + /// Sets up the streams and file header characters. + /// Uses multiply overloaded methods to call for the compress/decompress. + /// + public sealed class BZip2 + { + public static void Decompress(Stream instream, Stream outstream) + { + BufferedStream bos = new BufferedStream(outstream); + BufferedStream bis = new BufferedStream(instream); + int b = bis.ReadByte(); + if (b != 'B') { + return; + } + b = bis.ReadByte(); + if (b != 'Z') { + return; + } + BZip2InputStream bzis = new BZip2InputStream(bis); + int ch = bzis.ReadByte(); + while (ch != -1) { + bos.WriteByte((byte)ch); + ch = bzis.ReadByte(); + } + bos.Flush(); + } + + public static void Compress(Stream instream, Stream outstream, int blockSize) + { + BufferedStream bos = new BufferedStream(outstream); + bos.WriteByte((byte)'B'); + bos.WriteByte((byte)'Z'); + BufferedStream bis = new BufferedStream(instream); + int ch = bis.ReadByte(); + BZip2OutputStream bzos = new BZip2OutputStream(bos); + while(ch != -1) { + bzos.WriteByte((byte)ch); + ch = bis.ReadByte(); + } + bis.Close(); + bzos.Close(); + } + } +} +/* derived from a file which contained this license : + * Copyright (c) 1999-2001 Keiron Liddle, Aftex Software + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * +*/ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2Constants.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2Constants.cs new file mode 100644 index 000000000..e9f2c691f --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2Constants.cs @@ -0,0 +1,155 @@ +// BZip2Constants.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +namespace BCG.Utilities.SharpZipLib.BZip2 { + + /// + /// Base class for both the compress and decompress classes. + /// Holds common arrays, and static data. + /// + public sealed class BZip2Constants + { + public readonly static int[] rNums = { + 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, + 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, + 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, + 419, 436, 278, 496, 867, 210, 399, 680, 480, 51, + 878, 465, 811, 169, 869, 675, 611, 697, 867, 561, + 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, + 150, 238, 59, 379, 684, 877, 625, 169, 643, 105, + 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, + 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, + 909, 545, 703, 919, 874, 474, 882, 500, 594, 612, + 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, + 161, 604, 958, 533, 221, 400, 386, 867, 600, 782, + 382, 596, 414, 171, 516, 375, 682, 485, 911, 276, + 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, + 227, 730, 475, 186, 263, 647, 537, 686, 600, 224, + 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, + 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, + 715, 67, 618, 276, 204, 918, 873, 777, 604, 560, + 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, + 652, 934, 970, 447, 318, 353, 859, 672, 112, 785, + 645, 863, 803, 350, 139, 93, 354, 99, 820, 908, + 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, + 653, 282, 762, 623, 680, 81, 927, 626, 789, 125, + 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, + 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, + 857, 956, 358, 619, 580, 124, 737, 594, 701, 612, + 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, + 944, 375, 748, 52, 600, 747, 642, 182, 862, 81, + 344, 805, 988, 739, 511, 655, 814, 334, 249, 515, + 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, + 433, 837, 553, 268, 926, 240, 102, 654, 459, 51, + 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, + 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, + 978, 321, 576, 617, 626, 502, 894, 679, 243, 440, + 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, + 707, 151, 457, 449, 797, 195, 791, 558, 945, 679, + 297, 59, 87, 824, 713, 663, 412, 693, 342, 606, + 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, + 343, 97, 430, 751, 497, 314, 983, 374, 822, 928, + 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, + 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, + 369, 970, 294, 750, 807, 827, 150, 790, 288, 923, + 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, + 896, 831, 547, 261, 524, 462, 293, 465, 502, 56, + 661, 821, 976, 991, 658, 869, 905, 758, 745, 193, + 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, + 61, 688, 793, 644, 986, 403, 106, 366, 905, 644, + 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, + 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, + 920, 176, 193, 713, 857, 265, 203, 50, 668, 108, + 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, + 936, 638 + }; + + public readonly static int baseBlockSize = 100000; + + public readonly static int MAX_ALPHA_SIZE = 258; + public readonly static int MAX_CODE_LEN = 23; + + public readonly static int RUNA = 0; + public readonly static int RUNB = 1; + + public readonly static int N_GROUPS = 6; + public readonly static int G_SIZE = 50; + public readonly static int N_ITERS = 4; + + public readonly static int MAX_SELECTORS = (2 + (900000 / G_SIZE)); + + public readonly static int NUM_OVERSHOOT_BYTES = 20; + } +} + +/* This file was derived from a file containing under this license: + * + * This file is a part of bzip2 and/or libbzip2, a program and + * library for lossless, block-sorting data compression. + * + * Copyright (C) 1996-1998 Julian R Seward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. The origin of this software must not be misrepresented; you must + * not claim that you wrote the original software. If you use this + * software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 3. Altered source versions must be plainly marked as such, and must + * not be misrepresented as being the original software. + * + * 4. The name of the author may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Java version ported by Keiron Liddle, Aftex Software 1999-2001 + */ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2InputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2InputStream.cs new file mode 100644 index 000000000..bdf18a007 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2InputStream.cs @@ -0,0 +1,945 @@ +// BZip2InputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; + +using BCG.Utilities.SharpZipLib.Checksums; + +namespace BCG.Utilities.SharpZipLib.BZip2 { + + /// + /// An input stream that decompresses from the BZip2 format (without the file + /// header chars) to be read as any other stream. + /// + public class BZip2InputStream : Stream + { + /// + /// I needed to implement the abstract member. + /// + public override bool CanRead { + get { + return baseStream.CanRead; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanSeek { + get { + return baseStream.CanSeek; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanWrite { + get { + return baseStream.CanWrite; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Length { + get { + return baseStream.Length; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Position { + get { + return baseStream.Position; + } + set { + baseStream.Position = value; + } + } + + /// + /// Flushes the baseInputStream + /// + public override void Flush() + { + if (baseStream != null) { + baseStream.Flush(); + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Seek(long offset, SeekOrigin origin) + { + return baseStream.Seek(offset, origin); + } + + /// + /// I needed to implement the abstract member. + /// + public override void SetLength(long val) + { + baseStream.SetLength(val); + } + + /// + /// I needed to implement the abstract member. + /// + public override void Write(byte[] array, int offset, int count) + { + baseStream.Write(array, offset, count); + } + + /// + /// I needed to implement the abstract member. + /// + public override void WriteByte(byte val) + { + baseStream.WriteByte(val); + } + + public override int Read(byte[] b, int off, int len) + { + for (int i = 0; i < len; ++i) { + int rb = ReadByte(); + if (rb == -1) { + return i; + } + b[off + i] = (byte)rb; + } + return len; + } + + /// + /// Closes the input stream + /// + public override void Close() + { + if (baseStream != null) { + baseStream.Close(); + } + } + + static void Cadvise() + { + Console.WriteLine("CRC Error"); + //throw new CCoruptionError(); + } + + static void BadBGLengths() + { + Console.WriteLine("bad BG lengths"); + } + + static void bitStreamEOF() + { + Console.WriteLine("bit stream eof"); + } + + static void CompressedStreamEOF() + { + Console.WriteLine("compressed stream eof"); + } + + void MakeMaps() + { + nInUse = 0; + for (int i = 0; i < 256; ++i) { + if (inUse[i]) { + seqToUnseq[nInUse] = (byte)i; + unseqToSeq[i] = (byte)nInUse; + nInUse++; + } + } + } + + /*-- + index of the last char in the block, so + the block size == last + 1. + --*/ + int last; + + /*-- + index in zptr[] of original string after sorting. + --*/ + int origPtr; + + /*-- + always: in the range 0 .. 9. + The current block size is 100000 * this number. + --*/ + int blockSize100k; + + bool blockRandomised; + +// private int bytesIn; +// private int bytesOut; + int bsBuff; + int bsLive; + IChecksum mCrc = new StrangeCRC(); + + bool[] inUse = new bool[256]; + int nInUse; + + byte[] seqToUnseq = new byte[256]; + byte[] unseqToSeq = new byte[256]; + + byte[] selector = new byte[BZip2Constants.MAX_SELECTORS]; + byte[] selectorMtf = new byte[BZip2Constants.MAX_SELECTORS]; + + int[] tt; + byte[] ll8; + + /*-- + freq table collected to save a pass over the data + during decompression. + --*/ + int[] unzftab = new int[256]; + + int[][] limit = new int[BZip2Constants.N_GROUPS][]; + int[][] baseArray = new int[BZip2Constants.N_GROUPS][]; + int[][] perm = new int[BZip2Constants.N_GROUPS][]; + int[] minLens = new int[BZip2Constants.N_GROUPS]; + + Stream baseStream; + bool streamEnd = false; + + int currentChar = -1; + + const int START_BLOCK_STATE = 1; + const int RAND_PART_A_STATE = 2; + const int RAND_PART_B_STATE = 3; + const int RAND_PART_C_STATE = 4; + const int NO_RAND_PART_A_STATE = 5; + const int NO_RAND_PART_B_STATE = 6; + const int NO_RAND_PART_C_STATE = 7; + + int currentState = START_BLOCK_STATE; + + int storedBlockCRC, storedCombinedCRC; + int computedBlockCRC; + uint computedCombinedCRC; + + int count, chPrev, ch2; + int tPos; + int rNToGo = 0; + int rTPos = 0; + int i2, j2; + byte z; + + public BZip2InputStream(Stream zStream) + { + // init arrays + for (int i = 0; i < BZip2Constants.N_GROUPS; ++i) { + limit[i] = new int[BZip2Constants.MAX_ALPHA_SIZE]; + baseArray[i] = new int[BZip2Constants.MAX_ALPHA_SIZE]; + perm[i] = new int[BZip2Constants.MAX_ALPHA_SIZE]; + } + + ll8 = null; + tt = null; + BsSetStream(zStream); + Initialize(); + InitBlock(); + SetupBlock(); + } + + public override int ReadByte() + { + if (streamEnd) { + return -1; + } + + int retChar = currentChar; + switch (currentState) { + case RAND_PART_B_STATE: + SetupRandPartB(); + break; + case RAND_PART_C_STATE: + SetupRandPartC(); + break; + case NO_RAND_PART_B_STATE: + SetupNoRandPartB(); + break; + case NO_RAND_PART_C_STATE: + SetupNoRandPartC(); + break; + case START_BLOCK_STATE: + case NO_RAND_PART_A_STATE: + case RAND_PART_A_STATE: + break; + default: + break; + } + return retChar; + } + + void Initialize() + { + char magic3 = BsGetUChar(); + char magic4 = BsGetUChar(); + + if (magic3 != 'h' || magic4 < '1' || magic4 > '9') { + streamEnd = true; + return; + } + + SetDecompressStructureSizes(magic4 - '0'); + computedCombinedCRC = 0; + } + + void InitBlock() + { + char magic1 = BsGetUChar(); + char magic2 = BsGetUChar(); + char magic3 = BsGetUChar(); + char magic4 = BsGetUChar(); + char magic5 = BsGetUChar(); + char magic6 = BsGetUChar(); + + if (magic1 == 0x17 && magic2 == 0x72 && magic3 == 0x45 && magic4 == 0x38 && magic5 == 0x50 && magic6 == 0x90) { + Complete(); + return; + } + + if (magic1 != 0x31 || magic2 != 0x41 || magic3 != 0x59 || magic4 != 0x26 || magic5 != 0x53 || magic6 != 0x59) { + BadBlockHeader(); + streamEnd = true; + return; + } + + storedBlockCRC = BsGetInt32(); + + blockRandomised = (BsR(1) == 1); + + // currBlockNo++; + GetAndMoveToFrontDecode(); + + mCrc.Reset(); + currentState = START_BLOCK_STATE; + } + + void EndBlock() + { + computedBlockCRC = (int)mCrc.Value; + + /*-- A bad CRC is considered a fatal error. --*/ + if (storedBlockCRC != computedBlockCRC) { + CrcError(); + } + + // 1528150659 + computedCombinedCRC = ((computedCombinedCRC << 1) & 0xFFFFFFFF) | (computedCombinedCRC >> 31); + computedCombinedCRC = computedCombinedCRC ^ (uint)computedBlockCRC; + } + + void Complete() + { + storedCombinedCRC = BsGetInt32(); + if (storedCombinedCRC != (int)computedCombinedCRC) { + CrcError(); + } + + streamEnd = true; + } + + static void BlockOverrun() + { + Console.WriteLine("Block overrun"); + } + + static void BadBlockHeader() + { + Console.WriteLine("Bad block header"); + } + + static void CrcError() + { + Console.WriteLine("crc error"); + } + + + void BsSetStream(Stream f) + { + baseStream = f; + bsLive = 0; + bsBuff = 0; + } + + void FillBuffer() + { + int thech = 0; + + try { + thech = baseStream.ReadByte(); + } catch (Exception) { + CompressedStreamEOF(); + } + + if (thech == -1) { + CompressedStreamEOF(); + } + + bsBuff = (bsBuff << 8) | (thech & 0xFF); + bsLive += 8; + } + + int BsR(int n) + { + while (bsLive < n) { + FillBuffer(); + } + + int v = (bsBuff >> (bsLive - n)) & ((1 << n) - 1); + bsLive -= n; + return v; + } + + char BsGetUChar() + { + return (char)BsR(8); + } + + int BsGetint() + { + int u = 0; + u = (u << 8) | BsR(8); + u = (u << 8) | BsR(8); + u = (u << 8) | BsR(8); + u = (u << 8) | BsR(8); + return u; + } + + int BsGetIntVS(int numBits) + { + return (int)BsR(numBits); + } + + int BsGetInt32() + { + return (int)BsGetint(); + } + + void HbCreateDecodeTables(int[] limit, int[] baseArray, int[] perm, char[] length, int minLen, int maxLen, int alphaSize) + { + int pp = 0; + + for (int i = minLen; i <= maxLen; ++i) { + for (int j = 0; j < alphaSize; ++j) { + if (length[j] == i) { + perm[pp] = j; + ++pp; + } + } + } + + for (int i = 0; i < BZip2Constants.MAX_CODE_LEN; i++) { + baseArray[i] = 0; + } + + for (int i = 0; i < alphaSize; i++) { + ++baseArray[length[i] + 1]; + } + + for (int i = 1; i < BZip2Constants.MAX_CODE_LEN; i++) { + baseArray[i] += baseArray[i - 1]; + } + + for (int i = 0; i < BZip2Constants.MAX_CODE_LEN; i++) { + limit[i] = 0; + } + + int vec = 0; + + for (int i = minLen; i <= maxLen; i++) { + vec += (baseArray[i + 1] - baseArray[i]); + limit[i] = vec - 1; + vec <<= 1; + } + + for (int i = minLen + 1; i <= maxLen; i++) { + baseArray[i] = ((limit[i - 1] + 1) << 1) - baseArray[i]; + } + } + + void RecvDecodingTables() + { + char[][] len = new char[BZip2Constants.N_GROUPS][]; + for (int i = 0; i < BZip2Constants.N_GROUPS; ++i) { + len[i] = new char[BZip2Constants.MAX_ALPHA_SIZE]; + } + + bool[] inUse16 = new bool[16]; + + /*--- Receive the mapping table ---*/ + for (int i = 0; i < 16; i++) { + inUse16[i] = (BsR(1) == 1); + } + + for (int i = 0; i < 16; i++) { + if (inUse16[i]) { + for (int j = 0; j < 16; j++) { + inUse[i * 16 + j] = (BsR(1) == 1); + } + } + } + + MakeMaps(); + int alphaSize = nInUse + 2; + + /*--- Now the selectors ---*/ + int nGroups = BsR(3); + int nSelectors = BsR(15); + + for (int i = 0; i < nSelectors; i++) { + int j = 0; + while (BsR(1) == 1) { + j++; + } + selectorMtf[i] = (byte)j; + } + + /*--- Undo the MTF values for the selectors. ---*/ + byte[] pos = new byte[BZip2Constants.N_GROUPS]; + for (int v = 0; v < nGroups; v++) { + pos[v] = (byte)v; + } + + for (int i = 0; i < nSelectors; i++) { + int v = selectorMtf[i]; + byte tmp = pos[v]; + while (v > 0) { + pos[v] = pos[v - 1]; + v--; + } + pos[0] = tmp; + selector[i] = tmp; + } + + /*--- Now the coding tables ---*/ + for (int t = 0; t < nGroups; t++) { + int curr = BsR(5); + for (int i = 0; i < alphaSize; i++) { + while (BsR(1) == 1) { + if (BsR(1) == 0) { + curr++; + } else { + curr--; + } + } + len[t][i] = (char)curr; + } + } + + /*--- Create the Huffman decoding tables ---*/ + for (int t = 0; t < nGroups; t++) { + int minLen = 32; + int maxLen = 0; + for (int i = 0; i < alphaSize; i++) { + maxLen = Math.Max(maxLen, len[t][i]); + minLen = Math.Min(minLen, len[t][i]); + } + HbCreateDecodeTables(limit[t], baseArray[t], perm[t], len[t], minLen, maxLen, alphaSize); + minLens[t] = minLen; + } + } + + void GetAndMoveToFrontDecode() + { + byte[] yy = new byte[256]; + int nextSym; + + int limitLast = BZip2Constants.baseBlockSize * blockSize100k; + origPtr = BsGetIntVS(24); + + RecvDecodingTables(); + int EOB = nInUse+1; + int groupNo = -1; + int groupPos = 0; + + /*-- + Setting up the unzftab entries here is not strictly + necessary, but it does save having to do it later + in a separate pass, and so saves a block's worth of + cache misses. + --*/ + for (int i = 0; i <= 255; i++) { + unzftab[i] = 0; + } + + for (int i = 0; i <= 255; i++) { + yy[i] = (byte)i; + } + + last = -1; + + if (groupPos == 0) { + groupNo++; + groupPos = BZip2Constants.G_SIZE; + } + + groupPos--; + int zt = selector[groupNo]; + int zn = minLens[zt]; + int zvec = BsR(zn); + int zj; + + while (zvec > limit[zt][zn]) { + zn++; + while (bsLive < 1) { + FillBuffer(); + } + zj = (bsBuff >> (bsLive-1)) & 1; + bsLive--; + zvec = (zvec << 1) | zj; + } + nextSym = perm[zt][zvec - baseArray[zt][zn]]; + + while (true) { + + if (nextSym == EOB) { + break; + } + + if (nextSym == BZip2Constants.RUNA || nextSym == BZip2Constants.RUNB) { + int s = -1; + int n = 1; + do { + if (nextSym == BZip2Constants.RUNA) { + s += (0+1) * n; + } else if (nextSym == BZip2Constants.RUNB) { + s += (1+1) * n; + } + + n <<= 1; + + if (groupPos == 0) { + groupNo++; + groupPos = BZip2Constants.G_SIZE; + } + + groupPos--; + + zt = selector[groupNo]; + zn = minLens[zt]; + zvec = BsR(zn); + + while (zvec > limit[zt][zn]) { + zn++; + while (bsLive < 1) { + FillBuffer(); + } + zj = (bsBuff >> (bsLive - 1)) & 1; + bsLive--; + zvec = (zvec << 1) | zj; + } + nextSym = perm[zt][zvec - baseArray[zt][zn]]; + } while (nextSym == BZip2Constants.RUNA || nextSym == BZip2Constants.RUNB); + + s++; + byte ch = seqToUnseq[yy[0]]; + unzftab[ch] += s; + + while (s > 0) { + last++; + ll8[last] = ch; + s--; + } + + if (last >= limitLast) { + BlockOverrun(); + } + continue; + } else { + last++; + if (last >= limitLast) { + BlockOverrun(); + } + + byte tmp = yy[nextSym - 1]; + unzftab[seqToUnseq[tmp]]++; + ll8[last] = seqToUnseq[tmp]; + + for (int j = nextSym-1; j > 0; --j) { + yy[j] = yy[j - 1]; + } + yy[0] = tmp; + + if (groupPos == 0) { + groupNo++; + groupPos = BZip2Constants.G_SIZE; + } + + groupPos--; + zt = selector[groupNo]; + zn = minLens[zt]; + zvec = BsR(zn); + while (zvec > limit[zt][zn]) { + zn++; + while (bsLive < 1) { + FillBuffer(); + } + zj = (bsBuff >> (bsLive-1)) & 1; + bsLive--; + zvec = (zvec << 1) | zj; + } + nextSym = perm[zt][zvec - baseArray[zt][zn]]; + continue; + } + } + } + + void SetupBlock() + { + int[] cftab = new int[257]; + + cftab[0] = 0; + Array.Copy(unzftab, 0, cftab, 1, 256); + + for (int i = 1; i <= 256; i++) { + cftab[i] += cftab[i - 1]; + } + + for (int i = 0; i <= last; i++) { + byte ch = ll8[i]; + tt[cftab[ch]] = i; + cftab[ch]++; + } + + cftab = null; + + tPos = tt[origPtr]; + + count = 0; + i2 = 0; + ch2 = 256; /*-- not a char and not EOF --*/ + + if (blockRandomised) { + rNToGo = 0; + rTPos = 0; + SetupRandPartA(); + } else { + SetupNoRandPartA(); + } + } + + void SetupRandPartA() + { + if(i2 <= last) { + chPrev = ch2; + ch2 = ll8[tPos]; + tPos = tt[tPos]; + if (rNToGo == 0) { + rNToGo = BZip2Constants.rNums[rTPos]; + rTPos++; + if(rTPos == 512) { + rTPos = 0; + } + } + rNToGo--; + ch2 ^= (int)((rNToGo == 1) ? 1 : 0); + i2++; + + currentChar = ch2; + currentState = RAND_PART_B_STATE; + mCrc.Update(ch2); + } else { + EndBlock(); + InitBlock(); + SetupBlock(); + } + } + + void SetupNoRandPartA() + { + if (i2 <= last) { + chPrev = ch2; + ch2 = ll8[tPos]; + tPos = tt[tPos]; + i2++; + + currentChar = ch2; + currentState = NO_RAND_PART_B_STATE; + mCrc.Update(ch2); + } else { + EndBlock(); + InitBlock(); + SetupBlock(); + } + } + + void SetupRandPartB() + { + if (ch2 != chPrev) { + currentState = RAND_PART_A_STATE; + count = 1; + SetupRandPartA(); + } else { + count++; + if (count >= 4) { + z = ll8[tPos]; + tPos = tt[tPos]; + if (rNToGo == 0) { + rNToGo = BZip2Constants.rNums[rTPos]; + rTPos++; + if (rTPos == 512) { + rTPos = 0; + } + } + rNToGo--; + z ^= (byte)((rNToGo == 1) ? 1 : 0); + j2 = 0; + currentState = RAND_PART_C_STATE; + SetupRandPartC(); + } else { + currentState = RAND_PART_A_STATE; + SetupRandPartA(); + } + } + } + + void SetupRandPartC() + { + if (j2 < (int)z) { + currentChar = ch2; + mCrc.Update(ch2); + j2++; + } else { + currentState = RAND_PART_A_STATE; + i2++; + count = 0; + SetupRandPartA(); + } + } + + void SetupNoRandPartB() + { + if (ch2 != chPrev) { + currentState = NO_RAND_PART_A_STATE; + count = 1; + SetupNoRandPartA(); + } else { + count++; + if (count >= 4) { + z = ll8[tPos]; + tPos = tt[tPos]; + currentState = NO_RAND_PART_C_STATE; + j2 = 0; + SetupNoRandPartC(); + } else { + currentState = NO_RAND_PART_A_STATE; + SetupNoRandPartA(); + } + } + } + + void SetupNoRandPartC() + { + if (j2 < (int)z) { + currentChar = ch2; + mCrc.Update(ch2); + j2++; + } else { + currentState = NO_RAND_PART_A_STATE; + i2++; + count = 0; + SetupNoRandPartA(); + } + } + + void SetDecompressStructureSizes(int newSize100k) + { + if (!(0 <= newSize100k && newSize100k <= 9 && + 0 <= blockSize100k && blockSize100k <= 9)) { + throw new ApplicationException("Invalid block size"); + } + + blockSize100k = newSize100k; + + if (newSize100k == 0) { + return; + } + + int n = BZip2Constants.baseBlockSize * newSize100k; + ll8 = new byte[n]; + tt = new int[n]; + } + } +} +/* This file was derived from a file containing under this license: + * + * This file is a part of bzip2 and/or libbzip2, a program and + * library for lossless, block-sorting data compression. + * + * Copyright (C) 1996-1998 Julian R Seward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. The origin of this software must not be misrepresented; you must + * not claim that you wrote the original software. If you use this + * software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 3. Altered source versions must be plainly marked as such, and must + * not be misrepresented as being the original software. + * + * 4. The name of the author may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Java version ported by Keiron Liddle, Aftex Software 1999-2001 + */ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2OutputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2OutputStream.cs new file mode 100644 index 000000000..9a51e020d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/BZip2/BZip2OutputStream.cs @@ -0,0 +1,1746 @@ +// BZip2OutputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; + +using BCG.Utilities.SharpZipLib.Checksums; + +namespace BCG.Utilities.SharpZipLib.BZip2 { + + /// + /// An output stream that compresses into the BZip2 format (without the file + /// header chars) into another stream. + /// TODO: Update to BZip2 1.0.1, 1.0.2 + /// + public class BZip2OutputStream : Stream + { + /// + /// I needed to implement the abstract member. + /// + public override bool CanRead { + get { + return baseStream.CanRead; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanSeek { + get { + return baseStream.CanSeek; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanWrite { + get { + return baseStream.CanWrite; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Length { + get { + return baseStream.Length; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Position { + get { + return baseStream.Position; + } + set { + baseStream.Position = value; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Seek(long offset, SeekOrigin origin) + { + return baseStream.Seek(offset, origin); + } + + /// + /// I needed to implement the abstract member. + /// + public override void SetLength(long val) + { + baseStream.SetLength(val); + } + + /// + /// I needed to implement the abstract member. + /// + public override int ReadByte() + { + return baseStream.ReadByte(); + } + + /// + /// I needed to implement the abstract member. + /// + public override int Read(byte[] b, int off, int len) + { + return baseStream.Read(b, off, len); + } + + public override void Write(byte[] buf, int off, int len) + { + for (int i = 0; i < len; ++i) { + WriteByte(buf[off + i]); + } + } + + readonly static int SETMASK = (1 << 21); + readonly static int CLEARMASK = (~SETMASK); + readonly static int GREATER_ICOST = 15; + readonly static int LESSER_ICOST = 0; + readonly static int SMALL_THRESH = 20; + readonly static int DEPTH_THRESH = 10; + + /*-- + If you are ever unlucky/improbable enough + to get a stack overflow whilst sorting, + increase the following constant and try + again. In practice I have never seen the + stack go above 27 elems, so the following + limit seems very generous. + --*/ + readonly static int QSORT_STACK_SIZE = 1000; + + static void Panic() + { + Console.WriteLine("panic"); + } + + void MakeMaps() + { + int i; + nInUse = 0; + for (i = 0; i < 256; i++) { + if (inUse[i]) { + seqToUnseq[nInUse] = (char)i; + unseqToSeq[i] = (char)nInUse; + nInUse++; + } + } + } + + static void HbMakeCodeLengths(char[] len, int[] freq, int alphaSize, int maxLen) + { + /*-- + Nodes and heap entries run from 1. Entry 0 + for both the heap and nodes is a sentinel. + --*/ + int nNodes, nHeap, n1, n2, j, k; + bool tooLong; + + int[] heap = new int[BZip2Constants.MAX_ALPHA_SIZE + 2]; + int[] weight = new int[BZip2Constants.MAX_ALPHA_SIZE * 2]; + int[] parent = new int[BZip2Constants.MAX_ALPHA_SIZE * 2]; + + for (int i = 0; i < alphaSize; ++i) { + weight[i+1] = (freq[i] == 0 ? 1 : freq[i]) << 8; + } + + while (true) { + nNodes = alphaSize; + nHeap = 0; + + heap[0] = 0; + weight[0] = 0; + parent[0] = -2; + + for (int i = 1; i <= alphaSize; ++i) { + parent[i] = -1; + nHeap++; + heap[nHeap] = i; + int zz = nHeap; + int tmp = heap[zz]; + while (weight[tmp] < weight[heap[zz >> 1]]) { + heap[zz] = heap[zz >> 1]; + zz >>= 1; + } + heap[zz] = tmp; + } + if (!(nHeap < (BZip2Constants.MAX_ALPHA_SIZE+2))) { + Panic(); + } + + while (nHeap > 1) { + n1 = heap[1]; + heap[1] = heap[nHeap]; + nHeap--; + int zz = 1; + int yy = 0; + int tmp = heap[zz]; + while (true) { + yy = zz << 1; + if (yy > nHeap) { + break; + } + if (yy < nHeap && weight[heap[yy+1]] < weight[heap[yy]]) { + yy++; + } + if (weight[tmp] < weight[heap[yy]]) { + break; + } + + heap[zz] = heap[yy]; + zz = yy; + } + heap[zz] = tmp; + n2 = heap[1]; + heap[1] = heap[nHeap]; + nHeap--; + + zz = 1; + yy = 0; + tmp = heap[zz]; + while (true) { + yy = zz << 1; + if (yy > nHeap) + break; + if (yy < nHeap && + weight[heap[yy+1]] < weight[heap[yy]]) + yy++; + if (weight[tmp] < weight[heap[yy]]) + break; + heap[zz] = heap[yy]; + zz = yy; + } + heap[zz] = tmp; + nNodes++; + parent[n1] = parent[n2] = nNodes; + + weight[nNodes] = (int)((weight[n1] & 0xffffff00) + (weight[n2] & 0xffffff00)) | + (int)(1 + (((weight[n1] & 0x000000ff) > (weight[n2] & 0x000000ff)) ? (weight[n1] & 0x000000ff) : (weight[n2] & 0x000000ff))); + + parent[nNodes] = -1; + nHeap++; + heap[nHeap] = nNodes; + + zz = nHeap; + tmp = heap[zz]; + while (weight[tmp] < weight[heap[zz >> 1]]) { + heap[zz] = heap[zz >> 1]; + zz >>= 1; + } + heap[zz] = tmp; + } + if (!(nNodes < (BZip2Constants.MAX_ALPHA_SIZE * 2))) { + Panic(); + } + + tooLong = false; + for (int i = 1; i <= alphaSize; ++i) { + j = 0; + k = i; + while (parent[k] >= 0) { + k = parent[k]; + j++; + } + len[i - 1] = (char)j; + if (j > maxLen) { + tooLong = true; + } + } + + if (!tooLong) { + break; + } + + for (int i = 1; i < alphaSize; ++i) { + j = weight[i] >> 8; + j = 1 + (j / 2); + weight[i] = j << 8; + } + } + } + + /*-- + index of the last char in the block, so + the block size == last + 1. + --*/ + int last; + + /*-- + index in zptr[] of original string after sorting. + --*/ + int origPtr; + + /*-- + always: in the range 0 .. 9. + The current block size is 100000 * this number. + --*/ + int blockSize100k; + + bool blockRandomised; + +// int bytesIn; + int bytesOut; + int bsBuff; + int bsLive; + IChecksum mCrc = new StrangeCRC(); + + bool[] inUse = new bool[256]; + int nInUse; + + char[] seqToUnseq = new char[256]; + char[] unseqToSeq = new char[256]; + + char[] selector = new char[BZip2Constants.MAX_SELECTORS]; + char[] selectorMtf = new char[BZip2Constants.MAX_SELECTORS]; + + byte[] block; + int[] quadrant; + int[] zptr; + short[] szptr; + int[] ftab; + + int nMTF; + + int[] mtfFreq = new int[BZip2Constants.MAX_ALPHA_SIZE]; + + /* + * Used when sorting. If too many long comparisons + * happen, we stop sorting, randomise the block + * slightly, and try again. + */ + int workFactor; + int workDone; + int workLimit; + bool firstAttempt; + int nBlocksRandomised; + + int currentChar = -1; + int runLength = 0; + + public BZip2OutputStream(Stream inStream) : this(inStream, 9) + { + } + + public BZip2OutputStream(Stream inStream, int inBlockSize) + { + block = null; + quadrant = null; + zptr = null; + ftab = null; + + BsSetStream(inStream); + + workFactor = 50; + if(inBlockSize > 9) { + inBlockSize = 9; + } + if(inBlockSize < 1) { + inBlockSize = 1; + } + blockSize100k = inBlockSize; + AllocateCompressStructures(); + Initialize(); + InitBlock(); + } + + public override void WriteByte(byte bv) + { + int b = (256 + bv) % 256; + if (currentChar != -1) { + if (currentChar == b) { + runLength++; + if (runLength > 254) { + WriteRun(); + currentChar = -1; + runLength = 0; + } + } else { + WriteRun(); + runLength = 1; + currentChar = b; + } + } else { + currentChar = b; + runLength++; + } + } + + void WriteRun() + { + if (last < allowableBlockSize) { + inUse[currentChar] = true; + for (int i = 0; i < runLength; i++) { + mCrc.Update(currentChar); + } + + switch (runLength) { + case 1: + last++; + block[last + 1] = (byte)currentChar; + break; + case 2: + last++; + block[last + 1] = (byte)currentChar; + last++; + block[last + 1] = (byte)currentChar; + break; + case 3: + last++; + block[last + 1] = (byte)currentChar; + last++; + block[last + 1] = (byte)currentChar; + last++; + block[last + 1] = (byte)currentChar; + break; + default: + inUse[runLength - 4] = true; + last++; + block[last + 1] = (byte)currentChar; + last++; + block[last + 1] = (byte)currentChar; + last++; + block[last + 1] = (byte)currentChar; + last++; + block[last + 1] = (byte)currentChar; + last++; + block[last + 1] = (byte)(runLength - 4); + break; + } + } else { + EndBlock(); + InitBlock(); + WriteRun(); + } + } + + bool closed = false; + + ~BZip2OutputStream() + { + Close(); + } + + public override void Close() + { + if (closed) { + return; + } + + if (runLength > 0) { + WriteRun(); + } + + currentChar = -1; + EndBlock(); + EndCompression(); + closed = true; +// super.close(); + Flush(); + baseStream.Close(); + } + + public override void Flush() + { +// super.flush(); + baseStream.Flush(); + } + + uint blockCRC, combinedCRC; + + void Initialize() + { +// bytesIn = 0; + bytesOut = 0; + nBlocksRandomised = 0; + + /*--- Write `magic' bytes h indicating file-format == huffmanised, + followed by a digit indicating blockSize100k. + ---*/ + BsPutUChar('h'); + BsPutUChar('0' + blockSize100k); + + combinedCRC = 0; + } + + int allowableBlockSize; + + void InitBlock() + { + // blockNo++; + mCrc.Reset(); + last = -1; + // ch = 0; + + for (int i = 0; i < 256; i++) { + inUse[i] = false; + } + + /*--- 20 is just a paranoia constant ---*/ + allowableBlockSize = BZip2Constants.baseBlockSize * blockSize100k - 20; + } + + void EndBlock() + { + blockCRC = (uint)mCrc.Value; + combinedCRC = (combinedCRC << 1) | (combinedCRC >> 31); + combinedCRC ^= blockCRC; + + /*-- sort the block and establish posn of original string --*/ + DoReversibleTransformation(); + + /*-- + A 6-byte block header, the value chosen arbitrarily + as 0x314159265359 :-). A 32 bit value does not really + give a strong enough guarantee that the value will not + appear by chance in the compressed datastream. Worst-case + probability of this event, for a 900k block, is about + 2.0e-3 for 32 bits, 1.0e-5 for 40 bits and 4.0e-8 for 48 bits. + For a compressed file of size 100Gb -- about 100000 blocks -- + only a 48-bit marker will do. NB: normal compression/ + decompression do *not* rely on these statistical properties. + They are only important when trying to recover blocks from + damaged files. + --*/ + BsPutUChar(0x31); + BsPutUChar(0x41); + BsPutUChar(0x59); + BsPutUChar(0x26); + BsPutUChar(0x53); + BsPutUChar(0x59); + + /*-- Now the block's CRC, so it is in a known place. --*/ + BsPutint((int)blockCRC); + + /*-- Now a single bit indicating randomisation. --*/ + if (blockRandomised) { + BsW(1,1); + nBlocksRandomised++; + } else { + BsW(1,0); + } + + /*-- Finally, block's contents proper. --*/ + MoveToFrontCodeAndSend(); + } + + void EndCompression() + { + /*-- + Now another magic 48-bit number, 0x177245385090, to + indicate the end of the last block. (sqrt(pi), if + you want to know. I did want to use e, but it contains + too much repetition -- 27 18 28 18 28 46 -- for me + to feel statistically comfortable. Call me paranoid.) + --*/ + BsPutUChar(0x17); + BsPutUChar(0x72); + BsPutUChar(0x45); + BsPutUChar(0x38); + BsPutUChar(0x50); + BsPutUChar(0x90); + + BsPutint((int)combinedCRC); + + BsFinishedWithStream(); + } + + void HbAssignCodes (int[] code, char[] length, int minLen, int maxLen, int alphaSize) + { + int vec = 0; + for (int n = minLen; n <= maxLen; ++n) { + for (int i = 0; i < alphaSize; ++i) { + if (length[i] == n) { + code[i] = vec; + ++vec; + } + } + vec <<= 1; + } + } + + void BsSetStream(Stream f) + { + baseStream = f; + bsLive = 0; + bsBuff = 0; + bytesOut = 0; + } + + void BsFinishedWithStream() + { + while (bsLive > 0) { + int ch = (bsBuff >> 24); + baseStream.WriteByte((byte)ch); // write 8-bit + bsBuff <<= 8; + bsLive -= 8; + bytesOut++; + } + } + + void BsW(int n, int v) + { + while (bsLive >= 8) { + int ch = (bsBuff >> 24); + baseStream.WriteByte((byte)ch); // write 8-bit + bsBuff <<= 8; + bsLive -= 8; + ++bytesOut; + } + bsBuff |= (v << (32 - bsLive - n)); + bsLive += n; + } + + void BsPutUChar(int c) + { + BsW(8, c); + } + + void BsPutint(int u) + { + BsW(8, (u >> 24) & 0xFF); + BsW(8, (u >> 16) & 0xFF); + BsW(8, (u >> 8) & 0xFF); + BsW(8, u & 0xFF); + } + + void BsPutIntVS(int numBits, int c) + { + BsW(numBits, c); + } + + void SendMTFValues() + { + char[][] len = new char[BZip2Constants.N_GROUPS][]; + for (int i = 0; i < BZip2Constants.N_GROUPS; ++i) { + len[i] = new char[BZip2Constants.MAX_ALPHA_SIZE]; + } + + int gs, ge, totc, bt, bc, iter; + int nSelectors = 0, alphaSize, minLen, maxLen, selCtr; + int nGroups, nBytes; + + alphaSize = nInUse + 2; + for (int t = 0; t < BZip2Constants.N_GROUPS; t++) { + for (int v = 0; v < alphaSize; v++) { + len[t][v] = (char)GREATER_ICOST; + } + } + + /*--- Decide how many coding tables to use ---*/ + if (nMTF <= 0) { + Panic(); + } + + if (nMTF < 200) { + nGroups = 2; + } else if (nMTF < 600) { + nGroups = 3; + } else if (nMTF < 1200) { + nGroups = 4; + } else if (nMTF < 2400) { + nGroups = 5; + } else { + nGroups = 6; + } + + /*--- Generate an initial set of coding tables ---*/ + int nPart = nGroups; + int remF = nMTF; + gs = 0; + while (nPart > 0) { + int tFreq = remF / nPart; + int aFreq = 0; + ge = gs - 1; + while (aFreq < tFreq && ge < alphaSize - 1) { + ge++; + aFreq += mtfFreq[ge]; + } + + if (ge > gs && nPart != nGroups && nPart != 1 && ((nGroups - nPart) % 2 == 1)) { + aFreq -= mtfFreq[ge]; + ge--; + } + + for (int v = 0; v < alphaSize; v++) { + if (v >= gs && v <= ge) { + len[nPart - 1][v] = (char)LESSER_ICOST; + } else { + len[nPart - 1][v] = (char)GREATER_ICOST; + } + } + + nPart--; + gs = ge + 1; + remF -= aFreq; + } + + int[][] rfreq = new int[BZip2Constants.N_GROUPS][]; + for (int i = 0; i < BZip2Constants.N_GROUPS; ++i) { + rfreq[i] = new int[BZip2Constants.MAX_ALPHA_SIZE]; + } + + int[] fave = new int[BZip2Constants.N_GROUPS]; + short[] cost = new short[BZip2Constants.N_GROUPS]; + /*--- + Iterate up to N_ITERS times to improve the tables. + ---*/ + for (iter = 0; iter < BZip2Constants.N_ITERS; ++iter) { + for (int t = 0; t < nGroups; ++t) { + fave[t] = 0; + } + + for (int t = 0; t < nGroups; ++t) { + for (int v = 0; v < alphaSize; ++v) { + rfreq[t][v] = 0; + } + } + + nSelectors = 0; + totc = 0; + gs = 0; + while (true) { + + /*--- Set group start & end marks. --*/ + if (gs >= nMTF) { + break; + } + ge = gs + BZip2Constants.G_SIZE - 1; + if (ge >= nMTF) { + ge = nMTF - 1; + } + + /*-- + Calculate the cost of this group as coded + by each of the coding tables. + --*/ + for (int t = 0; t < nGroups; t++) { + cost[t] = 0; + } + + if (nGroups == 6) { + short cost0, cost1, cost2, cost3, cost4, cost5; + cost0 = cost1 = cost2 = cost3 = cost4 = cost5 = 0; + for (int i = gs; i <= ge; ++i) { + short icv = szptr[i]; + cost0 += (short)len[0][icv]; + cost1 += (short)len[1][icv]; + cost2 += (short)len[2][icv]; + cost3 += (short)len[3][icv]; + cost4 += (short)len[4][icv]; + cost5 += (short)len[5][icv]; + } + cost[0] = cost0; + cost[1] = cost1; + cost[2] = cost2; + cost[3] = cost3; + cost[4] = cost4; + cost[5] = cost5; + } else { + for (int i = gs; i <= ge; ++i) { + short icv = szptr[i]; + for (int t = 0; t < nGroups; t++) { + cost[t] += (short)len[t][icv]; + } + } + } + + /*-- + Find the coding table which is best for this group, + and record its identity in the selector table. + --*/ + bc = 999999999; + bt = -1; + for (int t = 0; t < nGroups; ++t) { + if (cost[t] < bc) { + bc = cost[t]; + bt = t; + } + } + totc += bc; + fave[bt]++; + selector[nSelectors] = (char)bt; + nSelectors++; + + /*-- + Increment the symbol frequencies for the selected table. + --*/ + for (int i = gs; i <= ge; ++i) { + ++rfreq[bt][szptr[i]]; + } + + gs = ge+1; + } + + /*-- + Recompute the tables based on the accumulated frequencies. + --*/ + for (int t = 0; t < nGroups; ++t) { + HbMakeCodeLengths(len[t], rfreq[t], alphaSize, 20); + } + } + + rfreq = null; + fave = null; + cost = null; + + if (!(nGroups < 8)) { + Panic(); + } + if (!(nSelectors < 32768 && nSelectors <= (2 + (900000 / BZip2Constants.G_SIZE)))) { + Panic(); + } + + + /*--- Compute MTF values for the selectors. ---*/ + char[] pos = new char[BZip2Constants.N_GROUPS]; + char ll_i, tmp2, tmp; + for (int i = 0; i < nGroups; i++) { + pos[i] = (char)i; + } + for (int i = 0; i < nSelectors; i++) { + ll_i = selector[i]; + int j = 0; + tmp = pos[j]; + while (ll_i != tmp) { + j++; + tmp2 = tmp; + tmp = pos[j]; + pos[j] = tmp2; + } + pos[0] = tmp; + selectorMtf[i] = (char)j; + } + + int[][] code = new int[BZip2Constants.N_GROUPS][]; + + for (int i = 0; i < BZip2Constants.N_GROUPS; ++i) { + code[i] = new int[BZip2Constants.MAX_ALPHA_SIZE]; + } + + /*--- Assign actual codes for the tables. --*/ + for (int t = 0; t < nGroups; t++) { + minLen = 32; + maxLen = 0; + for (int i = 0; i < alphaSize; i++) { + if (len[t][i] > maxLen) { + maxLen = len[t][i]; + } + if (len[t][i] < minLen) { + minLen = len[t][i]; + } + } + if (maxLen > 20) { + Panic(); + } + if (minLen < 1) { + Panic(); + } + HbAssignCodes(code[t], len[t], minLen, maxLen, alphaSize); + } + + /*--- Transmit the mapping table. ---*/ + bool[] inUse16 = new bool[16]; + for (int i = 0; i < 16; ++i) { + inUse16[i] = false; + for (int j = 0; j < 16; ++j) { + if (inUse[i * 16 + j]) { + inUse16[i] = true; + } // TODO : insert break; + } + } + + nBytes = bytesOut; + for (int i = 0; i < 16; ++i) { + if (inUse16[i]) { + BsW(1,1); + } else { + BsW(1,0); + } + } + + for (int i = 0; i < 16; ++i) { + if (inUse16[i]) { + for (int j = 0; j < 16; ++j) { + if (inUse[i * 16 + j]) { + BsW(1,1); + } else { + BsW(1,0); + } + } + } + } + + /*--- Now the selectors. ---*/ + nBytes = bytesOut; + BsW(3, nGroups); + BsW(15, nSelectors); + for (int i = 0; i < nSelectors; ++i) { + for (int j = 0; j < selectorMtf[i]; ++j) { + BsW(1,1); + } + BsW(1,0); + } + + /*--- Now the coding tables. ---*/ + nBytes = bytesOut; + + for (int t = 0; t < nGroups; ++t) { + int curr = len[t][0]; + BsW(5, curr); + for (int i = 0; i < alphaSize; ++i) { + while (curr < len[t][i]) { + BsW(2, 2); + curr++; /* 10 */ + } + while (curr > len[t][i]) { + BsW(2, 3); + curr--; /* 11 */ + } + BsW (1, 0); + } + } + + /*--- And finally, the block data proper ---*/ + nBytes = bytesOut; + selCtr = 0; + gs = 0; + while (true) { + if (gs >= nMTF) { + break; + } + ge = gs + BZip2Constants.G_SIZE - 1; + if (ge >= nMTF) { + ge = nMTF - 1; + } + + for (int i = gs; i <= ge; i++) { + BsW(len[selector[selCtr]][szptr[i]], code[selector[selCtr]][szptr[i]]); + } + + gs = ge + 1; + ++selCtr; + } + if (!(selCtr == nSelectors)) { + Panic(); + } + } + + void MoveToFrontCodeAndSend () + { + BsPutIntVS(24, origPtr); + GenerateMTFValues(); + SendMTFValues(); + } + + Stream baseStream; + + void SimpleSort(int lo, int hi, int d) + { + int i, j, h, bigN, hp; + int v; + + bigN = hi - lo + 1; + if (bigN < 2) { + return; + } + + hp = 0; + while (incs[hp] < bigN) { + hp++; + } + hp--; + + for (; hp >= 0; hp--) { + h = incs[hp]; + + i = lo + h; + while (true) { + /*-- copy 1 --*/ + if (i > hi) + break; + v = zptr[i]; + j = i; + while (FullGtU(zptr[j-h]+d, v+d)) { + zptr[j] = zptr[j-h]; + j = j - h; + if (j <= (lo + h - 1)) + break; + } + zptr[j] = v; + i++; + + /*-- copy 2 --*/ + if (i > hi) + break; + v = zptr[i]; + j = i; + while (FullGtU ( zptr[j-h]+d, v+d )) { + zptr[j] = zptr[j-h]; + j = j - h; + if (j <= (lo + h - 1)) + break; + } + zptr[j] = v; + i++; + + /*-- copy 3 --*/ + if (i > hi) + break; + v = zptr[i]; + j = i; + while (FullGtU ( zptr[j-h]+d, v+d)) { + zptr[j] = zptr[j-h]; + j = j - h; + if (j <= (lo + h - 1)) + break; + } + zptr[j] = v; + i++; + + if (workDone > workLimit && firstAttempt) + return; + } + } + } + + void Vswap(int p1, int p2, int n ) + { + int temp = 0; + while (n > 0) { + temp = zptr[p1]; + zptr[p1] = zptr[p2]; + zptr[p2] = temp; + p1++; + p2++; + n--; + } + } + + byte Med3(byte a, byte b, byte c ) + { + byte t; + if (a > b) { + t = a; + a = b; + b = t; + } + if (b > c) { + t = b; + b = c; + c = t; + } + if (a > b) { + b = a; + } + return b; + } + + class StackElem + { + public int ll; + public int hh; + public int dd; + } + + void QSort3(int loSt, int hiSt, int dSt) + { + int unLo, unHi, ltLo, gtHi, med, n, m; + int sp, lo, hi, d; + StackElem[] stack = new StackElem[QSORT_STACK_SIZE]; + for(int count = 0; count < QSORT_STACK_SIZE; count++) + stack[count] = new StackElem(); + + sp = 0; + + stack[sp].ll = loSt; + stack[sp].hh = hiSt; + stack[sp].dd = dSt; + sp++; + + while (sp > 0) { + if (sp >= QSORT_STACK_SIZE) { + Panic(); + } + + sp--; + lo = stack[sp].ll; + hi = stack[sp].hh; + d = stack[sp].dd; + + if (hi - lo < SMALL_THRESH || d > DEPTH_THRESH) { + SimpleSort(lo, hi, d); + if (workDone > workLimit && firstAttempt) + return; + continue; + } + + med = Med3(block[zptr[lo] + d + 1], + block[zptr[hi ] + d + 1], + block[zptr[(lo + hi) >> 1] + d + 1]); + + unLo = ltLo = lo; + unHi = gtHi = hi; + + while (true) { + while (true) { + if (unLo > unHi) { + break; + } + n = ((int)block[zptr[unLo]+d + 1]) - med; + if (n == 0) { + int temp = 0; + temp = zptr[unLo]; + zptr[unLo] = zptr[ltLo]; + zptr[ltLo] = temp; + ltLo++; + unLo++; + continue; + } + if (n > 0) { + break; + } + unLo++; + } + while (true) { + if (unLo > unHi) + break; + n = ((int)block[zptr[unHi]+d + 1]) - med; + if (n == 0) { + int temp = 0; + temp = zptr[unHi]; + zptr[unHi] = zptr[gtHi]; + zptr[gtHi] = temp; + gtHi--; + unHi--; + continue; + }; + if (n < 0) + break; + unHi--; + } + if (unLo > unHi) + break; + { + int temp = zptr[unLo]; + zptr[unLo] = zptr[unHi]; + zptr[unHi] = temp; + unLo++; + unHi--; + } + } + + if (gtHi < ltLo) { + stack[sp].ll = lo; + stack[sp].hh = hi; + stack[sp].dd = d+1; + sp++; + continue; + } + + n = ((ltLo-lo) < (unLo-ltLo)) ? (ltLo-lo) : (unLo-ltLo); + Vswap(lo, unLo-n, n); + m = ((hi-gtHi) < (gtHi-unHi)) ? (hi-gtHi) : (gtHi-unHi); + Vswap(unLo, hi-m+1, m); + + n = lo + unLo - ltLo - 1; + m = hi - (gtHi - unHi) + 1; + + stack[sp].ll = lo; + stack[sp].hh = n; + stack[sp].dd = d; + sp++; + + stack[sp].ll = n + 1; + stack[sp].hh = m - 1; + stack[sp].dd = d+1; + sp++; + + stack[sp].ll = m; + stack[sp].hh = hi; + stack[sp].dd = d; + sp++; + } + } + + void MainSort() + { + int i, j, ss, sb; + int[] runningOrder = new int[256]; + int[] copy = new int[256]; + bool[] bigDone = new bool[256]; + int c1, c2; + int numQSorted; + + /*-- + In the various block-sized structures, live data runs + from 0 to last+NUM_OVERSHOOT_BYTES inclusive. First, + set up the overshoot area for block. + --*/ + + // if (verbosity >= 4) fprintf ( stderr, " sort initialise ...\n" ); + for (i = 0; i < BZip2Constants.NUM_OVERSHOOT_BYTES; i++) { + block[last + i + 2] = block[(i % (last + 1)) + 1]; + } + for (i = 0; i <= last + BZip2Constants.NUM_OVERSHOOT_BYTES; i++) { + quadrant[i] = 0; + } + + block[0] = (byte)(block[last + 1]); + + if (last < 4000) { + /*-- + Use simpleSort(), since the full sorting mechanism + has quite a large constant overhead. + --*/ + for (i = 0; i <= last; i++) + zptr[i] = i; + firstAttempt = false; + workDone = workLimit = 0; + SimpleSort(0, last, 0); + } else { + numQSorted = 0; + for (i = 0; i <= 255; i++) + bigDone[i] = false; + + for (i = 0; i <= 65536; i++) + ftab[i] = 0; + + c1 = block[0]; + for (i = 0; i <= last; i++) { + c2 = block[i + 1]; + ftab[(c1 << 8) + c2]++; + c1 = c2; + } + + for (i = 1; i <= 65536; i++) + ftab[i] += ftab[i - 1]; + + c1 = block[1]; + for (i = 0; i < last; i++) { + c2 = block[i + 2]; + j = (c1 << 8) + c2; + c1 = c2; + ftab[j]--; + zptr[ftab[j]] = i; + } + + j = ((block[last + 1]) << 8) + (block[1]); + ftab[j]--; + zptr[ftab[j]] = last; + + /*-- + Now ftab contains the first loc of every small bucket. + Calculate the running order, from smallest to largest + big bucket. + --*/ + + for (i = 0; i <= 255; i++) { + runningOrder[i] = i; + } + + int vv; + int h = 1; + do { + h = 3 * h + 1; + } while (h <= 256); + do { + h = h / 3; + for (i = h; i <= 255; i++) { + vv = runningOrder[i]; + j = i; + while ((ftab[((runningOrder[j-h])+1) << 8] - ftab[(runningOrder[j-h]) << 8]) > (ftab[((vv)+1) << 8] - ftab[(vv) << 8])) { + runningOrder[j] = runningOrder[j-h]; + j = j - h; + if (j <= (h - 1)) + break; + } + runningOrder[j] = vv; + } + } while (h != 1); + + /*-- + The main sorting loop. + --*/ + for (i = 0; i <= 255; i++) { + + /*-- + Process big buckets, starting with the least full. + --*/ + ss = runningOrder[i]; + + /*-- + Complete the big bucket [ss] by quicksorting + any unsorted small buckets [ss, j]. Hopefully + previous pointer-scanning phases have already + completed many of the small buckets [ss, j], so + we don't have to sort them at all. + --*/ + for (j = 0; j <= 255; j++) { + sb = (ss << 8) + j; + if(!((ftab[sb] & SETMASK) == SETMASK) ) { + int lo = ftab[sb] & CLEARMASK; + int hi = (ftab[sb+1] & CLEARMASK) - 1; + if (hi > lo) { + QSort3(lo, hi, 2); + numQSorted += (hi - lo + 1); + if (workDone > workLimit && firstAttempt) { + return; + } + } + ftab[sb] |= SETMASK; + } + } + + /*-- + The ss big bucket is now done. Record this fact, + and update the quadrant descriptors. Remember to + update quadrants in the overshoot area too, if + necessary. The "if (i < 255)" test merely skips + this updating for the last bucket processed, since + updating for the last bucket is pointless. + --*/ + bigDone[ss] = true; + + if (i < 255) { + int bbStart = ftab[ss << 8] & CLEARMASK; + int bbSize = (ftab[(ss+1) << 8] & CLEARMASK) - bbStart; + int shifts = 0; + + while ((bbSize >> shifts) > 65534) { + shifts++; + } + + for (j = 0; j < bbSize; j++) { + int a2update = zptr[bbStart + j]; + int qVal = (j >> shifts); + quadrant[a2update] = qVal; + if (a2update < BZip2Constants.NUM_OVERSHOOT_BYTES) { + quadrant[a2update + last + 1] = qVal; + } + } + + if (!(((bbSize-1) >> shifts) <= 65535)) { + Panic(); + } + } + + /*-- + Now scan this big bucket so as to synthesise the + sorted order for small buckets [t, ss] for all t != ss. + --*/ + for (j = 0; j <= 255; j++) { + copy[j] = ftab[(j << 8) + ss] & CLEARMASK; + } + + for (j = ftab[ss << 8] & CLEARMASK; j < (ftab[(ss+1) << 8] & CLEARMASK); j++) { + c1 = block[zptr[j]]; + if (!bigDone[c1]) { + zptr[copy[c1]] = zptr[j] == 0 ? last : zptr[j] - 1; + copy[c1] ++; + } + } + + for (j = 0; j <= 255; j++) { + ftab[(j << 8) + ss] |= SETMASK; + } + } + } + } + + void RandomiseBlock() + { + int i; + int rNToGo = 0; + int rTPos = 0; + for (i = 0; i < 256; i++) + inUse[i] = false; + + for (i = 0; i <= last; i++) { + if (rNToGo == 0) { + rNToGo = (int)BZip2Constants.rNums[rTPos]; + rTPos++; + if(rTPos == 512) + rTPos = 0; + } + rNToGo--; + block[i + 1] ^= (byte)((rNToGo == 1) ? 1 : 0); + // handle 16 bit signed numbers + block[i + 1] &= 0xFF; + + inUse[block[i + 1]] = true; + } + } + + void DoReversibleTransformation() + { + workLimit = workFactor * last; + workDone = 0; + blockRandomised = false; + firstAttempt = true; + + MainSort(); + + if (workDone > workLimit && firstAttempt) { + RandomiseBlock(); + workLimit = workDone = 0; + blockRandomised = true; + firstAttempt = false; + MainSort(); + } + + origPtr = -1; + for (int i = 0; i <= last; i++) { + if (zptr[i] == 0) { + origPtr = i; + break; + } + } + + if (origPtr == -1) { + Panic(); + } + } + + bool FullGtU(int i1, int i2) + { + int k; + byte c1, c2; + int s1, s2; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + i1++; + i2++; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + i1++; + i2++; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + i1++; + i2++; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + i1++; + i2++; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) + return c1 > c2; + i1++; + i2++; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + i1++; + i2++; + + k = last + 1; + + do { + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + s1 = quadrant[i1]; + s2 = quadrant[i2]; + if (s1 != s2) { + return s1 > s2; + } + i1++; + i2++; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + s1 = quadrant[i1]; + s2 = quadrant[i2]; + if (s1 != s2) { + return s1 > s2; + } + i1++; + i2++; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + s1 = quadrant[i1]; + s2 = quadrant[i2]; + if (s1 != s2) { + return s1 > s2; + } + i1++; + i2++; + + c1 = block[i1 + 1]; + c2 = block[i2 + 1]; + if (c1 != c2) { + return c1 > c2; + } + s1 = quadrant[i1]; + s2 = quadrant[i2]; + if (s1 != s2) { + return s1 > s2; + } + i1++; + i2++; + + if (i1 > last) { + i1 -= last; + i1--; + } + if (i2 > last) { + i2 -= last; + i2--; + } + + k -= 4; + ++workDone; + } while (k >= 0); + + return false; + } + + /*-- + Knuth's increments seem to work better + than Incerpi-Sedgewick here. Possibly + because the number of elems to sort is + usually small, typically <= 20. + --*/ + readonly int[] incs = new int[] { 1, 4, 13, 40, 121, 364, 1093, 3280, + 9841, 29524, 88573, 265720, + 797161, 2391484 }; + + void AllocateCompressStructures() + { + int n = BZip2Constants.baseBlockSize * blockSize100k; + block = new byte[(n + 1 + BZip2Constants.NUM_OVERSHOOT_BYTES)]; + quadrant = new int[(n + BZip2Constants.NUM_OVERSHOOT_BYTES)]; + zptr = new int[n]; + ftab = new int[65537]; + + if (block == null || quadrant == null || zptr == null || ftab == null) { + // int totalDraw = (n + 1 + NUM_OVERSHOOT_BYTES) + (n + NUM_OVERSHOOT_BYTES) + n + 65537; + // compressOutOfMemory ( totalDraw, n ); + } + + /* + The back end needs a place to store the MTF values + whilst it calculates the coding tables. We could + put them in the zptr array. However, these values + will fit in a short, so we overlay szptr at the + start of zptr, in the hope of reducing the number + of cache misses induced by the multiple traversals + of the MTF values when calculating coding tables. + Seems to improve compression speed by about 1%. + */ + // szptr = zptr; + + + szptr = new short[2 * n]; + } + + void GenerateMTFValues() + { + char[] yy = new char[256]; + int i, j; + char tmp; + char tmp2; + int zPend; + int wr; + int EOB; + + MakeMaps(); + EOB = nInUse+1; + + for (i = 0; i <= EOB; i++) { + mtfFreq[i] = 0; + } + + wr = 0; + zPend = 0; + for (i = 0; i < nInUse; i++) { + yy[i] = (char) i; + } + + + for (i = 0; i <= last; i++) { + char ll_i; + + ll_i = unseqToSeq[block[zptr[i]]]; + + j = 0; + tmp = yy[j]; + while ( ll_i != tmp ) { + j++; + tmp2 = tmp; + tmp = yy[j]; + yy[j] = tmp2; + } + yy[0] = tmp; + + if (j == 0) { + zPend++; + } else { + if (zPend > 0) { + zPend--; + while (true) { + switch (zPend % 2) { + case 0: + szptr[wr] = (short)BZip2Constants.RUNA; + wr++; + mtfFreq[BZip2Constants.RUNA]++; + break; + case 1: + szptr[wr] = (short)BZip2Constants.RUNB; + wr++; + mtfFreq[BZip2Constants.RUNB]++; + break; + }; + if (zPend < 2) + break; + zPend = (zPend - 2) / 2; + }; + zPend = 0; + } + szptr[wr] = (short)(j + 1); + wr++; + mtfFreq[j + 1]++; + } + } + + if (zPend > 0) { + zPend--; + while (true) { + switch (zPend % 2) { + case 0: + szptr[wr] = (short)BZip2Constants.RUNA; + wr++; + mtfFreq[BZip2Constants.RUNA]++; + break; + case 1: + szptr[wr] = (short)BZip2Constants.RUNB; + wr++; + mtfFreq[BZip2Constants.RUNB]++; + break; + } + if (zPend < 2) + break; + zPend = (zPend - 2) / 2; + } + } + + szptr[wr] = (short)EOB; + wr++; + mtfFreq[EOB]++; + + nMTF = wr; + } + } +} + +/* This file was derived from a file containing under this license: + * + * This file is a part of bzip2 and/or libbzip2, a program and + * library for lossless, block-sorting data compression. + * + * Copyright (C) 1996-1998 Julian R Seward. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. The origin of this software must not be misrepresented; you must + * not claim that you wrote the original software. If you use this + * software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 3. Altered source versions must be plainly marked as such, and must + * not be misrepresented as being the original software. + * + * 4. The name of the author may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Java version ported by Keiron Liddle, Aftex Software 1999-2001 + */ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/Adler32.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/Adler32.cs new file mode 100644 index 000000000..14ae55de7 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/Adler32.cs @@ -0,0 +1,199 @@ +// Adler32.cs - Computes Adler32 data checksum of a data stream +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Checksums { + + /// + /// Computes Adler32 checksum for a stream of data. An Adler32 + /// checksum is not as reliable as a CRC32 checksum, but a lot faster to + /// compute. + /// + /// The specification for Adler32 may be found in RFC 1950. + /// ZLIB Compressed Data Format Specification version 3.3) + /// + /// + /// From that document: + /// + /// "ADLER32 (Adler-32 checksum) + /// This contains a checksum value of the uncompressed data + /// (excluding any dictionary data) computed according to Adler-32 + /// algorithm. This algorithm is a 32-bit extension and improvement + /// of the Fletcher algorithm, used in the ITU-T X.224 / ISO 8073 + /// standard. + /// + /// Adler-32 is composed of two sums accumulated per byte: s1 is + /// the sum of all bytes, s2 is the sum of all s1 values. Both sums + /// are done modulo 65521. s1 is initialized to 1, s2 to zero. The + /// Adler-32 checksum is stored as s2*65536 + s1 in most- + /// significant-byte first (network) order." + /// + /// "8.2. The Adler-32 algorithm + /// + /// The Adler-32 algorithm is much faster than the CRC32 algorithm yet + /// still provides an extremely low probability of undetected errors. + /// + /// The modulo on unsigned long accumulators can be delayed for 5552 + /// bytes, so the modulo operation time is negligible. If the bytes + /// are a, b, c, the second sum is 3a + 2b + c + 3, and so is position + /// and order sensitive, unlike the first sum, which is just a + /// checksum. That 65521 is prime is important to avoid a possible + /// large class of two-byte errors that leave the check unchanged. + /// (The Fletcher checksum uses 255, which is not prime and which also + /// makes the Fletcher check insensitive to single byte changes 0 - + /// 255.) + /// + /// The sum s1 is initialized to 1 instead of zero to make the length + /// of the sequence part of s2, so that the length does not have to be + /// checked separately. (Any sequence of zeroes has a Fletcher + /// checksum of zero.)" + /// + /// + /// + public sealed class Adler32 : IChecksum + { + /// + /// largest prime smaller than 65536 + /// + readonly static uint BASE = 65521; + + uint checksum; + + /// + /// Returns the Adler32 data checksum computed so far. + /// + public long Value { + get { + return checksum; + } + } + + /// + /// Creates a new instance of the Adler32 class. + /// The checksum starts off with a value of 1. + /// + public Adler32() + { + Reset(); + } + + /// + /// Resets the Adler32 checksum to the initial value. + /// + public void Reset() + { + checksum = 1; //Initialize to 1 + } + + /// + /// Updates the checksum with the byte b. + /// + /// + /// the data value to add. The high byte of the int is ignored. + /// + public void Update(int bval) + { + //We could make a length 1 byte array and call update again, but I + //would rather not have that overhead + uint s1 = checksum & 0xFFFF; + uint s2 = checksum >> 16; + + s1 = (s1 + ((uint)bval & 0xFF)) % BASE; + s2 = (s1 + s2) % BASE; + + checksum = (s2 << 16) + s1; + } + + /// + /// Updates the checksum with the bytes taken from the array. + /// + /// + /// buffer an array of bytes + /// + public void Update(byte[] buffer) + { + Update(buffer, 0, buffer.Length); + } + + /// + /// Updates the checksum with the bytes taken from the array. + /// + /// + /// an array of bytes + /// + /// + /// the start of the data used for this update + /// + /// + /// the number of bytes to use for this update + /// + public void Update(byte[] buf, int off, int len) + { + if (buf == null) { + throw new ArgumentNullException("buf"); + } + + if (off < 0 || len < 0 || off + len > buf.Length) { + throw new ArgumentOutOfRangeException(); + } + + //(By Per Bothner) + uint s1 = checksum & 0xFFFF; + uint s2 = checksum >> 16; + + while (len > 0) { + // We can defer the modulo operation: + // s1 maximally grows from 65521 to 65521 + 255 * 3800 + // s2 maximally grows by 3800 * median(s1) = 2090079800 < 2^31 + int n = 3800; + if (n > len) { + n = len; + } + len -= n; + while (--n >= 0) { + s1 = s1 + (uint)(buf[off++] & 0xFF); + s2 = s2 + s1; + } + s1 %= BASE; + s2 %= BASE; + } + + checksum = (s2 << 16) | s1; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/Crc32.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/Crc32.cs new file mode 100644 index 000000000..1a6f7f55a --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/Crc32.cs @@ -0,0 +1,202 @@ +// CRC32.cs - Computes CRC32 data checksum of a data stream +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Checksums { + + /// + /// Generate a table for a byte-wise 32-bit CRC calculation on the polynomial: + /// x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. + /// + /// Polynomials over GF(2) are represented in binary, one bit per coefficient, + /// with the lowest powers in the most significant bit. Then adding polynomials + /// is just exclusive-or, and multiplying a polynomial by x is a right shift by + /// one. If we call the above polynomial p, and represent a byte as the + /// polynomial q, also with the lowest power in the most significant bit (so the + /// byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) mod p, + /// where a mod b means the remainder after dividing a by b. + /// + /// This calculation is done using the shift-register method of multiplying and + /// taking the remainder. The register is initialized to zero, and for each + /// incoming bit, x^32 is added mod p to the register if the bit is a one (where + /// x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by + /// x (which is shifting right by one and adding x^32 mod p if the bit shifted + /// out is a one). We start with the highest power (least significant bit) of + /// q and repeat for all eight bits of q. + /// + /// The table is simply the CRC of all possible eight bit values. This is all + /// the information needed to generate CRC's on data a byte at a time for all + /// combinations of CRC register values and incoming bytes. + /// + public sealed class Crc32 : IChecksum + { + readonly static uint CrcSeed = 0xFFFFFFFF; + + readonly static uint[] CrcTable = new uint[] { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, + 0x706AF48F, 0xE963A535, 0x9E6495A3, 0x0EDB8832, 0x79DCB8A4, + 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, + 0x90BF1D91, 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, 0x136C9856, + 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, + 0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4, + 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, + 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, 0x26D930AC, 0x51DE003A, + 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, + 0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190, + 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, + 0x9FBFE4A5, 0xE8B8D433, 0x7807C9A2, 0x0F00F934, 0x9609A88E, + 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, + 0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950, + 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, + 0xFBD44C65, 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A, + 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, + 0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA, 0xBE0B1010, + 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, + 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, + 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, + 0x73DC1683, 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, 0xF00F9344, + 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, + 0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, + 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, + 0xA6BC5767, 0x3FB506DD, 0x48B2364B, 0xD80D2BDA, 0xAF0A1B4C, + 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, + 0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE, + 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, + 0x2CD99E8B, 0x5BDEAE1D, 0x9B64C2B0, 0xEC63F226, 0x756AA39C, + 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, + 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242, + 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, + 0x18B74777, 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, 0xA00AE278, + 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, + 0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, + 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, + 0xCDD70693, 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, + 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, + 0x2D02EF8D + }; + + /// + /// The crc data checksum so far. + /// + uint crc = 0; + + /// + /// Returns the CRC32 data checksum computed so far. + /// + public long Value { + get { + return (long)crc; + } + } + + /// + /// Resets the CRC32 data checksum as if no update was ever called. + /// + public void Reset() + { + crc = 0; + } + + /// + /// Updates the checksum with the int bval. + /// + /// + /// the byte is taken as the lower 8 bits of bval + /// + public void Update(int bval) + { + crc ^= CrcSeed; + crc = CrcTable[(crc ^ bval) & 0xFF] ^ (crc >> 8); + crc ^= CrcSeed; + } + + /// + /// Updates the checksum with the bytes taken from the array. + /// + /// + /// buffer an array of bytes + /// + public void Update(byte[] buffer) + { + Update(buffer, 0, buffer.Length); + } + + /// + /// Adds the byte array to the data checksum. + /// + /// + /// the buffer which contains the data + /// + /// + /// the offset in the buffer where the data starts + /// + /// + /// the length of the data + /// + public void Update(byte[] buf, int off, int len) + { + if (buf == null) { + throw new ArgumentNullException("buf"); + } + + if (off < 0 || len < 0 || off + len > buf.Length) { + throw new ArgumentOutOfRangeException(); + } + + crc ^= CrcSeed; + + while (--len >= 0) { + crc = CrcTable[(crc ^ buf[off++]) & 0xFF] ^ (crc >> 8); + } + + crc ^= CrcSeed; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/IChecksum.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/IChecksum.cs new file mode 100644 index 000000000..906462a54 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/IChecksum.cs @@ -0,0 +1,91 @@ +// IChecksum.cs - Interface to compute a data checksum +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +namespace BCG.Utilities.SharpZipLib.Checksums { + + /// + /// Interface to compute a data checksum used by checked input/output streams. + /// A data checksum can be updated by one byte or with a byte array. After each + /// update the value of the current checksum can be returned by calling + /// getValue. The complete checksum object can also be reset + /// so it can be used again with new data. + /// + public interface IChecksum + { + /// + /// Returns the data checksum computed so far. + /// + long Value { + get; + } + + /// + /// Resets the data checksum as if no update was ever called. + /// + void Reset(); + + /// + /// Adds one byte to the data checksum. + /// + /// + /// the data value to add. The high byte of the int is ignored. + /// + void Update(int bval); + + /// + /// Updates the data checksum with the bytes taken from the array. + /// + /// + /// buffer an array of bytes + /// + void Update(byte[] buffer); + + /// + /// Adds the byte array to the data checksum. + /// + /// + /// the buffer which contains the data + /// + /// + /// the offset in the buffer where the data starts + /// + /// + /// the length of the data + /// + void Update(byte[] buf, int off, int len); + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/StrangeCrc.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/StrangeCrc.cs new file mode 100644 index 000000000..5dc885293 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Checksums/StrangeCrc.cs @@ -0,0 +1,158 @@ +// StrangeCRC.cs - computes a crc used in the bziplib ... I don't think that +// this is the 'standard' crc, please correct me, if I'm wrong +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Checksums { + + public class StrangeCRC : IChecksum + { + readonly static uint[] crc32Table = { + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, + 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, + 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, + 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, + 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, + 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, + 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, + 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, + 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, + 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, + 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, + 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, + 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, + 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, + 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, + 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, + 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, + 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, + 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, + 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, + 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, + 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, + 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, + 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, + 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, + 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, + 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, + 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, + 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, + 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, + 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, + 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, + 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, + 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, + 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, + 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, + 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, + 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, + 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, + 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, + 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, + 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, + 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + }; + int globalCrc; + + public StrangeCRC() + { + Reset(); + } + + public void Reset() + { + globalCrc = -1; + } + + public long Value { + get { + return ~globalCrc; + } + } + + public void Update(int inCh) + { + int temp = (globalCrc >> 24) ^ inCh; + if (temp < 0) { + temp = 256 + temp; + } + globalCrc = (int)((globalCrc << 8) ^ crc32Table[temp]); + } + + public void Update(byte[] buf) + { + Update(buf, 0, buf.Length); + } + + public void Update(byte[] buf, int off, int len) + { + if (buf == null) { + throw new ArgumentNullException("buf"); + } + + if (off < 0 || len < 0 || off + len > buf.Length) { + throw new ArgumentOutOfRangeException(); + } + + for (int i = 0; i < len; ++i) { + Update(buf[off++]); + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipConstants.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipConstants.cs new file mode 100644 index 000000000..0862ba5a5 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipConstants.cs @@ -0,0 +1,67 @@ +// GZIPConstants.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +namespace BCG.Utilities.SharpZipLib.GZip { + + /// + /// This class contains constants used for gzip. + /// + public class GZipConstants + { + /// + /// Magic number found at start of GZIP header + /// + public static readonly int GZIP_MAGIC = 0x1F8B; + + /* The flag byte is divided into individual bits as follows: + + bit 0 FTEXT + bit 1 FHCRC + bit 2 FEXTRA + bit 3 FNAME + bit 4 FCOMMENT + bit 5 reserved + bit 6 reserved + bit 7 reserved + */ + public const int FTEXT = 0x1; + public const int FHCRC = 0x2; + public const int FEXTRA = 0x4; + public const int FNAME = 0x8; + public const int FCOMMENT = 0x10; + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipInputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipInputStream.cs new file mode 100644 index 000000000..2a4f663ee --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipInputStream.cs @@ -0,0 +1,337 @@ +// GzipInputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; + +using BCG.Utilities.SharpZipLib.Checksums; +using BCG.Utilities.SharpZipLib.Zip.Compression; +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; + +namespace BCG.Utilities.SharpZipLib.GZip { + + /// + /// This filter stream is used to decompress a "GZIP" format stream. + /// The "GZIP" format is described baseInputStream RFC 1952. + /// + /// author of the original java version : John Leuner + /// + /// This sample shows how to unzip a gzipped file + /// + /// using System; + /// using System.IO; + /// + /// using NZlib.GZip; + /// + /// class MainClass + /// { + /// public static void Main(string[] args) + /// { + /// Stream s = new GZipInputStream(File.OpenRead(args[0])); + /// FileStream fs = File.Create(Path.GetFileNameWithoutExtension(args[0])); + /// int size = 2048; + /// byte[] writeData = new byte[2048]; + /// while (true) { + /// size = s.Read(writeData, 0, size); + /// if (size > 0) { + /// fs.Write(writeData, 0, size); + /// } else { + /// break; + /// } + /// } + /// s.Close(); + /// } + /// } + /// + /// + public class GZipInputStream : InflaterInputStream + { + + //Variables + + /// + /// CRC-32 value for uncompressed data + /// + protected Crc32 crc = new Crc32(); + + /// + /// Indicates end of stream + /// + protected bool eos; + + /// + /// Creates a GzipInputStream with the default buffer size + /// + /// + /// The stream to read compressed data from (baseInputStream GZIP format) + /// + public GZipInputStream(Stream baseInputStream) : this(baseInputStream, 4096) + { + } + + /// + /// Creates a GZIPInputStream with the specified buffer size + /// + /// + /// The stream to read compressed data from (baseInputStream GZIP format) + /// + /// + /// Size of the buffer to use + /// + public GZipInputStream(Stream baseInputStream, int size) : base(baseInputStream, new Inflater(true), size) + { + } + + /// + /// Reads uncompressed data into an array of bytes + /// + /// + /// the buffer to read uncompressed data into + /// + /// + /// the offset indicating where the data should be placed + /// + /// + /// the number of uncompressed bytes to be read + /// + public override int Read(byte[] buf, int offset, int len) + { + // We first have to slurp baseInputStream the GZIP header, then we feed all the + // rest of the data to the superclass. + // + // As we do that we continually update the CRC32. Once the data is + // finished, we check the CRC32 + // + // This means we don't need our own buffer, as everything is done + // baseInputStream the superclass. + if (!readGZIPHeader) { + ReadHeader(); + } + + if (eos) { + return -1; + } + + // System.err.println("GZIPIS.read(byte[], off, len ... " + offset + " and len " + len); + //We don't have to read the header, so we just grab data from the superclass + int numRead = base.Read(buf, offset, len); + if (numRead > 0) { + crc.Update(buf, offset, numRead); + } + + if (inf.IsFinished) { + ReadFooter(); + } + return numRead; + } + + private void ReadHeader() + { + /* 1. Check the two magic bytes */ + Crc32 headCRC = new Crc32(); + int magic = baseInputStream.ReadByte(); + if (magic < 0) { + eos = true; + return; + } + headCRC.Update(magic); + if (magic != (GZipConstants.GZIP_MAGIC >> 8)) { + throw new IOException("Error baseInputStream GZIP header, first byte doesn't match"); + } + + magic = baseInputStream.ReadByte(); + if (magic != (GZipConstants.GZIP_MAGIC & 0xFF)) { + throw new IOException("Error baseInputStream GZIP header, second byte doesn't match"); + } + headCRC.Update(magic); + + /* 2. Check the compression type (must be 8) */ + int CM = baseInputStream.ReadByte(); + if (CM != 8) { + throw new IOException("Error baseInputStream GZIP header, data not baseInputStream deflate format"); + } + headCRC.Update(CM); + + /* 3. Check the flags */ + int flags = baseInputStream.ReadByte(); + if (flags < 0) { + throw new Exception("Early EOF baseInputStream GZIP header"); + } + headCRC.Update(flags); + + /* This flag byte is divided into individual bits as follows: + + bit 0 FTEXT + bit 1 FHCRC + bit 2 FEXTRA + bit 3 FNAME + bit 4 FCOMMENT + bit 5 reserved + bit 6 reserved + bit 7 reserved + */ + + /* 3.1 Check the reserved bits are zero */ + + if ((flags & 0xd0) != 0) { + throw new IOException("Reserved flag bits baseInputStream GZIP header != 0"); + } + + /* 4.-6. Skip the modification time, extra flags, and OS type */ + for (int i=0; i< 6; i++) { + int readByte = baseInputStream.ReadByte(); + if (readByte < 0) { + throw new Exception("Early EOF baseInputStream GZIP header"); + } + headCRC.Update(readByte); + } + + /* 7. Read extra field */ + if ((flags & GZipConstants.FEXTRA) != 0) { + /* Skip subfield id */ + for (int i=0; i< 2; i++) { + int readByte = baseInputStream.ReadByte(); + if (readByte < 0) { + throw new Exception("Early EOF baseInputStream GZIP header"); + } + headCRC.Update(readByte); + } + if (baseInputStream.ReadByte() < 0 || baseInputStream.ReadByte() < 0) { + throw new Exception("Early EOF baseInputStream GZIP header"); + } + + int len1, len2, extraLen; + len1 = baseInputStream.ReadByte(); + len2 = baseInputStream.ReadByte(); + if ((len1 < 0) || (len2 < 0)) { + throw new Exception("Early EOF baseInputStream GZIP header"); + } + headCRC.Update(len1); + headCRC.Update(len2); + + extraLen = (len1 << 8) | len2; + for (int i = 0; i < extraLen;i++) { + int readByte = baseInputStream.ReadByte(); + if (readByte < 0) { + throw new Exception("Early EOF baseInputStream GZIP header"); + } + headCRC.Update(readByte); + } + } + + /* 8. Read file name */ + if ((flags & GZipConstants.FNAME) != 0) { + int readByte; + while ( (readByte = baseInputStream.ReadByte()) > 0) { + headCRC.Update(readByte); + } + if (readByte < 0) { + throw new Exception("Early EOF baseInputStream GZIP file name"); + } + headCRC.Update(readByte); + } + + /* 9. Read comment */ + if ((flags & GZipConstants.FCOMMENT) != 0) { + int readByte; + while ( (readByte = baseInputStream.ReadByte()) > 0) { + headCRC.Update(readByte); + } + + if (readByte < 0) { + throw new Exception("Early EOF baseInputStream GZIP comment"); + } + headCRC.Update(readByte); + } + + /* 10. Read header CRC */ + if ((flags & GZipConstants.FHCRC) != 0) { + int tempByte; + int crcval = baseInputStream.ReadByte(); + if (crcval < 0) { + throw new Exception("Early EOF baseInputStream GZIP header"); + } + + tempByte = baseInputStream.ReadByte(); + if (tempByte < 0) { + throw new Exception("Early EOF baseInputStream GZIP header"); + } + + crcval = (crcval << 8) | tempByte; + if (crcval != ((int) headCRC.Value & 0xffff)) { + throw new IOException("Header CRC value mismatch"); + } + } + + readGZIPHeader = true; + //System.err.println("Read GZIP header"); + } + + private void ReadFooter() + { + byte[] footer = new byte[8]; + int avail = inf.RemainingInput; + if (avail > 8) { + avail = 8; + } + System.Array.Copy(buf, len - inf.RemainingInput, footer, 0, avail); + int needed = 8 - avail; + while (needed > 0) { + int count = baseInputStream.Read(footer, 8-needed, needed); + if (count <= 0) { + throw new Exception("Early EOF baseInputStream GZIP footer"); + } + needed -= count; //Jewel Jan 16 + } + int crcval = (footer[0] & 0xff) | ((footer[1] & 0xff) << 8) | ((footer[2] & 0xff) << 16) | (footer[3] << 24); + if (crcval != (int) crc.Value) { + throw new IOException("GZIP crc sum mismatch, theirs \"" + crcval + "\" and ours \"" + (int) crc.Value); + } + int total = (footer[4] & 0xff) | ((footer[5] & 0xff) << 8) | ((footer[6] & 0xff) << 16) | (footer[7] << 24); + if (total != inf.TotalOut) { + throw new IOException("Number of bytes mismatch"); + } + /* XXX Should we support multiple members. + * Difficult, since there may be some bytes still baseInputStream buf + */ + eos = true; + } + + /* Have we read the GZIP header yet? */ + private bool readGZIPHeader; + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipOutputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipOutputStream.cs new file mode 100644 index 000000000..434957096 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/GZip/GZipOutputStream.cs @@ -0,0 +1,170 @@ +// GzipOutputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; + +using BCG.Utilities.SharpZipLib.Checksums; +using BCG.Utilities.SharpZipLib.Zip.Compression; +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; + +namespace BCG.Utilities.SharpZipLib.GZip { + + /// + /// This filter stream is used to compress a stream into a "GZIP" stream. + /// The "GZIP" format is described in RFC 1952. + /// + /// author of the original java version : John Leuner + /// + /// This sample shows how to gzip a file + /// + /// using System; + /// using System.IO; + /// + /// using NZlib.GZip; + /// + /// class MainClass + /// { + /// public static void Main(string[] args) + /// { + /// Stream s = new GZipOutputStream(File.Create(args[0] + ".gz")); + /// FileStream fs = File.OpenRead(args[0]); + /// byte[] writeData = new byte[fs.Length]; + /// fs.Read(writeData, 0, (int)fs.Length); + /// s.Write(writeData, 0, writeData.Length); + /// s.Close(); + /// } + /// } + /// + /// + public class GZipOutputStream : DeflaterOutputStream + { + //Variables + + /// + /// CRC-32 value for uncompressed data + /// + protected Crc32 crc = new Crc32(); + + // Constructors + + /// + /// Creates a GzipOutputStream with the default buffer size + /// + /// + /// The stream to read data (to be compressed) from + /// + public GZipOutputStream(Stream baseOutputStream) : this(baseOutputStream, 4096) + { + } + + /// + /// Creates a GZIPOutputStream with the specified buffer size + /// + /// + /// The stream to read data (to be compressed) from + /// + /// + /// Size of the buffer to use + /// + public GZipOutputStream(Stream baseOutputStream, int size) : base(baseOutputStream, new Deflater(Deflater.DEFAULT_COMPRESSION, true), size) + { + // TODO : find out correctness, orgininally this was : (int) (System.currentTimeMillis() / 1000L); + int mod_time = (int)(DateTime.Now.Ticks / 10000L); // Ticks give back 100ns intervals + byte[] gzipHeader = { + /* The two magic bytes */ + (byte) (GZipConstants.GZIP_MAGIC >> 8), (byte) GZipConstants.GZIP_MAGIC, + + /* The compression type */ + (byte) Deflater.DEFLATED, + + /* The flags (not set) */ + 0, + + /* The modification time */ + (byte) mod_time, (byte) (mod_time >> 8), + (byte) (mod_time >> 16), (byte) (mod_time >> 24), + + /* The extra flags */ + 0, + + /* The OS type (unknown) */ + (byte) 255 + }; + + baseOutputStream.Write(gzipHeader, 0, gzipHeader.Length); + // System.err.println("wrote GZIP header (" + gzipHeader.length + " bytes )"); + } + + public override void Write(byte[] buf, int off, int len) + { + crc.Update(buf, off, len); + base.Write(buf, off, len); + } + + /// + /// Writes remaining compressed output data to the output stream + /// and closes it. + /// + public override void Close() + { + Finish(); + baseOutputStream.Close(); + } + + public override void Finish() + { + base.Finish(); + + int totalin = def.TotalIn; + int crcval = (int) (crc.Value & 0xffffffff); + + // System.err.println("CRC val is " + Integer.toHexString( crcval ) + " and length " + Integer.toHexString(totalin)); + + byte[] gzipFooter = { + (byte) crcval, (byte) (crcval >> 8), + (byte) (crcval >> 16), (byte) (crcval >> 24), + + (byte) totalin, (byte) (totalin >> 8), + (byte) (totalin >> 16), (byte) (totalin >> 24) + }; + + baseOutputStream.Write(gzipFooter, 0, gzipFooter.Length); + // System.err.println("wrote GZIP trailer (" + gzipFooter.length + " bytes )"); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/InvalidHeaderException.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/InvalidHeaderException.cs new file mode 100644 index 000000000..960704630 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/InvalidHeaderException.cs @@ -0,0 +1,72 @@ +// BZip2OutputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; + +using BCG.Utilities.SharpZipLib.Checksums; + +namespace BCG.Utilities.SharpZipLib.Tar { + + /// + /// This exception is used to indicate that there is a problem + /// with a TAR archive header. + /// + public class InvalidHeaderException : System.IO.IOException + { + public InvalidHeaderException() + { + } + + public InvalidHeaderException(string msg) : base(msg) + { + } + } +} + +/* The original Java file had this header: +** Authored by Timothy Gerard Endres +** +** +** This work has been placed into the public domain. +** You may use this work in any way and for any purpose you wish. +** +** THIS SOFTWARE IS PROVIDED AS-IS WITHOUT WARRANTY OF ANY KIND, +** NOT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY. THE AUTHOR +** OF THIS SOFTWARE, ASSUMES _NO_ RESPONSIBILITY FOR ANY +** CONSEQUENCE RESULTING FROM THE USE, MODIFICATION, OR +** REDISTRIBUTION OF THIS SOFTWARE. +** +*/ + diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarArchive.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarArchive.cs new file mode 100644 index 000000000..adc4bd93c --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarArchive.cs @@ -0,0 +1,693 @@ +// TarInputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; +using System.Text; + +namespace BCG.Utilities.SharpZipLib.Tar { + + public delegate void ProgressMessageHandler(TarArchive archive, string message); + + /// + /// The TarArchive class implements the concept of a + /// tar archive. A tar archive is a series of entries, each of + /// which represents a file system object. Each entry in + /// the archive consists of a header record. Directory entries + /// consist only of the header record, and are followed by entries + /// for the directory's contents. File entries consist of a + /// header record followed by the number of records needed to + /// contain the file's contents. All entries are written on + /// record boundaries. Records are 512 bytes long. + /// + /// TarArchives are instantiated in either read or write mode, + /// based upon whether they are instantiated with an InputStream + /// or an OutputStream. Once instantiated TarArchives read/write + /// mode can not be changed. + /// + /// There is currently no support for random access to tar archives. + /// However, it seems that subclassing TarArchive, and using the + /// TarBuffer.getCurrentRecordNum() and TarBuffer.getCurrentBlockNum() + /// methods, this would be rather trvial. + /// + public class TarArchive + { + protected bool verbose; + protected bool debug; + protected bool keepOldFiles; + protected bool asciiTranslate; + + protected int userId; + protected string userName; + protected int groupId; + protected string groupName; + + protected string rootPath; + protected string pathPrefix; + + protected int recordSize; + protected byte[] recordBuf; + + protected TarInputStream tarIn; + protected TarOutputStream tarOut; + + public event ProgressMessageHandler ProgressMessageEvent; + + protected virtual void OnProgressMessageEvent(string message) + { + if (ProgressMessageEvent != null) { + ProgressMessageEvent(this, message); + } + } + + protected TarArchive() + { + } + + /// + /// The InputStream based constructors create a TarArchive for the + /// purposes of e'x'tracting or lis't'ing a tar archive. Thus, use + /// these constructors when you wish to extract files from or list + /// the contents of an existing tar archive. + /// + public static TarArchive CreateInputTarArchive(Stream inputStream) + { + return CreateInputTarArchive(inputStream, TarBuffer.DEFAULT_BLKSIZE); + } + + public static TarArchive CreateInputTarArchive(Stream inputStream, int blockSize) + { + return CreateInputTarArchive(inputStream, blockSize, TarBuffer.DEFAULT_RCDSIZE); + } + + public static TarArchive CreateInputTarArchive(Stream inputStream, int blockSize, int recordSize) + { + TarArchive archive = new TarArchive(); + archive.tarIn = new TarInputStream(inputStream, blockSize, recordSize); + archive.Initialize(recordSize); + return archive; + } + + /// + /// The OutputStream based constructors create a TarArchive for the + /// purposes of 'c'reating a tar archive. Thus, use these constructors + /// when you wish to create a new tar archive and write files into it. + /// + public static TarArchive CreateOutputTarArchive(Stream outputStream) + { + return CreateOutputTarArchive(outputStream, TarBuffer.DEFAULT_BLKSIZE); + } + + public static TarArchive CreateOutputTarArchive(Stream outputStream, int blockSize) + { + return CreateOutputTarArchive(outputStream, blockSize, TarBuffer.DEFAULT_RCDSIZE); + } + + public static TarArchive CreateOutputTarArchive(Stream outputStream, int blockSize, int recordSize) + { + TarArchive archive = new TarArchive(); + archive.tarOut = new TarOutputStream(outputStream, blockSize, recordSize); + archive.Initialize(recordSize); + return archive; + } + + + /// + /// Common constructor initialization code. + /// + void Initialize(int recordSize) + { + this.rootPath = null; + this.pathPrefix = null; + +// this.tempPath = System.getProperty( "user.dir" ); + + this.userId = 0; + this.userName = String.Empty; + this.groupId = 0; + this.groupName = String.Empty; + + this.debug = false; + this.verbose = false; + this.keepOldFiles = false; + + this.recordBuf = new byte[RecordSize]; + } + + /** + * Set the debugging flag. + * + * @param debugF The new debug setting. + */ + public void SetDebug(bool debugF) + { + this.debug = debugF; + if (this.tarIn != null) { + this.tarIn.SetDebug(debugF); + } + if (this.tarOut != null) { + this.tarOut.SetDebug(debugF); + } + } + + /// + /// Get/Set the verbosity setting. + /// + public bool IsVerbose { + get { + return verbose; + } + set { + verbose = value; + } + } + + /// + /// Set the flag that determines whether existing files are + /// kept, or overwritten during extraction. + /// + /// + /// If true, do not overwrite existing files. + /// + public void SetKeepOldFiles(bool keepOldFiles) + { + this.keepOldFiles = keepOldFiles; + } + + /// + /// Set the ascii file translation flag. If ascii file translatio + /// is true, then the MIME file type will be consulted to determine + /// if the file is of type 'text/*'. If the MIME type is not found, + /// then the TransFileTyper is consulted if it is not null. If + /// either of these two checks indicates the file is an ascii text + /// file, it will be translated. The translation converts the local + /// operating system's concept of line ends into the UNIX line end, + /// '\n', which is the defacto standard for a TAR archive. This makes + /// text files compatible with UNIX, and since most tar implementations + /// text files compatible with UNIX, and since most tar implementations + /// + /// + /// If true, translate ascii text files. + /// + public void SetAsciiTranslation(bool asciiTranslate) + { + this.asciiTranslate = asciiTranslate; + } + + /* + /// + /// Set the object that will determine if a file is of type + /// ascii text for translation purposes. + /// + /// + /// The new TransFileTyper object. + /// + public void SetTransFileTyper(TarTransFileTyper transTyper) + { + this.transTyper = transTyper; + }*/ + + /// + /// Set user and group information that will be used to fill in the + /// tar archive's entry headers. Since Java currently provides no means + /// of determining a user name, user id, group name, or group id for + /// a given File, TarArchive allows the programmer to specify values + /// to be used in their place. + /// + /// + /// The user Id to use in the headers. + /// + /// + /// The user name to use in the headers. + /// + /// + /// The group id to use in the headers. + /// + /// + /// The group name to use in the headers. + /// + public void SetUserInfo(int userId, string userName, int groupId, string groupName) + { + this.userId = userId; + this.userName = userName; + this.groupId = groupId; + this.groupName = groupName; + } + + /// + /// Get the user id being used for archive entry headers. + /// + /// + /// The current user id. + /// + public int UserId { + get { + return this.userId; + } + } + + /// + /// Get the user name being used for archive entry headers. + /// + /// + /// The current user name. + /// + public string UserName { + get { + return this.userName; + } + } + + /// + /// Get the group id being used for archive entry headers. + /// + /// + /// The current group id. + /// + public int GroupId { + get { + return this.groupId; + } + } + + /// + /// Get the group name being used for archive entry headers. + /// + /// + /// The current group name. + /// + public string GroupName { + get { + return this.groupName; + } + } + + /// + /// Get the archive's record size. Because of its history, tar + /// supports the concept of buffered IO consisting of BLOCKS of + /// RECORDS. This allowed tar to match the IO characteristics of + /// the physical device being used. Of course, in the Java world, + /// this makes no sense, WITH ONE EXCEPTION - archives are expected + /// to be propertly "blocked". Thus, all of the horrible TarBuffer + /// support boils down to simply getting the "boundaries" correct. + /// + /// + /// The record size this archive is using. + /// + public int RecordSize { + get { + if (this.tarIn != null) { + return this.tarIn.GetRecordSize(); + } else if (this.tarOut != null) { + return this.tarOut.GetRecordSize(); + } + return TarBuffer.DEFAULT_RCDSIZE; + } + } + + /// + /// Close the archive. This simply calls the underlying + /// tar stream's close() method. + /// + public void CloseArchive() + { + if (this.tarIn != null) { + this.tarIn.Close(); + } else if (this.tarOut != null) { + this.tarOut.Flush(); + this.tarOut.Close(); + } + } + + /// + /// Perform the "list" command and list the contents of the archive. + /// + /// NOTE That this method uses the progress display to actually list + /// the conents. If the progress display is not set, nothing will be + /// listed! + /// + public void ListContents() + { + while (true) { + TarEntry entry = this.tarIn.GetNextEntry(); + + if (entry == null) { + if (this.debug) { + Console.Error.WriteLine("READ EOF RECORD"); + } + break; + } + OnProgressMessageEvent(entry.Name); + } + } + + /// + /// Perform the "extract" command and extract the contents of the archive. + /// + /// + /// The destination directory into which to extract. + /// + public void ExtractContents(string destDir) + { + while (true) { + TarEntry entry = this.tarIn.GetNextEntry(); + + if (entry == null) { + if (this.debug) { + Console.Error.WriteLine("READ EOF RECORD"); + } + break; + } + + this.ExtractEntry(destDir, entry); + } + } + + void EnsureDirectoryExists(string directoryName) + { + if (!Directory.Exists(directoryName)) { + try { + Directory.CreateDirectory(directoryName); + } catch (Exception e) { + throw new IOException("error making directory path '" + directoryName + "', " + e.Message); + } + } + } + + + bool IsBinary(string filename) + { + FileStream fs = File.OpenRead(filename); + + byte[] content = new byte[fs.Length]; + + fs.Read(content, 0, (int)fs.Length); + fs.Close(); + + // assume that ascii 0 or + // ascii 255 are only found in non text files. + // and that all non text files contain 0 and 255 + foreach (byte b in content) { + if (b == 0 || b == 255) { + return true; + } + } + + return false; + } + + /// + /// Extract an entry from the archive. This method assumes that the + /// tarIn stream has been properly set with a call to getNextEntry(). + /// + /// + /// The destination directory into which to extract. + /// + /// + /// The TarEntry returned by tarIn.getNextEntry(). + /// + void ExtractEntry(string destDir, TarEntry entry) + { + if (this.verbose) { + OnProgressMessageEvent(entry.Name); + } + + string name = entry.Name; + name = name.Replace('/', Path.DirectorySeparatorChar); + + if (!destDir.EndsWith(Path.DirectorySeparatorChar.ToString())) { + destDir += Path.DirectorySeparatorChar; + } + + string destFile = destDir + name; + + if (entry.IsDirectory) { + EnsureDirectoryExists(destFile); + } else { + string parentDirectory = Path.GetDirectoryName(destFile); + EnsureDirectoryExists(parentDirectory); + + if (this.keepOldFiles && File.Exists(destFile)) { + if (this.verbose) { + OnProgressMessageEvent("not overwriting " + entry.Name); + } + } else { + bool asciiTrans = false; + Stream outputStream = File.Create(destFile); + if (this.asciiTranslate) { + asciiTrans = !IsBinary(destFile); +// original java sourcecode : +// MimeType mime = null; +// string contentType = null; +// try { +// contentType = FileTypeMap.getDefaultFileTypeMap().getContentType( destFile ); +// +// mime = new MimeType(contentType); +// +// if (mime.getPrimaryType().equalsIgnoreCase( "text" )) { +// asciiTrans = true; +// } else if ( this.transTyper != null ) { +// if ( this.transTyper.isAsciiFile( entry.getName() ) ) { +// asciiTrans = true; +// } +// } +// } catch (MimeTypeParseException ex) { +// } +// +// if (this.debug) { +// Console.Error.WriteLine(("EXTRACT TRANS? '" + asciiTrans + "' ContentType='" + contentType + "' PrimaryType='" + mime.getPrimaryType() + "'" ); +// } + } + + StreamWriter outw = null; + if (asciiTrans) { + outw = new StreamWriter(outputStream); + } + + byte[] rdbuf = new byte[32 * 1024]; + + while (true) { + int numRead = this.tarIn.Read(rdbuf, 0, rdbuf.Length); + + if (numRead <= 0) { + break; + } + + if (asciiTrans) { + for (int off = 0, b = 0; b < numRead; ++b) { + if (rdbuf[b] == 10) { + string s = Encoding.ASCII.GetString(rdbuf, off, (b - off)); + outw.WriteLine(s); + off = b + 1; + } + } + } else { + outputStream.Write(rdbuf, 0, numRead); + } + } + + if (asciiTrans) { + outw.Close(); + } else { + outputStream.Close(); + } + } + } + } + + /// + /// Write an entry to the archive. This method will call the putNextEntry + /// and then write the contents of the entry, and finally call closeEntry()() + /// for entries that are files. For directories, it will call putNextEntry(), + /// and then, if the recurse flag is true, process each entry that is a + /// child of the directory. + /// + /// + /// The TarEntry representing the entry to write to the archive. + /// + /// + /// If true, process the children of directory entries. + /// + public void WriteEntry(TarEntry entry, bool recurse) + { + bool asciiTrans = false; + + string tempFileName = null; + string eFile = entry.File; + + // Work on a copy of the entry so we can manipulate it. + // Note that we must distinguish how the entry was constructed. + // + if (eFile == null || eFile.Length == 0) { + entry = TarEntry.CreateTarEntry(entry.Name); + } else { + // + // The user may have explicitly set the entry's name to + // something other than the file's path, so we must save + // and restore it. This should work even when the name + // was set from the File's name. + // + string saveName = entry.Name; + entry = TarEntry.CreateEntryFromFile(eFile); + entry.Name = saveName; + } + + if (this.verbose) { + OnProgressMessageEvent(entry.Name); + } + + if (this.asciiTranslate && !entry.IsDirectory) { + asciiTrans = !IsBinary(eFile); +// original java source : +// MimeType mime = null; +// string contentType = null; +// +// try { +// contentType = FileTypeMap.getDefaultFileTypeMap(). getContentType( eFile ); +// +// mime = new MimeType( contentType ); +// +// if ( mime.getPrimaryType(). +// equalsIgnoreCase( "text" ) ) +// { +// asciiTrans = true; +// } +// else if ( this.transTyper != null ) +// { +// if ( this.transTyper.isAsciiFile( eFile ) ) +// { +// asciiTrans = true; +// } +// } +// } catch ( MimeTypeParseException ex ) +// { +// // IGNORE THIS ERROR... +// } +// +// if (this.debug) { +// Console.Error.WriteLine("CREATE TRANS? '" + asciiTrans + "' ContentType='" + contentType + "' PrimaryType='" + mime.getPrimaryType()+ "'" ); +// } + + if (asciiTrans) { + tempFileName = Path.GetTempFileName(); + + StreamReader inStream = File.OpenText(eFile); + Stream outStream = new BufferedStream(File.Create(tempFileName)); + + while (true) { + string line = inStream.ReadLine(); + if (line == null) { + break; + } + byte[] data = Encoding.ASCII.GetBytes(line); + outStream.Write(data, 0, data.Length); + outStream.WriteByte((byte)'\n'); + } + + inStream.Close(); + outStream.Flush(); + outStream.Close(); + + entry.Size = new FileInfo(tempFileName).Length; + + eFile = tempFileName; + } + } + + string newName = null; + + if (this.rootPath != null) { + if (entry.Name.StartsWith(this.rootPath)) { + newName = entry.Name.Substring(this.rootPath.Length + 1 ); + } + } + + if (this.pathPrefix != null) { + newName = (newName == null) ? this.pathPrefix + "/" + entry.Name : this.pathPrefix + "/" + newName; + } + + if (newName != null) { + entry.Name = newName; + } + + this.tarOut.PutNextEntry(entry); + + if (entry.IsDirectory) { + if (recurse) { + TarEntry[] list = entry.GetDirectoryEntries(); + for (int i = 0; i < list.Length; ++i) { + this.WriteEntry(list[i], recurse); + } + } + } else { + Stream inputStream = File.OpenRead(eFile); + int numWritten = 0; + byte[] eBuf = new byte[32 * 1024]; + while (true) { + int numRead = inputStream.Read(eBuf, 0, eBuf.Length); + + if (numRead <=0) { + break; + } + + this.tarOut.Write(eBuf, 0, numRead); + numWritten += numRead; + } + Console.WriteLine("written " + numWritten + " bytes"); + + inputStream.Close(); + + if (tempFileName != null && tempFileName.Length > 0) { + File.Delete(tempFileName); + } + + this.tarOut.CloseEntry(); + } + } + } +} +/* The original Java file had this header: + ** Authored by Timothy Gerard Endres + ** + ** + ** This work has been placed into the public domain. + ** You may use this work in any way and for any purpose you wish. + ** + ** THIS SOFTWARE IS PROVIDED AS-IS WITHOUT WARRANTY OF ANY KIND, + ** NOT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY. THE AUTHOR + ** OF THIS SOFTWARE, ASSUMES _NO_ RESPONSIBILITY FOR ANY + ** CONSEQUENCE RESULTING FROM THE USE, MODIFICATION, OR + ** REDISTRIBUTION OF THIS SOFTWARE. + ** + */ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarBuffer.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarBuffer.cs new file mode 100644 index 000000000..e9396168e --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarBuffer.cs @@ -0,0 +1,437 @@ +// TarBuffer.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; +using System.Text; + +namespace BCG.Utilities.SharpZipLib.Tar { + + /// + /// The TarBuffer class implements the tar archive concept + /// of a buffered input stream. This concept goes back to the + /// days of blocked tape drives and special io devices. In the + /// C# universe, the only real function that this class + /// performs is to ensure that files have the correct "block" + /// size, or other tars will complain. + ///

+ /// You should never have a need to access this class directly. + /// TarBuffers are created by Tar IO Streams. + ///

+ ///
+ public class TarBuffer + { + public static readonly int DEFAULT_RCDSIZE = 512; + public static readonly int DEFAULT_BLKSIZE = DEFAULT_RCDSIZE * 20; + + Stream inputStream; + Stream outputStream; + + byte[] blockBuffer; + int currBlkIdx; + int currRecIdx; + int blockSize; + int recordSize; + int recsPerBlock; + + bool debug; + + protected TarBuffer() + { + } + + public static TarBuffer CreateInputTarBuffer(Stream inputStream) + { + return CreateInputTarBuffer(inputStream, TarBuffer.DEFAULT_BLKSIZE); + } + public static TarBuffer CreateInputTarBuffer(Stream inputStream, int blockSize ) + { + return CreateInputTarBuffer(inputStream, blockSize, TarBuffer.DEFAULT_RCDSIZE); + } + public static TarBuffer CreateInputTarBuffer(Stream inputStream, int blockSize, int recordSize) + { + TarBuffer tarBuffer = new TarBuffer(); + tarBuffer.inputStream = inputStream; + tarBuffer.outputStream = null; + tarBuffer.Initialize(blockSize, recordSize); + + return tarBuffer; + } + + public static TarBuffer CreateOutputTarBuffer(Stream outputStream) + { + return CreateOutputTarBuffer(outputStream, TarBuffer.DEFAULT_BLKSIZE); + } + public static TarBuffer CreateOutputTarBuffer(Stream outputStream, int blockSize ) + { + return CreateOutputTarBuffer(outputStream, blockSize, TarBuffer.DEFAULT_RCDSIZE); + } + public static TarBuffer CreateOutputTarBuffer(Stream outputStream, int blockSize, int recordSize) + { + TarBuffer tarBuffer = new TarBuffer(); + tarBuffer.inputStream = null; + tarBuffer.outputStream = outputStream; + tarBuffer.Initialize(blockSize, recordSize); + + return tarBuffer; + } + + /// + /// Initialization common to all constructors. + /// + void Initialize(int blockSize, int recordSize) + { + this.debug = false; + this.blockSize = blockSize; + this.recordSize = recordSize; + this.recsPerBlock = this.blockSize / this.recordSize; + this.blockBuffer = new byte[this.blockSize]; + + if (inputStream != null) { + this.currBlkIdx = -1; + this.currRecIdx = this.recsPerBlock; + } else { + this.currBlkIdx = 0; + this.currRecIdx = 0; + } + } + + /// + /// Get the TAR Buffer's block size. Blocks consist of multiple records. + /// + public int GetBlockSize() + { + return this.blockSize; + } + + /// + /// Get the TAR Buffer's record size. + /// + public int GetRecordSize() + { + return this.recordSize; + } + + /// + /// Set the debugging flag for the buffer. + /// + public void SetDebug(bool debug) + { + this.debug = debug; + } + + /// + /// Determine if an archive record indicate End of Archive. End of + /// archive is indicated by a record that consists entirely of null bytes. + /// + /// + /// The record data to check. + /// + public bool IsEOFRecord(byte[] record) + { + for (int i = 0, sz = this.GetRecordSize(); i < sz; ++i) { + if (record[i] != 0) { + return false; + } + } + + return true; + } + + /// + /// Skip over a record on the input stream. + /// + public void SkipRecord() + { + if (this.debug) { + Console.Error.WriteLine("SkipRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx); + } + + if (this.inputStream == null) { + throw new System.IO.IOException("no input stream defined"); + } + + if (this.currRecIdx >= this.recsPerBlock) { + if (!this.ReadBlock()) { + return; // UNDONE + } + } + + this.currRecIdx++; + } + + /// + /// Read a record from the input stream and return the data. + /// + /// + /// The record data. + /// + public byte[] ReadRecord() + { + if (this.debug) { + Console.Error.WriteLine( "ReadRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx ); + } + + if (this.inputStream == null) { + throw new System.IO.IOException("no input stream defined"); + } + + if (this.currRecIdx >= this.recsPerBlock) { + if (!this.ReadBlock()) { + return null; + } + } + + byte[] result = new byte[this.recordSize]; + + Array.Copy(this.blockBuffer, (this.currRecIdx * this.recordSize), result, 0, this.recordSize ); + this.currRecIdx++; + return result; + } + + /// + /// false if End-Of-File, else true + /// + bool ReadBlock() + { + Console.WriteLine(this.debug); + if (this.debug) { + Console.Error.WriteLine("ReadBlock: blkIdx = " + this.currBlkIdx); + } + + if (this.inputStream == null) { + throw new System.IO.IOException("no input stream stream defined"); + } + + this.currRecIdx = 0; + + int offset = 0; + int bytesNeeded = this.blockSize; + for (; bytesNeeded > 0 ;) { + long numBytes = this.inputStream.Read(this.blockBuffer, offset, bytesNeeded); + + // + // NOTE + // We have fit EOF, and the block is not full! + // + // This is a broken archive. It does not follow the standard + // blocking algorithm. However, because we are generous, and + // it requires little effort, we will simply ignore the error + // and continue as if the entire block were read. This does + // not appear to break anything upstream. We used to return + // false in this case. + // + // Thanks to 'Yohann.Roussel@alcatel.fr' for this fix. + // + if (numBytes <= 0) { + break; + } + + offset += (int)numBytes; + bytesNeeded -= (int)numBytes; + if (numBytes != this.blockSize) { + if (this.debug) { + Console.Error.WriteLine("ReadBlock: INCOMPLETE READ " + numBytes + " of " + this.blockSize + " bytes read."); + } + } + } + + this.currBlkIdx++; + return true; + } + + /// + /// Get the current block number, zero based. + /// + /// + /// The current zero based block number. + /// + public int GetCurrentBlockNum() + { + return this.currBlkIdx; + } + + /// + /// Get the current record number, within the current block, zero based. + /// Thus, current offset = (currentBlockNum * recsPerBlk) + currentRecNum. + /// + /// + /// The current zero based record number. + /// + public int GetCurrentRecordNum() + { + return this.currRecIdx - 1; + } + + /// + /// Write an archive record to the archive. + /// + /// + /// The record data to write to the archive. + /// + /// + public void WriteRecord(byte[] record) + { + if (this.debug) { + Console.Error.WriteLine("WriteRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx ); + } + + if (this.outputStream == null) { + throw new System.IO.IOException("no output stream defined"); + } + + if (record.Length != this.recordSize) { + throw new IOException("record to write has length '" + record.Length + "' which is not the record size of '" + this.recordSize + "'" ); + } + + if (this.currRecIdx >= this.recsPerBlock) { + this.WriteBlock(); + } + Array.Copy(record, 0, this.blockBuffer, (this.currRecIdx * this.recordSize), this.recordSize ); + this.currRecIdx++; + } + + /// + /// Write an archive record to the archive, where the record may be + /// inside of a larger array buffer. The buffer must be "offset plus + /// record size" long. + /// + /// + /// The buffer containing the record data to write. + /// + /// + /// The offset of the record data within buf. + /// + public void WriteRecord(byte[] buf, int offset) + { + if (this.debug) { + Console.Error.WriteLine("WriteRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx ); + } + + if (this.outputStream == null) { + throw new System.IO.IOException("no output stream stream defined"); + } + + if ((offset + this.recordSize) > buf.Length) { + throw new IOException("record has length '" + buf.Length + "' with offset '" + offset + "' which is less than the record size of '" + this.recordSize + "'" ); + } + + if (this.currRecIdx >= this.recsPerBlock) { + this.WriteBlock(); + } + + Array.Copy(buf, offset, this.blockBuffer, (this.currRecIdx * this.recordSize), this.recordSize ); + + this.currRecIdx++; + } + + /// + /// Write a TarBuffer block to the archive. + /// + void WriteBlock() + { + if (this.debug) { + Console.Error.WriteLine("WriteBlock: blkIdx = " + this.currBlkIdx); + } + + if (this.outputStream == null) { + throw new System.IO.IOException("no output stream defined"); + } + + this.outputStream.Write(this.blockBuffer, 0, this.blockSize); + this.outputStream.Flush(); + + this.currRecIdx = 0; + this.currBlkIdx++; + } + + /// + /// Flush the current data block if it has any data in it. + /// + void Flush() + { + if (this.debug) { + Console.Error.WriteLine("TarBuffer.FlushBlock() called."); + } + + if (this.outputStream == null) { + throw new System.IO.IOException("no output base stream defined"); + } + + if (this.currRecIdx > 0) { + this.WriteBlock(); + } + outputStream.Flush(); + } + + /// + /// Close the TarBuffer. If this is an output buffer, also flush the + /// current block before closing. + /// + public void Close() + { + if (this.debug) { + Console.Error.WriteLine("TarBuffer.Close()."); + } + + if (outputStream != null ) { + Flush(); + +// if ( this.outStream != System.out +// && this.outStream != System.err ) { + outputStream.Close(); + outputStream = null; + } else if (inputStream != null) { +// if (this.inStream != System.in ) { + inputStream.Close(); + inputStream = null; + } + } + } +} +/* The original Java file had this header: + * + ** Authored by Timothy Gerard Endres + ** + ** + ** This work has been placed into the public domain. + ** You may use this work in any way and for any purpose you wish. + ** + ** THIS SOFTWARE IS PROVIDED AS-IS WITHOUT WARRANTY OF ANY KIND, + ** NOT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY. THE AUTHOR + ** OF THIS SOFTWARE, ASSUMES _NO_ RESPONSIBILITY FOR ANY + ** CONSEQUENCE RESULTING FROM THE USE, MODIFICATION, OR + ** REDISTRIBUTION OF THIS SOFTWARE. + ** + */ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarEntry.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarEntry.cs new file mode 100644 index 000000000..de4de9779 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarEntry.cs @@ -0,0 +1,672 @@ +// TarEntry.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; +using System.Text; + +namespace BCG.Utilities.SharpZipLib.Tar { + + /// + /// This class represents an entry in a Tar archive. It consists + /// of the entry's header, as well as the entry's File. Entries + /// can be instantiated in one of three ways, depending on how + /// they are to be used. + ///

+ /// TarEntries that are created from the header bytes read from + /// an archive are instantiated with the TarEntry( byte[] ) + /// constructor. These entries will be used when extracting from + /// or listing the contents of an archive. These entries have their + /// header filled in using the header bytes. They also set the File + /// to null, since they reference an archive entry not a file.

+ ///

+ /// TarEntries that are created from Files that are to be written + /// into an archive are instantiated with the TarEntry( File ) + /// constructor. These entries have their header filled in using + /// the File's information. They also keep a reference to the File + /// for convenience when writing entries.

+ ///

+ /// Finally, TarEntries can be constructed from nothing but a name. + /// This allows the programmer to construct the entry by hand, for + /// instance when only an InputStream is available for writing to + /// the archive, and the header information is constructed from + /// other information. In this case the header fields are set to + /// defaults and the File is set to null.

+ /// + ///

+ /// The C structure for a Tar Entry's header is: + ///

+	/// struct header {
+	/// 	char	name[NAMSIZ];
+	/// 	char	mode[8];
+	/// 	char	uid[8];
+	/// 	char	gid[8];
+	/// 	char	size[12];
+	/// 	char	mtime[12];
+	/// 	char	chksum[8];
+	/// 	char	linkflag;
+	/// 	char	linkname[NAMSIZ];
+	/// 	char	magic[8];
+	/// 	char	uname[TUNMLEN];
+	/// 	char	gname[TGNMLEN];
+	/// 	char	devmajor[8];
+	/// 	char	devminor[8];
+	/// 	} header;
+	/// 
+ ///

+ /// + ///
+ public class TarEntry + { + /// + /// If this entry represents a File, this references it. + /// + protected string file; + + /// + /// This is the entry's header information. + /// + protected TarHeader header; + + /// + /// Only Create Entries with the static CreateXYZ methods or a headerBuffer. + /// + private TarEntry() + { + } + + /// + /// Construct an entry from an archive's header bytes. File is set + /// to null. + /// + /// + /// The header bytes from a tar archive entry. + /// + public TarEntry(byte[] headerBuf) + { + this.Initialize(); + this.ParseTarHeader(this.header, headerBuf); + } + + + + /// + /// Construct an entry with only a name. This allows the programmer + /// to construct the entry's header "by hand". File is set to null. + /// + public static TarEntry CreateTarEntry(string name) + { + TarEntry entry = new TarEntry(); + entry.Initialize(); + entry.NameTarHeader(entry.header, name); + return entry; + } + + /// + /// Construct an entry for a file. File is set to file, and the + /// header is constructed from information from the file. + /// + /// + /// The file that the entry represents. + /// + public static TarEntry CreateEntryFromFile(string fileName) + { + TarEntry entry = new TarEntry(); + entry.Initialize(); + entry.GetFileTarHeader(entry.header, fileName); + return entry; + } + + /// + /// Initialization code common to all constructors. + /// + void Initialize() + { + this.file = null; + this.header = new TarHeader(); + } + + /// + /// Determine if the two entries are equal. Equality is determined + /// by the header names being equal. + /// + /// + /// True if the entries are equal. + /// + public override bool Equals(object it) + { + if (!(it is TarEntry)) { + return false; + } + return this.header.name.ToString().Equals(((TarEntry)it).header.name.ToString()); + } + + /// + /// Must be overridden when you override Equals. + /// + public override int GetHashCode() + { + return this.header.name.ToString().GetHashCode(); + } + + + /// + /// Determine if the given entry is a descendant of this entry. + /// Descendancy is determined by the name of the descendant + /// starting with this entry's name. + /// + /// + /// Entry to be checked as a descendent of this. + /// + /// + /// True if entry is a descendant of this. + /// + public bool IsDescendent(TarEntry desc) + { + return desc.header.name.ToString().StartsWith(this.header.name.ToString()); + } + + /// + /// Get this entry's header. + /// + /// + /// This entry's TarHeader. + /// + public TarHeader TarHeader { + get { + return this.header; + } + } + + /// + /// Get/Set this entry's name. + /// + public string Name { + get { + return this.header.name.ToString(); + } + set { + this.header.name = new StringBuilder(value); + } + } + + /// + /// Get/set this entry's user id. + /// + public int UserId { + get { + return this.header.userId; + } + set { + this.header.userId = value; + } + } + + /// + /// Get/set this entry's group id. + /// + public int GroupId { + get { + return this.header.groupId; + } + set { + this.header.groupId = value; + } + } + + /// + /// Get/set this entry's user name. + /// + public string UserName { + get { + return this.header.userName.ToString(); + } + set { + this.header.userName = new StringBuilder(value); + } + } + + /// + /// Get/set this entry's group name. + /// + public string GroupName { + get { + return this.header.groupName.ToString(); + } + set { + this.header.groupName = new StringBuilder(value); + } + } + + /// + /// Convenience method to set this entry's group and user ids. + /// + /// + /// This entry's new user id. + /// + /// + /// This entry's new group id. + /// + public void SetIds(int userId, int groupId) + { + UserId = userId; + GroupId = groupId; + } + + /// + /// Convenience method to set this entry's group and user names. + /// + /// + /// This entry's new user name. + /// + /// + /// This entry's new group name. + /// + public void SetNames(string userName, string groupName) + { + UserName = userName; + GroupName = groupName; + } + +// TODO : +// /** +// * Set this entry's modification time. The parameter passed +// * to this method is in "Java time". +// * +// * @param time This entry's new modification time. +// */ +// public void setModTime( long time ) +// { +// this.header.modTime = time / 1000; +// } + + /// Convert time to DateTimes + /** + * Get/Set this entry's modification time. + * + * @param time This entry's new modification time. + */ + public DateTime ModTime { + get { + return this.header.modTime; + } + set { + this.header.modTime = value; + } + } + + /// + /// Get this entry's file. + /// + /// + /// This entry's file. + /// + public string File { + get { + return this.file; + } + } + + /// + /// Get/set this entry's file size. + /// + public long Size { + get { + return this.header.size; + } + set { + this.header.size = value; + } + } + + /// + /// Convenience method that will modify an entry's name directly + /// in place in an entry header buffer byte array. + /// + /// + /// The buffer containing the entry header to modify. + /// + /// + /// The new name to place into the header buffer. + /// + public void AdjustEntryName(byte[] outbuf, string newName) + { + int offset = 0; + offset = TarHeader.GetNameBytes(new StringBuilder(newName), outbuf, offset, TarHeader.NAMELEN); + } + + /// + /// Return whether or not this entry represents a directory. + /// + /// + /// True if this entry is a directory. + /// + public bool IsDirectory + { + get { + if (this.file != null) { + return Directory.Exists(file); + } + + if (this.header != null) { + if (this.header.linkFlag == TarHeader.LF_DIR || this.header.name.ToString().EndsWith( "/" )) { + return true; + } + } + return false; + } + } + + /// + /// Fill in a TarHeader with information from a File. + /// + /// + /// The TarHeader to fill in. + /// + /// + /// The file from which to get the header information. + /// + public void GetFileTarHeader(TarHeader hdr, string file) + { + this.file = file; + + string name = Path.GetDirectoryName(file); + + if (Path.DirectorySeparatorChar == '\\') { // check if the OS is a windows + // Strip off drive letters! + if (name.Length > 2) { + char ch1 = name[0]; + char ch2 = name[1]; + + if (ch2 == ':' && Char.IsLetter(ch1)) { + name = name.Substring(2); + } + } + } + + name = name.Replace(Path.DirectorySeparatorChar, '/'); + + // No absolute pathnames + // Windows (and Posix?) paths can start with "\\NetworkDrive\", + // so we loop on starting /'s. + while (name.StartsWith("/")) { + name = name.Substring(1); + } + + hdr.linkName = new StringBuilder(String.Empty); + hdr.name = new StringBuilder(name); + + if (Directory.Exists(file)) { + hdr.mode = 040755; // TODO : what does this magic number ?? Mike + hdr.linkFlag = TarHeader.LF_DIR; + if (hdr.name[hdr.name.Length - 1] != '/') { + hdr.name.Append("/"); + } + hdr.size = 0; + } else { + hdr.mode = 0100644; // TODO : again a magic number + hdr.linkFlag = TarHeader.LF_NORMAL; + Console.WriteLine(file.Replace('/', Path.DirectorySeparatorChar)); + hdr.size = new FileInfo(file.Replace('/', Path.DirectorySeparatorChar)).Length; + } + + // UNDONE When File lets us get the userName, use it! + hdr.modTime = System.IO.File.GetLastAccessTime(file.Replace('/', Path.DirectorySeparatorChar)); + hdr.checkSum = 0; + hdr.devMajor = 0; + hdr.devMinor = 0; + } + + /// + /// If this entry represents a file, and the file is a directory, return + /// an array of TarEntries for this entry's children. + /// + /// + /// An array of TarEntry's for this entry's children. + /// + public TarEntry[] GetDirectoryEntries() + { + if (this.file == null || !Directory.Exists(this.file)) { + return new TarEntry[0]; + } + + string[] list = Directory.GetFileSystemEntries(this.file); + TarEntry[] result = new TarEntry[list.Length]; + + string dirName = file; + if (!dirName.EndsWith(Path.DirectorySeparatorChar.ToString())) { + dirName += Path.DirectorySeparatorChar; + } + + for (int i = 0; i < list.Length; ++i) { + result[i] = TarEntry.CreateEntryFromFile(list[i]); + } + + return result; + } + + /// + /// Compute the checksum of a tar entry header. + /// + /// + /// The tar entry's header buffer. + /// + /// + /// The computed checksum. + /// + public long ComputeCheckSum(byte[] buf) + { + long sum = 0; + for (int i = 0; i < buf.Length; ++i) { + sum += 255 & buf[i]; // TODO : I think the 255 & x isn't neccessary += buf[i] should be enough. CHECK IT! + } + return sum; + } + + /// + /// Write an entry's header information to a header buffer. + /// + /// + /// The tar entry header buffer to fill in. + /// + public void WriteEntryHeader(byte[] outbuf) + { + int offset = 0; + + offset = TarHeader.GetNameBytes(this.header.name, outbuf, offset, TarHeader.NAMELEN); + offset = TarHeader.GetOctalBytes(this.header.mode, outbuf, offset, TarHeader.MODELEN); + offset = TarHeader.GetOctalBytes(this.header.userId, outbuf, offset, TarHeader.UIDLEN); + offset = TarHeader.GetOctalBytes(this.header.groupId, outbuf, offset, TarHeader.GIDLEN); + + long size = this.header.size; + + offset = TarHeader.GetLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN); + offset = TarHeader.GetLongOctalBytes(GetCTime(this.header.modTime), outbuf, offset, TarHeader.MODTIMELEN); + + int csOffset = offset; + for (int c = 0; c < TarHeader.CHKSUMLEN; ++c) { + outbuf[offset++] = (byte)' '; + } + + outbuf[offset++] = this.header.linkFlag; + + offset = TarHeader.GetNameBytes(this.header.linkName, outbuf, offset, TarHeader.NAMELEN); + offset = TarHeader.GetNameBytes(this.header.magic, outbuf, offset, TarHeader.MAGICLEN); + offset = TarHeader.GetNameBytes(this.header.userName, outbuf, offset, TarHeader.UNAMELEN); + offset = TarHeader.GetNameBytes(this.header.groupName, outbuf, offset, TarHeader.GNAMELEN); + + offset = TarHeader.GetOctalBytes(this.header.devMajor, outbuf, offset, TarHeader.DEVLEN); + offset = TarHeader.GetOctalBytes(this.header.devMinor, outbuf, offset, TarHeader.DEVLEN); + + for (; offset < outbuf.Length;) { + outbuf[offset++] = 0; + } + + long checkSum = this.ComputeCheckSum(outbuf); + + TarHeader.GetCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN); + } + + // time conversion functions + readonly static long timeConversionFactor = 10000000L; + readonly static DateTime datetTime1970 = new DateTime(1970, 1, 1, 0, 0, 0, 0); + + static int GetCTime(System.DateTime dateTime) + { + return (int)((dateTime.Ticks - datetTime1970.Ticks) / timeConversionFactor); + } + + static DateTime GetDateTimeFromCTime(long ticks) + { + return new DateTime(datetTime1970.Ticks + ticks * timeConversionFactor); + } + + /// + /// Parse an entry's TarHeader information from a header buffer. + /// + /// + /// Parse an entry's TarHeader information from a header buffer. + /// + /// + /// The tar entry header buffer to get information from. + /// + public void ParseTarHeader(TarHeader hdr, byte[] header) + { + int offset = 0; + + hdr.name = TarHeader.ParseName(header, offset, TarHeader.NAMELEN); + + offset += TarHeader.NAMELEN; + + hdr.mode = (int)TarHeader.ParseOctal(header, offset, TarHeader.MODELEN); + + offset += TarHeader.MODELEN; + + hdr.userId = (int)TarHeader.ParseOctal(header, offset, TarHeader.UIDLEN); + + offset += TarHeader.UIDLEN; + + hdr.groupId = (int)TarHeader.ParseOctal(header, offset, TarHeader.GIDLEN); + + offset += TarHeader.GIDLEN; + + hdr.size = TarHeader.ParseOctal(header, offset, TarHeader.SIZELEN); + + offset += TarHeader.SIZELEN; + + hdr.modTime = GetDateTimeFromCTime(TarHeader.ParseOctal(header, offset, TarHeader.MODTIMELEN)); + + offset += TarHeader.MODTIMELEN; + + hdr.checkSum = (int)TarHeader.ParseOctal(header, offset, TarHeader.CHKSUMLEN); + + offset += TarHeader.CHKSUMLEN; + + hdr.linkFlag = header[ offset++ ]; + + hdr.linkName = TarHeader.ParseName(header, offset, TarHeader.NAMELEN); + + offset += TarHeader.NAMELEN; + + hdr.magic = TarHeader.ParseName(header, offset, TarHeader.MAGICLEN); + + offset += TarHeader.MAGICLEN; + + hdr.userName = TarHeader.ParseName(header, offset, TarHeader.UNAMELEN); + + offset += TarHeader.UNAMELEN; + + hdr.groupName = TarHeader.ParseName(header, offset, TarHeader.GNAMELEN); + + offset += TarHeader.GNAMELEN; + + hdr.devMajor = (int)TarHeader.ParseOctal(header, offset, TarHeader.DEVLEN); + + offset += TarHeader.DEVLEN; + + hdr.devMinor = (int)TarHeader.ParseOctal(header, offset, TarHeader.DEVLEN); + } + + /// + /// Fill in a TarHeader given only the entry's name. + /// + /// + /// The TarHeader to fill in. + /// + /// + /// The tar entry name. + /// + public void NameTarHeader(TarHeader hdr, string name) + { + bool isDir = name.EndsWith("/"); + + hdr.checkSum = 0; + hdr.devMajor = 0; + hdr.devMinor = 0; + + hdr.name = new StringBuilder(name); + hdr.mode = isDir ? 040755 : 0100644; // TODO : I think I've seen these magics before ... + hdr.userId = 0; + hdr.groupId = 0; + hdr.size = 0; + hdr.checkSum = 0; + + hdr.modTime = DateTime.Now;//(new java.util.Date()).getTime() / 1000; + + hdr.linkFlag = isDir ? TarHeader.LF_DIR : TarHeader.LF_NORMAL; + + hdr.linkName = new StringBuilder(String.Empty); + hdr.userName = new StringBuilder(String.Empty); + hdr.groupName = new StringBuilder(String.Empty); + + hdr.devMajor = 0; + hdr.devMinor = 0; + } + } +} +/* The original Java file had this header: + * + ** Authored by Timothy Gerard Endres + ** + ** + ** This work has been placed into the public domain. + ** You may use this work in any way and for any purpose you wish. + ** + ** THIS SOFTWARE IS PROVIDED AS-IS WITHOUT WARRANTY OF ANY KIND, + ** NOT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY. THE AUTHOR + ** OF THIS SOFTWARE, ASSUMES _NO_ RESPONSIBILITY FOR ANY + ** CONSEQUENCE RESULTING FROM THE USE, MODIFICATION, OR + ** REDISTRIBUTION OF THIS SOFTWARE. + ** + */ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarHeader.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarHeader.cs new file mode 100644 index 000000000..072a5d899 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarHeader.cs @@ -0,0 +1,500 @@ +// TarHeader.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.Text; + +namespace BCG.Utilities.SharpZipLib.Tar { + + + /// + /// This class encapsulates the Tar Entry Header used in Tar Archives. + /// The class also holds a number of tar constants, used mostly in headers. + /// + public class TarHeader : ICloneable + { + /// + /// The length of the name field in a header buffer. + /// + public readonly static int NAMELEN = 100; + + /// + /// The length of the mode field in a header buffer. + /// + public readonly static int MODELEN = 8; + + /// + /// The length of the user id field in a header buffer. + /// + public readonly static int UIDLEN = 8; + + /// + /// The length of the group id field in a header buffer. + /// + public readonly static int GIDLEN = 8; + + /// + /// The length of the checksum field in a header buffer. + /// + public readonly static int CHKSUMLEN = 8; + + /// + /// The length of the size field in a header buffer. + /// + public readonly static int SIZELEN = 12; + + /// + /// The length of the magic field in a header buffer. + /// + public readonly static int MAGICLEN = 8; + + /// + /// The length of the modification time field in a header buffer. + /// + public readonly static int MODTIMELEN = 12; + + /// + /// The length of the user name field in a header buffer. + /// + public readonly static int UNAMELEN = 32; + + /// + /// The length of the group name field in a header buffer. + /// + public readonly static int GNAMELEN = 32; + + /// + /// The length of the devices field in a header buffer. + /// + public readonly static int DEVLEN = 8; + + /// + /// LF_ constants represent the "link flag" of an entry, or more commonly, + /// the "entry type". This is the "old way" of indicating a normal file. + /// + public readonly static byte LF_OLDNORM = 0; + + /// + /// Normal file type. + /// + public readonly static byte LF_NORMAL = (byte) '0'; + + /// + /// Link file type. + /// + public readonly static byte LF_LINK = (byte) '1'; + + /// + /// Symbolic link file type. + /// + public readonly static byte LF_SYMLINK = (byte) '2'; + + /// + /// Character device file type. + /// + public readonly static byte LF_CHR = (byte) '3'; + + /// + /// Block device file type. + /// + public readonly static byte LF_BLK = (byte) '4'; + + /// + /// Directory file type. + /// + public readonly static byte LF_DIR = (byte) '5'; + + /// + /// FIFO (pipe) file type. + /// + public readonly static byte LF_FIFO = (byte) '6'; + + /// + /// Contiguous file type. + /// + public readonly static byte LF_CONTIG = (byte) '7'; + + /// + /// The magic tag representing a POSIX tar archive. + /// + public readonly static string TMAGIC = "ustar"; + + /// + /// The magic tag representing a GNU tar archive. + /// + public readonly static string GNU_TMAGIC = "ustar "; + + /// + /// The entry's name. + /// + public StringBuilder name; + + /// + /// The entry's permission mode. + /// + public int mode; + + /// + /// The entry's user id. + /// + public int userId; + + /// + /// The entry's group id. + /// + public int groupId; + + /// + /// The entry's size. + /// + public long size; + + /// + /// The entry's modification time. + /// + public DateTime modTime; + + /// + /// The entry's checksum. + /// + public int checkSum; + + /// + /// The entry's link flag. + /// + public byte linkFlag; + + /// + /// The entry's link name. + /// + public StringBuilder linkName; + + /// + /// The entry's magic tag. + /// + public StringBuilder magic; + + /// + /// The entry's user name. + /// + public StringBuilder userName; + + /// + /// The entry's group name. + /// + public StringBuilder groupName; + + /// + /// The entry's major device number. + /// + public int devMajor; + + /// + /// The entry's minor device number. + /// + public int devMinor; + + public TarHeader() + { + this.magic = new StringBuilder(TarHeader.TMAGIC); + + this.name = new StringBuilder(); + this.linkName = new StringBuilder(); + + string user = Environment.UserName; + + if (user.Length > 31) { + user = user.Substring(0, 31); + } + + this.userId = 0; + this.groupId = 0; + this.userName = new StringBuilder(user); + this.groupName = new StringBuilder(String.Empty); + } + + /// + /// TarHeaders can be cloned. + /// + public object Clone() + { + TarHeader hdr = new TarHeader(); + + hdr.name = (this.name == null) ? null : new StringBuilder(this.name.ToString()); + hdr.mode = this.mode; + hdr.userId = this.userId; + hdr.groupId = this.groupId; + hdr.size = this.size; + hdr.modTime = this.modTime; + hdr.checkSum = this.checkSum; + hdr.linkFlag = this.linkFlag; + hdr.linkName = (this.linkName == null) ? null : new StringBuilder(this.linkName.ToString()); + hdr.magic = (this.magic == null) ? null : new StringBuilder(this.magic.ToString()); + hdr.userName = (this.userName == null) ? null : new StringBuilder(this.userName.ToString()); + hdr.groupName = (this.groupName == null) ? null : new StringBuilder(this.groupName.ToString()); + hdr.devMajor = this.devMajor; + hdr.devMinor = this.devMinor; + + return hdr; + } + + /// + /// Get the name of this entry. + /// + /// + /// The entry's name. + /// + public string GetName() + { + return this.name.ToString(); + } + + /// + /// Parse an octal string from a header buffer. This is used for the + /// file permission mode value. + /// + /// + /// The header buffer from which to parse. + /// + /// + /// The offset into the buffer from which to parse. + /// + /// + /// The number of header bytes to parse. + /// + /// + /// The long value of the octal string. + /// + public static long ParseOctal(byte[] header, int offset, int length) + { + long result = 0; + bool stillPadding = true; + + int end = offset + length; + for (int i = offset; i < end ; ++i) { + if (header[i] == 0) { + break; + } + + if (header[i] == (byte)' ' || header[i] == '0') { + if (stillPadding) { + continue; + } + + if (header[i] == (byte)' ') { + break; + } + } + + stillPadding = false; + + result = (result << 3) + (header[i] - '0'); + } + + return result; + } + + /// + /// Parse an entry name from a header buffer. + /// + /// + /// The header buffer from which to parse. + /// + /// + /// The offset into the buffer from which to parse. + /// + /// + /// The number of header bytes to parse. + /// + /// + /// The header's entry name. + /// + public static StringBuilder ParseName(byte[] header, int offset, int length) + { + StringBuilder result = new StringBuilder(length); + + for (int i = offset; i < offset + length; ++i) { + if (header[i] == 0) { + break; + } + result.Append((char)header[i]); + } + + return result; + } + + /// + /// Determine the number of bytes in an entry name. + /// + /// + /// + /// + /// The header buffer from which to parse. + /// + /// + /// The offset into the buffer from which to parse. + /// + /// + /// The number of header bytes to parse. + /// + /// + /// The number of bytes in a header's entry name. + /// + public static int GetNameBytes(StringBuilder name, byte[] buf, int offset, int length) + { + int i; + + for (i = 0 ; i < length && i < name.Length; ++i) { + buf[offset + i] = (byte)name[i]; + } + + for (; i < length ; ++i) { + buf[offset + i] = 0; + } + + return offset + length; + } + + /// + /// Parse an octal integer from a header buffer. + /// + /// + /// + /// + /// The header buffer from which to parse. + /// + /// + /// The offset into the buffer from which to parse. + /// + /// + /// The number of header bytes to parse. + /// + /// + /// The integer value of the octal bytes. + /// + public static int GetOctalBytes(long val, byte[] buf, int offset, int length) + { + byte[] result = new byte[length]; + + int idx = length - 1; + + buf[offset + idx] = 0; + --idx; + buf[offset + idx] = (byte)' '; + --idx; + + if (val == 0) { + buf[offset + idx] = (byte)'0'; + --idx; + } else { + for (long v = val; idx >= 0 && v > 0; --idx) { + buf[offset + idx] = (byte)((byte)'0' + (byte)(v & 7)); + v >>= 3; + } + } + + for (; idx >= 0; --idx) { + buf[offset + idx] = (byte)' '; + } + + return offset + length; + } + + /// + /// Parse an octal long integer from a header buffer. + /// + /// + /// + /// + /// The header buffer from which to parse. + /// + /// + /// The offset into the buffer from which to parse. + /// + /// + /// The number of header bytes to parse. + /// + /// + /// The long value of the octal bytes. + /// + public static int GetLongOctalBytes(long val, byte[] buf, int offset, int length) + { + byte[] temp = new byte[length + 1]; + TarHeader.GetOctalBytes(val, temp, 0, length + 1); + Array.Copy(temp, 0, buf, offset, length); + return offset + length; + } + + /// + /// Parse the checksum octal integer from a header buffer. + /// + /// + /// + /// + /// The header buffer from which to parse. + /// + /// + /// The offset into the buffer from which to parse. + /// + /// + /// The number of header bytes to parse. + /// + /// + /// The integer value of the entry's checksum. + /// + public static int GetCheckSumOctalBytes(long val, byte[] buf, int offset, int length) + { + TarHeader.GetOctalBytes(val, buf, offset, length); + buf[offset + length - 1] = (byte)' '; + buf[offset + length - 2] = 0; + return offset + length; + } + } +} + +/* The original Java file had this header: + * +** Authored by Timothy Gerard Endres +** +** +** This work has been placed into the public domain. +** You may use this work in any way and for any purpose you wish. +** +** THIS SOFTWARE IS PROVIDED AS-IS WITHOUT WARRANTY OF ANY KIND, +** NOT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY. THE AUTHOR +** OF THIS SOFTWARE, ASSUMES _NO_ RESPONSIBILITY FOR ANY +** CONSEQUENCE RESULTING FROM THE USE, MODIFICATION, OR +** REDISTRIBUTION OF THIS SOFTWARE. +** +*/ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarInputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarInputStream.cs new file mode 100644 index 000000000..0e2d771f3 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarInputStream.cs @@ -0,0 +1,522 @@ +// TarInputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; +using System.Text; + +namespace BCG.Utilities.SharpZipLib.Tar { + + /// + /// The TarInputStream reads a UNIX tar archive as an InputStream. + /// methods are provided to position at each successive entry in + /// the archive, and the read each entry as a normal input stream + /// using read(). + /// + public class TarInputStream : Stream + { + protected bool debug; + protected bool hasHitEOF; + + protected int entrySize; + protected int entryOffset; + + protected byte[] readBuf; + + protected TarBuffer buffer; + protected TarEntry currEntry; + protected IEntryFactory eFactory; + + Stream inputStream; + /// + /// I needed to implement the abstract member. + /// + public override bool CanRead { + get { + return inputStream.CanRead; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanSeek { + get { + return inputStream.CanSeek; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanWrite { + get { + return inputStream.CanWrite; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Length { + get { + return inputStream.Length; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Position { + get { + return inputStream.Position; + } + set { + inputStream.Position = value; + } + } + + /// + /// Flushes the baseInputStream + /// + public override void Flush() + { + inputStream.Flush(); + } + + /// + /// I needed to implement the abstract member. + /// + public override long Seek(long offset, SeekOrigin origin) + { + return inputStream.Seek(offset, origin); + } + + /// + /// I needed to implement the abstract member. + /// + public override void SetLength(long val) + { + inputStream.SetLength(val); + } + + /// + /// I needed to implement the abstract member. + /// + public override void Write(byte[] array, int offset, int count) + { + inputStream.Write(array, offset, count); + } + + /// + /// I needed to implement the abstract member. + /// + public override void WriteByte(byte val) + { + inputStream.WriteByte(val); + } + + + public TarInputStream(Stream inputStream) : this(inputStream, TarBuffer.DEFAULT_BLKSIZE, TarBuffer.DEFAULT_RCDSIZE) + { + } + + public TarInputStream(Stream inputStream, int blockSize) : this(inputStream, blockSize, TarBuffer.DEFAULT_RCDSIZE) + { + } + + public TarInputStream(Stream inputStream, int blockSize, int recordSize) + { + this.inputStream = inputStream; + this.buffer = TarBuffer.CreateInputTarBuffer(inputStream, blockSize, recordSize); + + this.readBuf = null; + this.debug = false; + this.hasHitEOF = false; + this.eFactory = null; + } + + public void SetDebug(bool debugF) + { + this.debug = debugF; + SetBufferDebug(debugF); + } + public void SetBufferDebug(bool debug) + { + this.buffer.SetDebug(debug); + } + + + + public void SetEntryFactory(IEntryFactory factory) + { + this.eFactory = factory; + } + + /// + /// Closes this stream. Calls the TarBuffer's close() method. + /// The underlying stream is closed by the TarBuffer. + /// + public override void Close() + { + this.buffer.Close(); + } + + /// + /// Get the record size being used by this stream's TarBuffer. + /// + /// + /// TarBuffer record size. + /// + public int GetRecordSize() + { + return this.buffer.GetRecordSize(); + } + + /// + /// Get the available data that can be read from the current + /// entry in the archive. This does not indicate how much data + /// is left in the entire archive, only in the current entry. + /// This value is determined from the entry's size header field + /// and the amount of data already read from the current entry. + /// + /// + /// The number of available bytes for the current entry. + /// + public int Available { + get { + return this.entrySize - this.entryOffset; + } + } + + /// + /// Skip bytes in the input buffer. This skips bytes in the + /// current entry's data, not the entire archive, and will + /// stop at the end of the current entry's data if the number + /// to skip extends beyond that point. + /// + /// + /// The number of bytes to skip. + /// + public void Skip(int numToSkip) + { + // REVIEW + // This is horribly inefficient, but it ensures that we + // properly skip over bytes via the TarBuffer... + // + byte[] skipBuf = new byte[8 * 1024]; + + for (int num = numToSkip; num > 0;){ + int numRead = this.Read(skipBuf, 0, (num > skipBuf.Length ? skipBuf.Length : num)); + + if (numRead == -1) { + break; + } + + num -= numRead; + } + } + + /// + /// Since we do not support marking just yet, we return false. + /// + public bool IsMarkSupported { + get { + return false; + } + } + + /// + /// Since we do not support marking just yet, we do nothing. + /// + /// + /// The limit to mark. + /// + public void Mark(int markLimit) + { + } + + /// + /// Since we do not support marking just yet, we do nothing. + /// + public void Reset() + { + } + + /// + /// Get the next entry in this tar archive. This will skip + /// over any remaining data in the current entry, if there + /// is one, and place the input stream at the header of the + /// next entry, and read the header and instantiate a new + /// TarEntry from the header bytes and return that entry. + /// If there are no more entries in the archive, null will + /// be returned to indicate that the end of the archive has + /// been reached. + /// + /// + /// The next TarEntry in the archive, or null. + /// + public TarEntry GetNextEntry() + { + if (this.hasHitEOF) { + return null; + } + + if (this.currEntry != null) { + int numToSkip = this.entrySize - this.entryOffset; + + if (this.debug) { + Console.Error.WriteLine("TarInputStream: SKIP currENTRY '" + this.currEntry.Name + "' SZ " + this.entrySize + " OFF " + this.entryOffset + " skipping " + numToSkip + " bytes"); + } + + if (numToSkip > 0) { + this.Skip(numToSkip); + } + + this.readBuf = null; + } + + byte[] headerBuf = this.buffer.ReadRecord(); + + if (headerBuf == null) { + if (this.debug) { + Console.Error.WriteLine("READ NULL RECORD"); + } + + this.hasHitEOF = true; + } else if (this.buffer.IsEOFRecord(headerBuf)) { + if (this.debug) { + Console.Error.WriteLine( "READ EOF RECORD" ); + } + + this.hasHitEOF = true; + } + + if (this.hasHitEOF) { + this.currEntry = null; + } else { + try { + if (this.eFactory == null) { + this.currEntry = new TarEntry(headerBuf); + } else { + this.currEntry = this.eFactory.CreateEntry(headerBuf); + } + + if (!(headerBuf[257] == 'u' && headerBuf[258] == 's' && headerBuf[259] == 't' && headerBuf[260] == 'a' && headerBuf[261] == 'r')) { + throw new InvalidHeaderException("header magic is not 'ustar', but '" + headerBuf[257] + headerBuf[258] + headerBuf[259] + headerBuf[260] + headerBuf[261] + + "', or (dec) " + ((int)headerBuf[257]) + ", " + ((int)headerBuf[258]) + ", " + ((int)headerBuf[259]) + ", " + ((int)headerBuf[260]) + ", " + ((int)headerBuf[261])); + } + + if (this.debug) { + Console.Error.WriteLine("TarInputStream: SET CURRENTRY '" + this.currEntry.Name + "' size = " + this.currEntry.Size); + } + + this.entryOffset = 0; + + // REVIEW How do we resolve this discrepancy?! + this.entrySize = (int) this.currEntry.Size; + } catch (InvalidHeaderException ex) { + this.entrySize = 0; + this.entryOffset = 0; + this.currEntry = null; + throw new InvalidHeaderException("bad header in block " + this.buffer.GetCurrentBlockNum() + " record " + this.buffer.GetCurrentRecordNum() + ", " + ex.Message); + } + } + return this.currEntry; + } + + /// + /// Reads a byte from the current tar archive entry. + /// This method simply calls read(byte[], int, int). + /// + public override int ReadByte() + { + byte[] oneByteBuffer = new byte[1]; + int num = this.Read(oneByteBuffer, 0, 1); + if (num <= 0) { // return -1 to indicate that no byte was read. + return -1; + } + return (int)oneByteBuffer[0]; + } + + /// + /// Reads bytes from the current tar archive entry. + /// + /// This method is aware of the boundaries of the current + /// entry in the archive and will deal with them as if they + /// entry in the archive and will deal with them as if they + /// + /// + /// The buffer into which to place bytes read. + /// + /// + /// The offset at which to place bytes read. + /// + /// + /// The number of bytes to read. + /// + /// + /// The number of bytes read, or -1 at EOF. + /// + public override int Read(byte[] buf, int offset, int numToRead) + { + int totalRead = 0; + + if (this.entryOffset >= this.entrySize) { + return -1; + } + + if ((numToRead + this.entryOffset) > this.entrySize) { + numToRead = this.entrySize - this.entryOffset; + } + + if (this.readBuf != null) { + int sz = (numToRead > this.readBuf.Length) ? this.readBuf.Length : numToRead; + + Array.Copy(this.readBuf, 0, buf, offset, sz); + + if (sz >= this.readBuf.Length) { + this.readBuf = null; + } else { + int newLen = this.readBuf.Length - sz; + byte[] newBuf = new byte[ newLen ]; + Array.Copy(this.readBuf, sz, newBuf, 0, newLen); + this.readBuf = newBuf; + } + + totalRead += sz; + numToRead -= sz; + offset += sz; + } + + while (numToRead > 0) { + byte[] rec = this.buffer.ReadRecord(); + if (rec == null) { + // Unexpected EOF! + throw new IOException("unexpected EOF with " + numToRead + " bytes unread"); + } + + int sz = numToRead; + int recLen = rec.Length; + + if (recLen > sz) { + Array.Copy(rec, 0, buf, offset, sz); + this.readBuf = new byte[recLen - sz]; + Array.Copy(rec, sz, this.readBuf, 0, recLen - sz); + } else { + sz = recLen; + Array.Copy(rec, 0, buf, offset, recLen); + } + + totalRead += sz; + numToRead -= sz; + offset += sz; + } + + this.entryOffset += totalRead; + + return totalRead; + } + + /// + /// Copies the contents of the current tar archive entry directly into + /// an output stream. + /// + /// + /// The OutputStream into which to write the entry's data. + /// + public void CopyEntryContents(Stream outputStream) + { + byte[] buf = new byte[32 * 1024]; + + while (true) { + int numRead = this.Read(buf, 0, buf.Length); + if (numRead <= 0) { + break; + } + outputStream.Write(buf, 0, numRead); + } + } + + /// + /// This interface is provided, with the method setEntryFactory(), to allow + /// the programmer to have their own TarEntry subclass instantiated for the + /// entries return from getNextEntry(). + /// + public interface IEntryFactory + { + TarEntry CreateEntry(string name); + + TarEntry CreateEntryFromFile(string fileName); + + TarEntry CreateEntry(byte[] headerBuf); + } + + public class EntryFactoryAdapter : IEntryFactory + { + public TarEntry CreateEntry(string name) + { + return TarEntry.CreateTarEntry(name); + } + + public TarEntry CreateEntryFromFile(string fileName) + { + return TarEntry.CreateEntryFromFile(fileName); + } + + public TarEntry CreateEntry(byte[] headerBuf) + { + return new TarEntry(headerBuf); + } + } + } + + +} + +/* The original Java file had this header: + ** Authored by Timothy Gerard Endres + ** + ** + ** This work has been placed into the public domain. + ** You may use this work in any way and for any purpose you wish. + ** + ** THIS SOFTWARE IS PROVIDED AS-IS WITHOUT WARRANTY OF ANY KIND, + ** NOT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY. THE AUTHOR + ** OF THIS SOFTWARE, ASSUMES _NO_ RESPONSIBILITY FOR ANY + ** CONSEQUENCE RESULTING FROM THE USE, MODIFICATION, OR + ** REDISTRIBUTION OF THIS SOFTWARE. + ** + */ + diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarOutputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarOutputStream.cs new file mode 100644 index 000000000..aef1708a8 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarOutputStream.cs @@ -0,0 +1,379 @@ +// TarOutputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; +using System.Text; + +namespace BCG.Utilities.SharpZipLib.Tar { + + /// + /// The TarOutputStream writes a UNIX tar archive as an OutputStream. + /// Methods are provided to put entries, and then write their contents + /// by writing to this stream using write(). + /// + /// public + public class TarOutputStream : Stream + { + protected bool debug; + protected int currSize; + protected int currBytes; + protected byte[] recordBuf; + protected int assemLen; + protected byte[] assemBuf; + + protected TarBuffer buffer; + protected Stream outputStream; + + /// + /// I needed to implement the abstract member. + /// + public override bool CanRead { + get { + return outputStream.CanRead; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanSeek { + get { + return outputStream.CanSeek; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanWrite { + get { + return outputStream.CanWrite; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Length { + get { + return outputStream.Length; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Position { + get { + return outputStream.Position; + } + set { + outputStream.Position = value; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Seek(long offset, SeekOrigin origin) + { + return outputStream.Seek(offset, origin); + } + + /// + /// I needed to implement the abstract member. + /// + public override void SetLength(long val) + { + outputStream.SetLength(val); + } + + /// + /// I needed to implement the abstract member. + /// + public override int ReadByte() + { + return outputStream.ReadByte(); + } + + /// + /// I needed to implement the abstract member. + /// + public override int Read(byte[] b, int off, int len) + { + return outputStream.Read(b, off, len); + } + + public override void Flush() + { + outputStream.Flush(); + } + + public TarOutputStream(Stream outputStream) : this(outputStream, TarBuffer.DEFAULT_BLKSIZE, TarBuffer.DEFAULT_RCDSIZE) + { + } + + public TarOutputStream(Stream outputStream, int blockSize) : this(outputStream, blockSize, TarBuffer.DEFAULT_RCDSIZE) + { + } + + public TarOutputStream(Stream outputStream, int blockSize, int recordSize) + { + this.outputStream = outputStream; + this.buffer = TarBuffer.CreateOutputTarBuffer(outputStream, blockSize, recordSize); + + this.debug = false; + this.assemLen = 0; + this.assemBuf = new byte[recordSize]; + this.recordBuf = new byte[recordSize]; + } + + /// + /// Sets the debugging flag. + /// + /// + /// True to turn on debugging. + /// + public void SetDebug(bool debugF) + { + this.debug = debugF; + SetBufferDebug(debugF); + } + + public void SetBufferDebug(bool debug) + { + this.buffer.SetDebug(debug); + } + + /// + /// Ends the TAR archive without closing the underlying OutputStream. + /// The result is that the EOF record of nulls is written. + /// + public void Finish() + { + this.WriteEOFRecord(); + } + + /// + /// Ends the TAR archive and closes the underlying OutputStream. + /// This means that finish() is called followed by calling the + /// TarBuffer's close(). + /// + public override void Close() + { + this.Finish(); + this.buffer.Close(); + } + + /// + /// Get the record size being used by this stream's TarBuffer. + /// + /// + /// The TarBuffer record size. + /// + public int GetRecordSize() + { + return this.buffer.GetRecordSize(); + } + + /// + /// Put an entry on the output stream. This writes the entry's + /// header record and positions the output stream for writing + /// the contents of the entry. Once this method is called, the + /// stream is ready for calls to write() to write the entry's + /// contents. Once the contents are written, closeEntry() + /// MUST be called to ensure that all buffered data + /// is completely written to the output stream. + /// + /// + /// The TarEntry to be written to the archive. + /// + public void PutNextEntry(TarEntry entry) + { + if (entry.TarHeader.name.Length > TarHeader.NAMELEN) { + throw new InvalidHeaderException("file name '" + entry.TarHeader.name + "' is too long ( > " + TarHeader.NAMELEN + " bytes )"); + } + + entry.WriteEntryHeader(this.recordBuf); + this.buffer.WriteRecord(this.recordBuf); + + this.currBytes = 0; + + this.currSize = entry.IsDirectory ? 0 : (int)entry.Size; + } + + /// + /// Close an entry. This method MUST be called for all file + /// entries that contain data. The reason is that we must + /// buffer data written to the stream in order to satisfy + /// the buffer's record based writes. Thus, there may be + /// data fragments still being assembled that must be written + /// to the output stream before this entry is closed and the + /// next entry written. + /// + public void CloseEntry() + { + if (this.assemLen > 0) { + for (int i = this.assemLen; i < this.assemBuf.Length; ++i) { + this.assemBuf[i] = 0; + } + + this.buffer.WriteRecord(this.assemBuf); + + this.currBytes += this.assemLen; + this.assemLen = 0; + } + + if (this.currBytes < this.currSize) { + throw new IOException("entry closed at '" + this.currBytes + "' before the '" + this.currSize + "' bytes specified in the header were written"); + } + } + + /// + /// Writes a byte to the current tar archive entry. + /// This method simply calls Write(byte[], int, int). + /// + /// + /// The byte written. + /// + public override void WriteByte(byte b) + { + this.Write(new byte[] { b }, 0, 1); + } + + /// + /// Writes bytes to the current tar archive entry. This method + /// is aware of the current entry and will throw an exception if + /// you attempt to write bytes past the length specified for the + /// current entry. The method is also (painfully) aware of the + /// record buffering required by TarBuffer, and manages buffers + /// that are not a multiple of recordsize in length, including + /// assembling records from small buffers. + /// + /// + /// The buffer to write to the archive. + /// + /// + /// The offset in the buffer from which to get bytes. + /// + /// + /// The number of bytes to write. + /// + public override void Write(byte[] wBuf, int wOffset, int numToWrite) + { + if ((this.currBytes + numToWrite) > this.currSize) { + throw new IOException("request to write '" + numToWrite + "' bytes exceeds size in header of '" + this.currSize + "' bytes"); + } + + // + // We have to deal with assembly!!! + // The programmer can be writing little 32 byte chunks for all + // we know, and we must assemble complete records for writing. + // REVIEW Maybe this should be in TarBuffer? Could that help to + // eliminate some of the buffer copying. + // + if (this.assemLen > 0) { + if ((this.assemLen + numToWrite ) >= this.recordBuf.Length) { + int aLen = this.recordBuf.Length - this.assemLen; + + Array.Copy(this.assemBuf, 0, this.recordBuf, 0, this.assemLen); + + Array.Copy(wBuf, wOffset, this.recordBuf, this.assemLen, aLen); + + this.buffer.WriteRecord(this.recordBuf); + + this.currBytes += this.recordBuf.Length; + + wOffset += aLen; + numToWrite -= aLen; + + this.assemLen = 0; + } else {// ( (this.assemLen + numToWrite ) < this.recordBuf.length ) + Array.Copy(wBuf, wOffset, this.assemBuf, this.assemLen, numToWrite); + wOffset += numToWrite; + this.assemLen += numToWrite; + numToWrite -= numToWrite; + } + } + + // + // When we get here we have EITHER: + // o An empty "assemble" buffer. + // o No bytes to write (numToWrite == 0) + // + while (numToWrite > 0) { + if (numToWrite < this.recordBuf.Length) { + Array.Copy(wBuf, wOffset, this.assemBuf, this.assemLen, numToWrite); + this.assemLen += numToWrite; + break; + } + + this.buffer.WriteRecord(wBuf, wOffset); + + int num = this.recordBuf.Length; + this.currBytes += num; + numToWrite -= num; + wOffset += num; + } + } + + /// + /// Write an EOF (end of archive) record to the tar archive. + /// An EOF record consists of a record of all zeros. + /// + void WriteEOFRecord() + { + for (int i = 0; i < this.recordBuf.Length; ++i) { + this.recordBuf[i] = 0; + } + this.buffer.WriteRecord(this.recordBuf); + } + } +} + +/* The original Java file had this header: + ** Authored by Timothy Gerard Endres + ** + ** + ** This work has been placed into the public domain. + ** You may use this work in any way and for any purpose you wish. + ** + ** THIS SOFTWARE IS PROVIDED AS-IS WITHOUT WARRANTY OF ANY KIND, + ** NOT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY. THE AUTHOR + ** OF THIS SOFTWARE, ASSUMES _NO_ RESPONSIBILITY FOR ANY + ** CONSEQUENCE RESULTING FROM THE USE, MODIFICATION, OR + ** REDISTRIBUTION OF THIS SOFTWARE. + ** + */ diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Deflater.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Deflater.cs new file mode 100644 index 000000000..2221c626e --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Deflater.cs @@ -0,0 +1,542 @@ +// Deflater.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + /// + /// This is the Deflater class. The deflater class compresses input + /// with the deflate algorithm described in RFC 1951. It has several + /// compression levels and three different strategies described below. + /// + /// This class is not thread safe. This is inherent in the API, due + /// to the split of deflate and setInput. + /// + /// author of the original java version : Jochen Hoenicke + /// + public class Deflater + { + /// + /// The best and slowest compression level. This tries to find very + /// long and distant string repetitions. + /// + public static int BEST_COMPRESSION = 9; + + /// + /// The worst but fastest compression level. + /// + public static int BEST_SPEED = 1; + + /// + /// The default compression level. + /// + public static int DEFAULT_COMPRESSION = -1; + + /// + /// This level won't compress at all but output uncompressed blocks. + /// + public static int NO_COMPRESSION = 0; + + /// + /// The compression method. This is the only method supported so far. + /// There is no need to use this constant at all. + /// + public static int DEFLATED = 8; + + /* + * The Deflater can do the following state transitions: + * + * (1) -> INIT_STATE ----> INIT_FINISHING_STATE ---. + * / | (2) (5) | + * / v (5) | + * (3)| SETDICT_STATE ---> SETDICT_FINISHING_STATE |(3) + * \ | (3) | ,-------' + * | | | (3) / + * v v (5) v v + * (1) -> BUSY_STATE ----> FINISHING_STATE + * | (6) + * v + * FINISHED_STATE + * \_____________________________________/ + * | (7) + * v + * CLOSED_STATE + * + * (1) If we should produce a header we start in INIT_STATE, otherwise + * we start in BUSY_STATE. + * (2) A dictionary may be set only when we are in INIT_STATE, then + * we change the state as indicated. + * (3) Whether a dictionary is set or not, on the first call of deflate + * we change to BUSY_STATE. + * (4) -- intentionally left blank -- :) + * (5) FINISHING_STATE is entered, when flush() is called to indicate that + * there is no more INPUT. There are also states indicating, that + * the header wasn't written yet. + * (6) FINISHED_STATE is entered, when everything has been flushed to the + * internal pending output buffer. + * (7) At any time (7) + * + */ + + private static int IS_SETDICT = 0x01; + private static int IS_FLUSHING = 0x04; + private static int IS_FINISHING = 0x08; + + private static int INIT_STATE = 0x00; + private static int SETDICT_STATE = 0x01; +// private static int INIT_FINISHING_STATE = 0x08; +// private static int SETDICT_FINISHING_STATE = 0x09; + private static int BUSY_STATE = 0x10; + private static int FLUSHING_STATE = 0x14; + private static int FINISHING_STATE = 0x1c; + private static int FINISHED_STATE = 0x1e; + private static int CLOSED_STATE = 0x7f; + + /// + /// Compression level. + /// + private int level; + + /// + /// should we include a header. + /// + private bool noHeader; + +// /// +// /// Compression strategy. +// /// +// private int strategy; + + /// + /// The current state. + /// + private int state; + + /// + /// The total bytes of output written. + /// + private int totalOut; + + /// + /// The pending output. + /// + private DeflaterPending pending; + + /// + /// The deflater engine. + /// + private DeflaterEngine engine; + + /// + /// Creates a new deflater with default compression level. + /// + public Deflater() : this(DEFAULT_COMPRESSION, false) + { + + } + + /// + /// Creates a new deflater with given compression level. + /// + /// + /// the compression level, a value between NO_COMPRESSION + /// and BEST_COMPRESSION, or DEFAULT_COMPRESSION. + /// + /// if lvl is out of range. + public Deflater(int lvl) : this(lvl, false) + { + + } + + /// + /// Creates a new deflater with given compression level. + /// + /// + /// the compression level, a value between NO_COMPRESSION + /// and BEST_COMPRESSION. + /// + /// + /// true, if we should suppress the deflate header at the + /// beginning and the adler checksum at the end of the output. This is + /// useful for the GZIP format. + /// + /// if lvl is out of range. + public Deflater(int lvl, bool nowrap) + { + if (lvl == DEFAULT_COMPRESSION) { + lvl = 6; + } else if (lvl < NO_COMPRESSION || lvl > BEST_COMPRESSION) { + throw new ArgumentOutOfRangeException("lvl"); + } + + pending = new DeflaterPending(); + engine = new DeflaterEngine(pending); + this.noHeader = nowrap; + SetStrategy(DeflateStrategy.Default); + SetLevel(lvl); + Reset(); + } + + + /// + /// Resets the deflater. The deflater acts afterwards as if it was + /// just created with the same compression level and strategy as it + /// had before. + /// + public void Reset() + { + state = (noHeader ? BUSY_STATE : INIT_STATE); + totalOut = 0; + pending.Reset(); + engine.Reset(); + } + + /// + /// Gets the current adler checksum of the data that was processed so far. + /// + public int Adler { + get { + return engine.Adler; + } + } + + /// + /// Gets the number of input bytes processed so far. + /// + public int TotalIn { + get { + return engine.TotalIn; + } + } + + /// + /// Gets the number of output bytes so far. + /// + public int TotalOut { + get { + return totalOut; + } + } + + /// + /// Flushes the current input block. Further calls to deflate() will + /// produce enough output to inflate everything in the current input + /// block. This is not part of Sun's JDK so I have made it package + /// private. It is used by DeflaterOutputStream to implement + /// flush(). + /// + public void Flush() + { + state |= IS_FLUSHING; + } + + /// + /// Finishes the deflater with the current input block. It is an error + /// to give more input after this method was called. This method must + /// be called to force all bytes to be flushed. + /// + public void Finish() + { + state |= IS_FLUSHING | IS_FINISHING; + } + + /// + /// Returns true if the stream was finished and no more output bytes + /// are available. + /// + public bool IsFinished { + get { + return state == FINISHED_STATE && pending.IsFlushed; + } + } + + /// + /// Returns true, if the input buffer is empty. + /// You should then call setInput(). + /// NOTE: This method can also return true when the stream + /// was finished. + /// + public bool IsNeedingInput { + get { + return engine.NeedsInput(); + } + } + + /// + /// Sets the data which should be compressed next. This should be only + /// called when needsInput indicates that more input is needed. + /// If you call setInput when needsInput() returns false, the + /// previous input that is still pending will be thrown away. + /// The given byte array should not be changed, before needsInput() returns + /// true again. + /// This call is equivalent to setInput(input, 0, input.length). + /// + /// + /// the buffer containing the input data. + /// + /// + /// if the buffer was finished() or ended(). + /// + public void SetInput(byte[] input) + { + SetInput(input, 0, input.Length); + } + + /// + /// Sets the data which should be compressed next. This should be + /// only called when needsInput indicates that more input is needed. + /// The given byte array should not be changed, before needsInput() returns + /// true again. + /// + /// + /// the buffer containing the input data. + /// + /// + /// the start of the data. + /// + /// + /// the length of the data. + /// + /// + /// if the buffer was finished() or ended() or if previous input is still pending. + /// + public void SetInput(byte[] input, int off, int len) + { + if ((state & IS_FINISHING) != 0) { + throw new InvalidOperationException("finish()/end() already called"); + } + engine.SetInput(input, off, len); + } + + /// + /// Sets the compression level. There is no guarantee of the exact + /// position of the change, but if you call this when needsInput is + /// true the change of compression level will occur somewhere near + /// before the end of the so far given input. + /// + /// + /// the new compression level. + /// + public void SetLevel(int lvl) + { + if (lvl == DEFAULT_COMPRESSION) { + lvl = 6; + } else if (lvl < NO_COMPRESSION || lvl > BEST_COMPRESSION) { + throw new ArgumentOutOfRangeException("lvl"); + } + + + if (level != lvl) { + level = lvl; + engine.SetLevel(lvl); + } + } + + /// + /// Sets the compression strategy. Strategy is one of + /// DEFAULT_STRATEGY, HUFFMAN_ONLY and FILTERED. For the exact + /// position where the strategy is changed, the same as for + /// setLevel() applies. + /// + /// + /// the new compression strategy. + /// + public void SetStrategy(DeflateStrategy stgy) + { + engine.Strategy = stgy; + } + + /// + /// Deflates the current input block to the given array. It returns + /// the number of bytes compressed, or 0 if either + /// needsInput() or finished() returns true or length is zero. + /// + /// + /// the buffer where to write the compressed data. + /// + public int Deflate(byte[] output) + { + return Deflate(output, 0, output.Length); + } + + /// + /// Deflates the current input block to the given array. It returns + /// the number of bytes compressed, or 0 if either + /// needsInput() or finished() returns true or length is zero. + /// + /// + /// the buffer where to write the compressed data. + /// + /// + /// the offset into the output array. + /// + /// + /// the maximum number of bytes that may be written. + /// + /// + /// if end() was called. + /// + /// + /// if offset and/or length don't match the array length. + /// + public int Deflate(byte[] output, int offset, int length) + { + int origLength = length; + + if (state == CLOSED_STATE) { + throw new InvalidOperationException("Deflater closed"); + } + + if (state < BUSY_STATE) { + /* output header */ + int header = (DEFLATED + + ((DeflaterConstants.MAX_WBITS - 8) << 4)) << 8; + int level_flags = (level - 1) >> 1; + if (level_flags < 0 || level_flags > 3) { + level_flags = 3; + } + header |= level_flags << 6; + if ((state & IS_SETDICT) != 0) { + /* Dictionary was set */ + header |= DeflaterConstants.PRESET_DICT; + } + header += 31 - (header % 31); + + + pending.WriteShortMSB(header); + if ((state & IS_SETDICT) != 0) { + int chksum = engine.Adler; + engine.ResetAdler(); + pending.WriteShortMSB(chksum >> 16); + pending.WriteShortMSB(chksum & 0xffff); + } + + state = BUSY_STATE | (state & (IS_FLUSHING | IS_FINISHING)); + } + + for (;;) { + int count = pending.Flush(output, offset, length); + offset += count; + totalOut += count; + length -= count; + + if (length == 0 || state == FINISHED_STATE) { + break; + } + + if (!engine.Deflate((state & IS_FLUSHING) != 0, (state & IS_FINISHING) != 0)) { + if (state == BUSY_STATE) { + /* We need more input now */ + return origLength - length; + } else if (state == FLUSHING_STATE) { + if (level != NO_COMPRESSION) { + /* We have to supply some lookahead. 8 bit lookahead + * are needed by the zlib inflater, and we must fill + * the next byte, so that all bits are flushed. + */ + int neededbits = 8 + ((-pending.BitCount) & 7); + while (neededbits > 0) { + /* write a static tree block consisting solely of + * an EOF: + */ + pending.WriteBits(2, 10); + neededbits -= 10; + } + } + state = BUSY_STATE; + } else if (state == FINISHING_STATE) { + pending.AlignToByte(); + /* We have completed the stream */ + if (!noHeader) { + int adler = engine.Adler; + pending.WriteShortMSB(adler >> 16); + pending.WriteShortMSB(adler & 0xffff); + } + state = FINISHED_STATE; + } + } + } + return origLength - length; + } + + /// + /// Sets the dictionary which should be used in the deflate process. + /// This call is equivalent to setDictionary(dict, 0, dict.Length). + /// + /// + /// the dictionary. + /// + /// + /// if setInput () or deflate () were already called or another dictionary was already set. + /// + public void SetDictionary(byte[] dict) + { + SetDictionary(dict, 0, dict.Length); + } + + /// + /// Sets the dictionary which should be used in the deflate process. + /// The dictionary should be a byte array containing strings that are + /// likely to occur in the data which should be compressed. The + /// dictionary is not stored in the compressed output, only a + /// checksum. To decompress the output you need to supply the same + /// dictionary again. + /// + /// + /// the dictionary. + /// + /// + /// an offset into the dictionary. + /// + /// + /// the length of the dictionary. + /// + /// + /// if setInput () or deflate () were already called or another dictionary was already set. + /// + public void SetDictionary(byte[] dict, int offset, int length) + { + if (state != INIT_STATE) { + throw new InvalidOperationException(); + } + + state = SETDICT_STATE; + engine.SetDictionary(dict, offset, length); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterConstants.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterConstants.cs new file mode 100644 index 000000000..0f423da16 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterConstants.cs @@ -0,0 +1,84 @@ +// DeflaterConstants.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + /// + /// This class contains constants used for the deflater. + /// + public class DeflaterConstants + { + public const bool DEBUGGING = false; + + public const int STORED_BLOCK = 0; + public const int STATIC_TREES = 1; + public const int DYN_TREES = 2; + public const int PRESET_DICT = 0x20; + + public const int DEFAULT_MEM_LEVEL = 8; + + public const int MAX_MATCH = 258; + public const int MIN_MATCH = 3; + + public const int MAX_WBITS = 15; + public const int WSIZE = 1 << MAX_WBITS; + public const int WMASK = WSIZE - 1; + + public const int HASH_BITS = DEFAULT_MEM_LEVEL + 7; + public const int HASH_SIZE = 1 << HASH_BITS; + public const int HASH_MASK = HASH_SIZE - 1; + public const int HASH_SHIFT = (HASH_BITS + MIN_MATCH - 1) / MIN_MATCH; + + public const int MIN_LOOKAHEAD = MAX_MATCH + MIN_MATCH + 1; + public const int MAX_DIST = WSIZE - MIN_LOOKAHEAD; + + public const int PENDING_BUF_SIZE = 1 << (DEFAULT_MEM_LEVEL + 8); + public static int MAX_BLOCK_SIZE = Math.Min(65535, PENDING_BUF_SIZE-5); + + public const int DEFLATE_STORED = 0; + public const int DEFLATE_FAST = 1; + public const int DEFLATE_SLOW = 2; + + public static int[] GOOD_LENGTH = { 0, 4, 4, 4, 4, 8, 8, 8, 32, 32 }; + public static int[] MAX_LAZY = { 0, 4, 5, 6, 4,16, 16, 32, 128, 258 }; + public static int[] NICE_LENGTH = { 0, 8,16,32,16,32,128,128, 258, 258 }; + public static int[] MAX_CHAIN = { 0, 4, 8,32,16,32,128,256,1024,4096 }; + public static int[] COMPR_FUNC = { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2 }; + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterEngine.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterEngine.cs new file mode 100644 index 000000000..5264015c0 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterEngine.cs @@ -0,0 +1,654 @@ +// DeflaterEngine.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +using BCG.Utilities.SharpZipLib.Checksums; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + public enum DeflateStrategy { + // The default strategy. + Default = 0, + + // This strategy will only allow longer string repetitions. It is + // useful for random data with a small character set. + Filtered = 1, + + // This strategy will not look for string repetitions at all. It + // only encodes with Huffman trees (which means, that more common + // characters get a smaller encoding. + HuffmanOnly = 2 + } + + public class DeflaterEngine : DeflaterConstants + { + static int TOO_FAR = 4096; + + int ins_h; +// private byte[] buffer; + short[] head; + short[] prev; + + int matchStart, matchLen; + bool prevAvailable; + int blockStart; + int strstart, lookahead; + byte[] window; + + DeflateStrategy strategy; + int max_chain, max_lazy, niceLength, goodLength; + + /// + /// The current compression function. + /// + int comprFunc; + + /// + /// The input data for compression. + /// + byte[] inputBuf; + + /// + /// The total bytes of input read. + /// + int totalIn; + + /// + /// The offset into inputBuf, where input data starts. + /// + int inputOff; + + /// + /// The end offset of the input data. + /// + int inputEnd; + + DeflaterPending pending; + DeflaterHuffman huffman; + + /// + /// The adler checksum + /// + Adler32 adler; + + public DeflaterEngine(DeflaterPending pending) + { + this.pending = pending; + huffman = new DeflaterHuffman(pending); + adler = new Adler32(); + + window = new byte[2 * WSIZE]; + head = new short[HASH_SIZE]; + prev = new short[WSIZE]; + + /* We start at index 1, to avoid a implementation deficiency, that + * we cannot build a repeat pattern at index 0. + */ + blockStart = strstart = 1; + } + + public void Reset() + { + huffman.Reset(); + adler.Reset(); + blockStart = strstart = 1; + lookahead = 0; + totalIn = 0; + prevAvailable = false; + matchLen = MIN_MATCH - 1; + + for (int i = 0; i < HASH_SIZE; i++) { + head[i] = 0; + } + + for (int i = 0; i < WSIZE; i++) { + prev[i] = 0; + } + } + + public void ResetAdler() + { + adler.Reset(); + } + + public int Adler { + get { + return (int)adler.Value; + } + } + + public int TotalIn { + get { + return totalIn; + } + } + + public DeflateStrategy Strategy { + get { + return strategy; + } + set { + strategy = value; + } + } + + public void SetLevel(int lvl) + { + goodLength = DeflaterConstants.GOOD_LENGTH[lvl]; + max_lazy = DeflaterConstants.MAX_LAZY[lvl]; + niceLength = DeflaterConstants.NICE_LENGTH[lvl]; + max_chain = DeflaterConstants.MAX_CHAIN[lvl]; + + if (DeflaterConstants.COMPR_FUNC[lvl] != comprFunc) { +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("Change from "+comprFunc +" to " +// + DeflaterConstants.COMPR_FUNC[lvl]); +// } + switch (comprFunc) { + case DEFLATE_STORED: + if (strstart > blockStart) { + huffman.FlushStoredBlock(window, blockStart, + strstart - blockStart, false); + blockStart = strstart; + } + UpdateHash(); + break; + case DEFLATE_FAST: + if (strstart > blockStart) { + huffman.FlushBlock(window, blockStart, strstart - blockStart, + false); + blockStart = strstart; + } + break; + case DEFLATE_SLOW: + if (prevAvailable) { + huffman.TallyLit(window[strstart-1] & 0xff); + } + if (strstart > blockStart) { + huffman.FlushBlock(window, blockStart, strstart - blockStart, + false); + blockStart = strstart; + } + prevAvailable = false; + matchLen = MIN_MATCH - 1; + break; + } + comprFunc = COMPR_FUNC[lvl]; + } + } + + private void UpdateHash() + { +// if (DEBUGGING) { +// Console.WriteLine("updateHash: "+strstart); +// } + ins_h = (window[strstart] << HASH_SHIFT) ^ window[strstart + 1]; + } + + private int InsertString() + { + short match; + int hash = ((ins_h << HASH_SHIFT) ^ window[strstart + (MIN_MATCH -1)]) & HASH_MASK; + +// if (DEBUGGING) { +// if (hash != (((window[strstart] << (2*HASH_SHIFT)) ^ +// (window[strstart + 1] << HASH_SHIFT) ^ +// (window[strstart + 2])) & HASH_MASK)) { +// throw new Exception("hash inconsistent: "+hash+"/" +// +window[strstart]+"," +// +window[strstart+1]+"," +// +window[strstart+2]+","+HASH_SHIFT); +// } +// } + + prev[strstart & WMASK] = match = head[hash]; + head[hash] = (short)strstart; + ins_h = hash; + return match & 0xffff; + } + + void SlideWindow() + { + Array.Copy(window, WSIZE, window, 0, WSIZE); + matchStart -= WSIZE; + strstart -= WSIZE; + blockStart -= WSIZE; + + /* Slide the hash table (could be avoided with 32 bit values + * at the expense of memory usage). + */ + for (int i = 0; i < HASH_SIZE; ++i) { + int m = head[i] & 0xffff; + head[i] = (short)(m >= WSIZE ? (m - WSIZE) : 0); + } + + /* Slide the prev table. */ + for (int i = 0; i < WSIZE; i++) { + int m = prev[i] & 0xffff; + prev[i] = (short)(m >= WSIZE ? (m - WSIZE) : 0); + } + } + + public void FillWindow() + { + /* If the window is almost full and there is insufficient lookahead, + * move the upper half to the lower one to make room in the upper half. + */ + if (strstart >= WSIZE + MAX_DIST) { + SlideWindow(); + } + + /* If there is not enough lookahead, but still some input left, + * read in the input + */ + while (lookahead < DeflaterConstants.MIN_LOOKAHEAD && inputOff < inputEnd) { + int more = 2 * WSIZE - lookahead - strstart; + + if (more > inputEnd - inputOff) { + more = inputEnd - inputOff; + } + + System.Array.Copy(inputBuf, inputOff, window, strstart + lookahead, more); + adler.Update(inputBuf, inputOff, more); + + inputOff += more; + totalIn += more; + lookahead += more; + } + + if (lookahead >= MIN_MATCH) { + UpdateHash(); + } + } + + private bool FindLongestMatch(int curMatch) + { + int chainLength = this.max_chain; + int niceLength = this.niceLength; + short[] prev = this.prev; + int scan = this.strstart; + int match; + int best_end = this.strstart + matchLen; + int best_len = Math.Max(matchLen, MIN_MATCH - 1); + + int limit = Math.Max(strstart - MAX_DIST, 0); + + int strend = strstart + MAX_MATCH - 1; + byte scan_end1 = window[best_end - 1]; + byte scan_end = window[best_end]; + + /* Do not waste too much time if we already have a good match: */ + if (best_len >= this.goodLength) { + chainLength >>= 2; + } + + /* Do not look for matches beyond the end of the input. This is necessary + * to make deflate deterministic. + */ + if (niceLength > lookahead) { + niceLength = lookahead; + } + + /* + if (DeflaterConstants.DEBUGGING && strstart > 2 * WSIZE - MIN_LOOKAHEAD) { + throw new InvalidOperationException("need lookahead"); + }*/ + + do { + /* + if (DeflaterConstants.DEBUGGING && curMatch >= strstart) { + throw new InvalidOperationException("future match"); + }*/ + if (window[curMatch + best_len] != scan_end || + window[curMatch + best_len - 1] != scan_end1 || + window[curMatch] != window[scan] || + window[curMatch + 1] != window[scan + 1]) { + continue; + } + + match = curMatch + 2; + scan += 2; + + /* We check for insufficient lookahead only every 8th comparison; + * the 256th check will be made at strstart+258. + */ + while (window[++scan] == window[++match] && + window[++scan] == window[++match] && + window[++scan] == window[++match] && + window[++scan] == window[++match] && + window[++scan] == window[++match] && + window[++scan] == window[++match] && + window[++scan] == window[++match] && + window[++scan] == window[++match] && scan < strend) ; + + if (scan > best_end) { + // if (DeflaterConstants.DEBUGGING && ins_h == 0) + // System.err.println("Found match: "+curMatch+"-"+(scan-strstart)); + matchStart = curMatch; + best_end = scan; + best_len = scan - strstart; + if (best_len >= niceLength) { + break; + } + + scan_end1 = window[best_end - 1]; + scan_end = window[best_end]; + } + scan = strstart; + } while ((curMatch = (prev[curMatch & WMASK] & 0xffff)) > limit && --chainLength != 0); + + matchLen = Math.Min(best_len, lookahead); + return matchLen >= MIN_MATCH; + } + + public void SetDictionary(byte[] buffer, int offset, int length) + { + /* + if (DeflaterConstants.DEBUGGING && strstart != 1) { + throw new InvalidOperationException("strstart not 1"); + }*/ + adler.Update(buffer, offset, length); + if (length < MIN_MATCH) { + return; + } + if (length > MAX_DIST) { + offset += length - MAX_DIST; + length = MAX_DIST; + } + + System.Array.Copy(buffer, offset, window, strstart, length); + + UpdateHash(); + --length; + while (--length > 0) { + InsertString(); + strstart++; + } + strstart += 2; + blockStart = strstart; + } + + private bool DeflateStored(bool flush, bool finish) + { + if (!flush && lookahead == 0) { + return false; + } + + strstart += lookahead; + lookahead = 0; + + int storedLen = strstart - blockStart; + + if ((storedLen >= DeflaterConstants.MAX_BLOCK_SIZE) || /* Block is full */ + (blockStart < WSIZE && storedLen >= MAX_DIST) || /* Block may move out of window */ + flush) { + bool lastBlock = finish; + if (storedLen > DeflaterConstants.MAX_BLOCK_SIZE) { + storedLen = DeflaterConstants.MAX_BLOCK_SIZE; + lastBlock = false; + } + +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("storedBlock["+storedLen+","+lastBlock+"]"); +// } + + huffman.FlushStoredBlock(window, blockStart, storedLen, lastBlock); + blockStart += storedLen; + return !lastBlock; + } + return true; + } + + private bool DeflateFast(bool flush, bool finish) + { + if (lookahead < MIN_LOOKAHEAD && !flush) { + return false; + } + + while (lookahead >= MIN_LOOKAHEAD || flush) { + if (lookahead == 0) { + /* We are flushing everything */ + huffman.FlushBlock(window, blockStart, strstart - blockStart, finish); + blockStart = strstart; + return false; + } + + if (strstart > 2 * WSIZE - MIN_LOOKAHEAD) + { + /* slide window, as findLongestMatch need this. + * This should only happen when flushing and the window + * is almost full. + */ + SlideWindow(); + } + + int hashHead; + if (lookahead >= MIN_MATCH && + (hashHead = InsertString()) != 0 && + strategy != DeflateStrategy.HuffmanOnly && + strstart - hashHead <= MAX_DIST && + FindLongestMatch(hashHead)) { + /* longestMatch sets matchStart and matchLen */ +// if (DeflaterConstants.DEBUGGING) { +// for (int i = 0 ; i < matchLen; i++) { +// if (window[strstart+i] != window[matchStart + i]) { +// throw new Exception(); +// } +// } +// } + + huffman.TallyDist(strstart - matchStart, matchLen); + + lookahead -= matchLen; + if (matchLen <= max_lazy && lookahead >= MIN_MATCH) { + while (--matchLen > 0) { + ++strstart; + InsertString(); + } + ++strstart; + } else { + strstart += matchLen; + if (lookahead >= MIN_MATCH - 1) { + UpdateHash(); + } + } + matchLen = MIN_MATCH - 1; + continue; + } else { + /* No match found */ + huffman.TallyLit(window[strstart] & 0xff); + ++strstart; + --lookahead; + } + + if (huffman.IsFull()) { + bool lastBlock = finish && lookahead == 0; + huffman.FlushBlock(window, blockStart, strstart - blockStart, + lastBlock); + blockStart = strstart; + return !lastBlock; + } + } + return true; + } + + private bool DeflateSlow(bool flush, bool finish) + { + if (lookahead < MIN_LOOKAHEAD && !flush) { + return false; + } + + while (lookahead >= MIN_LOOKAHEAD || flush) { + if (lookahead == 0) { + if (prevAvailable) { + huffman.TallyLit(window[strstart-1] & 0xff); + } + prevAvailable = false; + + /* We are flushing everything */ + /* + if (DeflaterConstants.DEBUGGING && !flush) { + throw new Exception("Not flushing, but no lookahead"); + }*/ + huffman.FlushBlock(window, blockStart, strstart - blockStart, + finish); + blockStart = strstart; + return false; + } + + if (strstart >= 2 * WSIZE - MIN_LOOKAHEAD) + { + /* slide window, as findLongestMatch need this. + * This should only happen when flushing and the window + * is almost full. + */ + SlideWindow(); + } + + int prevMatch = matchStart; + int prevLen = matchLen; + if (lookahead >= MIN_MATCH) { + int hashHead = InsertString(); + if (strategy != DeflateStrategy.HuffmanOnly && hashHead != 0 && strstart - hashHead <= MAX_DIST && FindLongestMatch(hashHead)) + { + /* longestMatch sets matchStart and matchLen */ + + /* Discard match if too small and too far away */ + if (matchLen <= 5 && (strategy == DeflateStrategy.Filtered || (matchLen == MIN_MATCH && strstart - matchStart > TOO_FAR))) { + matchLen = MIN_MATCH - 1; + } + } + } + + /* previous match was better */ + if (prevLen >= MIN_MATCH && matchLen <= prevLen) { +// if (DeflaterConstants.DEBUGGING) { +// for (int i = 0 ; i < matchLen; i++) { +// if (window[strstart-1+i] != window[prevMatch + i]) +// throw new Exception(); +// } +// } + huffman.TallyDist(strstart - 1 - prevMatch, prevLen); + prevLen -= 2; + do { + strstart++; + lookahead--; + if (lookahead >= MIN_MATCH) { + InsertString(); + } + } while (--prevLen > 0); + strstart ++; + lookahead--; + prevAvailable = false; + matchLen = MIN_MATCH - 1; + } else { + if (prevAvailable) { + huffman.TallyLit(window[strstart-1] & 0xff); + } + prevAvailable = true; + strstart++; + lookahead--; + } + + if (huffman.IsFull()) { + int len = strstart - blockStart; + if (prevAvailable) { + len--; + } + bool lastBlock = (finish && lookahead == 0 && !prevAvailable); + huffman.FlushBlock(window, blockStart, len, lastBlock); + blockStart += len; + return !lastBlock; + } + } + return true; + } + + public bool Deflate(bool flush, bool finish) + { + bool progress; + do { + FillWindow(); + bool canFlush = flush && inputOff == inputEnd; +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("window: ["+blockStart+","+strstart+"," +// +lookahead+"], "+comprFunc+","+canFlush); +// } + switch (comprFunc) { + case DEFLATE_STORED: + progress = DeflateStored(canFlush, finish); + break; + case DEFLATE_FAST: + progress = DeflateFast(canFlush, finish); + break; + case DEFLATE_SLOW: + progress = DeflateSlow(canFlush, finish); + break; + default: + throw new InvalidOperationException("unknown comprFunc"); + } + } while (pending.IsFlushed && progress); /* repeat while we have no pending output and progress was made */ + return progress; + } + + public void SetInput(byte[] buf, int off, int len) + { + if (inputOff < inputEnd) { + throw new InvalidOperationException("Old input was not completely processed"); + } + + int end = off + len; + + /* We want to throw an ArrayIndexOutOfBoundsException early. The + * check is very tricky: it also handles integer wrap around. + */ + if (0 > off || off > end || end > buf.Length) { + throw new ArgumentOutOfRangeException(); + } + + inputBuf = buf; + inputOff = off; + inputEnd = end; + } + + public bool NeedsInput() + { + return inputEnd == inputOff; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterHuffman.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterHuffman.cs new file mode 100644 index 000000000..2f1590b43 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterHuffman.cs @@ -0,0 +1,779 @@ +// DeflaterHuffman.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + /// + /// This is the DeflaterHuffman class. + /// + /// This class is not thread safe. This is inherent in the API, due + /// to the split of deflate and setInput. + /// + /// author of the original java version : Jochen Hoenicke + /// + public class DeflaterHuffman + { + private static int BUFSIZE = 1 << (DeflaterConstants.DEFAULT_MEM_LEVEL + 6); + private static int LITERAL_NUM = 286; + private static int DIST_NUM = 30; + private static int BITLEN_NUM = 19; + private static int REP_3_6 = 16; + private static int REP_3_10 = 17; + private static int REP_11_138 = 18; + private static int EOF_SYMBOL = 256; + private static int[] BL_ORDER = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; + + private static byte[] bit4Reverse = { + 0, + 8, + 4, + 12, + 2, + 10, + 6, + 14, + 1, + 9, + 5, + 13, + 3, + 11, + 7, + 15 + }; + + + public class Tree + { + public short[] freqs; + public short[] codes; + public byte[] length; + public int[] bl_counts; + public int minNumCodes, numCodes; + public int maxLength; + DeflaterHuffman dh; + + public Tree(DeflaterHuffman dh, int elems, int minCodes, int maxLength) + { + this.dh = dh; + this.minNumCodes = minCodes; + this.maxLength = maxLength; + freqs = new short[elems]; + bl_counts = new int[maxLength]; + } + + public void Reset() + { + for (int i = 0; i < freqs.Length; i++) { + freqs[i] = 0; + } + codes = null; + length = null; + } + + public void WriteSymbol(int code) + { +// if (DeflaterConstants.DEBUGGING) { +// freqs[code]--; +// // Console.Write("writeSymbol("+freqs.length+","+code+"): "); +// } + dh.pending.WriteBits(codes[code] & 0xffff, length[code]); + } + + public void CheckEmpty() + { + bool empty = true; + for (int i = 0; i < freqs.Length; i++) { + if (freqs[i] != 0) { + Console.WriteLine("freqs["+i+"] == "+freqs[i]); + empty = false; + } + } + if (!empty) { + throw new Exception(); + } + Console.WriteLine("checkEmpty suceeded!"); + } + + public void SetStaticCodes(short[] stCodes, byte[] stLength) + { + codes = stCodes; + length = stLength; + } + + public void BuildCodes() + { + int numSymbols = freqs.Length; + int[] nextCode = new int[maxLength]; + int code = 0; + codes = new short[freqs.Length]; + +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("buildCodes: "+freqs.Length); +// } + + for (int bits = 0; bits < maxLength; bits++) { + nextCode[bits] = code; + code += bl_counts[bits] << (15 - bits); +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("bits: "+(bits+1)+" count: "+bl_counts[bits] +// +" nextCode: "+code); // HACK : Integer.toHexString( +// } + } + /* + if (DeflaterConstants.DEBUGGING && code != 65536) { + throw new Exception("Inconsistent bl_counts!"); + }*/ + + for (int i=0; i < numCodes; i++) { + int bits = length[i]; + if (bits > 0) { +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("codes["+i+"] = rev(" + nextCode[bits-1]+")," // HACK : Integer.toHexString( +// +bits); +// } + codes[i] = BitReverse(nextCode[bits-1]); + nextCode[bits-1] += 1 << (16 - bits); + } + } + } + + private void BuildLength(int[] childs) + { + this.length = new byte [freqs.Length]; + int numNodes = childs.Length / 2; + int numLeafs = (numNodes + 1) / 2; + int overflow = 0; + + for (int i = 0; i < maxLength; i++) { + bl_counts[i] = 0; + } + + /* First calculate optimal bit lengths */ + int[] lengths = new int[numNodes]; + lengths[numNodes-1] = 0; + + for (int i = numNodes - 1; i >= 0; i--) { + if (childs[2*i+1] != -1) { + int bitLength = lengths[i] + 1; + if (bitLength > maxLength) { + bitLength = maxLength; + overflow++; + } + lengths[childs[2*i]] = lengths[childs[2*i+1]] = bitLength; + } else { + /* A leaf node */ + int bitLength = lengths[i]; + bl_counts[bitLength - 1]++; + this.length[childs[2*i]] = (byte) lengths[i]; + } + } + +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("Tree "+freqs.Length+" lengths:"); +// for (int i=0; i < numLeafs; i++) { +// Console.WriteLine("Node "+childs[2*i]+" freq: "+freqs[childs[2*i]] +// + " len: "+length[childs[2*i]]); +// } +// } + + if (overflow == 0) { + return; + } + + int incrBitLen = maxLength - 1; + do { + /* Find the first bit length which could increase: */ + while (bl_counts[--incrBitLen] == 0) + ; + + /* Move this node one down and remove a corresponding + * amount of overflow nodes. + */ + do { + bl_counts[incrBitLen]--; + bl_counts[++incrBitLen]++; + overflow -= 1 << (maxLength - 1 - incrBitLen); + } while (overflow > 0 && incrBitLen < maxLength - 1); + } while (overflow > 0); + + /* We may have overshot above. Move some nodes from maxLength to + * maxLength-1 in that case. + */ + bl_counts[maxLength-1] += overflow; + bl_counts[maxLength-2] -= overflow; + + /* Now recompute all bit lengths, scanning in increasing + * frequency. It is simpler to reconstruct all lengths instead of + * fixing only the wrong ones. This idea is taken from 'ar' + * written by Haruhiko Okumura. + * + * The nodes were inserted with decreasing frequency into the childs + * array. + */ + int nodePtr = 2 * numLeafs; + for (int bits = maxLength; bits != 0; bits--) { + int n = bl_counts[bits-1]; + while (n > 0) { + int childPtr = 2*childs[nodePtr++]; + if (childs[childPtr + 1] == -1) { + /* We found another leaf */ + length[childs[childPtr]] = (byte) bits; + n--; + } + } + } +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("*** After overflow elimination. ***"); +// for (int i=0; i < numLeafs; i++) { +// Console.WriteLine("Node "+childs[2*i]+" freq: "+freqs[childs[2*i]] +// + " len: "+length[childs[2*i]]); +// } +// } + } + + public void BuildTree() + { + int numSymbols = freqs.Length; + + /* heap is a priority queue, sorted by frequency, least frequent + * nodes first. The heap is a binary tree, with the property, that + * the parent node is smaller than both child nodes. This assures + * that the smallest node is the first parent. + * + * The binary tree is encoded in an array: 0 is root node and + * the nodes 2*n+1, 2*n+2 are the child nodes of node n. + */ + int[] heap = new int[numSymbols]; + int heapLen = 0; + int maxCode = 0; + for (int n = 0; n < numSymbols; n++) { + int freq = freqs[n]; + if (freq != 0) { + /* Insert n into heap */ + int pos = heapLen++; + int ppos; + while (pos > 0 && freqs[heap[ppos = (pos - 1) / 2]] > freq) { + heap[pos] = heap[ppos]; + pos = ppos; + } + heap[pos] = n; + + maxCode = n; + } + } + + /* We could encode a single literal with 0 bits but then we + * don't see the literals. Therefore we force at least two + * literals to avoid this case. We don't care about order in + * this case, both literals get a 1 bit code. + */ + while (heapLen < 2) { + int node = maxCode < 2 ? ++maxCode : 0; + heap[heapLen++] = node; + } + + numCodes = Math.Max(maxCode + 1, minNumCodes); + + int numLeafs = heapLen; + int[] childs = new int[4*heapLen - 2]; + int[] values = new int[2*heapLen - 1]; + int numNodes = numLeafs; + for (int i = 0; i < heapLen; i++) { + int node = heap[i]; + childs[2*i] = node; + childs[2*i+1] = -1; + values[i] = freqs[node] << 8; + heap[i] = i; + } + + /* Construct the Huffman tree by repeatedly combining the least two + * frequent nodes. + */ + do { + int first = heap[0]; + int last = heap[--heapLen]; + + /* Propagate the hole to the leafs of the heap */ + int ppos = 0; + int path = 1; + while (path < heapLen) { + if (path + 1 < heapLen && values[heap[path]] > values[heap[path+1]]) { + path++; + } + + heap[ppos] = heap[path]; + ppos = path; + path = path * 2 + 1; + } + + /* Now propagate the last element down along path. Normally + * it shouldn't go too deep. + */ + int lastVal = values[last]; + while ((path = ppos) > 0 && values[heap[ppos = (path - 1)/2]] > lastVal) { + heap[path] = heap[ppos]; + } + heap[path] = last; + + + int second = heap[0]; + + /* Create a new node father of first and second */ + last = numNodes++; + childs[2*last] = first; + childs[2*last+1] = second; + int mindepth = Math.Min(values[first] & 0xff, values[second] & 0xff); + values[last] = lastVal = values[first] + values[second] - mindepth + 1; + + /* Again, propagate the hole to the leafs */ + ppos = 0; + path = 1; + while (path < heapLen) { + if (path + 1 < heapLen && values[heap[path]] > values[heap[path+1]]) { + path++; + } + + heap[ppos] = heap[path]; + ppos = path; + path = ppos * 2 + 1; + } + + /* Now propagate the new element down along path */ + while ((path = ppos) > 0 && values[heap[ppos = (path - 1)/2]] > lastVal) { + heap[path] = heap[ppos]; + } + heap[path] = last; + } while (heapLen > 1); + + if (heap[0] != childs.Length / 2 - 1) { + throw new Exception("Weird!"); + } + BuildLength(childs); + } + + public int GetEncodedLength() + { + int len = 0; + for (int i = 0; i < freqs.Length; i++) { + len += freqs[i] * length[i]; + } + return len; + } + + public void CalcBLFreq(Tree blTree) + { + int max_count; /* max repeat count */ + int min_count; /* min repeat count */ + int count; /* repeat count of the current code */ + int curlen = -1; /* length of current code */ + + int i = 0; + while (i < numCodes) { + count = 1; + int nextlen = length[i]; + if (nextlen == 0) { + max_count = 138; + min_count = 3; + } else { + max_count = 6; + min_count = 3; + if (curlen != nextlen) { + blTree.freqs[nextlen]++; + count = 0; + } + } + curlen = nextlen; + i++; + + while (i < numCodes && curlen == length[i]) { + i++; + if (++count >= max_count) { + break; + } + } + + if (count < min_count) { + blTree.freqs[curlen] += (short)count; + } else if (curlen != 0) { + blTree.freqs[REP_3_6]++; + } else if (count <= 10) { + blTree.freqs[REP_3_10]++; + } else { + blTree.freqs[REP_11_138]++; + } + } + } + + public void WriteTree(Tree blTree) + { + int max_count; /* max repeat count */ + int min_count; /* min repeat count */ + int count; /* repeat count of the current code */ + int curlen = -1; /* length of current code */ + + int i = 0; + while (i < numCodes) { + count = 1; + int nextlen = length[i]; + if (nextlen == 0) { + max_count = 138; + min_count = 3; + } else { + max_count = 6; + min_count = 3; + if (curlen != nextlen) { + blTree.WriteSymbol(nextlen); + count = 0; + } + } + curlen = nextlen; + i++; + + while (i < numCodes && curlen == length[i]) { + i++; + if (++count >= max_count) { + break; + } + } + + if (count < min_count) { + while (count-- > 0) { + blTree.WriteSymbol(curlen); + } + } + else if (curlen != 0) { + blTree.WriteSymbol(REP_3_6); + dh.pending.WriteBits(count - 3, 2); + } else if (count <= 10) { + blTree.WriteSymbol(REP_3_10); + dh.pending.WriteBits(count - 3, 3); + } else { + blTree.WriteSymbol(REP_11_138); + dh.pending.WriteBits(count - 11, 7); + } + } + } + } + + public DeflaterPending pending; + private Tree literalTree, distTree, blTree; + + private short[] d_buf; + private byte[] l_buf; + private int last_lit; + private int extra_bits; + + private static short[] staticLCodes; + private static byte[] staticLLength; + private static short[] staticDCodes; + private static byte[] staticDLength; + + /// + /// Reverse the bits of a 16 bit value. + /// + public static short BitReverse(int value) + { + return (short) (bit4Reverse[value & 0xF] << 12 + | bit4Reverse[(value >> 4) & 0xF] << 8 + | bit4Reverse[(value >> 8) & 0xF] << 4 + | bit4Reverse[value >> 12]); + } + + + static DeflaterHuffman() + { + /* See RFC 1951 3.2.6 */ + /* Literal codes */ + staticLCodes = new short[LITERAL_NUM]; + staticLLength = new byte[LITERAL_NUM]; + int i = 0; + while (i < 144) { + staticLCodes[i] = BitReverse((0x030 + i) << 8); + staticLLength[i++] = 8; + } + while (i < 256) { + staticLCodes[i] = BitReverse((0x190 - 144 + i) << 7); + staticLLength[i++] = 9; + } + while (i < 280) { + staticLCodes[i] = BitReverse((0x000 - 256 + i) << 9); + staticLLength[i++] = 7; + } + while (i < LITERAL_NUM) { + staticLCodes[i] = BitReverse((0x0c0 - 280 + i) << 8); + staticLLength[i++] = 8; + } + + /* Distant codes */ + staticDCodes = new short[DIST_NUM]; + staticDLength = new byte[DIST_NUM]; + for (i = 0; i < DIST_NUM; i++) { + staticDCodes[i] = BitReverse(i << 11); + staticDLength[i] = 5; + } + } + + public DeflaterHuffman(DeflaterPending pending) + { + this.pending = pending; + + literalTree = new Tree(this, LITERAL_NUM, 257, 15); + distTree = new Tree(this, DIST_NUM, 1, 15); + blTree = new Tree(this, BITLEN_NUM, 4, 7); + + d_buf = new short[BUFSIZE]; + l_buf = new byte [BUFSIZE]; + } + + public void Reset() + { + last_lit = 0; + extra_bits = 0; + literalTree.Reset(); + distTree.Reset(); + blTree.Reset(); + } + + private int L_code(int len) + { + if (len == 255) { + return 285; + } + + int code = 257; + while (len >= 8) { + code += 4; + len >>= 1; + } + return code + len; + } + + private int D_code(int distance) + { + int code = 0; + while (distance >= 4) { + code += 2; + distance >>= 1; + } + return code + distance; + } + + public void SendAllTrees(int blTreeCodes) + { + blTree.BuildCodes(); + literalTree.BuildCodes(); + distTree.BuildCodes(); + pending.WriteBits(literalTree.numCodes - 257, 5); + pending.WriteBits(distTree.numCodes - 1, 5); + pending.WriteBits(blTreeCodes - 4, 4); + for (int rank = 0; rank < blTreeCodes; rank++) { + pending.WriteBits(blTree.length[BL_ORDER[rank]], 3); + } + literalTree.WriteTree(blTree); + distTree.WriteTree(blTree); +// if (DeflaterConstants.DEBUGGING) { +// blTree.CheckEmpty(); +// } + } + + public void CompressBlock() + { + for (int i = 0; i < last_lit; i++) { + int litlen = l_buf[i] & 0xff; + int dist = d_buf[i]; + if (dist-- != 0) { +// if (DeflaterConstants.DEBUGGING) { +// Console.Write("["+(dist+1)+","+(litlen+3)+"]: "); +// } + + int lc = L_code(litlen); + literalTree.WriteSymbol(lc); + + int bits = (lc - 261) / 4; + if (bits > 0 && bits <= 5) { + pending.WriteBits(litlen & ((1 << bits) - 1), bits); + } + + int dc = D_code(dist); + distTree.WriteSymbol(dc); + + bits = dc / 2 - 1; + if (bits > 0) { + pending.WriteBits(dist & ((1 << bits) - 1), bits); + } + } else { +// if (DeflaterConstants.DEBUGGING) { +// if (litlen > 32 && litlen < 127) { +// Console.Write("("+(char)litlen+"): "); +// } else { +// Console.Write("{"+litlen+"}: "); +// } +// } + literalTree.WriteSymbol(litlen); + } + } +// if (DeflaterConstants.DEBUGGING) { +// Console.Write("EOF: "); +// } + literalTree.WriteSymbol(EOF_SYMBOL); +// if (DeflaterConstants.DEBUGGING) { +// literalTree.CheckEmpty(); +// distTree.CheckEmpty(); +// } + } + + public void FlushStoredBlock(byte[] stored, int stored_offset, int stored_len, bool lastBlock) + { +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("Flushing stored block "+ stored_len); +// } + pending.WriteBits((DeflaterConstants.STORED_BLOCK << 1) + + (lastBlock ? 1 : 0), 3); + pending.AlignToByte(); + pending.WriteShort(stored_len); + pending.WriteShort(~stored_len); + pending.WriteBlock(stored, stored_offset, stored_len); + Reset(); + } + + public void FlushBlock(byte[] stored, int stored_offset, int stored_len, bool lastBlock) + { + literalTree.freqs[EOF_SYMBOL]++; + + /* Build trees */ + literalTree.BuildTree(); + distTree.BuildTree(); + + /* Calculate bitlen frequency */ + literalTree.CalcBLFreq(blTree); + distTree.CalcBLFreq(blTree); + + /* Build bitlen tree */ + blTree.BuildTree(); + + int blTreeCodes = 4; + for (int i = 18; i > blTreeCodes; i--) { + if (blTree.length[BL_ORDER[i]] > 0) { + blTreeCodes = i+1; + } + } + int opt_len = 14 + blTreeCodes * 3 + blTree.GetEncodedLength() + + literalTree.GetEncodedLength() + distTree.GetEncodedLength() + + extra_bits; + + int static_len = extra_bits; + for (int i = 0; i < LITERAL_NUM; i++) { + static_len += literalTree.freqs[i] * staticLLength[i]; + } + for (int i = 0; i < DIST_NUM; i++) { + static_len += distTree.freqs[i] * staticDLength[i]; + } + if (opt_len >= static_len) { + /* Force static trees */ + opt_len = static_len; + } + + if (stored_offset >= 0 && stored_len+4 < opt_len >> 3) { + /* Store Block */ +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("Storing, since " + stored_len + " < " + opt_len +// + " <= " + static_len); +// } + FlushStoredBlock(stored, stored_offset, stored_len, lastBlock); + } else if (opt_len == static_len) { + /* Encode with static tree */ + pending.WriteBits((DeflaterConstants.STATIC_TREES << 1) + (lastBlock ? 1 : 0), 3); + literalTree.SetStaticCodes(staticLCodes, staticLLength); + distTree.SetStaticCodes(staticDCodes, staticDLength); + CompressBlock(); + Reset(); + } else { + /* Encode with dynamic tree */ + pending.WriteBits((DeflaterConstants.DYN_TREES << 1) + (lastBlock ? 1 : 0), 3); + SendAllTrees(blTreeCodes); + CompressBlock(); + Reset(); + } + } + + public bool IsFull() + { + return last_lit + 16 >= BUFSIZE; // HACK: This was == 'last_lit == BUFSIZE', but errors occured with DeflateFast + } + + public bool TallyLit(int lit) + { +// if (DeflaterConstants.DEBUGGING) { +// if (lit > 32 && lit < 127) { +// Console.WriteLine("("+(char)lit+")"); +// } else { +// Console.WriteLine("{"+lit+"}"); +// } +// } + d_buf[last_lit] = 0; + l_buf[last_lit++] = (byte)lit; + literalTree.freqs[lit]++; + return IsFull(); + } + + public bool TallyDist(int dist, int len) + { +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("["+dist+","+len+"]"); +// } + + d_buf[last_lit] = (short)dist; + l_buf[last_lit++] = (byte)(len - 3); + + int lc = L_code(len - 3); + literalTree.freqs[lc]++; + if (lc >= 265 && lc < 285) { + extra_bits += (lc - 261) / 4; + } + + int dc = D_code(dist - 1); + distTree.freqs[dc]++; + if (dc >= 4) { + extra_bits += dc / 2 - 1; + } + return IsFull(); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterPending.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterPending.cs new file mode 100644 index 000000000..84f0077f5 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/DeflaterPending.cs @@ -0,0 +1,51 @@ +// DeflaterPending.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + /// + /// This class stores the pending output of the Deflater. + /// + /// author of the original java version : Jochen Hoenicke + /// + public class DeflaterPending : PendingBuffer + { + public DeflaterPending() : base(DeflaterConstants.PENDING_BUF_SIZE) + { + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Inflater.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Inflater.cs new file mode 100644 index 000000000..24d67c85e --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Inflater.cs @@ -0,0 +1,778 @@ +// Inflater.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +using BCG.Utilities.SharpZipLib.Checksums; +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + /// + /// Inflater is used to decompress data that has been compressed according + /// to the "deflate" standard described in rfc1950. + /// + /// The usage is as following. First you have to set some input with + /// setInput(), then inflate() it. If inflate doesn't + /// inflate any bytes there may be three reasons: + ///
    + ///
  • needsInput() returns true because the input buffer is empty. + /// You have to provide more input with setInput(). + /// NOTE: needsInput() also returns true when, the stream is finished. + ///
  • + ///
  • needsDictionary() returns true, you have to provide a preset + /// dictionary with setDictionary().
  • + ///
  • finished() returns true, the inflater has finished.
  • + ///
+ /// Once the first output byte is produced, a dictionary will not be + /// needed at a later stage. + /// + /// author of the original java version : John Leuner, Jochen Hoenicke + ///
+ public class Inflater + { + /// + /// Copy lengths for literal codes 257..285 + /// + private static int[] CPLENS = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258 + }; + + /// + /// Extra bits for literal codes 257..285 + /// + private static int[] CPLEXT = { + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, + 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 + }; + + /// + /// Copy offsets for distance codes 0..29 + /// + private static int[] CPDIST = { + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, + 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, + 8193, 12289, 16385, 24577 + }; + + /// + /// Extra bits for distance codes + /// + private static int[] CPDEXT = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, + 12, 12, 13, 13 + }; + + /// + /// This are the state in which the inflater can be. + /// + private const int DECODE_HEADER = 0; + private const int DECODE_DICT = 1; + private const int DECODE_BLOCKS = 2; + private const int DECODE_STORED_LEN1 = 3; + private const int DECODE_STORED_LEN2 = 4; + private const int DECODE_STORED = 5; + private const int DECODE_DYN_HEADER = 6; + private const int DECODE_HUFFMAN = 7; + private const int DECODE_HUFFMAN_LENBITS = 8; + private const int DECODE_HUFFMAN_DIST = 9; + private const int DECODE_HUFFMAN_DISTBITS = 10; + private const int DECODE_CHKSUM = 11; + private const int FINISHED = 12; + + /// + /// This variable contains the current state. + /// + private int mode; + + /// + /// The adler checksum of the dictionary or of the decompressed + /// stream, as it is written in the header resp. footer of the + /// compressed stream. + /// Only valid if mode is DECODE_DICT or DECODE_CHKSUM. + /// + private int readAdler; + + /// + /// The number of bits needed to complete the current state. This + /// is valid, if mode is DECODE_DICT, DECODE_CHKSUM, + /// DECODE_HUFFMAN_LENBITS or DECODE_HUFFMAN_DISTBITS. + /// + private int neededBits; + private int repLength, repDist; + private int uncomprLen; + + /// + /// True, if the last block flag was set in the last block of the + /// inflated stream. This means that the stream ends after the + /// current block. + /// + private bool isLastBlock; + + /// + /// The total number of inflated bytes. + /// + private int totalOut; + + /// + /// The total number of bytes set with setInput(). This is not the + /// value returned by getTotalIn(), since this also includes the + /// unprocessed input. + /// + private int totalIn; + + /// + /// This variable stores the nowrap flag that was given to the constructor. + /// True means, that the inflated stream doesn't contain a header nor the + /// checksum in the footer. + /// + private bool nowrap; + + private StreamManipulator input; + private OutputWindow outputWindow; + private InflaterDynHeader dynHeader; + private InflaterHuffmanTree litlenTree, distTree; + private Adler32 adler; + + /// + /// Creates a new inflater. + /// + public Inflater() : this(false) + { + } + + /// + /// Creates a new inflater. + /// + /// + /// true if no header and checksum field appears in the + /// stream. This is used for GZIPed input. For compatibility with + /// Sun JDK you should provide one byte of input more than needed in + /// this case. + /// + public Inflater(bool nowrap) + { + this.nowrap = nowrap; + this.adler = new Adler32(); + input = new StreamManipulator(); + outputWindow = new OutputWindow(); + mode = nowrap ? DECODE_BLOCKS : DECODE_HEADER; + } + + /// + /// Resets the inflater so that a new stream can be decompressed. All + /// pending input and output will be discarded. + /// + public void Reset() + { + mode = nowrap ? DECODE_BLOCKS : DECODE_HEADER; + totalIn = totalOut = 0; + input.Reset(); + outputWindow.Reset(); + dynHeader = null; + litlenTree = null; + distTree = null; + isLastBlock = false; + adler.Reset(); + } + + /// + /// Decodes the deflate header. + /// + /// + /// false if more input is needed. + /// + /// + /// if header is invalid. + /// + private bool DecodeHeader() + { + int header = input.PeekBits(16); + if (header < 0) { + return false; + } + input.DropBits(16); + /* The header is written in "wrong" byte order */ + header = ((header << 8) | (header >> 8)) & 0xffff; + if (header % 31 != 0) { + throw new FormatException("Header checksum illegal"); + } + + if ((header & 0x0f00) != (Deflater.DEFLATED << 8)) { + throw new FormatException("Compression Method unknown"); + } + + /* Maximum size of the backwards window in bits. + * We currently ignore this, but we could use it to make the + * inflater window more space efficient. On the other hand the + * full window (15 bits) is needed most times, anyway. + int max_wbits = ((header & 0x7000) >> 12) + 8; + */ + + if ((header & 0x0020) == 0) { // Dictionary flag? + mode = DECODE_BLOCKS; + } else { + mode = DECODE_DICT; + neededBits = 32; + } + return true; + } + + /// + /// Decodes the dictionary checksum after the deflate header. + /// + /// + /// false if more input is needed. + /// + private bool DecodeDict() + { + while (neededBits > 0) { + int dictByte = input.PeekBits(8); + if (dictByte < 0) { + return false; + } + input.DropBits(8); + readAdler = (readAdler << 8) | dictByte; + neededBits -= 8; + } + return false; + } + + /// + /// Decodes the huffman encoded symbols in the input stream. + /// + /// + /// false if more input is needed, true if output window is + /// full or the current block ends. + /// + /// + /// if deflated stream is invalid. + /// + private bool DecodeHuffman() + { + int free = outputWindow.GetFreeSpace(); + while (free >= 258) { + int symbol; + switch (mode) { + case DECODE_HUFFMAN: + /* This is the inner loop so it is optimized a bit */ + while (((symbol = litlenTree.GetSymbol(input)) & ~0xff) == 0) { + outputWindow.Write(symbol); + if (--free < 258) { + return true; + } + } + if (symbol < 257) { + if (symbol < 0) { + return false; + } else { + /* symbol == 256: end of block */ + distTree = null; + litlenTree = null; + mode = DECODE_BLOCKS; + return true; + } + } + + try { + repLength = CPLENS[symbol - 257]; + neededBits = CPLEXT[symbol - 257]; + } catch (Exception) { + throw new FormatException("Illegal rep length code"); + } + goto case DECODE_HUFFMAN_LENBITS;/* fall through */ + case DECODE_HUFFMAN_LENBITS: + if (neededBits > 0) { + mode = DECODE_HUFFMAN_LENBITS; + int i = input.PeekBits(neededBits); + if (i < 0) { + return false; + } + input.DropBits(neededBits); + repLength += i; + } + mode = DECODE_HUFFMAN_DIST; + goto case DECODE_HUFFMAN_DIST;/* fall through */ + case DECODE_HUFFMAN_DIST: + symbol = distTree.GetSymbol(input); + if (symbol < 0) { + return false; + } + try { + repDist = CPDIST[symbol]; + neededBits = CPDEXT[symbol]; + } catch (Exception) { + throw new FormatException("Illegal rep dist code"); + } + + goto case DECODE_HUFFMAN_DISTBITS;/* fall through */ + case DECODE_HUFFMAN_DISTBITS: + if (neededBits > 0) { + mode = DECODE_HUFFMAN_DISTBITS; + int i = input.PeekBits(neededBits); + if (i < 0) { + return false; + } + input.DropBits(neededBits); + repDist += i; + } + outputWindow.Repeat(repLength, repDist); + free -= repLength; + mode = DECODE_HUFFMAN; + break; + default: + throw new FormatException(); + } + } + return true; + } + + /// + /// Decodes the adler checksum after the deflate stream. + /// + /// + /// false if more input is needed. + /// + /// + /// DataFormatException, if checksum doesn't match. + /// + private bool DecodeChksum() + { + while (neededBits > 0) { + int chkByte = input.PeekBits(8); + if (chkByte < 0) { + return false; + } + input.DropBits(8); + readAdler = (readAdler << 8) | chkByte; + neededBits -= 8; + } + if ((int) adler.Value != readAdler) { + throw new FormatException("Adler chksum doesn't match: " + + (int)adler.Value + + " vs. " + readAdler); + } + mode = FINISHED; + return false; + } + + /// + /// Decodes the deflated stream. + /// + /// + /// false if more input is needed, or if finished. + /// + /// + /// DataFormatException, if deflated stream is invalid. + /// + private bool Decode() + { + switch (mode) { + case DECODE_HEADER: + return DecodeHeader(); + case DECODE_DICT: + return DecodeDict(); + case DECODE_CHKSUM: + return DecodeChksum(); + + case DECODE_BLOCKS: + if (isLastBlock) { + if (nowrap) { + mode = FINISHED; + return false; + } else { + input.SkipToByteBoundary(); + neededBits = 32; + mode = DECODE_CHKSUM; + return true; + } + } + + int type = input.PeekBits(3); + if (type < 0) { + return false; + } + input.DropBits(3); + + if ((type & 1) != 0) { + isLastBlock = true; + } + switch (type >> 1) { + case DeflaterConstants.STORED_BLOCK: + input.SkipToByteBoundary(); + mode = DECODE_STORED_LEN1; + break; + case DeflaterConstants.STATIC_TREES: + litlenTree = InflaterHuffmanTree.defLitLenTree; + distTree = InflaterHuffmanTree.defDistTree; + mode = DECODE_HUFFMAN; + break; + case DeflaterConstants.DYN_TREES: + dynHeader = new InflaterDynHeader(); + mode = DECODE_DYN_HEADER; + break; + default: + throw new FormatException("Unknown block type "+type); + } + return true; + + case DECODE_STORED_LEN1: { + if ((uncomprLen = input.PeekBits(16)) < 0) { + return false; + } + input.DropBits(16); + mode = DECODE_STORED_LEN2; + } + goto case DECODE_STORED_LEN2; /* fall through */ + case DECODE_STORED_LEN2: { + int nlen = input.PeekBits(16); + if (nlen < 0) { + return false; + } + input.DropBits(16); + if (nlen != (uncomprLen ^ 0xffff)) { + throw new FormatException("broken uncompressed block"); + } + mode = DECODE_STORED; + } + goto case DECODE_STORED;/* fall through */ + case DECODE_STORED: { + int more = outputWindow.CopyStored(input, uncomprLen); + uncomprLen -= more; + if (uncomprLen == 0) { + mode = DECODE_BLOCKS; + return true; + } + return !input.IsNeedingInput; + } + + case DECODE_DYN_HEADER: + if (!dynHeader.Decode(input)) { + return false; + } + + litlenTree = dynHeader.BuildLitLenTree(); + distTree = dynHeader.BuildDistTree(); + mode = DECODE_HUFFMAN; + goto case DECODE_HUFFMAN; /* fall through */ + case DECODE_HUFFMAN: + case DECODE_HUFFMAN_LENBITS: + case DECODE_HUFFMAN_DIST: + case DECODE_HUFFMAN_DISTBITS: + return DecodeHuffman(); + case FINISHED: + return false; + default: + throw new FormatException(); + } + } + + /// + /// Sets the preset dictionary. This should only be called, if + /// needsDictionary() returns true and it should set the same + /// dictionary, that was used for deflating. The getAdler() + /// function returns the checksum of the dictionary needed. + /// + /// + /// the dictionary. + /// + /// + /// if no dictionary is needed. + /// + /// + /// if the dictionary checksum is wrong. + /// + public void SetDictionary(byte[] buffer) + { + SetDictionary(buffer, 0, buffer.Length); + } + + /// + /// Sets the preset dictionary. This should only be called, if + /// needsDictionary() returns true and it should set the same + /// dictionary, that was used for deflating. The getAdler() + /// function returns the checksum of the dictionary needed. + /// + /// + /// the dictionary. + /// + /// + /// the offset into buffer where the dictionary starts. + /// + /// + /// the length of the dictionary. + /// + /// + /// if no dictionary is needed. + /// + /// + /// if the dictionary checksum is wrong. + /// + /// + /// if the off and/or len are wrong. + /// + public void SetDictionary(byte[] buffer, int off, int len) + { + if (!IsNeedingDictionary) { + throw new InvalidOperationException(); + } + + adler.Update(buffer, off, len); + if ((int) adler.Value != readAdler) { + throw new ArgumentException("Wrong adler checksum"); + } + adler.Reset(); + outputWindow.CopyDict(buffer, off, len); + mode = DECODE_BLOCKS; + } + + /// + /// Sets the input. This should only be called, if needsInput() + /// returns true. + /// + /// + /// the input. + /// + /// + /// if no input is needed. + /// + public void SetInput(byte[] buf) + { + SetInput(buf, 0, buf.Length); + } + + /// + /// Sets the input. This should only be called, if needsInput() + /// returns true. + /// + /// + /// the input. + /// + /// + /// the offset into buffer where the input starts. + /// + /// + /// the length of the input. + /// + /// + /// if no input is needed. + /// + /// + /// if the off and/or len are wrong. + /// + public void SetInput(byte[] buf, int off, int len) + { + input.SetInput(buf, off, len); + totalIn += len; + } + + /// + /// Inflates the compressed stream to the output buffer. If this + /// returns 0, you should check, whether needsDictionary(), + /// needsInput() or finished() returns true, to determine why no + /// further output is produced. + /// + /// + /// the output buffer. + /// + /// + /// the number of bytes written to the buffer, 0 if no further + /// output can be produced. + /// + /// + /// if buf has length 0. + /// + /// + /// if deflated stream is invalid. + /// + public int Inflate(byte[] buf) + { + return Inflate(buf, 0, buf.Length); + } + + /// + /// Inflates the compressed stream to the output buffer. If this + /// returns 0, you should check, whether needsDictionary(), + /// needsInput() or finished() returns true, to determine why no + /// further output is produced. + /// + /// + /// the output buffer. + /// + /// + /// the offset into buffer where the output should start. + /// + /// + /// the maximum length of the output. + /// + /// + /// the number of bytes written to the buffer, 0 if no further output can be produced. + /// + /// + /// if len is <= 0. + /// + /// + /// if the off and/or len are wrong. + /// + /// + /// if deflated stream is invalid. + /// + public int Inflate(byte[] buf, int off, int len) + { + if (len < 0) { + throw new ArgumentOutOfRangeException("len < 0"); + } + // Special case: len may be zero + if (len == 0) { + return 0; + } +/* // Check for correct buff, off, len triple + if (off < 0 || off + len >= buf.Length) { + throw new ArgumentException("off/len outside buf bounds"); + }*/ + int count = 0; + int more; + do { + if (mode != DECODE_CHKSUM) { + /* Don't give away any output, if we are waiting for the + * checksum in the input stream. + * + * With this trick we have always: + * needsInput() and not finished() + * implies more output can be produced. + */ + more = outputWindow.CopyOutput(buf, off, len); + adler.Update(buf, off, more); + off += more; + count += more; + totalOut += more; + len -= more; + if (len == 0) { + return count; + } + } + } while (Decode() || (outputWindow.GetAvailable() > 0 && + mode != DECODE_CHKSUM)); + return count; + } + + /// + /// Returns true, if the input buffer is empty. + /// You should then call setInput(). + /// NOTE: This method also returns true when the stream is finished. + /// + public bool IsNeedingInput { + get { + return input.IsNeedingInput; + } + } + + /// + /// Returns true, if a preset dictionary is needed to inflate the input. + /// + public bool IsNeedingDictionary { + get { + return mode == DECODE_DICT && neededBits == 0; + } + } + + /// + /// Returns true, if the inflater has finished. This means, that no + /// input is needed and no output can be produced. + /// + public bool IsFinished { + get { + return mode == FINISHED && outputWindow.GetAvailable() == 0; + } + } + + /// + /// Gets the adler checksum. This is either the checksum of all + /// uncompressed bytes returned by inflate(), or if needsDictionary() + /// returns true (and thus no output was yet produced) this is the + /// adler checksum of the expected dictionary. + /// + /// + /// the adler checksum. + /// + public int Adler { + get { + return IsNeedingDictionary ? readAdler : (int) adler.Value; + } + } + + /// + /// Gets the total number of output bytes returned by inflate(). + /// + /// + /// the total number of output bytes. + /// + public int TotalOut { + get { + return totalOut; + } + } + + /// + /// Gets the total number of processed compressed input bytes. + /// + /// + /// the total number of bytes of processed input bytes. + /// + public int TotalIn { + get { + return totalIn - RemainingInput; + } + } + + /// + /// Gets the number of unprocessed input. Useful, if the end of the + /// stream is reached and you want to further process the bytes after + /// the deflate stream. + /// + /// + /// the number of bytes of the input which were not processed. + /// + public int RemainingInput { + get { + return input.AvailableBytes; + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/InflaterDynHeader.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/InflaterDynHeader.cs new file mode 100644 index 000000000..e60cbb1eb --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/InflaterDynHeader.cs @@ -0,0 +1,203 @@ +// InflaterDynHeader.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + class InflaterDynHeader + { + const int LNUM = 0; + const int DNUM = 1; + const int BLNUM = 2; + const int BLLENS = 3; + const int LENS = 4; + const int REPS = 5; + + static readonly int[] repMin = { 3, 3, 11 }; + static readonly int[] repBits = { 2, 3, 7 }; + + byte[] blLens; + byte[] litdistLens; + + InflaterHuffmanTree blTree; + + int mode; + int lnum, dnum, blnum, num; + int repSymbol; + byte lastLen; + int ptr; + + static readonly int[] BL_ORDER = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; + + public InflaterDynHeader() + { + } + + public bool Decode(StreamManipulator input) + { + decode_loop: + for (;;) { + switch (mode) { + case LNUM: + lnum = input.PeekBits(5); + if (lnum < 0) { + return false; + } + lnum += 257; + input.DropBits(5); + // System.err.println("LNUM: "+lnum); + mode = DNUM; + goto case DNUM; // fall through + case DNUM: + dnum = input.PeekBits(5); + if (dnum < 0) { + return false; + } + dnum++; + input.DropBits(5); + // System.err.println("DNUM: "+dnum); + num = lnum+dnum; + litdistLens = new byte[num]; + mode = BLNUM; + goto case BLNUM; // fall through + case BLNUM: + blnum = input.PeekBits(4); + if (blnum < 0) { + return false; + } + blnum += 4; + input.DropBits(4); + blLens = new byte[19]; + ptr = 0; + // System.err.println("BLNUM: "+blnum); + mode = BLLENS; + goto case BLLENS; // fall through + case BLLENS: + while (ptr < blnum) { + int len = input.PeekBits(3); + if (len < 0) { + return false; + } + input.DropBits(3); + // System.err.println("blLens["+BL_ORDER[ptr]+"]: "+len); + blLens[BL_ORDER[ptr]] = (byte) len; + ptr++; + } + blTree = new InflaterHuffmanTree(blLens); + blLens = null; + ptr = 0; + mode = LENS; + goto case LENS; // fall through + case LENS: { + int symbol; + while (((symbol = blTree.GetSymbol(input)) & ~15) == 0) { + /* Normal case: symbol in [0..15] */ + + // System.err.println("litdistLens["+ptr+"]: "+symbol); + litdistLens[ptr++] = lastLen = (byte)symbol; + + if (ptr == num) { + /* Finished */ + return true; + } + } + + /* need more input ? */ + if (symbol < 0) + return false; + + /* otherwise repeat code */ + if (symbol >= 17) { + /* repeat zero */ + // System.err.println("repeating zero"); + lastLen = 0; + } else { + if (ptr == 0) { + throw new Exception(); + } + } + repSymbol = symbol-16; + } + mode = REPS; + goto case REPS; // fall through + case REPS: + { + int bits = repBits[repSymbol]; + int count = input.PeekBits(bits); + if (count < 0) { + return false; + } + input.DropBits(bits); + count += repMin[repSymbol]; + // System.err.println("litdistLens repeated: "+count); + + if (ptr + count > num) { + throw new Exception(); + } + while (count-- > 0) { + litdistLens[ptr++] = lastLen; + } + + if (ptr == num) { + /* Finished */ + return true; + } + } + mode = LENS; + goto decode_loop; + } + } + } + + public InflaterHuffmanTree BuildLitLenTree() + { + byte[] litlenLens = new byte[lnum]; + Array.Copy(litdistLens, 0, litlenLens, 0, lnum); + return new InflaterHuffmanTree(litlenLens); + } + + public InflaterHuffmanTree BuildDistTree() + { + byte[] distLens = new byte[dnum]; + Array.Copy(litdistLens, lnum, distLens, 0, dnum); + return new InflaterHuffmanTree(distLens); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/InflaterHuffmanTree.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/InflaterHuffmanTree.cs new file mode 100644 index 000000000..04954323d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/InflaterHuffmanTree.cs @@ -0,0 +1,208 @@ +// InflaterHuffmanTree.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + public class InflaterHuffmanTree + { + private static int MAX_BITLEN = 15; + private short[] tree; + + public static InflaterHuffmanTree defLitLenTree, defDistTree; + + static InflaterHuffmanTree() + { + try { + byte[] codeLengths = new byte[288]; + int i = 0; + while (i < 144) { + codeLengths[i++] = 8; + } + while (i < 256) { + codeLengths[i++] = 9; + } + while (i < 280) { + codeLengths[i++] = 7; + } + while (i < 288) { + codeLengths[i++] = 8; + } + defLitLenTree = new InflaterHuffmanTree(codeLengths); + + codeLengths = new byte[32]; + i = 0; + while (i < 32) { + codeLengths[i++] = 5; + } + defDistTree = new InflaterHuffmanTree(codeLengths); + } catch (Exception) { + throw new ApplicationException("InflaterHuffmanTree: static tree length illegal"); + } + } + + /// + /// Constructs a Huffman tree from the array of code lengths. + /// + /// + /// the array of code lengths + /// + public InflaterHuffmanTree(byte[] codeLengths) + { + BuildTree(codeLengths); + } + + private void BuildTree(byte[] codeLengths) + { + int[] blCount = new int[MAX_BITLEN + 1]; + int[] nextCode = new int[MAX_BITLEN + 1]; + + for (int i = 0; i < codeLengths.Length; i++) { + int bits = codeLengths[i]; + if (bits > 0) + blCount[bits]++; + } + + int code = 0; + int treeSize = 512; + for (int bits = 1; bits <= MAX_BITLEN; bits++) { + nextCode[bits] = code; + code += blCount[bits] << (16 - bits); + if (bits >= 10) { + /* We need an extra table for bit lengths >= 10. */ + int start = nextCode[bits] & 0x1ff80; + int end = code & 0x1ff80; + treeSize += (end - start) >> (16 - bits); + } + } + if (code != 65536) { + throw new Exception("Code lengths don't add up properly."); + } + /* Now create and fill the extra tables from longest to shortest + * bit len. This way the sub trees will be aligned. + */ + tree = new short[treeSize]; + int treePtr = 512; + for (int bits = MAX_BITLEN; bits >= 10; bits--) { + int end = code & 0x1ff80; + code -= blCount[bits] << (16 - bits); + int start = code & 0x1ff80; + for (int i = start; i < end; i += 1 << 7) { + tree[DeflaterHuffman.BitReverse(i)] = (short) ((-treePtr << 4) | bits); + treePtr += 1 << (bits-9); + } + } + + for (int i = 0; i < codeLengths.Length; i++) { + int bits = codeLengths[i]; + if (bits == 0) { + continue; + } + code = nextCode[bits]; + int revcode = DeflaterHuffman.BitReverse(code); + if (bits <= 9) { + do { + tree[revcode] = (short) ((i << 4) | bits); + revcode += 1 << bits; + } while (revcode < 512); + } else { + int subTree = tree[revcode & 511]; + int treeLen = 1 << (subTree & 15); + subTree = -(subTree >> 4); + do { + tree[subTree | (revcode >> 9)] = (short) ((i << 4) | bits); + revcode += 1 << bits; + } while (revcode < treeLen); + } + nextCode[bits] = code + (1 << (16 - bits)); + } + + } + + /// + /// Reads the next symbol from input. The symbol is encoded using the + /// huffman tree. + /// + /// + /// input the input source. + /// + /// + /// the next symbol, or -1 if not enough input is available. + /// + public int GetSymbol(StreamManipulator input) + { + int lookahead, symbol; + if ((lookahead = input.PeekBits(9)) >= 0) { + if ((symbol = tree[lookahead]) >= 0) { + input.DropBits(symbol & 15); + return symbol >> 4; + } + int subtree = -(symbol >> 4); + int bitlen = symbol & 15; + if ((lookahead = input.PeekBits(bitlen)) >= 0) { + symbol = tree[subtree | (lookahead >> 9)]; + input.DropBits(symbol & 15); + return symbol >> 4; + } else { + int bits = input.AvailableBits; + lookahead = input.PeekBits(bits); + symbol = tree[subtree | (lookahead >> 9)]; + if ((symbol & 15) <= bits) { + input.DropBits(symbol & 15); + return symbol >> 4; + } else { + return -1; + } + } + } else { + int bits = input.AvailableBits; + lookahead = input.PeekBits(bits); + symbol = tree[lookahead]; + if (symbol >= 0 && (symbol & 15) <= bits) { + input.DropBits(symbol & 15); + return symbol >> 4; + } else { + return -1; + } + } + } + } +} + diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/PendingBuffer.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/PendingBuffer.cs new file mode 100644 index 000000000..f52436710 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/PendingBuffer.cs @@ -0,0 +1,215 @@ +// PendingBuffer.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression { + + /// + /// This class is general purpose class for writing data to a buffer. + /// + /// It allows you to write bits as well as bytes + /// Based on DeflaterPending.java + /// + /// author of the original java version : Jochen Hoenicke + /// + public class PendingBuffer + { + protected byte[] buf; + int start; + int end; + + uint bits; + int bitCount; + + public PendingBuffer() : this( 4096 ) + { + + } + + public PendingBuffer(int bufsize) + { + buf = new byte[bufsize]; + } + + public void Reset() + { + start = end = bitCount = 0; + } + + public void WriteByte(int b) + { + /* + if (DeflaterConstants.DEBUGGING && start != 0) + throw new Exception();*/ + buf[end++] = (byte) b; + } + + public void WriteShort(int s) + { + /* + if (DeflaterConstants.DEBUGGING && start != 0) { + throw new Exception(); + }*/ + buf[end++] = (byte) s; + buf[end++] = (byte) (s >> 8); + } + + public void WriteInt(int s) + { + /* + if (DeflaterConstants.DEBUGGING && start != 0) { + throw new Exception(); + }*/ + buf[end++] = (byte) s; + buf[end++] = (byte) (s >> 8); + buf[end++] = (byte) (s >> 16); + buf[end++] = (byte) (s >> 24); + } + + public void WriteBlock(byte[] block, int offset, int len) + { + /* + if (DeflaterConstants.DEBUGGING && start != 0) { + throw new Exception(); + }*/ + System.Array.Copy(block, offset, buf, end, len); + end += len; + } + + public int BitCount { + get { + return bitCount; + } + } + + public void AlignToByte() + { + /* + if (DeflaterConstants.DEBUGGING && start != 0) { + throw new Exception(); + }*/ + if (bitCount > 0) { + buf[end++] = (byte) bits; + if (bitCount > 8) { + buf[end++] = (byte) (bits >> 8); + } + } + bits = 0; + bitCount = 0; + } + + public void WriteBits(int b, int count) + { + /* + if (DeflaterConstants.DEBUGGING && start != 0) { + throw new Exception(); + }*/ +// if (DeflaterConstants.DEBUGGING) { +// Console.WriteLine("writeBits("+b+","+count+")"); +// } + bits |= (uint)(b << bitCount); + bitCount += count; + if (bitCount >= 16) { + buf[end++] = (byte) bits; + buf[end++] = (byte) (bits >> 8); + bits >>= 16; + bitCount -= 16; + } + } + + public void WriteShortMSB(int s) + { + /* + if (DeflaterConstants.DEBUGGING && start != 0) { + throw new Exception(); + }*/ + buf[end++] = (byte) (s >> 8); + buf[end++] = (byte) s; + } + + public bool IsFlushed { + get { + return end == 0; + } + } + + /// + /// Flushes the pending buffer into the given output array. If the + /// output array is to small, only a partial flush is done. + /// + /// + /// the output array; + /// + /// + /// the offset into output array; + /// + /// + /// length the maximum number of bytes to store; + /// + /// + /// IndexOutOfBoundsException if offset or length are invalid. + /// + public int Flush(byte[] output, int offset, int length) + { + if (bitCount >= 8) { + buf[end++] = (byte) bits; + bits >>= 8; + bitCount -= 8; + } + if (length > end - start) { + length = end - start; + System.Array.Copy(buf, start, output, offset, length); + start = 0; + end = 0; + } else { + System.Array.Copy(buf, start, output, offset, length); + start += length; + } + return length; + } + + public byte[] ToByteArray() + { + byte[] ret = new byte[ end - start ]; + System.Array.Copy(buf, start, ret, 0, ret.Length); + start = 0; + end = 0; + return ret; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs new file mode 100644 index 000000000..eabcd59cf --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs @@ -0,0 +1,296 @@ +// DeflaterOutputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; + +using BCG.Utilities.SharpZipLib.Zip.Compression; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression.Streams { + + /// + /// This is a special FilterOutputStream deflating the bytes that are + /// written through it. It uses the Deflater for deflating. + /// + /// authors of the original java version : Tom Tromey, Jochen Hoenicke + /// + public class DeflaterOutputStream : Stream + { + /// + /// This buffer is used temporarily to retrieve the bytes from the + /// deflater and write them to the underlying output stream. + /// + protected byte[] buf; + + /// + /// The deflater which is used to deflate the stream. + /// + protected Deflater def; + + /// + /// base stream the deflater depends on. + /// + protected Stream baseOutputStream; + + /// + /// I needed to implement the abstract member. + /// + public override bool CanRead { + get { + return baseOutputStream.CanRead; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanSeek { + get { + return baseOutputStream.CanSeek; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanWrite { + get { + return baseOutputStream.CanWrite; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Length { + get { + return baseOutputStream.Length; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Position { + get { + return baseOutputStream.Position; + } + set { + baseOutputStream.Position = value; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Seek(long offset, SeekOrigin origin) + { + return baseOutputStream.Seek(offset, origin); + } + + /// + /// I needed to implement the abstract member. + /// + public override void SetLength(long val) + { + baseOutputStream.SetLength(val); + } + + /// + /// I needed to implement the abstract member. + /// + public override int ReadByte() + { + return baseOutputStream.ReadByte(); + } + + /// + /// I needed to implement the abstract member. + /// + public override int Read(byte[] b, int off, int len) + { + return baseOutputStream.Read(b, off, len); + } + + + /// + /// Deflates everything in the def's input buffers. This will call + /// def.deflate() until all bytes from the input buffers + /// are processed. + /// + protected void deflate() + { + while (!def.IsNeedingInput) { + int len = def.Deflate(buf, 0, buf.Length); + + // System.err.println("DOS deflated " + len + " baseOutputStream of " + buf.length); + if (len <= 0) { + break; + } + baseOutputStream.Write(buf, 0, len); + } + + if (!def.IsNeedingInput) { + throw new ApplicationException("Can't deflate all input?"); + } + } + + /// + /// Creates a new DeflaterOutputStream with a default Deflater and default buffer size. + /// + /// + /// the output stream where deflated output should be written. + /// + public DeflaterOutputStream(Stream baseOutputStream) : this(baseOutputStream, new Deflater(), 512) + { + + } + + /// + /// Creates a new DeflaterOutputStream with the given Deflater and + /// default buffer size. + /// + /// + /// the output stream where deflated output should be written. + /// + /// + /// the underlying deflater. + /// + public DeflaterOutputStream(Stream baseOutputStream, Deflater defl) :this(baseOutputStream, defl, 512) + { + } + + /// + /// Creates a new DeflaterOutputStream with the given Deflater and + /// buffer size. + /// + /// + /// the output stream where deflated output should be written. + /// + /// + /// the underlying deflater. + /// + /// + /// the buffer size. + /// + /// + /// if bufsize isn't positive. + /// + public DeflaterOutputStream(Stream baseOutputStream, Deflater defl, int bufsize) + { + this.baseOutputStream = baseOutputStream; + if (bufsize <= 0) { + throw new InvalidOperationException("bufsize <= 0"); + } + buf = new byte[bufsize]; + def = defl; + } + + /// + /// Flushes the stream by calling flush() on the deflater and then + /// on the underlying stream. This ensures that all bytes are + /// flushed. + /// + public override void Flush() + { + def.Flush(); + deflate(); + baseOutputStream.Flush(); + } + + /// + /// Finishes the stream by calling finish() on the deflater. + /// + public virtual void Finish() + { + def.Finish(); + while (!def.IsFinished) { + int len = def.Deflate(buf, 0, buf.Length); + if (len <= 0) { + break; + } + baseOutputStream.Write(buf, 0, len); + } + if (!def.IsFinished) { + throw new ApplicationException("Can't deflate all input?"); + } + baseOutputStream.Flush(); + } + + /// + /// Calls finish () and closes the stream. + /// + public override void Close() + { + Finish(); + baseOutputStream.Close(); + } + + /// + /// Writes a single byte to the compressed output stream. + /// + /// + /// the byte value. + /// + public override void WriteByte(byte bval) + { + byte[] b = new byte[1]; + b[0] = (byte) bval; + Write(b, 0, 1); + } + + /// + /// Writes a len bytes from an array to the compressed stream. + /// + /// + /// the byte array. + /// + /// + /// the offset into the byte array where to start. + /// + /// + /// the number of bytes to write. + /// + public override void Write(byte[] buf, int off, int len) + { + // System.err.println("DOS with off " + off + " and len " + len); + def.SetInput(buf, off, len); + deflate(); + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/InflaterInputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/InflaterInputStream.cs new file mode 100644 index 000000000..4fb38ac77 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/InflaterInputStream.cs @@ -0,0 +1,335 @@ +// InflaterInputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; + +using BCG.Utilities.SharpZipLib.Zip.Compression; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression.Streams { + + /// + /// This filter stream is used to decompress data compressed baseInputStream the "deflate" + /// format. The "deflate" format is described baseInputStream RFC 1951. + /// + /// This stream may form the basis for other decompression filters, such + /// as the GzipInputStream. + /// + /// author of the original java version : John Leuner + /// + public class InflaterInputStream : Stream + { + //Variables + + /// + /// Decompressor for this filter + /// + protected Inflater inf; + + + /// + /// Byte array used as a buffer + /// + protected byte[] buf; + + /// + /// Size of buffer + /// + protected int len; + + //We just use this if we are decoding one byte at a time with the read() call + private byte[] onebytebuffer = new byte[1]; + + /// + /// base stream the inflater depends on. + /// + protected Stream baseInputStream; + + /// + /// I needed to implement the abstract member. + /// + public override bool CanRead { + get { + return baseInputStream.CanRead; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanSeek { + get { + return baseInputStream.CanSeek; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override bool CanWrite { + get { + return baseInputStream.CanWrite; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Length { + get { + return len; + } + } + + /// + /// I needed to implement the abstract member. + /// + public override long Position { + get { + return baseInputStream.Position; + } + set { + baseInputStream.Position = value; + } + } + + /// + /// Flushes the baseInputStream + /// + public override void Flush() + { + baseInputStream.Flush(); + } + + /// + /// I needed to implement the abstract member. + /// + public override long Seek(long offset, SeekOrigin origin) + { + return baseInputStream.Seek(offset, origin); + } + + /// + /// I needed to implement the abstract member. + /// + public override void SetLength(long val) + { + baseInputStream.SetLength(val); + } + + /// + /// I needed to implement the abstract member. + /// + public override void Write(byte[] array, int offset, int count) + { + baseInputStream.Write(array, offset, count); + } + + /// + /// I needed to implement the abstract member. + /// + public override void WriteByte(byte val) + { + baseInputStream.WriteByte(val); + } + + //Constructors + + /// + /// Create an InflaterInputStream with the default decompresseor + /// and a default buffer size. + /// + /// + /// the InputStream to read bytes from + /// + public InflaterInputStream(Stream baseInputStream) : this(baseInputStream, new Inflater(), 4096) + { + + } + + /// + /// Create an InflaterInputStream with the specified decompresseor + /// and a default buffer size. + /// + /// + /// the InputStream to read bytes from + /// + /// + /// the decompressor used to decompress data read from baseInputStream + /// + public InflaterInputStream(Stream baseInputStream, Inflater inf) : this(baseInputStream, inf, 4096) + { + + } + + /// + /// Create an InflaterInputStream with the specified decompresseor + /// and a specified buffer size. + /// + /// + /// the InputStream to read bytes from + /// + /// + /// the decompressor used to decompress data read from baseInputStream + /// + /// + /// size of the buffer to use + /// + public InflaterInputStream(Stream baseInputStream, Inflater inf, int size) + { + this.baseInputStream = baseInputStream; + this.inf = inf; + try { + this.len = (int)baseInputStream.Length; + } catch (Exception) { // the stream may not support .Length + this.len = 0; + } + + if (size <= 0) { + throw new ArgumentOutOfRangeException("size <= 0"); + } + + if (len > size) + size = len; + buf = new byte[size]; //Create the buffer + } + + //Methods + + /// + /// Returns 0 once the end of the stream (EOF) has been reached. + /// Otherwise returns 1. + /// + public virtual int Available { + get { + return inf.IsFinished ? 0 : 1; + } + } + + /// + /// Closes the input stream + /// + public override void Close() + { + baseInputStream.Close(); + } + + /// + /// Fills the buffer with more data to decompress. + /// + protected void Fill() + { + len = baseInputStream.Read(buf, 0, buf.Length); + + if (len < 0) { + throw new ApplicationException("Deflated stream ends early."); + } + inf.SetInput(buf, 0, len); + } + + /// + /// Reads one byte of decompressed data. + /// + /// The byte is baseInputStream the lower 8 bits of the int. + /// + public override int ReadByte() + { + int nread = Read(onebytebuffer, 0, 1); //read one byte + if (nread > 0) { + return onebytebuffer[0] & 0xff; + } + return -1; + } + + /// + /// Decompresses data into the byte array + /// + /// + /// the array to read and decompress data into + /// + /// + /// the offset indicating where the data should be placed + /// + /// + /// the number of bytes to decompress + /// + public override int Read(byte[] b, int off, int len) + { + for (;;) { + int count; + try { + count = inf.Inflate(b, off, len); + } catch (Exception e) { + throw new ZipException(e.ToString()); + } + + if (count > 0) { + return count; + } + + if (inf.IsNeedingDictionary) { + throw new ZipException("Need a dictionary"); + } else if (inf.IsFinished) { + return 0; + } else if (inf.IsNeedingInput) { + Fill(); + } else { + throw new InvalidOperationException("Don't know what to do"); + } + } + } + + /// + /// Skip specified number of bytes of uncompressed data + /// + /// + /// number of bytes to skip + /// + public long Skip(long n) + { + if (n < 0) { + throw new ArgumentOutOfRangeException("n"); + } + int len = 2048; + if (n < len) { + len = (int) n; + } + byte[] tmp = new byte[len]; + return (long)baseInputStream.Read(tmp, 0, tmp.Length); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/OutputWindow.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/OutputWindow.cs new file mode 100644 index 000000000..2ed20430d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/OutputWindow.cs @@ -0,0 +1,176 @@ +// OutputWindow.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression.Streams { + + /// + /// Contains the output from the Inflation process. + /// We need to have a window so that we can refer backwards into the output stream + /// to repeat stuff. + /// + /// author of the original java version : John Leuner + /// + public class OutputWindow + { + private static int WINDOW_SIZE = 1 << 15; + private static int WINDOW_MASK = WINDOW_SIZE - 1; + + private byte[] window = new byte[WINDOW_SIZE]; //The window is 2^15 bytes + private int window_end = 0; + private int window_filled = 0; + + public void Write(int abyte) + { + if (window_filled++ == WINDOW_SIZE) { + throw new InvalidOperationException("Window full"); + } + window[window_end++] = (byte) abyte; + window_end &= WINDOW_MASK; + } + + + private void SlowRepeat(int rep_start, int len, int dist) + { + while (len-- > 0) { + window[window_end++] = window[rep_start++]; + window_end &= WINDOW_MASK; + rep_start &= WINDOW_MASK; + } + } + + public void Repeat(int len, int dist) + { + if ((window_filled += len) > WINDOW_SIZE) { + throw new InvalidOperationException("Window full"); + } + + int rep_start = (window_end - dist) & WINDOW_MASK; + int border = WINDOW_SIZE - len; + if (rep_start <= border && window_end < border) { + if (len <= dist) { + System.Array.Copy(window, rep_start, window, window_end, len); + window_end += len; + } else { + /* We have to copy manually, since the repeat pattern overlaps. + */ + while (len-- > 0) { + window[window_end++] = window[rep_start++]; + } + } + } else { + SlowRepeat(rep_start, len, dist); + } + } + + public int CopyStored(StreamManipulator input, int len) + { + len = Math.Min(Math.Min(len, WINDOW_SIZE - window_filled), input.AvailableBytes); + int copied; + + int tailLen = WINDOW_SIZE - window_end; + if (len > tailLen) { + copied = input.CopyBytes(window, window_end, tailLen); + if (copied == tailLen) { + copied += input.CopyBytes(window, 0, len - tailLen); + } + } else { + copied = input.CopyBytes(window, window_end, len); + } + + window_end = (window_end + copied) & WINDOW_MASK; + window_filled += copied; + return copied; + } + + public void CopyDict(byte[] dict, int offset, int len) + { + if (window_filled > 0) { + throw new InvalidOperationException(); + } + + if (len > WINDOW_SIZE) { + offset += len - WINDOW_SIZE; + len = WINDOW_SIZE; + } + System.Array.Copy(dict, offset, window, 0, len); + window_end = len & WINDOW_MASK; + } + + public int GetFreeSpace() + { + return WINDOW_SIZE - window_filled; + } + + public int GetAvailable() + { + return window_filled; + } + + public int CopyOutput(byte[] output, int offset, int len) + { + int copy_end = window_end; + if (len > window_filled) { + len = window_filled; + } else { + copy_end = (window_end - window_filled + len) & WINDOW_MASK; + } + + int copied = len; + int tailLen = len - copy_end; + + if (tailLen > 0) { + System.Array.Copy(window, WINDOW_SIZE - tailLen, + output, offset, tailLen); + offset += tailLen; + len = copy_end; + } + System.Array.Copy(window, copy_end - len, output, offset, len); + window_filled -= copied; + if (window_filled < 0) { + throw new InvalidOperationException(); + } + return copied; + } + + public void Reset() + { + window_filled = window_end = 0; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/StreamManipulator.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/StreamManipulator.cs new file mode 100644 index 000000000..9848d1539 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/Compression/Streams/StreamManipulator.cs @@ -0,0 +1,244 @@ +// StreamManipulator.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Zip.Compression.Streams { + + /// + /// This class allows us to retrieve a specified amount of bits from + /// the input buffer, as well as copy big byte blocks. + /// + /// It uses an int buffer to store up to 31 bits for direct + /// manipulation. This guarantees that we can get at least 16 bits, + /// but we only need at most 15, so this is all safe. + /// + /// There are some optimizations in this class, for example, you must + /// never peek more then 8 bits more than needed, and you must first + /// peek bits before you may drop them. This is not a general purpose + /// class but optimized for the behaviour of the Inflater. + /// + /// authors of the original java version : John Leuner, Jochen Hoenicke + /// + public class StreamManipulator + { + private byte[] window; + private int window_start = 0; + private int window_end = 0; + + private uint buffer = 0; + private int bits_in_buffer = 0; + + /// + /// Get the next n bits but don't increase input pointer. n must be + /// less or equal 16 and if you if this call succeeds, you must drop + /// at least n-8 bits in the next call. + /// + /// + /// the value of the bits, or -1 if not enough bits available. */ + /// + public int PeekBits(int n) + { + if (bits_in_buffer < n) { + if (window_start == window_end) { + return -1; + } + buffer |= (uint)((window[window_start++] & 0xff | + (window[window_start++] & 0xff) << 8) << bits_in_buffer); + bits_in_buffer += 16; + } + return (int)(buffer & ((1 << n) - 1)); + } + + /// + /// Drops the next n bits from the input. You should have called peekBits + /// with a bigger or equal n before, to make sure that enough bits are in + /// the bit buffer. + /// + public void DropBits(int n) + { + buffer >>= n; + bits_in_buffer -= n; + } + + /// + /// Gets the next n bits and increases input pointer. This is equivalent + /// to peekBits followed by dropBits, except for correct error handling. + /// + /// + /// the value of the bits, or -1 if not enough bits available. + /// + public int GetBits(int n) + { + int bits = PeekBits(n); + if (bits >= 0) { + DropBits(n); + } + return bits; + } + + /// + /// Gets the number of bits available in the bit buffer. This must be + /// only called when a previous peekBits() returned -1. + /// + /// + /// the number of bits available. + /// + public int AvailableBits { + get { + return bits_in_buffer; + } + } + + /// + /// Gets the number of bytes available. + /// + /// + /// the number of bytes available. + /// + public int AvailableBytes { + get { + return window_end - window_start + (bits_in_buffer >> 3); + } + } + + /// + /// Skips to the next byte boundary. + /// + public void SkipToByteBoundary() + { + buffer >>= (bits_in_buffer & 7); + bits_in_buffer &= ~7; + } + + public bool IsNeedingInput { + get { + return window_start == window_end; + } + } + + /// + /// Copies length bytes from input buffer to output buffer starting + /// at output[offset]. You have to make sure, that the buffer is + /// byte aligned. If not enough bytes are available, copies fewer + /// bytes. + /// + /// + /// the buffer. + /// + /// + /// the offset in the buffer. + /// + /// + /// the length to copy, 0 is allowed. + /// + /// + /// the number of bytes copied, 0 if no byte is available. + /// + public int CopyBytes(byte[] output, int offset, int length) + { + if (length < 0) { + throw new ArgumentOutOfRangeException("length negative"); + } + if ((bits_in_buffer & 7) != 0) { + /* bits_in_buffer may only be 0 or 8 */ + throw new InvalidOperationException("Bit buffer is not aligned!"); + } + + int count = 0; + while (bits_in_buffer > 0 && length > 0) { + output[offset++] = (byte) buffer; + buffer >>= 8; + bits_in_buffer -= 8; + length--; + count++; + } + if (length == 0) { + return count; + } + + int avail = window_end - window_start; + if (length > avail) { + length = avail; + } + System.Array.Copy(window, window_start, output, offset, length); + window_start += length; + + if (((window_start - window_end) & 1) != 0) { + /* We always want an even number of bytes in input, see peekBits */ + buffer = (uint)(window[window_start++] & 0xff); + bits_in_buffer = 8; + } + return count + length; + } + + public StreamManipulator() + { + } + + public void Reset() + { + buffer = (uint)(window_start = window_end = bits_in_buffer = 0); + } + + public void SetInput(byte[] buf, int off, int len) + { + if (window_start < window_end) { + throw new InvalidOperationException("Old input was not completely processed"); + } + + int end = off + len; + + /* We want to throw an ArrayIndexOutOfBoundsException early. The + * check is very tricky: it also handles integer wrap around. + */ + if (0 > off || off > end || end > buf.Length) { + throw new ArgumentOutOfRangeException(); + } + + if ((len & 1) != 0) { + /* We always want an even number of bytes in input, see peekBits */ + buffer |= (uint)((buf[off++] & 0xff) << bits_in_buffer); + bits_in_buffer += 8; + } + + window = buf; + window_start = off; + window_end = end; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipConstants.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipConstants.cs new file mode 100644 index 000000000..f9629c167 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipConstants.cs @@ -0,0 +1,124 @@ +// ZipConstants.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System.Text; + +namespace BCG.Utilities.SharpZipLib.Zip { + + /// + /// This class contains constants used for zip. + /// + public sealed class ZipConstants + { + /* The local file header */ + public const int LOCHDR = 30; + public const int LOCSIG = 'P' | ('K' << 8) | (3 << 16) | (4 << 24); + + public const int LOCVER = 4; + public const int LOCFLG = 6; + public const int LOCHOW = 8; + public const int LOCTIM = 10; + public const int LOCCRC = 14; + public const int LOCSIZ = 18; + public const int LOCLEN = 22; + public const int LOCNAM = 26; + public const int LOCEXT = 28; + + /* The Data descriptor */ + public const int EXTSIG = 'P' | ('K' << 8) | (7 << 16) | (8 << 24); + public const int EXTHDR = 16; + + public const int EXTCRC = 4; + public const int EXTSIZ = 8; + public const int EXTLEN = 12; + + /* The central directory file header */ + public const int CENSIG = 'P' | ('K' << 8) | (1 << 16) | (2 << 24); + + /* The central directory file header for 64bit ZIP*/ + public const int CENSIG64 = 0x06064b50; + + public const int CENHDR = 46; + + public const int CENVEM = 4; + public const int CENVER = 6; + public const int CENFLG = 8; + public const int CENHOW = 10; + public const int CENTIM = 12; + public const int CENCRC = 16; + public const int CENSIZ = 20; + public const int CENLEN = 24; + public const int CENNAM = 28; + public const int CENEXT = 30; + public const int CENCOM = 32; + public const int CENDSK = 34; + public const int CENATT = 36; + public const int CENATX = 38; + public const int CENOFF = 42; + + /* The entries in the end of central directory */ + public const int ENDSIG = 'P' | ('K' << 8) | (5 << 16) | (6 << 24); + public const int ENDHDR = 22; + + /* The following two fields are missing in SUN JDK */ + public const int ENDNRD = 4; + public const int ENDDCD = 6; + + public const int ENDSUB = 8; + public const int ENDTOT = 10; + public const int ENDSIZ = 12; + public const int ENDOFF = 16; + public const int ENDCOM = 20; + + /* Using the codepage 1252 doesn't solve the 8bit ASCII problem :/ + any help would be appreciated. + + // get encoding for latin characters (like ö, ü, ß or ô) + static Encoding ecp1252 = Encoding.GetEncoding(1252); + */ + public static string ConvertToString(byte[] data) + { + + return Encoding.ASCII.GetString(data); + } + + public static byte[] ConvertToArray(string str) + { + return Encoding.ASCII.GetBytes(str); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipEntry.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipEntry.cs new file mode 100644 index 000000000..a067bf5c8 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipEntry.cs @@ -0,0 +1,393 @@ +// ZipEntry.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib.Zip { + + public enum CompressionMethod + { + Stored = 0, + Deflated = 8, + } + + /// + /// This class represents a member of a zip archive. ZipFile and + /// ZipInputStream will give you instances of this class as information + /// about the members in an archive. On the other hand ZipOutputStream + /// needs an instance of this class to create a new member. + /// + /// author of the original java version : Jochen Hoenicke + /// + public class ZipEntry : ICloneable + { + static int KNOWN_SIZE = 1; + static int KNOWN_CSIZE = 2; + static int KNOWN_CRC = 4; + static int KNOWN_TIME = 8; + + DateTime cal = DateTime.Now; + + string name; + uint size; + ushort version; + uint compressedSize; + int crc; + + ushort known = 0; + CompressionMethod method = CompressionMethod.Deflated; + byte[] extra = null; + string comment = null; + + public int zipFileIndex = -1; /* used by ZipFile */ + public int flags; /* used by ZipOutputStream */ + public int offset; /* used by ZipFile and ZipOutputStream */ + + /// + /// Creates a zip entry with the given name. + /// + /// + /// the name. May include directory components separated by '/'. + /// + public ZipEntry(string name) + { + if (name == null) { + throw new System.ArgumentNullException("name"); + } + this.name = name; + } + + /// + /// Creates a copy of the given zip entry. + /// + /// + /// the entry to copy. + /// + public ZipEntry(ZipEntry e) + { + name = e.name; + known = e.known; + size = e.size; + compressedSize = e.compressedSize; + crc = e.crc; +// time = e.time; + method = e.method; + extra = e.extra; + comment = e.comment; + } + + public ushort Version { + get { + return version; + } + set { + version = value; + } + } + + public int DosTime { + get { +// if ((known & KNOWN_TIME) == 0) { +// return 0; +// } + lock (this) { + return (cal.Year - 1980 & 0x7f) << 25 | + (cal.Month + 1) << 21 | + (cal.Day ) << 16 | + (cal.Hour) << 11 | + (cal.Minute) << 5 | + (cal.Second) >> 1; + } + } + set { + // Guard against invalid or missing date causing + // IndexOutOfBoundsException. + try { + lock (this) { + cal = CalculateDateTime(value); +// time = (int) (cal.Millisecond / 1000L); + } + known |= (ushort)KNOWN_TIME; + } catch (Exception) { + /* Ignore illegal time stamp */ + known &= (ushort)~KNOWN_TIME; + } + } + } + + /// + /// Gets/Sets the time of last modification of the entry. + /// + public DateTime DateTime { + get { + return cal; + } + set { + lock (this) { + cal = value; +// time = (int) (cal.Millisecond / 1000L); + } + known |= (ushort)KNOWN_TIME; + } + } + + /// + /// Returns the entry name. The path components in the entry are + /// always separated by slashes ('/'). + /// + public string Name { + get { + return name; + } + } + +// /// +// /// Gets/Sets the time of last modification of the entry. +// /// +// /// +// /// the time of last modification of the entry, or -1 if unknown. +// /// +// public long Time { +// get { +// return (known & KNOWN_TIME) != 0 ? time * 1000L : -1; +// } +// set { +// this.time = (int) (value / 1000L); +// this.known |= (ushort)KNOWN_TIME; +// } +// } + + /// + /// Gets/Sets the size of the uncompressed data. + /// + /// + /// if size is not in 0..0xffffffffL + /// + /// + /// the size or -1 if unknown. + /// + public long Size { + get { + return (known & KNOWN_SIZE) != 0 ? (long)size : -1L; + } + set { + if (((ulong)value & 0xFFFFFFFF00000000L) != 0) { + throw new ArgumentOutOfRangeException("size"); + } + this.size = (uint)value; + this.known |= (ushort)KNOWN_SIZE; + } + } + + /// + /// Gets/Sets the size of the compressed data. + /// + /// + /// if csize is not in 0..0xffffffffL + /// + /// + /// the size or -1 if unknown. + /// + public long CompressedSize { + get { + return (known & KNOWN_CSIZE) != 0 ? (long)compressedSize : -1L; + } + set { + if (((ulong)value & 0xffffffff00000000L) != 0) { + throw new ArgumentOutOfRangeException(); + } + this.compressedSize = (uint)value; + this.known |= (ushort)KNOWN_CSIZE; + } + } + + /// + /// Gets/Sets the crc of the uncompressed data. + /// + /// + /// if crc is not in 0..0xffffffffL + /// + /// + /// the crc or -1 if unknown. + /// + public long Crc { + get { + return (known & KNOWN_CRC) != 0 ? crc & 0xffffffffL : -1L; + } + set { + if (((ulong)crc & 0xffffffff00000000L) != 0) { + throw new Exception(); + } + this.crc = (int)value; + this.known |= (ushort)KNOWN_CRC; + } + } + + /// + /// Gets/Sets the compression method. Only DEFLATED and STORED are supported. + /// + /// + /// if method is not supported. + /// + /// + /// the compression method or -1 if unknown. + /// + /// + /// + public CompressionMethod CompressionMethod { + get { + return method; + } + set { + this.method = value; + } + } + + /// + /// Gets/Sets the extra data. + /// + /// + /// if extra is longer than 0xffff bytes. + /// + /// + /// the extra data or null if not set. + /// + public byte[] ExtraData { + get { + return extra; + } + set { + if (value == null) { + this.extra = null; + return; + } + + if (value.Length > 0xffff) { + throw new System.ArgumentOutOfRangeException(); + } + this.extra = value; + try { + int pos = 0; + while (pos < extra.Length) { + int sig = (extra[pos++] & 0xff) | (extra[pos++] & 0xff) << 8; + int len = (extra[pos++] & 0xff) | (extra[pos++] & 0xff) << 8; + if (sig == 0x5455) { + /* extended time stamp, unix format by Rainer Prem */ + int flags = extra[pos]; + if ((flags & 1) != 0) { + int iTime = ((extra[pos+1] & 0xff) | + (extra[pos+2] & 0xff) << 8 | + (extra[pos+3] & 0xff) << 16 | + (extra[pos+4] & 0xff) << 24); + + cal = (new DateTime ( 1970, 1, 1, 0, 0, 0 ) + + new TimeSpan ( 0, 0, 0, iTime, 0 )).ToLocalTime (); + known |= (ushort)KNOWN_TIME; + } + } + pos += len; + } + } catch (Exception) { + /* be lenient */ + return; + } + } + } + + /// + /// Gets/Sets the entry comment. + /// + /// + /// if comment is longer than 0xffff. + /// + /// + /// the comment or null if not set. + /// + public string Comment { + get { + return comment; + } + set { + if (value.Length > 0xffff) { + throw new ArgumentOutOfRangeException(); + } + this.comment = value; + } + } + + /// + /// Gets true, if the entry is a directory. This is solely + /// determined by the name, a trailing slash '/' marks a directory. + /// + public bool IsDirectory { + get { + int nlen = name.Length; + return nlen > 0 && name[nlen - 1] == '/'; + } + } + + /// + /// Creates a copy of this zip entry. + /// + public object Clone() + { + return this.MemberwiseClone(); + } + + /// + /// Gets the string representation of this ZipEntry. This is just + /// the name as returned by getName(). + /// + public override string ToString() + { + return name; + } + + DateTime CalculateDateTime(int dosTime) + { + int sec = 2 * (dosTime & 0x1f); + int min = (dosTime >> 5) & 0x3f; + int hrs = (dosTime >> 11) & 0x1f; + int day = (dosTime >> 16) & 0x1f; + int mon = ((dosTime >> 21) & 0xf); + int year = ((dosTime >> 25) & 0x7f) + 1980; /* since 1900 */ + + return new DateTime(year, mon, day, hrs, min, sec); + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipFile.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipFile.cs new file mode 100644 index 000000000..c9ec2f7e7 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipFile.cs @@ -0,0 +1,521 @@ +// ZipFile.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.Collections; +using System.IO; +using System.Text; + +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; +using BCG.Utilities.SharpZipLib.Zip.Compression; + +namespace BCG.Utilities.SharpZipLib.Zip { + + /// + /// This class represents a Zip archive. You can ask for the contained + /// entries, or get an input stream for a file entry. The entry is + /// automatically decompressed. + /// + /// This class is thread safe: You can open input streams for arbitrary + /// entries in different threads. + /// + /// author of the original java version : Jochen Hoenicke + /// + /// + /// using System; + /// using System.Text; + /// using System.Collections; + /// using System.IO; + /// + /// using NZlib.Zip; + /// + /// class MainClass + /// { + /// static public void Main(string[] args) + /// { + /// ZipFile zFile = new ZipFile(args[0]); + /// Console.WriteLine("Listing of : " + zFile.Name); + /// Console.WriteLine(""); + /// Console.WriteLine("Raw Size Size Date Time Name"); + /// Console.WriteLine("-------- -------- -------- ------ ---------"); + /// foreach (ZipEntry e in zFile) { + /// DateTime d = e.DateTime; + /// Console.WriteLine("{0, -10}{1, -10}{2} {3} {4}", e.Size, e.CompressedSize, + /// d.ToString("dd-MM-yy"), d.ToString("t"), + /// e.Name); + /// } + /// } + /// } + /// + public class ZipFile : IEnumerable + { + string name; + Stream baseStream; + ZipEntry[] entries; + + /// + /// Opens a Zip file with the given name for reading. + /// + /// + /// IOException if a i/o error occured. + /// + /// + /// if the file doesn't contain a valid zip archive. + /// + public ZipFile(string name) : this(File.OpenRead(name)) + { + } + + /// + /// Opens a Zip file reading the given FileStream + /// + /// + /// IOException if a i/o error occured. + /// + /// + /// if the file doesn't contain a valid zip archive. + /// + public ZipFile(FileStream file) + { + this.baseStream = file; + this.name = file.Name; + ReadEntries(); + } + + /// + /// Opens a Zip file reading the given Stream + /// + /// + /// IOException if a i/o error occured. + /// + /// + /// if the file doesn't contain a valid zip archive. + /// + public ZipFile(Stream baseStream) + { + this.baseStream = baseStream; + this.name = null; + ReadEntries(); + } + + + /// + /// Read an unsigned short in little endian byte order. + /// + /// + /// if a i/o error occured. + /// + /// + /// if the file ends prematurely + /// + int ReadLeShort() + { + return baseStream.ReadByte() | baseStream.ReadByte() << 8; + } + + /// + /// Read an int in little endian byte order. + /// + /// + /// if a i/o error occured. + /// + /// + /// if the file ends prematurely + /// + int ReadLeInt() + { + return ReadLeShort() | ReadLeShort() << 16; + } + + /// + /// Read the central directory of a zip file and fill the entries + /// array. This is called exactly once by the constructors. + /// + /// + /// if a i/o error occured. + /// + /// + /// if the central directory is malformed + /// + void ReadEntries() + { + /* Search for the End Of Central Directory. When a zip comment is + * present the directory may start earlier. + * FIXME: This searches the whole file in a very slow manner if the + * file isn't a zip file. + */ + long pos = baseStream.Length - ZipConstants.ENDHDR; + do { + if (pos < 0) { + throw new ZipException("central directory not found, probably not a zip file"); + } + baseStream.Seek(pos--, SeekOrigin.Begin); + } while (ReadLeInt() != ZipConstants.ENDSIG); + + long oldPos = baseStream.Position; + baseStream.Position += ZipConstants.ENDTOT - ZipConstants.ENDNRD; + + if (baseStream.Position - oldPos != ZipConstants.ENDTOT - ZipConstants.ENDNRD) { + throw new EndOfStreamException(); + } + int count = ReadLeShort(); + + oldPos = baseStream.Position; + baseStream.Position += ZipConstants.ENDOFF - ZipConstants.ENDSIZ; + + if (baseStream.Position - oldPos != ZipConstants.ENDOFF - ZipConstants.ENDSIZ) { + throw new EndOfStreamException(); + } + + int centralOffset = ReadLeInt(); + + entries = new ZipEntry[count]; + baseStream.Seek(centralOffset, SeekOrigin.Begin); + for (int i = 0; i < count; i++) { + if (ReadLeInt() != ZipConstants.CENSIG) { + throw new ZipException("Wrong Central Directory signature"); + } + + oldPos = baseStream.Position; + baseStream.Position += ZipConstants.CENHOW - ZipConstants.CENVEM; + + if (baseStream.Position - oldPos != ZipConstants.CENHOW - ZipConstants.CENVEM) { + throw new EndOfStreamException(); + } + int method = ReadLeShort(); + int dostime = ReadLeInt(); + int crc = ReadLeInt(); + int csize = ReadLeInt(); + int size = ReadLeInt(); + int nameLen = ReadLeShort(); + int extraLen = ReadLeShort(); + int commentLen = ReadLeShort(); + + oldPos = baseStream.Position; + baseStream.Position += ZipConstants.CENOFF - ZipConstants.CENDSK; + if (baseStream.Position - oldPos != ZipConstants.CENOFF - ZipConstants.CENDSK) { + throw new EndOfStreamException(); + } + int offset = ReadLeInt(); + + byte[] buffer = new byte[Math.Max(nameLen, commentLen)]; + + baseStream.Read(buffer, 0, nameLen); + string name = ZipConstants.ConvertToString(buffer); + + ZipEntry entry = new ZipEntry(name); + entry.CompressionMethod = (CompressionMethod)method; + entry.Crc = crc & 0xffffffffL; + entry.Size = size & 0xffffffffL; + entry.CompressedSize = csize & 0xffffffffL; + entry.DosTime = dostime; + if (extraLen > 0) { + byte[] extra = new byte[extraLen]; + baseStream.Read(extra, 0, extraLen); + entry.ExtraData = extra; + } + if (commentLen > 0) { + baseStream.Read(buffer, 0, commentLen); + entry.Comment = ZipConstants.ConvertToString(buffer); + } + entry.zipFileIndex = i; + entry.offset = offset; + entries[i] = entry; + } + } + + /// + /// Closes the ZipFile. This also closes all input streams given by + /// this class. After this is called, no further method should be + /// called. + /// + /// + /// if a i/o error occured. + /// + public void Close() + { + entries = null; + lock(baseStream) { + baseStream.Close(); + } + } + + /// + /// Returns an IEnumerator of all Zip entries in this Zip file. + /// + public IEnumerator GetEnumerator() + { + if (entries == null) { + throw new InvalidOperationException("ZipFile has closed"); + } + + return new ZipEntryEnumeration(entries); + } + + int GetEntryIndex(string name) + { + for (int i = 0; i < entries.Length; i++) { + if (name.Equals(entries[i].Name)) { + return i; + } + } + return -1; + } + + /// + /// Searches for a zip entry in this archive with the given name. + /// + /// + /// the name. May contain directory components separated by slashes ('/'). + /// + /// + /// the zip entry, or null if no entry with that name exists. + /// + public ZipEntry GetEntry(string name) + { + if (entries == null) { + throw new InvalidOperationException("ZipFile has closed"); + } + int index = GetEntryIndex(name); + return index >= 0 ? (ZipEntry) entries[index].Clone() : null; + } + + /// + /// Checks, if the local header of the entry at index i matches the + /// central directory, and returns the offset to the data. + /// + /// + /// the start offset of the (compressed) data. + /// + /// + /// if a i/o error occured. + /// + /// + /// if the local header doesn't match the central directory header + /// + long CheckLocalHeader(ZipEntry entry) + { + lock(baseStream) { + baseStream.Seek(entry.offset, SeekOrigin.Begin); + if (ReadLeInt() != ZipConstants.LOCSIG) { + throw new ZipException("Wrong Local header signature"); + } + + /* skip version and flags */ + long oldPos = baseStream.Position; + baseStream.Position += ZipConstants.LOCHOW - ZipConstants.LOCVER; + if (baseStream.Position - oldPos != ZipConstants.LOCHOW - ZipConstants.LOCVER) { + throw new EndOfStreamException(); + } + + if (entry.CompressionMethod != (CompressionMethod)ReadLeShort()) { + throw new ZipException("Compression method mismatch"); + } + + /* Skip time, crc, size and csize */ + oldPos = baseStream.Position; + baseStream.Position += ZipConstants.LOCNAM - ZipConstants.LOCTIM; + + if (baseStream.Position - oldPos != ZipConstants.LOCNAM - ZipConstants.LOCTIM) { + throw new EndOfStreamException(); + } + + if (entry.Name.Length != ReadLeShort()) { + throw new ZipException("file name length mismatch"); + } + + int extraLen = entry.Name.Length + ReadLeShort(); + return entry.offset + ZipConstants.LOCHDR + extraLen; + } + } + + /// + /// Creates an input stream reading the given zip entry as + /// uncompressed data. Normally zip entry should be an entry + /// returned by GetEntry(). + /// + /// + /// the input stream. + /// + /// + /// if a i/o error occured. + /// + /// + /// if the Zip archive is malformed. + /// + public Stream GetInputStream(ZipEntry entry) + { + if (entries == null) { + throw new InvalidOperationException("ZipFile has closed"); + } + + int index = entry.zipFileIndex; + if (index < 0 || index >= entries.Length || entries[index].Name != entry.Name) { + index = GetEntryIndex(entry.Name); + if (index < 0) { + throw new IndexOutOfRangeException(); + } + } + + long start = CheckLocalHeader(entries[index]); + CompressionMethod method = entries[index].CompressionMethod; + Stream istr = new PartialInputStream(baseStream, start, entries[index].CompressedSize); + switch (method) { + case CompressionMethod.Stored: + return istr; + case CompressionMethod.Deflated: + return new InflaterInputStream(istr, new Inflater(true)); + default: + throw new ZipException("Unknown compression method " + method); + } + } + + /// + /// Returns the name of this zip file. + /// + public string Name { + get { + return name; + } + } + + /// + /// Returns the number of entries in this zip file. + /// + public int Size { + get { + try { + return entries.Length; + } catch (Exception) { + throw new InvalidOperationException("ZipFile has closed"); + } + } + } + + class ZipEntryEnumeration : IEnumerator + { + ZipEntry[] array; + int ptr = -1; + + public ZipEntryEnumeration(ZipEntry[] arr) + { + array = arr; + } + + public object Current { + get { + return array[ptr]; + } + } + + public void Reset() + { + ptr = -1; + } + + public bool MoveNext() + { + return (++ptr < array.Length); + } + } + + class PartialInputStream : InflaterInputStream + { + Stream baseStream; + long filepos, end; + + public PartialInputStream(Stream baseStream, long start, long len) : base(baseStream) + { + this.baseStream = baseStream; + filepos = start; + end = start + len; + } + + public override int Available { + get { + long amount = end - filepos; + if (amount > Int32.MaxValue) { + return Int32.MaxValue; + } + + return (int) amount; + } + } + + public override int ReadByte() + { + if (filepos == end) { + return -1; + } + lock(baseStream) { + baseStream.Seek(filepos++, SeekOrigin.Begin); + return baseStream.ReadByte(); + } + } + + public override int Read(byte[] b, int off, int len) + { + if (len > end - filepos) { + len = (int) (end - filepos); + if (len == 0) { + return -1; + } + } + lock(baseStream) { + baseStream.Seek(filepos, SeekOrigin.Begin); + int count = baseStream.Read(b, off, len); + if (count > 0) { + filepos += len; + } + return count; + } + } + + public long SkipBytes(long amount) + { + if (amount < 0) { + throw new ArgumentOutOfRangeException(); + } + if (amount > end - filepos) { + amount = end - filepos; + } + filepos += amount; + return amount; + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipInputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipInputStream.cs new file mode 100644 index 000000000..10f231efe --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipInputStream.cs @@ -0,0 +1,431 @@ +// ZipInputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.Text; +using System.IO; + +using BCG.Utilities.SharpZipLib.Checksums; +using BCG.Utilities.SharpZipLib.Zip.Compression; +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; + +namespace BCG.Utilities.SharpZipLib.Zip { + + /// + /// This is a FilterInputStream that reads the files baseInputStream an zip archive + /// one after another. It has a special method to get the zip entry of + /// the next file. The zip entry contains information about the file name + /// size, compressed size, CRC, etc. + /// It includes support for STORED and DEFLATED entries. + /// + /// author of the original java version : Jochen Hoenicke + /// + /// This sample shows how to read a zip file + /// + /// using System; + /// using System.Text; + /// using System.IO; + /// + /// using NZlib.Zip; + /// + /// class MainClass + /// { + /// public static void Main(string[] args) + /// { + /// ZipInputStream s = new ZipInputStream(File.OpenRead(args[0])); + /// + /// ZipEntry theEntry; + /// while ((theEntry = s.GetNextEntry()) != null) { + /// Console.WriteLine("File " + theEntry.Name); + /// int size = 2048; + /// byte[] data = new byte[2048]; + /// + /// Console.Write("Show contents (y/n) ?"); + /// if (Console.ReadLine() == "y") { + /// while (true) { + /// size = s.Read(data, 0, data.Length); + /// if (size > 0) { + /// Console.Write(new ASCIIEncoding().GetString(data, 0, size)); + /// } else { + /// break; + /// } + /// } + /// } + /// Console.WriteLine(); + /// } + /// s.Close(); + /// } + /// } + /// + /// + public class ZipInputStream : InflaterInputStream + { + private Crc32 crc = new Crc32(); + private ZipEntry entry = null; + + private long csize; + private long size; + private int method; + private int flags; + private long avail; + + /// + /// Creates a new Zip input stream, reading a zip archive. + /// + public ZipInputStream(Stream baseInputStream) : base(baseInputStream, new Inflater(true)) + { + + } + + private void FillBuf() + { + avail = len = baseInputStream.Read(buf, 0, buf.Length); + } + + private int ReadBuf(byte[] outBuf, int offset, int length) + { + if (avail <= 0) { + FillBuf(); + if (avail <= 0) { + return -1; + } + } + if (length > avail) { + length = (int)avail; + } + System.Array.Copy(buf, len - (int)avail, outBuf, offset, length); + avail -= length; + return length; + } + + private void ReadFully(byte[] outBuf) + { + int off = 0; + int len = outBuf.Length; + while (len > 0) { + int count = ReadBuf(outBuf, off, len); + if (count == -1) { + throw new Exception(); + } + off += count; + len -= count; + } + } + + private int ReadLeByte() + { + if (avail <= 0) { + FillBuf(); + if (avail <= 0) { + throw new ZipException("EOF in header"); + } + } + return buf[len - avail--] & 0xff; + } + + /// + /// Read an unsigned short baseInputStream little endian byte order. + /// + private int ReadLeShort() + { + return ReadLeByte() | (ReadLeByte() << 8); + } + + /// + /// Read an int baseInputStream little endian byte order. + /// + private int ReadLeInt() + { + return ReadLeShort() | (ReadLeShort() << 16); + } + + /// + /// Read an int baseInputStream little endian byte order. + /// + private long ReadLeLong() + { + return ReadLeInt() | (ReadLeInt() << 32); + } + + /// + /// Open the next entry from the zip archive, and return its description. + /// If the previous entry wasn't closed, this method will close it. + /// + public ZipEntry GetNextEntry() + { + if (crc == null) { + throw new InvalidOperationException("Closed."); + } + if (entry != null) { + CloseEntry(); + } + + int header = ReadLeInt(); + if (header == ZipConstants.CENSIG) { + /* Central Header reached. */ + Close(); + return null; + } + if (header != ZipConstants.LOCSIG) { + throw new ZipException("Wrong Local header signature" + header); + } + + short version = (short)ReadLeShort(); + + flags = ReadLeShort(); + method = ReadLeShort(); + int dostime = ReadLeInt(); + int crc2 = ReadLeInt(); + csize = ReadLeInt(); + size = ReadLeInt(); + int nameLen = ReadLeShort(); + int extraLen = ReadLeShort(); + + if (method == ZipOutputStream.STORED && csize != size) { + throw new ZipException("Stored, but compressed != uncompressed"); + } + + byte[] buffer = new byte[nameLen]; + ReadFully(buffer); + + string name = ZipConstants.ConvertToString(buffer); + + entry = new ZipEntry(name); + + entry.Version = (ushort)version; + if (method != 0 && method != 8) { + throw new ZipException("unknown compression method " + method); + } + entry.CompressionMethod = (CompressionMethod)method; + + if ((flags & 8) == 0) { + entry.Crc = crc2 & 0xFFFFFFFFL; + entry.Size = size & 0xFFFFFFFFL; + entry.CompressedSize = csize & 0xFFFFFFFFL; + } + entry.DosTime = dostime; + if (extraLen > 0) { + byte[] extra = new byte[extraLen]; + ReadFully(extra); + entry.ExtraData = extra; + } + + if (method == ZipOutputStream.DEFLATED && avail > 0) { + System.Array.Copy(buf, len - (int)avail, buf, 0, (int)avail); + len = (int)avail; + avail = 0; + inf.SetInput(buf, 0, len); + } + return entry; + } + + private void ReadDataDescr() + { + if (ReadLeInt() != ZipConstants.EXTSIG) { + throw new ZipException("Data descriptor signature not found"); + } + entry.Crc = ReadLeInt() & 0xFFFFFFFFL; + csize = ReadLeInt(); + size = ReadLeInt(); + entry.Size = size & 0xFFFFFFFFL; + entry.CompressedSize = csize & 0xFFFFFFFFL; + } + + /// + /// Closes the current zip entry and moves to the next one. + /// + public void CloseEntry() + { + if (crc == null) { + throw new InvalidOperationException("Closed."); + } + if (entry == null) { + return; + } + + if (method == ZipOutputStream.DEFLATED) { + if ((flags & 8) != 0) { + /* We don't know how much we must skip, read until end. */ + byte[] tmp = new byte[2048]; + while (Read(tmp, 0, tmp.Length) > 0) + ; + /* read will close this entry */ + return; + } + csize -= inf.TotalIn; + avail = inf.RemainingInput; + } + if (avail > csize && csize >= 0) { + avail -= csize; + } else { + csize -= avail; + avail = 0; + while (csize != 0) { + int skipped = (int)base.Skip(csize & 0xFFFFFFFFL); + + if (skipped <= 0) { + throw new ZipException("zip archive ends early."); + } + + csize -= skipped; + } + } + + size = 0; + crc.Reset(); + if (method == ZipOutputStream.DEFLATED) { + inf.Reset(); + } + entry = null; + } + + public override int Available { + get { + return entry != null ? 1 : 0; + } + } + + /// + /// Reads a byte from the current zip entry. + /// + /// + /// the byte or -1 on EOF. + /// + /// + /// IOException if a i/o error occured. + /// + /// + /// ZipException if the deflated stream is corrupted. + /// + public override int ReadByte() + { + byte[] b = new byte[1]; + if (Read(b, 0, 1) <= 0) { + return -1; + } + return b[0] & 0xff; + } + + /// + /// Reads a block of bytes from the current zip entry. + /// + /// + /// the number of bytes read (may be smaller, even before EOF), or -1 on EOF. + /// + /// + /// IOException if a i/o error occured. + /// ZipException if the deflated stream is corrupted. + /// + public override int Read(byte[] b, int off, int len) + { + if (crc == null) { + throw new InvalidOperationException("Closed."); + } + if (entry == null) { + return -1; + } + bool finished = false; + + switch (method) { + case ZipOutputStream.DEFLATED: + len = base.Read(b, off, len); + if (len < 0) { + if (!inf.IsFinished) { + throw new ZipException("Inflater not finished!?"); + } + avail = inf.RemainingInput; + if ((flags & 8) != 0) { + ReadDataDescr(); + } + + if (inf.TotalIn != csize || inf.TotalOut != size) { + throw new ZipException("size mismatch: " + csize + ";" + size + " <-> " + inf.TotalIn + ";" + inf.TotalOut); + } + inf.Reset(); + finished = true; + } + break; + + case ZipOutputStream.STORED: + + if (len > csize && csize >= 0) { + len = (int)csize; + } + len = ReadBuf(b, off, len); + if (len > 0) { + csize -= len; + size -= len; + } + + if (csize == 0) { + finished = true; + } else { + if (len < 0) { + throw new ZipException("EOF in stored block"); + } + } + break; + } + + if (len > 0) { + crc.Update(b, off, len); + } + + if (finished) { + if ((crc.Value & 0xFFFFFFFFL) != entry.Crc) { + throw new ZipException("CRC mismatch"); + } + crc.Reset(); + entry = null; + } + return len; + } + + /// + /// Closes the zip file. + /// + /// + /// if a i/o error occured. + /// + public override void Close() + { + base.Close(); + crc = null; + entry = null; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipOutputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipOutputStream.cs new file mode 100644 index 000000000..97676b06c --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipOutputStream.cs @@ -0,0 +1,509 @@ +// ZipOutputStream.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; +using System.IO; +using System.Collections; +using System.Text; + +using BCG.Utilities.SharpZipLib.Checksums; +using BCG.Utilities.SharpZipLib.Zip.Compression; +using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; + +namespace BCG.Utilities.SharpZipLib.Zip { + + /// + /// This is a FilterOutputStream that writes the files into a zip + /// archive one after another. It has a special method to start a new + /// zip entry. The zip entries contains information about the file name + /// size, compressed size, CRC, etc. + /// + /// It includes support for STORED and DEFLATED entries. + /// This class is not thread safe. + /// + /// author of the original java version : Jochen Hoenicke + /// + /// This sample shows how to create a zip file + /// + /// using System; + /// using System.IO; + /// + /// using NZlib.Zip; + /// + /// class MainClass + /// { + /// public static void Main(string[] args) + /// { + /// string[] filenames = Directory.GetFiles(args[0]); + /// + /// ZipOutputStream s = new ZipOutputStream(File.Create(args[1])); + /// + /// s.SetLevel(5); // 0 - store only to 9 - means best compression + /// + /// foreach (string file in filenames) { + /// FileStream fs = File.OpenRead(file); + /// + /// byte[] buffer = new byte[fs.Length]; + /// fs.Read(buffer, 0, buffer.Length); + /// + /// ZipEntry entry = new ZipEntry(file); + /// + /// s.PutNextEntry(entry); + /// + /// s.Write(buffer, 0, buffer.Length); + /// + /// } + /// + /// s.Finish(); + /// s.Close(); + /// } + /// } + /// + /// + public class ZipOutputStream : DeflaterOutputStream + { + private ArrayList entries = new ArrayList(); + private Crc32 crc = new Crc32(); + private ZipEntry curEntry = null; + + private CompressionMethod curMethod; + private int size; + private int offset = 0; + + private byte[] zipComment = new byte[0]; + private int defaultMethod = DEFLATED; + + + /// + /// Our Zip version is hard coded to 1.0 resp. 2.0 + /// + private const int ZIP_STORED_VERSION = 10; + private const int ZIP_DEFLATED_VERSION = 20; + + /// + /// Compression method. This method doesn't compress at all. + /// + public const int STORED = 0; + + /// + /// Compression method. This method uses the Deflater. + /// + public const int DEFLATED = 8; + + /// + /// Creates a new Zip output stream, writing a zip archive. + /// + /// + /// the output stream to which the zip archive is written. + /// + public ZipOutputStream(Stream baseOutputStream) : base(baseOutputStream, new Deflater(Deflater.DEFAULT_COMPRESSION, true)) + { + } + + /// + /// Set the zip file comment. + /// + /// + /// the comment. + /// + /// + /// if UTF8 encoding of comment is longer than 0xffff bytes. + /// + public void SetComment(string comment) + { + byte[] commentBytes = ZipConstants.ConvertToArray(comment); + if (commentBytes.Length > 0xffff) { + throw new ArgumentException("Comment too long."); + } + zipComment = commentBytes; + } + + /// + /// Sets default compression method. If the Zip entry specifies + /// another method its method takes precedence. + /// + /// + /// the method. + /// + /// + /// if method is not supported. + /// + public void SetMethod(int method) + { + if (method != STORED && method != DEFLATED) { + throw new ArgumentException("Method not supported."); + } + defaultMethod = method; + } + + /// + /// Sets default compression level. The new level will be activated + /// immediately. + /// + /// + /// if level is not supported. + /// + /// + public void SetLevel(int level) + { + def.SetLevel(level); + } + + /// + /// Write an unsigned short in little endian byte order. + /// + private void WriteLeShort(int value) + { + baseOutputStream.WriteByte((byte)value); + baseOutputStream.WriteByte((byte)(value >> 8)); + } + + /// + /// Write an int in little endian byte order. + /// + private void WriteLeInt(int value) + { + WriteLeShort(value); + WriteLeShort(value >> 16); + } + + /// + /// Write an int in little endian byte order. + /// + private void WriteLeLong(long value) + { + WriteLeInt((int)value); + WriteLeInt((int)(value >> 32)); + } + + + bool shouldWriteBack = false; + long seekPos = -1; + /// + /// Starts a new Zip entry. It automatically closes the previous + /// entry if present. If the compression method is stored, the entry + /// must have a valid size and crc, otherwise all elements (except + /// name) are optional, but must be correct if present. If the time + /// is not set in the entry, the current time is used. + /// + /// + /// the entry. + /// + /// + /// if an I/O error occured. + /// + /// + /// if stream was finished + /// + public void PutNextEntry(ZipEntry entry) + { + if (entries == null) { + throw new InvalidOperationException("ZipOutputStream was finished"); + } + + CompressionMethod method = entry.CompressionMethod; + int flags = 0; + + switch (method) { + case CompressionMethod.Stored: + if (entry.CompressedSize >= 0) { + if (entry.Size < 0) { + entry.Size = entry.CompressedSize; + } else if (entry.Size != entry.CompressedSize) { + throw new ZipException("Method STORED, but compressed size != size"); + } + } else { + entry.CompressedSize = entry.Size; + } + + if (entry.Size < 0) { + throw new ZipException("Method STORED, but size not set"); + } else if (entry.Crc < 0) { + throw new ZipException("Method STORED, but crc not set"); + } + break; + case CompressionMethod.Deflated: + if (entry.CompressedSize < 0 || entry.Size < 0 || entry.Crc < 0) { + flags |= 8; + } + break; + } + + if (curEntry != null) { + CloseEntry(); + } + +// if (entry.DosTime < 0) { +// entry.Time = System.Environment.TickCount; +// } + + entry.flags = flags; + entry.offset = offset; + entry.CompressionMethod = (CompressionMethod)method; + + curMethod = method; + // Write the local file header + WriteLeInt(ZipConstants.LOCSIG); + + // write ZIP version + WriteLeShort(method == CompressionMethod.Stored ? ZIP_STORED_VERSION : ZIP_DEFLATED_VERSION); + if ((flags & 8) == 0) { + WriteLeShort(flags); + WriteLeShort((byte)method); + WriteLeInt(entry.DosTime); + WriteLeInt((int)entry.Crc); + WriteLeInt((int)entry.CompressedSize); + WriteLeInt((int)entry.Size); + } else { + if (baseOutputStream.CanSeek) { + shouldWriteBack = true; + WriteLeShort((short)(flags & ~8)); + } else { + shouldWriteBack = false; + WriteLeShort(flags); + } + WriteLeShort((byte)method); + WriteLeInt(entry.DosTime); + seekPos = baseOutputStream.Position; + WriteLeInt(0); + WriteLeInt(0); + WriteLeInt(0); + } + byte[] name = ZipConstants.ConvertToArray(entry.Name); + + if (name.Length > 0xFFFF) { + throw new ZipException("Name too long."); + } + byte[] extra = entry.ExtraData; + if (extra == null) { + extra = new byte[0]; + } + if (extra.Length > 0xFFFF) { + throw new ZipException("Extra data too long."); + } + + WriteLeShort(name.Length); + WriteLeShort(extra.Length); + baseOutputStream.Write(name, 0, name.Length); + baseOutputStream.Write(extra, 0, extra.Length); + + offset += ZipConstants.LOCHDR + name.Length + extra.Length; + + /* Activate the entry. */ + curEntry = entry; + crc.Reset(); + if (method == CompressionMethod.Deflated) { + def.Reset(); + } + size = 0; + } + + /// + /// Closes the current entry. + /// + /// + /// if an I/O error occured. + /// + /// + /// if no entry is active. + /// + public void CloseEntry() + { + if (curEntry == null) { + throw new InvalidOperationException("No open entry"); + } + + /* First finish the deflater, if appropriate */ + if (curMethod == CompressionMethod.Deflated) { + base.Finish(); + } + + int csize = curMethod == CompressionMethod.Deflated ? def.TotalOut : size; + + if (curEntry.Size < 0) { + curEntry.Size = size; + } else if (curEntry.Size != size) { + throw new ZipException("size was " + size + + ", but I expected " + curEntry.Size); + } + + if (curEntry.CompressedSize < 0) { + curEntry.CompressedSize = csize; + } else if (curEntry.CompressedSize != csize) { + throw new ZipException("compressed size was " + csize + + ", but I expected " + curEntry.CompressedSize); + } + + if (curEntry.Crc < 0) { + curEntry.Crc = crc.Value; + } else if (curEntry.Crc != crc.Value) { + throw new ZipException("crc was " + crc.Value + + ", but I expected " + + curEntry.Crc); + } + + offset += csize; + + /* Now write the data descriptor entry if needed. */ + if (curMethod == CompressionMethod.Deflated && (curEntry.flags & 8) != 0) { + if (shouldWriteBack) { + long curPos = baseOutputStream.Position; + baseOutputStream.Seek(seekPos, SeekOrigin.Begin); + WriteLeInt((int)curEntry.Crc); + WriteLeInt((int)curEntry.CompressedSize); + WriteLeInt((int)curEntry.Size); + baseOutputStream.Seek(curPos, SeekOrigin.Begin); + shouldWriteBack = false; + } else { + WriteLeInt(ZipConstants.EXTSIG); + WriteLeInt((int)curEntry.Crc); + WriteLeInt((int)curEntry.CompressedSize); + WriteLeInt((int)curEntry.Size); + offset += ZipConstants.EXTHDR; + } + } + + entries.Add(curEntry); + curEntry = null; + } + + + /// + /// Writes the given buffer to the current entry. + /// + /// + /// if an I/O error occured. + /// + /// + /// if no entry is active. + /// + public override void Write(byte[] b, int off, int len) + { + if (curEntry == null) { + throw new InvalidOperationException("No open entry."); + } + + switch (curMethod) { + case CompressionMethod.Deflated: + base.Write(b, off, len); + break; + case CompressionMethod.Stored: + baseOutputStream.Write(b, off, len); + break; + } + + crc.Update(b, off, len); + size += len; + } + + + /// + /// Finishes the stream. This will write the central directory at the + /// end of the zip file and flush the stream. + /// + /// + /// if an I/O error occured. + /// + public override void Finish() + { + if (entries == null) { + return; + } + if (curEntry != null) { + CloseEntry(); + } + + int numEntries = 0; + int sizeEntries = 0; + + foreach (ZipEntry entry in entries) { + // TODO : check the appnote file for compilance with the central directory standard + CompressionMethod method = entry.CompressionMethod; + WriteLeInt(ZipConstants.CENSIG); + WriteLeShort(method == CompressionMethod.Stored ? ZIP_STORED_VERSION : ZIP_DEFLATED_VERSION); + WriteLeShort(method == CompressionMethod.Stored ? ZIP_STORED_VERSION : ZIP_DEFLATED_VERSION); + WriteLeShort(entry.flags); + WriteLeShort((short)method); + WriteLeInt(entry.DosTime); + WriteLeInt((int)entry.Crc); + WriteLeInt((int)entry.CompressedSize); + WriteLeInt((int)entry.Size); + + byte[] name = ZipConstants.ConvertToArray(entry.Name); + + if (name.Length > 0xffff) { + throw new ZipException("Name too long."); + } + byte[] extra = entry.ExtraData; + if (extra == null) { + extra = new byte[0]; + } + string strComment = entry.Comment; + byte[] comment = strComment != null ? ZipConstants.ConvertToArray(strComment) : new byte[0]; + if (comment.Length > 0xffff) { + throw new ZipException("Comment too long."); + } + + WriteLeShort(name.Length); + WriteLeShort(extra.Length); + WriteLeShort(comment.Length); + WriteLeShort(0); /* disk number */ + WriteLeShort(0); /* internal file attr */ + WriteLeInt(0); /* external file attr */ + WriteLeInt(entry.offset); + + baseOutputStream.Write(name, 0, name.Length); + baseOutputStream.Write(extra, 0, extra.Length); + baseOutputStream.Write(comment, 0, comment.Length); + ++numEntries; + sizeEntries += ZipConstants.CENHDR + name.Length + extra.Length + comment.Length; + } + + WriteLeInt(ZipConstants.ENDSIG); + WriteLeShort(0); /* disk number */ + WriteLeShort(0); /* disk with start of central dir */ + WriteLeShort(numEntries); + WriteLeShort(numEntries); + WriteLeInt(sizeEntries); + WriteLeInt(offset); + WriteLeShort(zipComment.Length); + baseOutputStream.Write(zipComment, 0, zipComment.Length); + baseOutputStream.Flush(); + entries = null; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/ZipException.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/ZipException.cs new file mode 100644 index 000000000..a0cbace7e --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/ZipException.cs @@ -0,0 +1,61 @@ +// ZipException.cs +// Copyright (C) 2001 Mike Krueger +// +// This file was translated from java, it was part of the GNU Classpath +// Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc. +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// +// Linking this library statically or dynamically with other modules is +// making a combined work based on this library. Thus, the terms and +// conditions of the GNU General Public License cover the whole +// combination. +// +// As a special exception, the copyright holders of this library give you +// permission to link this library with independent modules to produce an +// executable, regardless of the license terms of these independent +// modules, and to copy and distribute the resulting executable under +// terms of your choice, provided that you also meet, for each linked +// independent module, the terms and conditions of the license of that +// module. An independent module is a module which is not derived from +// or based on this library. If you modify this library, you may extend +// this exception to your version of the library, but you are not +// obligated to do so. If you do not wish to do so, delete this +// exception statement from your version. + +using System; + +namespace BCG.Utilities.SharpZipLib { + + /// + /// Is thrown during the creation or input of a zip file. + /// + public class ZipException : Exception + { + /// + /// Initializes a new instance of the ZipException class with default properties. + /// + public ZipException() + { + } + + /// + /// Initializes a new instance of the ZipException class with a specified error message. + /// + public ZipException(string msg) : base(msg) + { + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs b/src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs new file mode 100644 index 000000000..948e5765d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs @@ -0,0 +1,27 @@ +using System; +using System.IO; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; + +namespace BCG +{ + public class GenericFormatter : IGenericFormatter where F : IFormatter, new () + { + IFormatter myFormatter = new F (); + + public T Deserialize (Stream serializationStream) + { + #pragma warning disable + return (T) myFormatter.Deserialize (serializationStream); + #pragma warning restore SYSLIB0011 + } + public void Serialize (Stream serializationStream, T graph) + { + #pragma warning disable + myFormatter.Serialize (serializationStream, graph); + #pragma warning restore SYSLIB0011 + } + } + + public class GenericBinaryFormatter : GenericFormatter { } +} diff --git a/src/DataStreamSuite/PubSub/Misc/HighPerformanceTimer.cs b/src/DataStreamSuite/PubSub/Misc/HighPerformanceTimer.cs new file mode 100644 index 000000000..05e53e502 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/HighPerformanceTimer.cs @@ -0,0 +1,119 @@ +using System; +using System.Runtime.InteropServices; + +namespace mbts +{ + /// + /// Provides a timer, much more accurate than using timers based on System.DateTime. It does not accumulate time, as the + /// stress was on small time intervals. + /// + public class HighPerformanceTimer + { + [DllImport("kernel32.dll")] + public extern static short QueryPerformanceCounter(ref long x); + [DllImport("kernel32.dll")] + public extern static short QueryPerformanceFrequency(ref long x); + + private long myStartCount = 0, myEndCount = 0, myFreq = 0, myTotalCount = 0; + private long myZeroTime; + /// + /// This a static instance that can be shared across a running process. + /// The only member function that is thread-safe right now + /// is 'runningMilliseconds' + /// + public readonly static HighPerformanceTimer sharedTimer = new HighPerformanceTimer (); + + + public HighPerformanceTimer() + { + QueryPerformanceFrequency(ref myFreq); + if (myFreq == 0) + throw new ApplicationException ("High resolution timers not supported by this machine's hardware"); + QueryPerformanceCounter(ref myZeroTime); + } + /// + /// starts measuring the time + /// + public void start() + { + QueryPerformanceCounter (ref myStartCount); + } + + public double runningMilliseconds() + { + double rv = 0; + lock (this) + { + long t = 0; + QueryPerformanceCounter (ref t); + rv = (t - myZeroTime) * 1000.0 / myFreq; + } + return rv; + } + + public double toMilliseconds(long count) + { + lock (this) + return count * 1000.0 / myFreq; + } + public double getRunningMilliseconds (long count) + { + double rv = 0; + lock (this) + { + rv = (count - myZeroTime) * 1000.0 / myFreq; + } + return rv; + } + + public long nowCount + { + get + { + long t = 0; + QueryPerformanceCounter (ref t); + return t; + } + } + + /// + /// stops recording time + /// + public void stop() + { + QueryPerformanceCounter (ref myEndCount); + myTotalCount += myEndCount - myStartCount; + } + + /// + /// total milliseconds, ie aggregated time of all intervals + /// + public double totalMilliseconds {get{return myTotalCount * 1000.0 / myFreq ; }} + + public double getMilliseconds (long count) + { + return count * 1000.0 / myFreq; + } + + public long ticks {get{return myEndCount - myStartCount;}} + + /// + /// time between the last start-stop + /// + public double lastIntervalTime {get{return (myEndCount - myStartCount) * 1000.0 / myFreq ; }} + + /// + /// Resets the running time + /// + public void reset () + { + QueryPerformanceCounter(ref myZeroTime); + myTotalCount = 0; + } + + public long freq + { + get { return myFreq; } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/ISyncQueue.cs b/src/DataStreamSuite/PubSub/Misc/ISyncQueue.cs new file mode 100644 index 000000000..4eca273fb --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/ISyncQueue.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + /// + /// interface added to support the new implementation + /// of the sync queue (so that it can be switched back + /// n forth by a config switch) + /// + /// + public interface ISyncQueue + { + void enqueue(T t); + T dequeue(int timeout); + T dequeue(); + List dequeueMultipleElements(int timeout, int maxElements); + List dequeueMultipleElements(TimeSpan ts, int maxElements); + int getCount(); + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/IWorker.cs b/src/DataStreamSuite/PubSub/Misc/IWorker.cs new file mode 100644 index 000000000..10f46dc30 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/IWorker.cs @@ -0,0 +1,9 @@ +using System; + +namespace BCG.Utilities.General +{ + public interface IWorker + { + void doWork(); + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs new file mode 100644 index 000000000..c5d0aba4a --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Text; +using BCG.Utilities.General; +using System.Threading; +using mbts; +//using BCG.Utilities; + +namespace BCG +{ + public class MasterWorker : IWorker + { + + private DateTime myLastDoWorkTime = DateTime.MinValue, + myLastLogTime = DateTime.MinValue; + protected TimeSpan myPeriod = new TimeSpan(0, 0, 0, 0, 100), + myLogPeriod = new TimeSpan(0, 1, 0); + protected Dictionary> myWorkers = new Dictionary>(), + myJustAdded = new Dictionary>(), + myJustRemoved = new Dictionary>(); + private HighPerformanceTimer myTimer = new HighPerformanceTimer(); + private static MasterWorker myThreadDriven = null, + myDoWorkDriven = null; + internal MasterWorker() + { + myInit(); + } + + protected virtual void myInit() + { } + + public static MasterWorker getInstance (bool threadDriven) + { + lock (typeof(MasterWorker)) + { + if (threadDriven) + { + // TODO + myThreadDriven = myThreadDriven ?? new ThreadDrivenMasterWorker(); + return myThreadDriven; + return null; + } + else + { + myDoWorkDriven = myDoWorkDriven ?? new MasterWorker(); + return myDoWorkDriven; + } + } + } + + public virtual void addWorker(IWorker worker, TimeSpan period) + { + // BlueEventLog.logEvent(String.Format("MasterWorker : adding worker {0} with period {1} , worker count before adding this = {2}", + // worker.ToString(), period.ToString(), myWorkers.Count), + // PS3LogLevels.connection); + + myJustAdded.Add(worker, new Pair(period, DateTime.UtcNow)); + } + + public virtual void removeWorker(IWorker worker) + { + // BlueEventLog.logEvent(String.Format("MasterWorker : removing worker {0} , worker count before removing this = {1}", + // worker.ToString(), myWorkers.Count), + // PS3LogLevels.disconnection); + + myJustRemoved.Add(worker, null); + } + + public virtual void doWork() + { + + foreach (KeyValuePair> p in myJustAdded) + { + myWorkers.Add(p.Key, p.Value); + } + foreach (KeyValuePair> p in myJustRemoved) + { + myWorkers.Remove(p.Key); + } + myJustAdded.Clear(); + myJustRemoved.Clear(); + if (DateTime.UtcNow - myLastDoWorkTime >= myPeriod) + { + myTimer.start(); + //myLogEntries(); + myLastDoWorkTime = DateTime.UtcNow; + foreach (KeyValuePair> p in myWorkers) + { + if (DateTime.UtcNow - p.Value.second >= p.Value.first) + { + p.Key.doWork(); + p.Value.second = DateTime.UtcNow; + } + } + myTimer.stop(); + if (myTimer.lastIntervalTime > 100){ + //BlueEventLog.logEvent(String.Format("{0} : took {1} msecs", GetType().ToString(), myTimer.lastIntervalTime), PS3LogLevels.performance); + } + } + + } + + protected virtual void myLogEntries () + { + if (DateTime.UtcNow - myLastLogTime > myLogPeriod) + { + myLastLogTime = DateTime.UtcNow; + } + } + + } + + //class ThreadDrivenMasterWorker : MasterWorker + //{ + // private Timer myTimer; + // internal ThreadDrivenMasterWorker() + // : base() + // { } + + // protected override void myInit() + // { + // myTimer = new Timer(new TimerCallback(myDoWork), null, myPeriod, myPeriod); + // } + + // public override void addWorker(IWorker worker, TimeSpan period) + // { + // ///to avoid re-entrance, its added to a temporay dictionary, + // ///and will be moved over to the main one, in the doWork + // lock (this) + // { + // myJustAdded.Add(worker, new Pair(period, DateTime.UtcNow)); + // } + // } + + // public override void removeWorker(IWorker worker) + // { + // ///same as addWorker, added to a temporary store, before moving + // ///it to the main one, in the next doWork + // lock (this) + // { + // myJustRemoved.Add(worker, null); + // } + // } + + // private void myDoWork(object args) + // { + // lock (this) + // { + // foreach (IWorker w in myJustRemoved.Keys) + // { + // BlueEventLog.logEvent(String.Format("ThreadDrivenMasterWorker : removing worker {0} , worker count before removing this = {1}", + // w.ToString(), myWorkers.Count), PS3LogLevels.connection); + // myWorkers.Remove(w); + // } + // foreach (KeyValuePair> p in myJustAdded) + // { + // BlueEventLog.logEvent(String.Format("ThreadDrivenMasterWorker : adding worker {0} , worker count before adding this = {1}", + // p.Key.ToString(), myWorkers.Count), PS3LogLevels.connection); + // myWorkers.Add(p.Key, p.Value); + // } + // myJustRemoved.Clear(); + // myJustAdded.Clear(); + // } + // doWork(); + // } + + //} +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Misc/Pair.cs b/src/DataStreamSuite/PubSub/Misc/Pair.cs new file mode 100644 index 000000000..003e38764 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/Pair.cs @@ -0,0 +1,90 @@ +//using ProtoBuf; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +//using System.Windows; +//using System.Windows.Media; + +namespace mbts +{ + // TODO move it out or replace it with Tuple + [Serializable] + public class Pair + { + public F first; + public S second; + + public Pair() { } + public Pair(F f, S s) + { + first = f; + second = s; + } + + public Pair(KeyValuePair kvp) + : this(kvp.Key, kvp.Value) { } + + public override string ToString() + { + return String.Format("{0},{1}", first != null ? first.ToString() : "NULL", + second != null ? second.ToString() : "NULL"); + } + + public override int GetHashCode() + { + return ToString().GetHashCode(); + } + + public override bool Equals(object obj) + { + return obj != null && obj is Pair && ToString() == obj.ToString(); + } + } + + [Serializable] + public class Triplet + { + public F first; + public S second; + public T third; + + public Triplet() { } + + public Triplet(Tuple tp) + { + first = tp.Item1; + second = tp.Item2; + third = tp.Item3; + } + + public Triplet(F f, S s, T t) + { + first = f; + second = s; + third = t; + } + + public static Triplet from_tuple(Tuple t) { return new Triplet(t.Item1, t.Item2, t.Item3); } + + public override string ToString() + { + return String.Format("{0},{1},{2}", first != null ? first.ToString() : "NULL", + second != null ? second.ToString() : "NULL", + third != null ? third.ToString() : "NULL"); + } + + public override bool Equals(object obj) + { + return obj != null && obj is Triplet && ToString() == obj.ToString(); + } + + public override int GetHashCode() + { + return ToString().GetHashCode(); + } + } + +} diff --git a/src/DataStreamSuite/PubSub/Misc/StreamDecompressor.cs b/src/DataStreamSuite/PubSub/Misc/StreamDecompressor.cs new file mode 100644 index 000000000..4470cb1ec --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/StreamDecompressor.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace BCG +{ + /* + * This class is used in the PubSubMessageReactor and in the subscriber core. It takes as input a strea that's partially compressed, splits it apart, + * decompresses the compressed part, and then returns the decompressed version of the stream. + * + * The input should already be at the point where the compression starts. + * */ + class StreamDecompressor + { + + public static Stream decompressStream(Stream toDecompress) + { + SharpZipCompressor Kompressor = new SharpZipCompressor(); + MemoryStream decompressedStream = new MemoryStream(); + long dataStartPosition = toDecompress.Position; + Stream[] regularAndCompressedParts = StreamUtilityMethods.splitStream(toDecompress, (int)dataStartPosition); + + Stream compressedData = regularAndCompressedParts[1]; + + Kompressor.decompress(compressedData, decompressedStream); + + Stream regularPart = regularAndCompressedParts[0]; + + regularPart.Position = 0; + decompressedStream.Position = 0; + + + + return StreamUtilityMethods.getAggregatedStream(new Stream[] { regularPart, decompressedStream }); + + + + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs b/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs new file mode 100644 index 000000000..112ee52f3 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs @@ -0,0 +1,83 @@ +using System; +using System.IO; +using System.Collections; + +namespace BCG +{ + /// + /// Collection of utility methods used throughout + /// + public class StreamUtilityMethods + { + public StreamUtilityMethods() {} + + public static Stream getAggregatedStream (Stream[] streams) + { + if (streams.Length == 1) + return streams[0]; + else + { + BinaryWriter bw = new BinaryWriter (new MemoryStream ()); + foreach (Stream s in streams) + { + bw.Write (new BinaryReader(s).ReadBytes (Convert.ToInt32 (s.Length-s.Position))); + } + bw.BaseStream.Position = 0; + return bw.BaseStream; + } + } + + public static void copyTo(Stream[] sourceStreams, Stream target) + { + long initPos = target.Position; + BinaryWriter bw = new BinaryWriter(target); + foreach (Stream s in sourceStreams) + { + bw.Write(new BinaryReader(s).ReadBytes(Convert.ToInt32(s.Length - s.Position))); + } + bw.BaseStream.Position = initPos; + } + + public static byte[] getBytesFromStream (params Stream[] streams) + { + long totalSize = 0; + BinaryWriter bw = new BinaryWriter (new MemoryStream ()); + foreach (Stream s in streams) + totalSize += s.Length - s.Position; + bw.Write (Convert.ToInt32 (totalSize)); + foreach (Stream s in streams) + { + BinaryReader br = new BinaryReader (s); + int lengthToReadForThisStream = Convert.ToInt32(s.Length - s.Position); + bw.Write (br.ReadBytes (lengthToReadForThisStream)); + } + return (bw.BaseStream as MemoryStream).ToArray (); + + } + + /*takes the input stream and splits it into two streams at the given position. */ + public static Stream[] splitStream(Stream stream, int splitLocation) + { + stream.Position = 0; + + byte[] streamBytes = getBytesFromStream(new Stream[] { stream }); + + byte[] firstStreamBytes = new byte[splitLocation]; + byte[] secondStreamBytes = new byte[stream.Length - splitLocation]; + for (int i = 0; i < stream.Length; ++i) + { + //for some resaon, the 'getBytesFromStream' function prepends the length of the byte array to the bytes. This means + //we have to ignore those bytes here. + if (i < splitLocation) + firstStreamBytes[i] = streamBytes[i + 4]; + else + secondStreamBytes[i - splitLocation] = streamBytes[i + 4]; + + } + + return new Stream[] {new MemoryStream(firstStreamBytes), new MemoryStream(secondStreamBytes)}; + + + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/SyncQueue.cs b/src/DataStreamSuite/PubSub/Misc/SyncQueue.cs new file mode 100644 index 000000000..bc2eee446 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/SyncQueue.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Threading; + +using BCG; +//using BCG.Utilities; + +namespace BCG +{ + + /// + /// Use this class for inter thread communication. It assumed a thread safe + /// queue (this is right now implemented using locks, but for better performance + /// will be switched over to lock free code, using cas). + /// + public class SyncQueue : ISyncQueue + { + private ThreadSafeQueue myQueue = new ThreadSafeQueue(); + private AutoResetEvent myEvent = new AutoResetEvent(false); + + public SyncQueue() {} + + public void enqueue(T t) + { + myQueue.enqueue(t); + myEvent.Set(); + //return true; + } + + //public bool enqueue(List list) + //{ + // myQueue.enqueue(list); + // myEvent.Set(); + // return true; + //} + + public T dequeue() + { + return dequeue(Int32.MaxValue); + } + + public T dequeue (int timeout) + { + DateTime startTime = DateTime.Now; + T rv = myQueue.dequeue(); + if (rv != null || timeout <= 0) + { + return rv; + } + else + { + int timeRemaining = timeout; + while (true) + { + bool res = myEvent.WaitOne(timeRemaining, false); + if (res) + { + ///timeout has not expired, so thread has been signalled + rv = myQueue.dequeue(); + if (rv == null) + { + timeRemaining = timeout - Convert.ToInt32(((TimeSpan)(DateTime.Now - startTime)).TotalMilliseconds); + if (timeRemaining <= 0) + return rv; + else + continue; + } + else + return rv; + } + else + { + ///timeout expired, return default; + return default(T); + } + } + + } + } + + public List dequeueMultipleElements(int timeout, int maxElements) + { + if (timeout == 0) + { + return myQueue.dequeueLimited(maxElements); + } + else + { + DateTime startTime = DateTime.Now; + List rv = new List(); + rv = myQueue.dequeueLimited(maxElements); + if (rv.Count == maxElements) + return rv; + else + { + int timeRemaining = timeout; + while (true) + { + T t = dequeue(timeRemaining); + if (t != null) + rv.Add(t); + timeRemaining = timeout - Convert.ToInt32(((TimeSpan)(DateTime.Now - startTime)).TotalMilliseconds); + if (timeRemaining <= 0 || rv.Count == maxElements) + return rv; + } + } + } + } + + public List dequeueMultipleElements(TimeSpan ts, int maxElements) + { + return dequeueMultipleElements(Convert.ToInt32(ts.TotalMilliseconds), maxElements); + } + + public int getCount() + { + return myQueue.getCount(); + } + + + public void terminate() + { + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/SysInfoManager.cs b/src/DataStreamSuite/PubSub/Misc/SysInfoManager.cs new file mode 100644 index 000000000..40c89351a --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/SysInfoManager.cs @@ -0,0 +1,81 @@ +using System; +//using System.Windows.Forms; +using System.Collections; +//using System.Management; +using System.Net; + +namespace BCG.Utilities.General +{ + /// + /// Contains utlitiy functions for getting System info + /// + public class SysInfoManager + { + public SysInfoManager() + { + } + + // public static string getIPWMI() + // { + // string[] adds = getIPSWMI(); + // if (adds.Length > 1) + // { + // throw new Exception("Mulitple network cards"); + // } + // else return (string)adds[0]; + // } + + // public static string[] getIPSWMI() + // { + // ManagementObjectSearcher query = new ManagementObjectSearcher("SELECT * FROM Win32_NetworkAdapterConfiguration"); + // ManagementObjectCollection queryCollection = query.Get (); + // ArrayList adds = new ArrayList(); + // foreach( ManagementObject mo in queryCollection ) + // { + // string[] addresses = (string[])mo["IPAddress"]; + // if((bool)mo["IpEnabled"]) + // foreach(string s in addresses) + // adds.Add (s); + // } + // return (string[])adds.ToArray(typeof(string)); + // } + + public static string getIP() + { + + // TODOds + string hostName = Dns.GetHostName(); + + // Get the IP addresses associated with the host name + IPAddress[] ipAddresses = Dns.GetHostAddresses(hostName); + return ipAddresses[0].ToString(); + + + // string cName = SystemInformation.ComputerName; + // IPAddress[] ipas; + // try + // { + // ipas = Dns.GetHostEntry(cName).AddressList; + // } + // catch(ArgumentNullException ane) + // { + // throw new Exception(ane.Message); + // } + // catch(System.Net.Sockets.SocketException se) + // { + // throw new Exception(se.Message); + // } + // if(ipas == null || ipas.Length == 0) + // { + // string msg = "Could not retreive the IP address of this machine, please contact IT"; + // throw new Exception(msg); + // } + // return ipas[0].ToString(); + } + + public static string getHostNameFromIP (string ip) + { + return Dns.GetHostEntry (ip).HostName.Split('.')[0]; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/ThreadSafeQueue.cs b/src/DataStreamSuite/PubSub/Misc/ThreadSafeQueue.cs new file mode 100644 index 000000000..196dac78c --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/ThreadSafeQueue.cs @@ -0,0 +1,181 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BCG +{ + /// + /// Uses locks to provide thread safety. This turns out to be + /// a problem if there are a lot of threads trying to access + /// a queue. For this purpose this class will serve as the base + /// class for later lock free (wait free) implementations + /// + /// + public class ThreadSafeQueue + { + private readonly int _max_size = Int32.MaxValue; + private Queue _main_queue = new Queue(); + + private DateTime _last_dequeque_time = DateTime.MinValue; + private string _queue_name = String.Empty; + private bool _log_queue_size_warnings = false; + private int _warning_log_size = Int32.MaxValue; + private readonly TimeSpan _warning_log_period = TimeSpan.MaxValue, _first_log_period = TimeSpan.FromSeconds(30); + private DateTime _last_log_time = DateTime.MinValue; + private readonly DateTime _construction_time = DateTime.Now; + + public ThreadSafeQueue() + : this(Int32.MaxValue) + { } + + public ThreadSafeQueue(int max_size) + : this(max_size, String.Empty, false, Int32.MaxValue, TimeSpan.MaxValue) + { } + + public ThreadSafeQueue(int max_size, string queue_name, bool log_queue_size_warnings, int warning_log_size, TimeSpan warning_log_period) + { + _max_size = max_size; + _queue_name = queue_name; + _log_queue_size_warnings = log_queue_size_warnings; + _warning_log_size = warning_log_size; + _warning_log_period = warning_log_period; + } + + public virtual bool enqueue(T t) + { + lock (this) + { + int count = getCount(); + _check_size_and_warn(count); + if (count < _max_size) + { + _main_queue.Enqueue(t); + return true; + } + else + return false; + } + } + + public virtual int enqueue(ICollection list) + { + int nElements = 0; + lock (this) + { + foreach (T t in list) + if (enqueue(t)) + nElements++; + } + return nElements; + } + + public virtual int getCount() + { + int count; + lock (this) { count = _main_queue.Count; } + return count; + } + + public virtual bool clear_and_enqueue(T t) + { + bool rv; + lock (this) + { + _main_queue.Clear(); + rv = enqueue(t); + } + return rv; + } + + public virtual int clear_and_enqueue(ICollection list) + { + int rv = 0; + lock (this) + { + _main_queue.Clear(); + rv = enqueue(list); + } + return rv; + } + + public virtual T dequeue() + { + T rv; + lock (this) + { + if (_main_queue.Count > 0) + { + _last_dequeque_time = DateTime.Now; + rv = _main_queue.Dequeue(); + } + else + rv = default(T); + } + return rv; + } + + public virtual List dequeueAll() + { + List rv = new List(); + lock (this) + { + int sz = getCount(); + for (int i = 0; i < sz; i++) + rv.Add(dequeue()); + } + return rv; + } + + public virtual bool enqueue_and_crush(T new_elem, Comparison comp) + { + bool rv; + lock (this) + { + Dictionary marked_for_deletion = new Dictionary (); + List all_elements = dequeueAll(); + for (int i = 0; i < all_elements.Count; i++) + if (_compare (new_elem, all_elements[i], comp)) + marked_for_deletion.Add (i,true); + for (int i = 0; i < all_elements.Count; i++) + if (!marked_for_deletion.ContainsKey(i)) + enqueue(all_elements[i]); + rv = enqueue(new_elem); + } + return rv; + } + + private static bool _compare(T t1, T t2, Comparison comp) + { + if (comp != null) + return comp (t1, t2) == 0; + else + return t1 != null && t1.Equals(t2); + } + + public virtual List dequeueLimited(int max) + { + List rv = new List(); + lock (this) + { + int limit = Math.Min (max, _main_queue.Count); + for (int i = 0; i < limit; i++) + rv.Add(dequeue()); + } + return rv; + } + + + private void _check_size_and_warn(int count) + { + if (_log_queue_size_warnings) + { + if (count > _warning_log_size && (DateTime.Now - _last_log_time > _warning_log_period) && (DateTime.Now - _construction_time > _first_log_period)) + { + _last_log_time = DateTime.Now; + //mbts.Logger.log(bts.log.Level.Warn, bts.log.Subsystem.gen, String.Format("ThreadSafeQueue size breach queue_name : {0} type : {1} hash : {2} n of elems : {3}", _queue_name, typeof(T), GetHashCode(), count)); + } + } + } + + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/UtilityMethods.cs b/src/DataStreamSuite/PubSub/Misc/UtilityMethods.cs new file mode 100644 index 000000000..43a3a9374 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/UtilityMethods.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.IO.Compression; + +namespace BCG +{ + /// + /// Collection of utility methods used throughout + /// + public class UtilityMethods + { + public UtilityMethods() { } + + public static Stream getAggregatedStream(Stream[] streams) + { + if (streams.Length == 1) + return streams[0]; + else + { + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + foreach (Stream s in streams) + { + bw.Write(new BinaryReader(s).ReadBytes(Convert.ToInt32(s.Length - s.Position))); + } + bw.BaseStream.Position = 0; + return bw.BaseStream; + } + } + + public static byte[] compress(Stream str) + { + return compress_gzip(str); + //MemoryStream rv = new MemoryStream(); + //IBlueCompressor compressor = new SharpZipCompressor(); + //compressor.compress(str, rv); + //rv.Position = 0; + //return rv.ToArray(); + } + + public static Stream decompress(byte[] compressedData) + { + return decompress_gzip(compressedData); + //MemoryStream compressedStream = new MemoryStream(compressedData); + //compressedStream.Position = 0; + //IBlueCompressor compressor = new SharpZipCompressor(); + //MemoryStream decompressedStream = new MemoryStream (); + //compressor.decompress(compressedStream, decompressedStream); + //decompressedStream.Position = 0; + //return decompressedStream; + } + + + #region GZip Compression + public static byte[] compress_gzip(Stream in_str) + { + byte[] rv; + using (var out_str = new MemoryStream()) + { + using (var gzip_str = new GZipStream(out_str, CompressionLevel.Optimal)) + in_str.CopyTo(gzip_str); + rv = out_str.ToArray(); + } + return rv; + } + + public static Stream decompress_gzip(byte[] compressed_data) + { + Stream rv_str = new MemoryStream(); + using (var in_str = new MemoryStream(compressed_data)) + { + using (var gzip_str = new GZipStream(in_str, CompressionMode.Decompress)) + gzip_str.CopyTo(rv_str); + } + rv_str.Position = 0; + return rv_str; + } + #endregion + + #region Deflate Compression + public static byte[] compress_deflate(Stream in_str) + { + byte[] rv; + using (var out_str = new MemoryStream()) + { + using (var def_str = new DeflateStream(out_str, CompressionMode.Compress)) + in_str.CopyTo(def_str); + rv = out_str.ToArray(); + } + return rv; + } + + public static Stream decompress_deflate(byte[] compressed_data) + { + Stream rv_str = new MemoryStream(); + using (var in_str = new MemoryStream(compressed_data)) + { + using (var def_str = new DeflateStream(in_str, CompressionMode.Decompress)) + def_str.CopyTo(rv_str); + } + rv_str.Position = 0; + return rv_str; + } + #endregion + } +} diff --git a/src/DataStreamSuite/PubSub/PubSub.csproj b/src/DataStreamSuite/PubSub/PubSub.csproj index de3c61cba..772f6fb9a 100644 --- a/src/DataStreamSuite/PubSub/PubSub.csproj +++ b/src/DataStreamSuite/PubSub/PubSub.csproj @@ -1,6 +1,11 @@ + + + + + true net7.0 enable diff --git a/src/DataStreamSuite/PubSub/PubSubMessage.cs b/src/DataStreamSuite/PubSub/PubSubMessage.cs new file mode 100644 index 000000000..722769216 --- /dev/null +++ b/src/DataStreamSuite/PubSub/PubSubMessage.cs @@ -0,0 +1,34 @@ +using System; +using System.IO; + +using BCG; +using BCG.Utilities; + +namespace BCG +{ + // Tuple of + public class PubSubMessage + { + public PubSubMessage (MetaData metaData, Stream body) + {this.metaData = metaData; this.body = body;} + + public PubSubMessage (MetaData metaData) + {this.metaData = metaData;} + + public MetaData metaData; + public Stream body = new MemoryStream(0); + + public Stream toStream() + { + MemoryStream ms = new MemoryStream(); + metaData.save(ms); + if (body.Length == 0 || body.Length == body.Position) + Console.WriteLine("Error: Trying to publishing message with no content."); + StreamUtilityMethods.copyTo(new Stream[] { body }, ms); + + ms.Position = 0; + return ms; + } + + } +} diff --git a/src/DataStreamSuite/PubSub/PubSubUpdate.cs b/src/DataStreamSuite/PubSub/PubSubUpdate.cs new file mode 100644 index 000000000..be922b6e6 --- /dev/null +++ b/src/DataStreamSuite/PubSub/PubSubUpdate.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections; +using System.IO; + +namespace BCG +{ + /// + /// this is the data returned from the + /// pubsub server + /// + public class PubSubUpdate + { + + public PubSubUpdate(string tag, string tagType, Stream body) + { this.tag = tag; this.body = body; this.tagType = tagType; } + + public string tag; + public Stream body; + public string tagType; + + + }; +} diff --git a/src/DataStreamSuite/PubSub/PublisherCore.cs b/src/DataStreamSuite/PubSub/PublisherCore.cs new file mode 100644 index 000000000..793a020d3 --- /dev/null +++ b/src/DataStreamSuite/PubSub/PublisherCore.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Text.RegularExpressions; + +using BCG.Utilities; + +namespace BCG +{ + public class PublisherCore : IPublisherCore + { + //private static BlueCommManaged.IPubSub myBCPS; + + // TODO: what is this? + // public static void setPubSub(BlueCommManaged.IPubSub ps) + // { + // lock(typeof(PublisherCore)) + // { + // BlueEventLog2.instance().assert(myBCPS == null, + // "PublisherCore.setPubSub() should be called only once and before any use of PublisherCore"); + // myBCPS = ps; + // } + // } + + public static string pubSubBlueCommPrefix = "ps/"; + public static string pubSubBlueCommTransactionPrefix = "tgps/"; + public static string pubSubBlueCommTransactionComponentPrefix = "tcps/"; + // public static BlueCommManaged.IPubSub getPubSub() + // { + // lock (typeof(PublisherCore)) { + // return myBCPS; + // } + // } + + enum Destination + { + None, + BlueComm, + PubSub, + Both + } + + //private IPublisherCore myBlueCommPublisher = null; + private IPublisherCore myPubSubPublisher = null; + private Dictionary myPubDestinations = + new Dictionary(); + + public PublisherCore() + { + myPubSubPublisher = new StaticPublisherCore(); + //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub()); + } + + public PublisherCore(IPubSubRouter theRouter) + { + myPubSubPublisher = new StaticPublisherCore(theRouter); + //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub()); + } + + public PublisherCore(string URL) + { + myPubSubPublisher = new StaticPublisherCore(URL); + //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub()); + } + + #region IPublisherCore Methods + + public void Dispose() + { + myPubSubPublisher.Dispose(); + //myBlueCommPublisher.Dispose(); + } + public int getNumberOfWaitingMessages() { return myPubSubPublisher.getNumberOfWaitingMessages(); } + + private Destination myGetPublishDestionation (string tag) + { + return Destination.PubSub; + // lock (this) + // { + // if (PubSub3Config.instance.BlueComm_mustDie) + // return Destination.PubSub; + // else + // { + // Destination d; + // return Destination.PubSub; + // if (myPubDestinations.TryGetValue(tag, out d)) + // return d; + // else + // { + // foreach (string regexp in PubSub3Config.instance.BlueComm_PSTransition.tagsToPublishInPubSub) + // if (Regex.IsMatch(tag, regexp)) + // { + // d = Destination.PubSub; + // foreach (string regexp2 in PubSub3Config.instance.BlueComm_PSTransition.exceptionsToTagsToPublishInPubSub) + // if (Regex.IsMatch(tag, regexp2)) + // { + // d = Destination.None; + // break; + // } + // break; + // } + + // if (!tag.StartsWith("mon")) + // foreach (string regexp in PubSub3Config.instance.BlueComm_PSTransition.tagsToPublishInBlueComm) + // if (Regex.IsMatch(tag, regexp)) + // { + // if (d == Destination.PubSub) + // d = Destination.Both; + // else + // d = Destination.BlueComm; + // foreach (string regexp2 in PubSub3Config.instance.BlueComm_PSTransition.exceptionsToTagsToPublishInBlueComm) + // if (Regex.IsMatch(tag, regexp2)) + // { + // if (d == Destination.Both) + // d = Destination.PubSub; + // else if (d == Destination.BlueComm) + // d = Destination.None; + // break; + // } + // break; + // } + + // myPubDestinations[tag] = d; + // return d; + // } + // } + // } + } + + public void publish(string tag, Stream content) + { + publish(tag, "", content); + } + + public void publish(string tagType, string tag, Stream content) + { + Destination d = myGetPublishDestionation(tag); + if (d == Destination.PubSub || d == Destination.Both) + myPubSubPublisher.publish(tagType, tag, content); + if (d == Destination.BlueComm || d == Destination.Both){} + //myBlueCommPublisher.publish(tagType, tag, content); + } + + public void beginTransaction() + { + myPubSubPublisher.beginTransaction(); + //myBlueCommPublisher.beginTransaction(); + } + + public void commitTransaction() + { + myPubSubPublisher.commitTransaction(); + //myBlueCommPublisher.commitTransaction(); + } + + public void abortTransaction() + { + myPubSubPublisher.abortTransaction(); + //myBlueCommPublisher.abortTransaction(); + } + + public void clearState(string tagToClear) + { + Destination d = myGetPublishDestionation(tagToClear); + if (d == Destination.PubSub || d == Destination.Both) + myPubSubPublisher.clearState(tagToClear); + if (d == Destination.BlueComm || d == Destination.Both){} + //myBlueCommPublisher.clearState(tagToClear); + } + + public bool inTransaction + { get { return myPubSubPublisher.inTransaction; } } + + #endregion + + } +} diff --git a/src/DataStreamSuite/PubSub/PubsubDeserializationController.cs b/src/DataStreamSuite/PubSub/PubsubDeserializationController.cs new file mode 100644 index 000000000..89e405fbf --- /dev/null +++ b/src/DataStreamSuite/PubSub/PubsubDeserializationController.cs @@ -0,0 +1,72 @@ +using System; +using System.IO; +using System.Collections; +using BCG.Utilities; + +namespace BCG +{ + /// + /// Controls (ie creates dynamically) an IPubsubDeserializer based on the type name which is + /// part of the published data. Since its also a IPubsubDeserializer, all subsequent calls + /// are passed though to the interface + /// + public class PubsubDeserializationController : IPubsubDeserializer + { + private IPubsubDeserializer myDeserializer; + + public PubsubDeserializationController() { } + + public void update(Stream stream) + { + byte[] ops; + Array rows; + deserialize(new Stream[] { stream }, out ops, out rows); + } + public void update(byte[] bytes) + { + byte[] ops; + Array rows; + deserialize(new Stream[] { new MemoryStream(bytes, 0, bytes.Length, false, true) }, out ops, out rows); + } + + #region IPubsubDeserializer Members + + public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) + { + if (myDeserializer != null) + { + myDeserializer.deserialize(streams, out theOps, out theRows); + } + else + { + Stream totalStream = UtilityMethods.getAggregatedStream(streams); + long initialPos = totalStream.Position; + BinaryReader br = new BinaryReader(totalStream); + byte opCode = br.ReadByte(); + if (opCode == StreamOpCode.create) + { + string deserName = br.ReadString(); + if (deserName == "BCG.PubsubUniversalDeserializer") // so we can play old replay files which would not include the assembly name + deserName = "BCG.PubsubUniversalDeserializer, DataStreamGateway"; + myDeserializer = Activator.CreateInstance(Type.GetType(deserName)) + as IPubsubDeserializer; + totalStream.Position = initialPos; + myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows); + } + else + { + // BlueEventLog.logEvent("Received a non create before a create. Bad op code was " + opCode.ToString(), BlueEventLevel.Warning); + theOps = new byte[] { }; + theRows = new object[] { }; + } + } + } + + public byte[] getState() + { + return myDeserializer == null ? new byte[] { } : myDeserializer.getState(); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPClientData.cs b/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPClientData.cs new file mode 100644 index 000000000..0a7469bc5 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPClientData.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Runtime.InteropServices; + +namespace BCG +{ + + /// + /// Data from the clients. (once again, this can be used + /// by any server to communicate to clients and is not + /// pubsub specific) + /// + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + public struct BlutoTCPClientMessage + { + public enum BlutoTCPClientMsgType + { + connection = 0, + disconnection = 1, + data = 2 + } + //ip and port where the msg came from + public IntPtr ip; + public int port; + //msg type can be a client message, of client + //going up or coming down message + public int msgType; + //size of the msg + public int size; + //actual clent data itself + public IntPtr msg; + } +} diff --git a/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPClientsWrapper.cs b/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPClientsWrapper.cs new file mode 100644 index 000000000..13b8b46b9 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPClientsWrapper.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Runtime.InteropServices; + +namespace BCG +{ + /// + /// Wraps around the series of C functions to provide server level + /// communication with multiple clients connected to the server. + /// This code can be used by any server, listening to clients + /// communicationg via TCP. (does not have any pubsub specific code, + /// the pubsub specificity would be in the content of the messages) + /// + public class BlutoTCPClientsWrapper + { + /// + /// this opens the well known port to listen for + /// clients trying to connect. this should be done + /// in the server initialization phase + /// + [DllImport("BlutoInterop.dll")] + public unsafe static extern void initPubsub(int port); + + /// + /// Gets all the accumulataed messages from the clients + /// + [DllImport("BlutoInterop.dll")] + public unsafe static extern void getClientData(int timeout, BlutoTCPClientMessage*** pts, out int npts, int maxElements); + + /// + /// Sends data to a single client + /// + [DllImport("BlutoInterop.dll")] + public static extern void sendData(byte[] ip, int port, byte[] message, int size); + + } +} diff --git a/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPMessageHub.cs new file mode 100644 index 000000000..76c58f401 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Communication/BlutoTCP/BlutoTCPMessageHub.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Runtime.InteropServices; +using System.IO; + +using BCG.Utilities; +using BCG.Utilities.General; + +namespace BCG +{ + /// + /// Using the BlutoTCPClientsWrapper, receives and sends + /// messages from/to TCP clients. also watches out for new and dropped + /// connections and reports it via callbacks + /// + //class BlutoTCPMessageHub : IPubsubMessageHub + //{ + // private BlockingQueue myConnectionStatusQueue = new BlockingQueue(); + // private List myCallbacks = new List(); + // private HighPerformanceTimer myTimer = new HighPerformanceTimer(); + + // public BlutoTCPMessageHub() + // { + // BlutoTCPClientsWrapper.initPubsub(PubSub2ClientSettings.instance.PS2_ServerPort); + // string msg = "PubsubServer is ready to listen to port " + PubSub2ClientSettings.instance.PS2_ServerPort + + // " for incoming connections"; + // BlueEventLog.logEvent(msg, BlueEventLevel.Information); + // Console.WriteLine(msg); + // } + + // #region IPubsubMessageHub Members + + // public unsafe void getUpdates(out string sender, out byte[] data) + // { + // sender = null; + // data = null; + // int n = 0; + // BlutoTCPClientMessage** psClientPackets = null; + // BlutoTCPClientMessage* clientPacket; + // ///if theres waiting data, this function returns immediatly, + // ///otherwise waits for 50 ms and then returns + // BlutoTCPClientsWrapper.getClientData(50, &psClientPackets, out n, 1); + // BlueEventLog.logAssert(n <= 1, "only one message should be retreived at a time"); + // if (n > 0) + // { + // clientPacket = psClientPackets[0]; + // string ip = Marshal.PtrToStringAnsi(clientPacket->ip); + // int port = clientPacket->port; + // sender = String.Format(BlutoSubAddress.prefix + "://{0}:{1}", ip, port); + // ///data message + // if (clientPacket->msgType == (int)BlutoTCPClientMessage.BlutoTCPClientMsgType.data) + // { + // data = new byte[clientPacket->size]; + // Marshal.Copy(clientPacket->msg, data, 0, data.Length); + // } + // else if (clientPacket->msgType == (int)BlutoTCPClientMessage.BlutoTCPClientMsgType.disconnection) + // { + // BlueEventLog.logEvent("client " + ip + ":" + port + " disconnected from pubsub", BlueEventLevel.Information); + // myConnectionStatusQueue.enqueue(new BlutoSubAddress(ip, port).ToString()); + // } + // else if (clientPacket->msgType == (int)BlutoTCPClientMessage.BlutoTCPClientMsgType.connection) + // { + // BlueEventLog.logEvent("client " + ip + ":" + port + " connected to pubsub", BlueEventLevel.Information); + // } + // } + // } + + // public bool send(string receiver, byte[] message) + // { + // BlutoSubAddress bsa = new BlutoSubAddress(receiver); + // BlutoTCPClientsWrapper.sendData(myStringToBytes(bsa.ip), bsa.port, message, message.Length); + // return true; + // } + + // public bool send(string receiver, Stream str) + // { + // BinaryReader br = new BinaryReader(str); + // return send(receiver, br.ReadBytes((int)(str.Length - str.Position))); + // } + + // public void registerConnectionCallback(ConnectionStatusChange del, bool register) + // { + // if (register) + // myCallbacks.Add(del); + // else + // myCallbacks.Remove(del); + // } + + // #endregion + + // #region IWorker Members + + // public void doWork() + // { + // object conn = null; + // while ((conn = myConnectionStatusQueue.dequeue(0)) != null) + // { + // string subAddress = conn as string; + // foreach (ConnectionStatusChange del in myCallbacks) + // del(subAddress, false); + // } + // } + + // #endregion + + // private byte[] myStringToBytes(string s) + // { + // byte[] rv = new byte[s.Length+1]; + // for (int i = 0; i < s.Length; i++) + // rv[i] = Convert.ToByte (s[i]); + // rv[s.Length] = Convert.ToByte ('\0'); + // return rv; + // } + //} +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs b/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs new file mode 100644 index 000000000..9fbbb852e --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections; +using BCG.Utilities.General; +using System.IO; +using System.Collections.Generic; +using mbts; + +namespace BCG +{ + public delegate void ConnectionStatusChange(string peerURL, bool connection); + public delegate void DataDelegate(string sender, byte[] data); + /// + /// Encapsulates PubsubServer's communication with the + /// rest of the world. Has send and get methods, and also + /// an event to tell if a client connects or disconnects. + /// this event is prompted by the doWork. + /// + public interface IPubsubMessageHub : IWorker + { + + /// + /// The port that the message hub is listening on. Return -1 if you're not listening on a port, i.e. for hte case of multicast + /// + int port { get; } + /// + /// register or unregister callback for data. The callback when data + /// is available will be called in an arbitrary thread. Its an orthogonal + /// way to receive data, and can only be used exclusively with the + /// getUpdates. The delegate will be fired on an arbitrary thread. + /// + /// + void registerDataCallback(DataDelegate del); + /// + /// registers a connection callback. this callback + /// will be fired within the doWork call. + /// + /// + /// + void registerConnectionCallback(ConnectionStatusChange del); + /// + /// this is where this interface starts communication with the + /// rest of the world. Obvioulsy, the resgiters should be + /// called before the start, or else the user might miss + /// connections and data. + /// + void activate(); + /// + /// Gets the updates and the source of the data + /// from the clients. Returns either if timeout + /// expires or maxElements have been fetched. + /// + /// + /// + List> getUpdates(TimeSpan timeout, int maxElements); + /// + /// sends data to the receiver. + /// + /// + /// + bool send(string receiver, byte[] data); + /// + /// sends the data stored in the current position in the Stream. + /// + /// + /// + /// + bool send(string receiver, Stream str); + } +} diff --git a/src/DataStreamSuite/PubSub/Server/Communication/MSMQ/MSMQMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/MSMQ/MSMQMessageHub.cs new file mode 100644 index 000000000..79b5acebc --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Communication/MSMQ/MSMQMessageHub.cs @@ -0,0 +1,215 @@ +// using System; +// using System.Collections; +// using System.Collections.Generic; +// using BCG.Utilities; +// using System.Threading; +// using System.IO; +// using System.Messaging; + +// namespace BCG +// { +// /// +// /// Abstracts out the message queue communication part of the +// /// pubsub server. it pairs up with its client counterpart, +// /// 'PubSubServerMSMQConnection'. +// /// +// /// it receives messages from the main PubSub queue, +// /// and sends them off the individual queues. gets either data +// /// messages or system messages. system messages are hb +// /// messages indicating the wellbeing of a client. missing +// /// n hbs, and it drops the client connection. +// /// +// /// apart from getUpates, all calls are on the main thread. +// /// however the lock is applied only when reacting to the hbs +// /// (as data structures are updated), so contention should be +// /// minimized +// /// +// public class MSMQMessageHub : IPubsubMessageHub +// { +// /// +// /// this is the well known queue where clients +// /// send messages (pub, sub, unsub, tb, te, ta) +// /// +// private MSMQWrapper myMainQueueWrapper; +// private TimeSpan mySubscriberDropThreshold = +// new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_MissingHeartBeatsForGC * PubSub3Config.instance.PS3_HeartBeatInterval); +// private Dictionary> myActiveSubscribers = new Dictionary>(); +// private Dictionary myDeadSubscribers = new Dictionary(); +// private SyncQueue> myReceivedData = new SyncQueue>(); +// /// +// /// delegates that are interested in getting callbacks +// /// when a subscriber connects/disconnects +// /// +// private ConnectionStatusChange myConnCb; +// private DataDelegate myDataCb; + +// public MSMQMessageHub() {} + +// #region IPubsubMessageHub Members + +// //the concept of ports doesn't exist in the multicast world +// public int port +// { +// get { return -1; } +// } +// public void registerDataCallback(DataDelegate del) +// { +// myDataCb = del; +// } + +// public void registerConnectionCallback(ConnectionStatusChange del) +// { +// myConnCb = del; +// } + +// public void activate() +// { +// string pubSubQName = System.Environment.MachineName + "\\Private$\\" + +// PubSub3Config.instance.PS3_ServerQueue; +// string msg = String.Format("MSMQ Component is ready .. listening on queue {0} for messages", pubSubQName); +// Console.WriteLine(msg); +// myMainQueueWrapper = new MSMQWrapper(pubSubQName); +// Thread readThread = new Thread(new ThreadStart(myReadMessages)); +// readThread.IsBackground = true; +// readThread.Start(); +// } + +// public List> getUpdates(TimeSpan timeout, int maxElements) +// { +// return myReceivedData.dequeueMultipleElements(timeout, maxElements); +// } + +// /// +// /// called in the main thread +// /// +// /// +// /// +// /// +// public bool send(string receiver, byte[] data) +// { +// lock (this) +// { +// if (myActiveSubscribers.ContainsKey(receiver)) +// { +// ///receiver already exists +// myActiveSubscribers[receiver].first.send(data); +// return true; +// } +// else if (myDeadSubscribers.ContainsKey(receiver)) +// { +// string errMsg = receiver + " already dead, hence cant send data"; +// BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); +// Console.WriteLine(errMsg); +// return false; +// } +// else +// { +// ///this subscriber's hb was never received, hence it +// ///wasnt constructed then, so it will be constructed now +// myCreateNewClient(receiver); +// myActiveSubscribers[receiver].first.send(data); +// return true; +// } +// } +// } + +// public bool send(string receiver, Stream str) +// { +// BinaryReader br = new BinaryReader(str); +// return send (receiver, br.ReadBytes( (int)(str.Length - str.Position))); +// } + +// #endregion + +// #region IWorker Members + +// /// +// /// checks the connection status of every connection (sub). +// /// if a hb has not been received from a sub, it removes it +// /// from the active sub list and informs the delegates hooked +// /// on to receive that information. this is called in the main thread +// /// +// public void doWork() +// { +// List deadSubscribers = new List(); +// lock (this) +// { +// foreach (KeyValuePair> p in myActiveSubscribers) +// { +// if (DateTime.Now - p.Value.second > mySubscriberDropThreshold) +// { +// deadSubscribers.Add(p.Key); +// } +// } +// foreach (string ds in deadSubscribers) +// { +// myDeadSubscribers.Add(ds, true); +// myActiveSubscribers.Remove(ds); +// } +// } +// foreach (string ds in deadSubscribers) +// { +// string msg = "client " + ds + " disconnected , removing subscriptions"; +// BlueEventLog.logEvent(msg, BlueEventLevel.Warning); +// Console.WriteLine(msg); +// myConnCb(ds, false); +// } +// } + +// #endregion + +// private void myCreateNewClient(string clientURL) +// { +// string msg = "client " + clientURL + " is being created for the first time"; +// BlueEventLog.logEvent(msg, BlueEventLevel.Warning); +// Console.WriteLine(msg); +// MSMQWrapper sendWrapper = new MSMQWrapper(new MessageQueue(new MSMQSubAddress(clientURL).formatName)); +// myActiveSubscribers.Add(clientURL, new Pair(sendWrapper, DateTime.Now)); +// } + +// private void myReadMessages() +// { +// string sender; +// byte[] data; +// while (true) +// { +// try +// { +// while (true) +// { +// string msgType; +// ///waits until get a message +// data = myMainQueueWrapper.receive(out msgType); +// sender = myMainQueueWrapper.ToString(); +// if (msgType == "s") +// { +// ///heartbeat message +// lock (this) +// { +// string clientURL = new BinaryReader(new MemoryStream(data)).ReadString(); +// if (!myActiveSubscribers.ContainsKey(clientURL)) +// { +// myCreateNewClient(clientURL); +// } +// else +// myActiveSubscribers[clientURL].second = DateTime.Now; +// } +// } +// else +// { +// if (myDataCb != null) +// myDataCb(sender, data); +// else +// myReceivedData.enqueue(new Pair(sender, data)); +// } +// } +// } +// catch (Exception ex) +// { +// BlueEventLog.logEvent(ex); +// } +// } +// } + +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs new file mode 100644 index 000000000..df6d6146d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs @@ -0,0 +1,418 @@ +using System; +using System.Collections.Generic; +using System.Text; +using BCG.Utilities.General; +using BCG.Utilities; + +using System.Threading; +using System.IO; +using System.Diagnostics; +using mbts; + +namespace BCG +{ + + public class ManagedTCPMessageHub : IPubsubMessageHub, + ITCPConnectionHandler, + ITCPMessageHandler + { + private int myPort; + private bool myIsDisposed = false; + public int port + { + get { return myPort; } + //property is read only + + } + private bool myActivated = false; + private ConnectionStatusChange myConnCb; + private DataDelegate myDataCb; + + /// + /// client to connection + /// + private Dictionary myConnections = new Dictionary(); + /// + /// data that is received (on the onData thread) is queued in this + /// to be fetched by the client of this class in the getUpdates + /// + private SyncQueue> myReceivedData; + /// + /// data to be sent to the client is stored in here and is fetched by the + /// myDoItAll thread to be sent to the clients + /// + private SyncQueue> mySendData = new SyncQueue>(); + /// + /// connections that are found to be dead or are just connected + /// are queued in the following queues respectively, and are + /// picked up by the doWork thread. + /// + private ThreadSafeQueue myDeadConnectionURLs = new ThreadSafeQueue(), + myNewConnectionURLs = new ThreadSafeQueue(); + /// + /// new connections received (on the onConnection thread) is queued + /// in this and will be fetched by the doItAll thread + /// + private ThreadSafeQueue myNewConnections = new ThreadSafeQueue(); + /// + /// clients that sent heartbeats are stored in here + /// + private ThreadSafeQueue myReceivedHeartbeats = new ThreadSafeQueue(); + + private int myReadSendQueueTimeout = 100; + private TimeSpan myHeartbeatSendInterval; + private TimeSpan mySubscriberDropThreshold; + private TimeSpan myCollectDisposedConnectionsPeriod = new TimeSpan(0, 0, 0, 0, 1000); + private DateTime myLastHeartbeatDeadCollectionTime = DateTime.Now; + private DateTime myLastDisposedDeadCollectionTime = DateTime.Now; + + public ManagedTCPMessageHub(int port, TimeSpan hbSendInterval, int missingHBsForGC, TimeSpan collectDisposedConnectionsPeriod) + { + myPort = port; + myHeartbeatSendInterval = hbSendInterval; + mySubscriberDropThreshold = new TimeSpan(0, 0, 0, 0, Convert.ToInt32(hbSendInterval.TotalMilliseconds * missingHBsForGC)); + myCollectDisposedConnectionsPeriod = collectDisposedConnectionsPeriod; + //if (PubSub3Config.instance.PS3_PendingMessagesOnServerThreshold == 0) + // myReceivedData = new SyncQueue>(); + //else + // myReceivedData = new SyncQueueSimplified>(PubSub3Config.instance.PS3_PendingMessagesOnServerThreshold); + myReceivedData = new SyncQueue>(); + } + + #region IPubsubMessageHub Members + + public void registerDataCallback(DataDelegate del) + { + // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate"); + myDataCb = del; + } + + public void registerConnectionCallback(ConnectionStatusChange del) + { + //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate"); + myConnCb = del; + } + + public void activate() + { + if (PubSub3Config.instance.PS3_BindToRandomPort || myPort == 0) + myPort = ServerToClientManagedTCPConnectionAcceptor.activateWithRandomPort(this, myHeartbeatSendInterval); + else + myPort = ServerToClientManagedTCPConnectionAcceptor.activate(this, myPort, myHeartbeatSendInterval); + + string msg = String.Format("TCP Component is ready, listening on port {0} for incoming connections", myPort); + Console.WriteLine(msg); + myActivated = true; + Thread doItAllThread = new Thread(new ThreadStart(myDoItAll)); + doItAllThread.IsBackground = true; + doItAllThread.Start(); + } + + public List> getUpdates(TimeSpan timeout, int maxElements) + { + return myReceivedData.dequeueMultipleElements(timeout, maxElements); + } + + public bool send(string receiver, byte[] data) + { + mySendData.enqueue(new Pair(receiver, data)); + return true; + } + + public bool send(string receiver, System.IO.Stream str) + { + BinaryReader br = new BinaryReader(str); + return send(receiver, br.ReadBytes(Convert.ToInt32(str.Length - str.Position))); + } + + #endregion + + #region IWorker Members + + /// + /// Reports back connected and disconnected connections + /// via the ConnectionStatucChange callback + /// + public void doWork() + { + if (myConnCb != null) + { + foreach (string nc in myNewConnectionURLs.dequeueAll()) + { + //BlueEventLog.logEvent(String.Format("doWork : New client {0} connected, making connection callback", nc), PS3LogLevels.connection); + myConnCb(nc, true); + } + foreach (string dc in myDeadConnectionURLs.dequeueAll()) + { + //BlueEventLog.logEvent(String.Format("doWork : Client {0} disconnected, making disconnection callback", dc), PS3LogLevels.disconnection); + myConnCb(dc, false); + } + } + } + + #endregion + + #region ITCPConnectionHandler Members + + /// + /// Enqeues both the connection name (to be reported back + /// via ConnectionStatusChange) and connection (to be + /// stored in the hashtable and used for later communication) + /// + /// + public void onConnection(ITCPConnection connection) + { + myNewConnections.enqueue(connection); + } + + #endregion + + #region ITCPMessageHandler Members + + public void onData(RemoteMessage message) + { + Console.WriteLine("ManagedTCPMessgageHub.onData recv msg :" + message.remoteConnectionInfo.ToString()); + string sender = new BlutoSubAddress(message.remoteConnectionInfo.ip, message.remoteConnectionInfo.port).ToString(); + if (message.messageType == RemoteMessageType.data) + { + if (myDataCb != null) + myDataCb(sender, message.message); + else + myReceivedData.enqueue(new Pair(sender, message.message)); + } + else if (message.messageType == RemoteMessageType.heartbeat) + myReceivedHeartbeats.enqueue(sender); + else + { + //BlueEventLog.logEvent("Unknown message type from the client", BlueEventLevel.Error); + } + } + + #endregion + + class TimmingInfo + { + public TimmingInfo() + { + myTimer = new Stopwatch(); + reset(); + } + + public void start() + { + if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0) + { + myTimer.Reset(); + myTimer.Start(); + } + } + + public void stop() + { + if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0) + { + myTimer.Stop(); + myMin = Math.Min(myMin, myTimer.ElapsedMilliseconds); + myMax = Math.Max(myMax, myTimer.ElapsedMilliseconds); + myTotal += myTimer.ElapsedMilliseconds; + myNMeasurements++; + } + } + + public void reset() + { + myMin = Double.MaxValue; + myMax = Double.MinValue; + myTotal = 0; + myNMeasurements = 0; + } + + public override string ToString() + { + if (myNMeasurements > 0) + return String.Format("min={0} max={1} avg={2} nMeasurements={3}", myMin, myMax, myTotal / myNMeasurements, myNMeasurements); + else + return "No measurements"; + } + + private long myNMeasurements = 0; + private Stopwatch myTimer; + private double myMin; + private double myMax; + private double myTotal; + } + + private void myDoItAll() + { + Dictionary times = new Dictionary(); + times.Add("myHandleNewConnections", new TimmingInfo()); + times.Add("myCollectDisposedConnections", new TimmingInfo()); + times.Add("myCollectMissingHeartbeatConnections", new TimmingInfo()); + times.Add("send", new TimmingInfo()); + + DateTime lastTimmingLog = DateTime.Now; + + while (true) + { + try + { + while (true) + { + ///new connections + times["myHandleNewConnections"].start(); + myHandleNewConnections(); + times["myHandleNewConnections"].stop(); + + ///dead connections that are disposed + /// + times["myCollectDisposedConnections"].start(); + myCollectDisposedConnections(); + times["myCollectDisposedConnections"].stop(); + + ///dead connections due to missing hbs + times["myCollectMissingHeartbeatConnections"].start(); + myCollectMissingHeartbeatConnections(); + times["myCollectMissingHeartbeatConnections"].stop(); + + ///sending data - this is where the bulk of the time + ///in this thread should be spend + Pair p = mySendData.dequeue(myReadSendQueueTimeout); + + times["send"].start(); + if (p != null) + { + ITCPConnection conn; + myConnections.TryGetValue(p.first, out conn); + if (conn != null) + { + conn.send(p.second); + } + else + { + string warningMsg = String.Format("sending data to {0} failed, this client might have " + + "disconnected, in which case it will be garbage collected within 60 secs", p.first); + //BlueEventLog.logEvent(warningMsg, PS3LogLevels.disconnection); + } + } + times["send"].stop(); + + if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0 && + (DateTime.Now - lastTimmingLog).TotalSeconds > PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs) + { + lastTimmingLog = DateTime.Now; + + StringBuilder b = new StringBuilder(); + b.AppendFormat("myDoItAll stats:\n"); + b.AppendFormat("queue size: {0}\n", mySendData.getCount()); + + foreach (string timerName in times.Keys) + { + b.AppendFormat("{0}: {1}\n", timerName, times[timerName]); + times[timerName].reset(); + } + //BlueEventLog.logEvent(b.ToString(), PS3LogLevels.performance); + } + } + } + catch (Exception ex) + { + //BlueEventLog.logEvent(ex); + if (ex is OutOfMemoryException) + throw ex; + } + } + } + + private void myCollectDisposedConnections() + { + if (DateTime.Now - myLastDisposedDeadCollectionTime > myCollectDisposedConnectionsPeriod) + { + myLastDisposedDeadCollectionTime = DateTime.Now; + List disposedURLs = new List(); + foreach (KeyValuePair p in myConnections) + { + if (p.Value.isDisposed) + disposedURLs.Add(p.Key); + } + foreach (string url in disposedURLs) + { + string disposeMsg = String.Format("Collecting disposed connections : {0} is disposed by the connection itself, " + + "connection count before deleting this = {1}", url, myConnections.Count); + Console.WriteLine(disposeMsg); + //BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection); + myConnections.Remove(url); + myDeadConnectionURLs.enqueue(url); + } + } + } + + private void myCollectMissingHeartbeatConnections() + { + if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold) + { + myLastHeartbeatDeadCollectionTime = DateTime.Now; + ///these are all the hbs received bw the last + ///and current collectDeadConnection calls + ///if a connection did not send a heartbeat during + ///this interval (which is spanned over a few hb + ///intervals, its considered dead) + IList hbs = myReceivedHeartbeats.dequeueAll(); + Dictionary indexedHbs = new Dictionary(); + List deadClients = new List(); + foreach (string clientURL in hbs) + if (!indexedHbs.ContainsKey(clientURL)) + indexedHbs.Add(clientURL, true); + foreach (string clientURL in myConnections.Keys) + if (!indexedHbs.ContainsKey(clientURL)) + deadClients.Add(clientURL); + foreach (string deadClient in deadClients) + { + string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " + + "connection count before deleting this = {1}", deadClient, myConnections.Count); + //BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection); + Console.WriteLine(disposeMsg); + myConnections[deadClient].Dispose(); + myConnections.Remove(deadClient); + myDeadConnectionURLs.enqueue(deadClient); + } + } + } + + private void myHandleNewConnections() + { + IList newConns = myNewConnections.dequeueAll(); + foreach (ITCPConnection conn in newConns) + { + string ip; + int port; + conn.getPeerAddress(out ip, out port); + string clientURL = new BlutoSubAddress(ip, port).ToString(); + // BlueEventLog.logEvent(String.Format("Handling new connections : {0} connnected, connection count before adding this = {1}", + // clientURL, myConnections.Count), PS3LogLevels.connection); + ///to stop it from being garbabge collected immediatly + myReceivedHeartbeats.enqueue(clientURL); + conn.register(this); + Console.WriteLine($"ManagedTCPMessageHub.myHandleNewConnections clientURL: {clientURL} type: {conn.GetType()}"); + myConnections.Add(clientURL, conn); + myNewConnectionURLs.enqueue(clientURL); + } + } + + #region IDisposable Memebers + public void Dispose() + { + myIsDisposed = true; + + } + public bool isDisposed + { + get { return myIsDisposed; } + } + #endregion + + + } + + + +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs new file mode 100644 index 000000000..7a8fb319b --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.IO; +using mbts; + +namespace BCG +{ + /// + /// Wraps around all the IPubsubMessageHub's that pubsub supports. + /// ie MSMQ and TCP to begin with. + /// + public class ServerMessageHub : IPubsubMessageHub + { + private static ServerMessageHub mySelf = null; + private static object myInstanceLock = new object(); + private Dictionary myHubs = new Dictionary(); + private SyncQueue> myReceivingQueue = new SyncQueue>(); + private ConnectionStatusChange myConnectionCb; + private DataDelegate myDataCb; + private MasterWorker myMasterWorker = MasterWorker.getInstance(false); + + protected ServerMessageHub() : this(PubSub3Config.instance.PS3_ServerPort) { } + protected ServerMessageHub(int portNumber) + { + // TODO: no more MSMQ + // if (!PubSub3Config.instance.PS3_DisableMSMQ) + // myHubs.Add(MSMQSubAddress.prefix, new MSMQMessageHub()); + Console.WriteLine($"ServerMessageHub port: {portNumber}"); + myHubs.Add(BlutoSubAddress.prefix, + new ManagedTCPMessageHub(portNumber, + new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval), + PubSub3Config.instance.PS3_MissingHeartBeatsForGC, + new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_DisposedConnectionCheckPeriod)) + as IPubsubMessageHub); + } + + public int port + { + get { return myHubs[BlutoSubAddress.prefix].port; } + + } + public static ServerMessageHub instance + { + get + { + if (mySelf == null) + { + lock (myInstanceLock) + { + if (mySelf == null) + mySelf = new ServerMessageHub(); + } + + } + return mySelf; + } + } + + + /* Builds the server message hub with a specified port number. Only works if the server message hub has not already been created. + * */ + public static ServerMessageHub getInstance(int portNumber) + { + if (mySelf == null) + { + lock (myInstanceLock) + { + if (mySelf == null) + mySelf = new ServerMessageHub(portNumber); + } + + } + else + throw new ArgumentException("Error: Cannot specify a port number for the ServerMessageHub instance if it has already been created."); + return mySelf; + } + #region IPubsubMessageHub Members + + public void registerDataCallback(DataDelegate del) { myDataCb = del; } + + public void registerConnectionCallback(ConnectionStatusChange del) { myConnectionCb = del; } + + public void activate() + { + string msg = "PubSubServer started .. initializing TCP transport components .. "; + Console.WriteLine(msg); + foreach (IPubsubMessageHub hub in myHubs.Values) + { + hub.registerDataCallback(new DataDelegate(myOnData)); + hub.registerConnectionCallback(new ConnectionStatusChange(myOnConnection)); + hub.activate(); + } + myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, 250)); + } + + public List> getUpdates(TimeSpan timeout, int maxElements) + { + return myReceivingQueue.dequeueMultipleElements(timeout, maxElements); + } + + public bool send(string receiver, byte[] data) + { + if (receiver.StartsWith(BlutoSubAddress.prefix)) + return myHubs[BlutoSubAddress.prefix].send(receiver, data); + else if (receiver.StartsWith(MSMQSubAddress.prefix)) + return myHubs[MSMQSubAddress.prefix].send(receiver, data); + else + return false; + } + + public bool send(string receiver, Stream str) + { + if (receiver.StartsWith(BlutoSubAddress.prefix)) + return myHubs[BlutoSubAddress.prefix].send(receiver, str); + else if (receiver.StartsWith(MSMQSubAddress.prefix)) + return myHubs[MSMQSubAddress.prefix].send(receiver, str); + else + return false; + } + + #endregion + + #region IWorker Members + + /// + /// connection callback will be called within this doWork + /// + public void doWork() + { + foreach (IPubsubMessageHub hub in myHubs.Values) + hub.doWork(); + } + + #endregion + + + public static string internalPublisherConnection = "InternalPublisher"; + + + public void internalSend(byte[] data) + { + myReceivingQueue.enqueue(new Pair(internalPublisherConnection, data)); + } + + private void myOnData(string sender, byte[] msg) + { + if (myDataCb != null) + myOnData(sender, msg); + else + myReceivingQueue.enqueue(new Pair(sender, msg)); + } + + private void myOnConnection(string clientURL, bool connected) + { + if (myConnectionCb != null) + myConnectionCb(clientURL, connected); + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Server/Governors/IPubSubGovernor.cs b/src/DataStreamSuite/PubSub/Server/Governors/IPubSubGovernor.cs new file mode 100644 index 000000000..8c9bcd131 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Governors/IPubSubGovernor.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace BCG +{ + /* + * The PubSubGovernor accepts or rejects publishes to the pubsub + * if the governor rejects a publish attempt, the publish fails. + * Clients will not get the update and the state of the pubsub will not change. + * + */ + public enum PubSubGovernorResponse {Accepted, Rejected_ShouldBeImported, Rejected_ThisIsNotARootPubsub}; + public interface IPubSubGovernor + { + PubSubGovernorResponse requestToPublish(PubMetaData metaData, Stream pubData); + } +} diff --git a/src/DataStreamSuite/PubSub/Server/Governors/PermissiveGovernor.cs b/src/DataStreamSuite/PubSub/Server/Governors/PermissiveGovernor.cs new file mode 100644 index 000000000..0fc7477bf --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/Governors/PermissiveGovernor.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class PermissiveGovernor : IPubSubGovernor + { + public PermissiveGovernor() + { + + } + + public PubSubGovernorResponse requestToPublish(PubMetaData md, System.IO.Stream message) + { + return PubSubGovernorResponse.Accepted; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs new file mode 100644 index 000000000..af25d1432 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Text; +using BCG.Utilities.General; +using System.IO; + +namespace BCG +{ + /// + /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat. + /// A subscriber can subscribe to it, and find out of a PubSub is alive or + /// not. Infact more so, if the subsriptions are working or not. + /// + internal class SystemHeartbeatPublisher : IWorker + { + public static string tagType = "System_Heartbeat"; + public static string tagName = "System_Heartbeat"; + + private static SystemHeartbeatPublisher mySelf; + + private SystemHeartbeatPublisher () + { + + ///this is using a multithreaded MasterWorker since the + ///regular one is called within the MessageReactor, which + ///waits 2500 msecs before calling doWork. This 2500 msecs + ///is only interceded by an external event (publication or + ///subscription) + MasterWorker.getInstance(true).addWorker(this, new TimeSpan(0, 0, 0, 0, 500)); + } + + public static void init() + { + if (mySelf == null) + mySelf = new SystemHeartbeatPublisher(); + } + + #region IWorker Members + + public void doWork() + { + PubMetaData pub = new PubMetaData(tagType, tagName, 1, "local", ClientInfo.thisClient.processString); + MetaData md = new MetaData(pub); + MemoryStream ms = new MemoryStream(); + md.save(ms); + BinaryWriter bw = new BinaryWriter(ms); + bw.Write(true); + ServerMessageHub.instance.internalSend(ms.ToArray()); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs new file mode 100644 index 000000000..b079c924d --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using BCG.Utilities.General; + +namespace BCG +{ + /// + /// Hooks on to the onPublish event. Mantains the statistics + /// for each tag. This also is an IPubSubServerTagHandler. The reason + /// being that since it already holds all the tag statistics, + /// theres no reason to have another class do the same. Hence, + /// it doesnt do anything on the update as it, already has the state. + /// + /// On the doWork, it internaly publishes the delta. + /// + internal class SystemStatsPublisher : IPubSubServerTagHandler + { + public static string tagType = "System_Stats"; + public static string tagName = "System_AllStatistics"; + private Dictionary myTagsReadyToPublish = new Dictionary(); + private Dictionary myTagStatsMap = new Dictionary(); + private MasterWorker myMasterWorker = MasterWorker.getInstance(false); + private static SystemStatsPublisher mySelf = null; + + private SystemStatsPublisher(PubSubState state) + { + state.onPublishStats += new PubSubState.PubSubStatsDelegate(myHandlePublish); + myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_StatisticsUpdatePeriod)); + } + + public static void init(PubSubState state) + { + mySelf = new SystemStatsPublisher(state); + } + + public static IPubSubServerTagHandler getInstance() + { + return mySelf; + } + + void myHandlePublish(string tag, int size) + { + if (tag != tagName) + { + myTagsReadyToPublish[tag] = true; + if (!myTagStatsMap.ContainsKey(tag)) + myTagStatsMap.Add(tag, new TagStatsInfo(tag)); + myTagStatsMap[tag].update(size); + } + } + + #region IWorker Members + + public void doWork() + { + if (myTagsReadyToPublish.Count > 0) + { + PubMetaData pub = new PubMetaData(tagType, tagName, 0, "local", ClientInfo.thisClient.processString); + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + MetaData md = new MetaData(pub); + md.save(str); + new BinaryWriter (str).Write(myTagsReadyToPublish.Count); + foreach (string tag in myTagsReadyToPublish.Keys) + { + myTagStatsMap[tag].writeTo(str); + } + myTagsReadyToPublish.Clear(); + ServerMessageHub.instance.internalSend(str.ToArray()); + } + } + + #endregion + + #region IPubSubServerTagHandler Members + + public void update(string tag, Stream[] messageBodies) + { + } + + public byte[] getContent(string tag) + { + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + bw.Write(myTagStatsMap.Count); + foreach (TagStatsInfo t in myTagStatsMap.Values) + t.writeTo(bw.BaseStream); + return (bw.BaseStream as MemoryStream).ToArray(); + } + + public string[] supportedTagTypes + { + get { return new string[] { SystemStatsPublisher.tagType }; } + } + + public void register(InternalPublishDelegate callback) { } + + #endregion + } +} diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs new file mode 100644 index 000000000..dd1b02c63 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Text; +using BCG.Utilities.General; +using System.IO; +using System.Threading; +using mbts; + +namespace BCG +{ + /// + /// This class deals with the publication of System_Sub tag, + /// which is the information about all the subscribers and the + /// publisher(s) of every tag in the pubsub. this is accomplished + /// by listening to the PubsubState for the onPub, onSub and + /// onUnsub events. + /// A list of all the tags that have changed + /// is collected and a TagInfo for each tag is fired at doWork + /// (after getting data from the PubSubState) + /// This class doesnt have any permanant state except some config + /// type variables. It has a 'temporary' state myTagsReadyToPublish + /// which is flushed every doWork. + /// This also is a IPubSubServerTagHandler (just like SystemStatsPublisher). + /// It uses the PubSubState to publish the entire map on getUpdates + /// + class SystemSubscriptionMapPublisher : IPubSubServerTagHandler + { + private PubSubState myPubsubState; + private Dictionary myTagsReadyToPublish = new Dictionary(); + private readonly string myIp = SysInfoManager.getIP(); + public static string tagType = "System_Sub"; + public static string tagName = "System_AllSubscriptions"; + private MasterWorker myMasterWorker = MasterWorker.getInstance(false); + private static IPubSubServerTagHandler mySelf; + + private SystemSubscriptionMapPublisher(PubSubState state) + { + myPubsubState = state; + myPubsubState.onTagMapChanged += new PubSubState.PubSubStateChangeDelegate(myOnTagStateChanges); + myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_StateUpdatePeriod)); + } + + public static void init(PubSubState state) + { + mySelf = new SystemSubscriptionMapPublisher(state); + } + + public static IPubSubServerTagHandler getInstance() + { + return mySelf; + } + + private void myOnTagStateChanges(string[] tags) + { + foreach (string t in tags) + myTagsReadyToPublish[t] = true; + } + + private void myInternalPublish() + { + if (myTagsReadyToPublish.Count > 0) + { + ///serialize and write a PubMetaData + PubMetaData pub = new PubMetaData(tagType, tagName, 0, "local", ClientInfo.thisClient.processString); + MemoryStream str = new MemoryStream(); + MetaData md = new MetaData(pub); + md.save(str); + BinaryWriter bw = new BinaryWriter(str); + ///count of the following items + bw.Write(myTagsReadyToPublish.Count); + ///TagPubSubInfo serialized one by one + foreach (string tag in myTagsReadyToPublish.Keys) + { + Dictionary subscribers, activePubslishers; + ICollection> deadPublishers; + myPubsubState.getAllConnections(tag, out subscribers, out activePubslishers, out deadPublishers); + TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, myIp); + ti.writeTo(str); + } + myTagsReadyToPublish.Clear(); + ServerMessageHub.instance.internalSend(str.ToArray()); + } + } + + #region IWorker Members + + public void doWork() + { + myInternalPublish(); + } + + #endregion + + #region IPubSubServerTagHandler Members + + public void update(string tag, Stream[] messageBodies) { } + + public byte[] getContent(string t) + { + ICollection allTags = myPubsubState.allTags; + MemoryStream str = new MemoryStream (); + BinaryWriter bw = new BinaryWriter(str); + bw.Write(allTags.Count); + foreach (string tag in allTags) + { + Dictionary subscribers, activePubslishers; + ICollection> deadPublishers; + myPubsubState.getAllConnections(tag, out subscribers, out activePubslishers, out deadPublishers); + TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, myIp); + ti.writeTo(str); + } + return str.ToArray(); + } + + public string[] supportedTagTypes + { + get { return new string[] { SystemSubscriptionMapPublisher.tagType }; } + } + + public void register(InternalPublishDelegate callback) { } + + #endregion + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs new file mode 100644 index 000000000..3152d5387 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -0,0 +1,444 @@ +using System; +//using System.Messaging; +using System.Collections; +using System.Net; +using System.Net.Sockets; +using System.IO; +using System.Text; +using System.Threading; +using System.Reflection; +using System.Runtime.Serialization.Formatters.Binary; +using System.Xml.Serialization; +using System.Collections.Generic; + +//using BCG.Utilities; +//using BCG.Utilities.General; +using mbts; + +namespace BCG +{ + public class PubSubMessageReactor + { + private ServerMessageHub myMessageHub; + private delegate void MyMsgHandler(string sender, Stream msg, MetaData m); + private Dictionary myOperationToHandler = new Dictionary(); + private Thread myReceivingProcThread = null; + private PubSubState myState = new PubSubState(); + private bool myStop = false; + private MasterWorker myMasterWorker = MasterWorker.getInstance(false); + private double myProcessingWarningThreshold = + Convert.ToDouble(PubSub3Config.instance.PS3_MessageProcessingWarningThreshold); + private HighPerformanceTimer myTimer = new HighPerformanceTimer(); + + public delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds); + public SingleTagTimeSliceDelegate onSingleTagPub; + private HighPerformanceTimer mySingleTagTimeSliceTimer = new HighPerformanceTimer(); + //this thing keeps track of whether a given tag is compressed or not. It is used for logging messages. + + /* This delegate allows pubsub components to be notified when the identity of the pubsub is known*/ + public delegate void IdentityFoundDelegate(string identity, bool isRoot); + public event IdentityFoundDelegate onIdentityFound; + public void fireIdentityFound(string identity, bool isRoot) + { + if (onIdentityFound != null) + onIdentityFound(identity, isRoot); + } + + private IPubSubGovernor myGovernor; + + + public Dictionary myTagCompressionState = new Dictionary(); + + public PubSubMessageReactor() : this(false, false, string.Empty, string.Empty) { } + public PubSubMessageReactor(string q) : this(false, false, string.Empty, q) { } + + public PubSubMessageReactor(bool clearQueue, bool bootEvent, string oldIP, string psq) + { + myOperationToHandler.Add(typeof(PubMetaData).Name, new MyMsgHandler(myHandlePub)); + myOperationToHandler.Add(typeof(SubMetaData).Name, new MyMsgHandler(myHandleSub)); + myOperationToHandler.Add(typeof(UnSubMetaData).Name, new MyMsgHandler(myHandleUnsub)); + myOperationToHandler.Add(typeof(TransactionMetaData).Name, new MyMsgHandler(myHandleTransaction)); + + // TODO, governor needs to be removed since its subststaion retired code + // if (PubSub3Config.instance.PS3_Components.governor != null) + // myGovernor = PubSubComponentFactory.buildGovernor(this); + + //BlueEventLog.logEvent("System Initialized", PS3LogLevels.info); + + myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc)); + myMessageHub = ServerMessageHub.instance; + myMessageHub.registerConnectionCallback(new ConnectionStatusChange(myHandleConnections)); + + + } + + #region message handlers + + public PubSubState state + { + get { return myState; } + //this variable is read only + + } + + private void myHandlePub(string sender, Stream msg, MetaData m) + { + + mySingleTagTimeSliceTimer.start(); + ///the position in the stream is set to where the data starts with, + ///since the meta data was already read in the myProcessMsg + PubMetaData pub = new PubMetaData(); + m.copyTo(pub); + pub.connection = sender; + Console.WriteLine("PubSubMessageReactor.myHandlePub sender: " + sender + " tag: " + pub.tag); + + //check to see if this publish took a while to get here + pub.warnIfLatencyExceedsThreshold(); + + //check the governor to see if this publish is allowed, if we have a governor + PubSubGovernorResponse response = PubSubGovernorResponse.Accepted; + if (myGovernor != null) + response = myGovernor.requestToPublish(pub, msg); + + if (response != PubSubGovernorResponse.Accepted) + { + // BlueEventLog2.instance().logEvent(string.Concat("PubSub Governor ", + // myGovernor.GetType().Name, + // " returned response ", + // response.ToString(), + // " publish on tag", + // pub.tag, + // "was ignored."), BlueEventLog2Level.Warning); + return; + } + + + //report the fact that we have a publish on this tag, if it's a debug tag + if (PubSub3Config.instance.isTagDebugTag(pub.tag)) + { + // BlueEventLog.logEvent(string.Concat(GetType().Name, + // " got publish on tag ", + // pub.tag, + // " from app ", pub.publishingApp, + // " over connection ", + // pub.connection, ".\n", + // pub.isCompressed ? "Data was compressed.\n" : "Data was not compresed.\n", + // "total size: ", + // pub.size), + + // PS3LogLevels.debug); + + + } + + ///for all the subscribers, sends the messages over straightaway + foreach (string address in myState.getSubscriberConnectionsForTag(pub.tag)) + { + ///the position is adjusted to zero, so as to include + ///the meta data as part of the message + msg.Position = 0; + myMessageHub.send(address, msg); + } + + + + //reset the position of the message so we can decompress it appropriately + msg.Position = 0; + msg.Position = m.size(); + + if (pub.isCompressed) + { + long mdLength = msg.Position; + long compressedSize = msg.Length - mdLength; + if (compressedSize == 0) + { + string errorMsg = "Error: tag " + pub.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update."; + //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error); + return; + } + msg = StreamDecompressor.decompressStream(msg); + long decompressedSize = msg.Length - mdLength; + + if (decompressedSize != pub.decompressedSize || compressedSize != pub.size) + { + string compressionWarning = string.Format("Warning: Stream for tag {0} decompressed from {1} to {2} bytes, with {3} byes of metadata. Metadata expected {4} to {5}. Tag state not updated.", + pub.tag, + compressedSize, + decompressedSize, + mdLength, + pub.size, + pub.decompressedSize); + + //BlueEventLog.logEvent(compressionWarning, BlueEventLevel.Error); + return; + + } + + } + + //give the user a notification of the compression status of the tag has changed. + if ((!(myTagCompressionState.ContainsKey(pub.tag)) || + myTagCompressionState[pub.tag] != pub.isCompressed)) + { + string compressionMessage; + myTagCompressionState[pub.tag] = pub.isCompressed; + + if (pub.isCompressed) + { + compressionMessage = "Tag " + pub.tag + " is compressed."; + //BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio); + } + else + { + compressionMessage = "Tag " + pub.tag + " is not compressed."; + //BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio); + + } + + } + + + ///updating state + msg.Position = 0; + msg.Position += m.size(); + myState.updatePubData(pub, msg); + + mySingleTagTimeSliceTimer.stop(); + + double thisTagProcessingTime = mySingleTagTimeSliceTimer.lastIntervalTime; + if (onSingleTagPub != null) + onSingleTagPub(pub.tag, thisTagProcessingTime); + + + } + + private void myHandleSub(string sender, Stream msg, MetaData m) + { + SubMetaData sub = new SubMetaData(); + m.copyTo(sub); + sub.connection = sender; + + ///update the state + myState.sub(sub); + ///to all these new subscriber s, send the existing data + foreach (string tag in sub.tags) + { + byte[] data = myState.getContent(tag); + //ie data has been published once + if (data != null) + { + string tagType = myState.getTagType(tag); + ///passing "" as the 4th arg, as PubMeta is used both by the + ///publisher to publish data to the server (where the publisher + ///indicates its identification as the 4th arg) as well as by + ///the server to send data to the client (like here). we can + ///identify the publisher by filing in the 4th arg, but its not + ///needed as of now, so its left black to save a few bytes + PubMetaData pub; + MemoryStream realContent; + if (PubSub3Config.instance.isTagCompressed(tag)) + { + SharpZipCompressor Kompessor = new SharpZipCompressor(); + realContent = new MemoryStream(); + Kompessor.compress(new MemoryStream(data), realContent); + + pub = new PubMetaData(tagType, tag, (int)realContent.Length, (int)data.Length, "", ""); + + } + else + { + realContent = new MemoryStream(data); + pub = new PubMetaData(tagType, tag, data.Length, "", ""); + } + + MemoryStream toSend = new MemoryStream(); + new MetaData(pub).save(toSend); + toSend.Write(realContent.ToArray(), 0, (int)realContent.Length); + + toSend.Position = 0; + myMessageHub.send(sub.connection, toSend); + } + } + } + + private void myHandleUnsub(string sender, Stream msg, MetaData m) + { + UnSubMetaData unsub = new UnSubMetaData(); + m.copyTo(unsub); + if (unsub.connection.Trim().Length == 0) + unsub.connection = sender; + myState.unsub(unsub); + } + + private void myHandleTransaction(string sender, Stream msg, MetaData md) + { + mySingleTagTimeSliceTimer.start(); + TransactionMetaData t = new TransactionMetaData(); + md.copyTo(t); + int nMessages = t.nMessages; + ///form the structure tag->byte[]* + Dictionary tagToMessages = new Dictionary(); + BinaryReader br = new BinaryReader(msg); + for (int i = 0; i < nMessages; i++) + { + MetaData pubMd = new MetaData(msg); + PubMetaData pub = new PubMetaData(); + pubMd.copyTo(pub); + pub.connection = sender; + byte[] tagData = new byte[pub.size]; + msg.Read(tagData, 0, tagData.Length); + //report the fact that we have a publish on this tag, if it's a debug tag + if (PubSub3Config.instance.isTagDebugTag(pub.tag)) + { + // BlueEventLog.logEvent(string.Concat(GetType().Name, + // " got publish on tag ", + // pub.tag, + // "\nfrom connection ", + // pub.connection, ".\n", + // pub.isCompressed ? "Data was compressed.\n" : "Data was not compressed.\n", + // "total size: ", + // tagData.Length, + // "\nMessage ", + // i, + // " of a transaction with ", + // nMessages, + // " total messages."), + + // PS3LogLevels.debug); + + + } + + myState.updatePubData(pub, new MemoryStream(tagData)); + tagToMessages.Add(pub.tag, tagData); + } + ///this gives subscriber->tag* + Dictionary> subscriberToTags = myState.getSubscriberConnectionsForTags(tagToMessages.Keys); + foreach (KeyValuePair> kvp in subscriberToTags) + { + MemoryStream subStream = new MemoryStream(); + TransactionMetaData subTransaction = new TransactionMetaData(kvp.Value.Count); + new MetaData(subTransaction).save(subStream); + foreach (string tag in kvp.Value) + { + byte[] tagData = tagToMessages[tag]; + PubMetaData pub = new PubMetaData(myState.getTagType(tag), tag, tagData.Length, "", ""); + new MetaData(pub).save(subStream); + subStream.Write(tagData, 0, tagData.Length); + } + subStream.Position = 0; + //myState.send(kvp.Key, subStream); + myMessageHub.send(kvp.Key, subStream); + } + + mySingleTagTimeSliceTimer.stop(); + double timeExpended = mySingleTagTimeSliceTimer.lastIntervalTime; + //Don't try to divide by zero + int numTags = tagToMessages.Keys.Count; + if (numTags != 0) + { + double timeExpendedPerTag = timeExpended / tagToMessages.Keys.Count; + if (onSingleTagPub != null) + foreach (string tag in tagToMessages.Keys) + onSingleTagPub(tag, timeExpendedPerTag); + } + } + + /// + /// Extracts the meta data out, and based on the meta data calls + /// the appropriate handler. also calls the SystemTagPublisher's doWork + /// + /// + /// + private void myProcessMsg(string sender, Stream msg, out MetaData md) + { + md = new MetaData(msg); + MyMsgHandler op = myOperationToHandler[md["op-code"]]; + //BlueEventLog.logEvent(string.Format("process message {0}", m)); + op(sender, msg, md); + } + + #endregion + + private void myHandleConnections(string client, bool connected) + { + Console.WriteLine("PubSubMessageReactor.myHandleConnections client: " + client + " connected: " + connected); + ///interested in disconnections only, as connections of no + ///significance until they subscribe, in which case an entry + ///will be made + if (!connected) + myState.removeConnection(client); + } + + public void stop() + { + myStop = true; + } + + public void start() + { + myReceivingProcThread.Start(); + } + + /// + /// loops around and reads messages from the messageReceiver. calls processMsg + /// passing in the IClientMessage. also logs stuff about the message. + /// + private void myReceivingThreadProc() + { + TimeSpan receiveTimeout = new TimeSpan(0, 0, 0, 0, 2500); + myMessageHub.activate(); + + //once you've activated the message hub, you're OK to create the watchers + //(some of the watchers depend upon the message hub) + // TODO + // PubSubComponentFactory.buildPubSubWatchers(this); + + string sender; + Stream msg; + MetaData meta; + while (!myStop) + { + sender = null; + msg = null; + meta = null; + try + { + List> receivedData = myMessageHub.getUpdates(receiveTimeout, 1); + if (receivedData.Count > 0) + { + sender = receivedData[0].first; + msg = new MemoryStream(receivedData[0].second); + myTimer.start(); + myProcessMsg(sender, msg, out meta); + myTimer.stop(); + if (myTimer.lastIntervalTime > myProcessingWarningThreshold) + { + string warningMsg = String.Format("processing metadata {0} took {1} msecs", meta, myTimer.lastIntervalTime); + //BlueEventLog.logEvent(warningMsg, PS3LogLevels.performance); + } + } + myMasterWorker.doWork(); + } + catch (Exception e) + { + //BlueEventLog.logEvent(e, BlueEventLevel.Error); + Console.WriteLine("{0} : exception handling message: {1}\n", DateTime.Now, e.Message); + string errMsg = String.Format("Following information about the meta data and sender is available : " + + "\n" + + "sender = {0}" + + "\n" + + "meta data = {1}" + + "\n" + + "stack trace:\n" + + e.StackTrace, sender == null ? "Unknown" : sender, + meta == null ? "Unknown" : meta.ToString()); + //BlueEventLog.logEvent(errMsg, BlueEventLevel.Error); + //BlueEventLog.logEvent(errMsg, BlueEventLog2Level.Console); + if (e is OutOfMemoryException) + throw e; + } + } + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs b/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs new file mode 100644 index 000000000..36e3fc67f --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +/* + * This class contains the recent chagnes of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. + * It has a list of new subscription events. + * + */ +namespace BCG +{ + public class PubSubSingleTagState + { + private string myTag; + public string tag + { + get { return myTag; } + set { myTag = value; } + + } + //we have this here as SortedList so that we can compare for equality + private SortedList myPublishers; + private SortedList mySubscribers; + private Dictionary myNewSubscribers; + private Dictionary myRemovedSubscribers; + private Dictionary myNewPublishers; + private Dictionary myRemovedPublishers; + + + + public PubSubSingleTagState() + { + //the total state of the tag + myPublishers = new SortedList(); + mySubscribers = new SortedList(); + + //how has the state changed recentlty + myNewSubscribers = new Dictionary(); + myNewPublishers = new Dictionary(); + myRemovedPublishers= new Dictionary(); + myRemovedSubscribers = new Dictionary(); + } + + //returns a list of all the events that need to be published. + public PubSubSingleTagStateEvent[] getNewEventsForPublishing() + { + PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count]; + + myNewSubscribers.Values.CopyTo(newEvents, 0); + int index = myNewSubscribers.Count; + myNewPublishers.Values.CopyTo(newEvents, index); + index += myNewPublishers.Count; + myRemovedSubscribers.Values.CopyTo(newEvents, index); + index += myRemovedSubscribers.Count; + myRemovedPublishers.Values.CopyTo(newEvents, index); + + return newEvents; + + } + + //call this after you have published the new events + //this is made public for code readability reasons + //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily + public void resetNewEvents() + { + myNewSubscribers = new Dictionary(); + myNewPublishers = new Dictionary(); + myRemovedSubscribers = new Dictionary(); + myRemovedPublishers = new Dictionary(); + } + + public void newPublish(PubSubSingleTagStateEvent newPublisher) + { + //don't do anything here if we already knew about the publishers because we were already dealing with this publishers + if(!myPublishers.ContainsKey(newPublisher.app)) + { + myPublishers.Add(newPublisher.app, newPublisher.app); + + //if we were going to delete it, that means it was already being published by this app + //that means we should no logner delete it, but we don't have to announce that it was published again + if (myRemovedPublishers.ContainsKey(newPublisher.app)) + myRemovedPublishers.Remove(newPublisher.app); + else + myNewPublishers[newPublisher.app] = newPublisher; + + + } + } + + public void removePublisher(PubSubSingleTagStateEvent removedPublisher) + { + if(myPublishers.ContainsKey(removedPublisher.app)) + { + myPublishers.Remove(removedPublisher.app); + + //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it + if (myNewPublishers.ContainsKey(removedPublisher.app)) + myNewPublishers.Remove(removedPublisher.app); + else + myRemovedPublishers[removedPublisher.app] = removedPublisher; + } + + } + + public void newSubscription(PubSubSingleTagStateEvent newSubscriber) + { + if (!mySubscribers.ContainsKey(newSubscriber.app)) + { + mySubscribers.Add(newSubscriber.app, newSubscriber.app); + + if (myRemovedSubscribers.ContainsKey(newSubscriber.app)) + myRemovedSubscribers.Remove(newSubscriber.app); + else + myNewSubscribers[newSubscriber.app] = newSubscriber; + + + } + } + + public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) + { + if(mySubscribers.ContainsKey(removedSubscriber.app)) + { + mySubscribers.Remove(removedSubscriber.app); + //if we were going to add this subscriber, we no logner want to + //this means no event needs to be published + if (myNewSubscribers.ContainsKey(removedSubscriber.app)) + myNewSubscribers.Remove(removedSubscriber.app); + else + //we have to remove this guy in this updated + myRemovedSubscribers[removedSubscriber.app] = removedSubscriber; + } + + + } + + + } +} diff --git a/src/DataStreamSuite/PubSub/Server/PubSubSingleTagStateEvent.cs b/src/DataStreamSuite/PubSub/Server/PubSubSingleTagStateEvent.cs new file mode 100644 index 000000000..2f2d2803a --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/PubSubSingleTagStateEvent.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +/* + * This class represents an event happening to a single tag in a pubsub (or substation.) It could be that a tag is being published, + * or it could be a subscription to a tag or an unsubscription to a tag. + * + */ +namespace BCG +{ + public class PubSubSingleTagStateEvent + { + + public enum EventType { publish, subscription, unsubscription, publisherDie }; + + public string tag; + public EventType eventType; + public string connection; + public string app; + + public static PubSubSingleTagStateEvent fromStream(Stream theStream) + { + BinaryReader bw = new BinaryReader(theStream); + + PubSubSingleTagStateEvent thisEvent = new PubSubSingleTagStateEvent(); + + thisEvent.tag = bw.ReadString(); + thisEvent.eventType = (EventType) bw.ReadInt32(); + thisEvent.connection = bw.ReadString(); + thisEvent.app = bw.ReadString(); + + + return thisEvent; + + } + public static PubSubSingleTagStateEvent fromBytes(byte[] bytes) + { + MemoryStream mem = new MemoryStream(bytes); + return fromStream(mem); + + } + //for debugging + public override string ToString() + { + StringBuilder theBuilder = new StringBuilder(); + theBuilder.Append(tag+" "); + switch (eventType) + { + case EventType.publish: + theBuilder.Append("published"); + break; + case EventType.publisherDie: + theBuilder.Append("publisher died"); + break; + case EventType.subscription: + theBuilder.Append("was subscribed to"); + break; + case EventType.unsubscription: + theBuilder.Append("was unsubscribed to"); + break; + + } + theBuilder.Append(" app: " + app); + theBuilder.Append(" connection: " + connection); + + return theBuilder.ToString(); + + } + public byte[] toBytes() + { + MemoryStream mem = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(mem); + + bw.Write(tag); + bw.Write((int) eventType); + bw.Write(connection); + bw.Write(app); + + return mem.GetBuffer(); + + } + + + public PubSubSingleTagStateEvent(string argTag, EventType type, string theConnection, string theApp) + { + + tag = argTag; + eventType = type; + connection = theConnection; + app = theApp; + + } + + private PubSubSingleTagStateEvent() {} + + } +} diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs new file mode 100644 index 000000000..8b600b932 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs @@ -0,0 +1,261 @@ +using System; +//using System.Messaging; +using System.Collections; +using System.Collections.Generic; +using System.Reflection; +using System.IO; + +using mbts; + +namespace BCG +{ + /// + /// Mantains all the state that there is in the server. + /// This information is keyed off the tags (rather than + /// keyed off subscribers or publishers) + /// State information consists of 2 distints sets : + /// myTagToData consists of all the data published to + /// this pubsub. + /// myTagToTagInfo consists of all the subscription and + /// publishing information + /// + /// + public class PubSubState + { + + public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent); + public event PubSubStateSingleTagChangeDelegate onSingleTagChange; + public delegate void PubSubStateChangeDelegate(string[] tags); + public event PubSubStateChangeDelegate onTagMapChanged; + public delegate void PubSubStatsDelegate(string tag, int size); + public event PubSubStatsDelegate onPublishStats; + + private Dictionary myTagToData = new Dictionary(); + private Dictionary myTagToPubsubMap = new Dictionary(); + + public PubSubState() + { + SystemSubscriptionMapPublisher.init(this); + SystemStatsPublisher.init(this); + SystemHeartbeatPublisher.init(); + } + + #region subscription based state changes + + /// + /// everytime a subscription is made, this gets called + /// by PubsubServer2. Adds a + /// TODO : subscibers sending sub twice from the client + /// + /// + /// + public void sub(SubMetaData sub) + { + ///tag based key ds update + foreach (string tag in sub.tags) + { + if (!myTagToPubsubMap.ContainsKey(tag)) + { + myTagToPubsubMap.Add(tag, new SingleTagPubSubMap(tag)); + } + myTagToPubsubMap[tag].onSub(sub); + + if (onSingleTagChange != null) + onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribingApp)); + } + if (onTagMapChanged != null) + onTagMapChanged(sub.tags); + } + + public void unsub(UnSubMetaData unsub) + { + foreach (string tag in unsub.tags) + { + if (myTagToPubsubMap.ContainsKey(tag)) + { + ///maybe check here, that if there are no subscribers + ///and publishers for the tag then remove it. + myTagToPubsubMap[tag].onUnsub(unsub); + if (onSingleTagChange != null) + onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp)); + } + else + { + string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" + + " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp); + //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); + } + } + if (onTagMapChanged != null) + onTagMapChanged(unsub.tags); + } + + public void removeConnection(string connection) + { + List tagsEffected = new List(); + foreach (KeyValuePair p in myTagToPubsubMap) + { + ///maybe check here, that if there are no subscribers + ///and publishers for the tag then remove it. + bool wasPublisher = false; + bool wasSubscriber = false; + string tag = p.Key; + if (p.Value.onConnectionRemoveIfExists(connection, out wasPublisher, out wasSubscriber)) + tagsEffected.Add(tag); + + if (onSingleTagChange != null) + { + //we don't know the app or key qualifier or anything for these connection. that is unfortunate. + if (wasPublisher) + onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.publisherDie, connection, "")); + if (wasSubscriber) + onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, connection, "")); + + } + } + if (onTagMapChanged != null) + onTagMapChanged(tagsEffected.ToArray()); + + + } + + #endregion + + #region data based state changes + /// + /// this is the only place in the state class where the + /// data is modified ; happens when data is published + /// + /// + /// + public void updatePubData(PubMetaData pub, Stream msg) + { + int sz = (int)(msg.Length - msg.Position); + ///this is the only place where SingleTagData should be created + if (!myTagToData.ContainsKey(pub.tag)) + { + ///first time data is published for this tag + myTagToData.Add(pub.tag, new SingleTagData(pub.tag, pub.type)); + } + myTagToData[pub.tag].pubUpdate(msg); + + if (!myTagToPubsubMap.ContainsKey(pub.tag)) + { + myTagToPubsubMap.Add(pub.tag, new SingleTagPubSubMap(pub.tag)); + } + bool pubChanged = false; + myTagToPubsubMap[pub.tag].onPub(pub, out pubChanged); + if (pubChanged) + { + if (onTagMapChanged != null) + onTagMapChanged(new string[] { pub.tag }); + + if (onSingleTagChange != null) + onSingleTagChange( + new PubSubSingleTagStateEvent(pub.tag, + PubSubSingleTagStateEvent.EventType.publish, + pub.connection, pub.publishingApp)); + } + if (onPublishStats != null) + onPublishStats(pub.tag, sz); + + } + + #endregion + + public ICollection getSubscriberConnectionsForTag(string tag) + { + if (myTagToPubsubMap.ContainsKey(tag)) + return myTagToPubsubMap[tag].subscriberConnections; + else + return new string[] { }; + } + + public ICollection getSubscribingAppsForTag(string tag) + { + if (myTagToPubsubMap.ContainsKey(tag)) + return myTagToPubsubMap[tag].subscripingApps; + else + return new string[] { }; + } + + public byte[] getContent(string tag) + { + if (!myTagToData.ContainsKey(tag)) + return null; + else + return myTagToData[tag].getCurrentContent(); + } + + public string getTagType(string tag) + { + if (!myTagToData.ContainsKey(tag)) + return null; + else + return myTagToData[tag].tagType; + } + + /// + /// returns all the applications 'connected' to one tag, which + /// includes a Dictionary of connection vs subscribers, + /// Dictionary of connections vs publishers and a connection, app + /// pair of all dead publishers + /// + /// + /// + /// + /// + public void getAllConnections(string tag, out Dictionary subscribers, + out Dictionary activePublishers, + out ICollection> deadPublishers) + { + if (myTagToPubsubMap.ContainsKey(tag)) + { + SingleTagPubSubMap map = myTagToPubsubMap[tag]; + subscribers = map.subscribers; + activePublishers = map.activePublishers; + deadPublishers = map.deadPublishers; + } + else + { + subscribers = activePublishers = new Dictionary(); + deadPublishers = new List>(); + } + } + + public ICollection allTags + { + get { return myTagToPubsubMap.Keys; } + } + + public ICollection tagsWithData + { + get { return myTagToData.Keys; } + } + + /// + /// given a list of tags, constructs a Dictionary of + /// subscribers vs tags + /// + /// + /// + public Dictionary> getSubscriberConnectionsForTags(IEnumerable tags) + { + Dictionary> retVal = new Dictionary>(); + foreach (string tag in tags) + { + if (myTagToPubsubMap.ContainsKey(tag)) + { + foreach (string sub in myTagToPubsubMap[tag].subscriberConnections) + { + if (!retVal.ContainsKey(sub)) + retVal.Add(sub, new List()); + retVal[sub].Add(tag); + } + } + } + return retVal; + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs new file mode 100644 index 000000000..12eb85c70 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs @@ -0,0 +1,97 @@ +//using System.Messaging; +using System; +using System.IO; +using System.Collections; +using System.Text; +using System.Collections.Generic; +using BCG.Utilities; +using mbts; + +namespace BCG +{ + /// + /// SingleTagData consists of (as the name implies) + /// the data published for a tag + /// + internal class SingleTagData + { + private string myTagType = null; + private string myTag; + private byte[] myLastPublishedMsg; + private IPubSubServerTagHandler myHandler; + + public SingleTagData(string tag, string tagType) + { + myTag = tag; + myTagType = tagType; + if (myTagType == SystemSubscriptionMapPublisher.tagType) + myHandler = SystemSubscriptionMapPublisher.getInstance(); + else if (myTagType == SystemStatsPublisher.tagType) + myHandler = SystemStatsPublisher.getInstance(); + else + myHandler = TagType2HandlerMap.getHandler(myTagType, true); + if (myHandler != null) + myHandler.register(new InternalPublishDelegate(myHandleHandlerCallbacks)); + } + + public string tag { get { return myTag; } } + + public string tagType { get { return myTagType; } } + + internal void pubUpdate(Stream stream) + { + if (myHandler != null) + { + try + { + myHandler.update(myTag, new Stream[] { stream }); + } + catch (Exception e) + { + // BlueEventLog.logEvent(e); + string errMsg = "tag handler threw an exception " + e.Message + " for tag type = " + myTagType + + " tag = " + myTag + " length = " + stream.Length + "\n" + + "tag stated not updated. Stack Trace : " + e.StackTrace; + Console.WriteLine(errMsg); + if (e is OutOfMemoryException) + throw e; + else + throw new ApplicationException(errMsg, e); + } + } + else + { + myLastPublishedMsg = new byte[stream.Length - stream.Position]; + stream.Read(myLastPublishedMsg, 0, myLastPublishedMsg.Length); + } + } + + public byte[] getCurrentContent() + { + if (myHandler != null) + { + return myHandler.getContent(myTag); + } + else + { + return myLastPublishedMsg; + } + } + + private void myHandleHandlerCallbacks(ICollection> cbItems) + { + foreach (Triplet p in cbItems) + { + string tagName = p.first, tagType = p.second; + byte[] data = p.third; + PubMetaData pub = new PubMetaData(tagType, tagName, data.Length, "local", ClientInfo.thisClient.processString); + MemoryStream str = new MemoryStream(); + MetaData md = new MetaData(pub); + md.save(str); + new BinaryWriter(str).Write(data); + ServerMessageHub.instance.internalSend(str.ToArray()); + } + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs b/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs new file mode 100644 index 000000000..a466fa71b --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.Text; +using BCG.Utilities; +using mbts; + +namespace BCG +{ + /// + /// SingleTagPubSubMap consists of all the state w.r.t to + /// publishing and subscription of a single tag. this information + /// consists of the publisher, and a map of the subscriber + /// connection vs the apps they're coming from + /// + internal class SingleTagPubSubMap + { + private string myTag; + /// + /// map of clientURL to client app (machine;app;pid;clientURL) + /// + Dictionary mySubConnToApp = new Dictionary(); + /// + /// map of all the active publishers (generally unique) to + /// the publishing app + /// + Dictionary myActivePubConnToApp = new Dictionary(); + /// + /// list of dead publishers to the conn + /// + List> myDeadPubConnToApp = new List>(); + + public SingleTagPubSubMap(string tag) + { + myTag = tag; + } + + /// + /// Returns a list all the active connections that + /// have subscribed to this tag + /// + public ICollection subscriberConnections + { + get { return mySubConnToApp.Keys; } + } + + /// + /// Returns a list of all the applications that have + /// subscribed to this tag + /// + public ICollection subscripingApps + { + get { return mySubConnToApp.Values; } + } + + /// + /// Returns a dictionary for all subsriber + /// connections as key and the application + /// as value + /// + public Dictionary subscribers + { + get { return mySubConnToApp; } + } + + /// + /// Returns a dictionary for all active publishing + /// connection as key and the application as value + /// + public Dictionary activePublishers + { + get { return myActivePubConnToApp; } + } + + /// + /// Returns a list of pair of dead connections and + /// theur corresponding applications + /// + public ICollection> deadPublishers + { + get { return myDeadPubConnToApp; } + } + + /// + /// Called by PubsubState when a subscription is made to this tag. + /// Returns a true, is the subscriber was not present, else logs + /// a warning and returns a false + /// + /// + /// + public bool onSub(SubMetaData sub) + { + if (mySubConnToApp.ContainsKey(sub.connection)) + { + string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}" + + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribingApp); + //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); + return false; + } + else + { + mySubConnToApp.Add(sub.connection, sub.subscribingApp); + return true; + } + } + + /// + /// Called by PubsubState when a subscriber unsubscribes to this tag. + /// Returns a true when a subscriber exists, else logs a warning and + /// returns a false + /// + /// + /// + public bool onUnsub(UnSubMetaData unsub) + { + if (mySubConnToApp.ContainsKey(unsub.connection)) + { + mySubConnToApp.Remove(unsub.connection); + return true; + } + else + { + string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}", + myTag, unsub.connection, unsub.unsubscribingApp); + //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); + return false; + } + } + + /// + /// Called by PubsubState when a client is disconnected. The client + /// might not have anything to do with this tag, in which a false is + /// returned, else true + /// + /// + /// + public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber) + { + bool rv = false; + if (mySubConnToApp.ContainsKey(connection)) + { + mySubConnToApp.Remove(connection); + wasSubscriber = true; + rv = true; + } + else + wasSubscriber = false; + if (myActivePubConnToApp.ContainsKey(connection)) + { + myDeadPubConnToApp.Add(new Pair(connection, myActivePubConnToApp[connection])); + myActivePubConnToApp.Remove(connection); + rv = true; + wasPublisher = true; + } + else + wasPublisher = false; + return rv; + } + + /// + /// Called by pubsubState, when this tag is published. if publisher + /// has changed, the out paramater is set to true + /// + /// + /// + public void onPub(PubMetaData pub, out bool newPublisher) + { + if (myActivePubConnToApp.ContainsKey(pub.connection)) + { + newPublisher = false; + } + else + { + myActivePubConnToApp.Add(pub.connection, pub.publishingApp); + newPublisher = true; + } + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs new file mode 100644 index 000000000..a76f31c23 --- /dev/null +++ b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs @@ -0,0 +1,217 @@ +using System; +using System.IO; +using System.Collections.Generic; +using System.Text; +using BCG.Utilities; +using BCG.Utilities.General; +using mbts; + +namespace BCG +{ + /// + /// Deals with the subscriptions (and getting data) from one + /// pubsub server. Once again, theres no sharing of conenctions + /// from this point. Every new instance of SingleServerSubscriberCore + /// will open its new dedicated connection to the serverURL that + /// its concerned with (passed as a conctructor arg). + /// + + internal interface ISingleServerSubscriberCore : IDisposable + { + IList getUpdates(int timeout); + event PubSubConnectionChange onPubSubConnectionChange; + void subscribe(params string[] tags); + void unsubscribe(params string[] tags); + } + internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore + { + private HighPerformanceTimer myTimer = new HighPerformanceTimer(); + private PubSubServerConnectionFactory myConnectionFactory; + private IPubSubServerConnection myConnection; + private List myTags = new List(); + + public event PubSubConnectionChange onPubSubConnectionChange; + + public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) + { + myConnectionFactory = connectionFactory; + myConnection = myConnectionFactory.getConnection(serverURL); + myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot)); + } + + public IList getUpdates() + { + return getUpdates(1); + } + + /// + /// gets as many data items that it can get in + /// timeout milliseconds + /// + /// + /// + public IList getUpdates(int timeout) + { + myConnection.doWork(); + List retVal = new List(); + IList dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue); + foreach (byte[] data in dataItems) + { + processMessage(data, retVal); + } + return retVal; + + } + + public List getSubscribedTags() + { + return myTags; + } + + public void subscribe(params string[] tags) + { + foreach (string tag in tags) + { + if (myTags.Contains(tag)) + { + string errMsg = String.Format("tag {0} was already subscribed from server {1}", + tag, myConnection.serverURL); + // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); + } + else + { + myTags.Add(tag); + } + } + mySubscribe(tags); + } + + public void unsubscribe(params string[] tags) + { + List unsubTags = new List(); + foreach (string tag in tags) + { + if (myTags.Contains(tag)) + { + myTags.Remove(tag); + unsubTags.Add(tag); + } + else + { + string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " + + "it was never subscribed", tag, myConnection.serverURL); + // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); + } + } + UnSubMetaData unsub = new UnSubMetaData(myConnection.clientURL, unsubTags.ToArray()); + Stream str = new MemoryStream(); + new MetaData(unsub).save(str); + str.Position = 0; + myConnection.send(str); + } + + #region IDisposable Members + + public void Dispose() + { + ///remove the subscriber + //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream()); + myConnection.Dispose(); + + //clear the list of tags you've subscribed to. This way, if you're ever used again, + //you have to send all the subscription metadata out again. + myTags.Clear(); + } + + #endregion + + private void mySubscribe(params string[] tags) + { + SubMetaData sub = new SubMetaData(myConnection.clientURL, tags); + Stream str = new MemoryStream(); + new MetaData(sub).save(str); + str.Position = 0; + myConnection.send(str); + } + + private void myHandleServerReboot(string serverURL, bool connection) + { + if (connection) + { + //ie connection established again, so has to + //resubscribe to all the tags + // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags", + // PS3LogLevels.disconnection); + mySubscribe(myTags.ToArray()); + + + } + else + { + // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...", + // PS3LogLevels.disconnection); + } + + //let anyone watching you know that your connection has changed. + if (onPubSubConnectionChange != null) + onPubSubConnectionChange(connection, serverURL); + } + + internal static void processMessage(byte[] data, List updates) + { + if (data != null && data.Length > 0) + { + MemoryStream str = new MemoryStream(data); + MetaData md = new MetaData(str); + MemoryStream bodyStream = str; + long bodyPosition = str.Position; + object op = Activator.CreateInstance (Type.GetType("BCG." + md["op-code"])); + md.copyTo(op); + if (op is PubMetaData) + { + PubMetaData p = op as PubMetaData; + //BCG.BlueTrace.WriteBlueCommReceiveW(p.tag, p.publishTimeTicks); + + //check to see if this publish took a while to get here + p.warnIfLatencyExceedsThreshold(); + //find out if we need to decompress this data + Stream realBody = bodyStream; + if (p.isCompressed) + { + long compressedSize = bodyStream.Length - bodyStream.Position; + if (compressedSize == 0) + { + string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update."; + Console.WriteLine(errorMsg); + //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error); + return; + + + } + + realBody = StreamDecompressor.decompressStream(bodyStream); + realBody.Position = bodyPosition; + } + + + + updates.Add(new PubSubUpdate(p.tag, p.type, realBody)); + } + else if (op is TransactionMetaData) + { + TransactionMetaData tr = op as TransactionMetaData; + for (int i = 0; i < tr.nMessages; i++) + { + PubMetaData pub = new PubMetaData(); + new MetaData(bodyStream).copyTo(pub); + byte[] trConstituent = new byte[pub.size]; + bodyStream.Read(trConstituent, 0, trConstituent.Length); + updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent))); + } + } + else{} + //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal"); + } + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/StaticPublisherCore.cs b/src/DataStreamSuite/PubSub/StaticPublisherCore.cs new file mode 100644 index 000000000..57ff7459b --- /dev/null +++ b/src/DataStreamSuite/PubSub/StaticPublisherCore.cs @@ -0,0 +1,571 @@ +using System; +//using System.Messaging; +using System.Xml.Serialization; +using System.Text; +using System.IO; +using System.Collections; +using System.Collections.Generic; +using System.Threading; + +using BCG; +//using BCG.Utilities.BlueConfiguration; +using BCG.Utilities; +using mbts; +/* StaticPublisherCore.cs + * This file was originally called 'PublisherCore.cs'. It was renamed so that the publisher core could be replaced. + * + * */ +namespace BCG +{ + public class CrushedTagManager + { + private static CrushedTagManager mySelf; + private Dictionary myTags = new Dictionary(); + public static CrushedTagManager instance + { + get + { + if (mySelf == null) + mySelf = new CrushedTagManager(); + return mySelf; + } + } + public bool isTagCrushable(string tag) + { + tag = tag.ToLower(); + lock (typeof(CrushedTagManager)) + { + if (!myTags.ContainsKey(tag)) + { + foreach (string r in PubSub3Config.instance.PS3_CrushableTags) + if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag)) + { + myTags.Add(tag, true); + break; + } + if (!myTags.ContainsKey(tag)) + myTags.Add(tag, false); + } + } + return myTags[tag]; + } + } + /// + /// All the data for a single tag, thats published within + /// a transaction + /// + public class TagWaitingInTransaction + { + private bool myIsCrushable; + private List myData = new List(); + private int myTotalSize = 0; + private string myTag, myTagType; + public TagWaitingInTransaction(string theTagType, string theTag) + { + myTag = theTag; + myTagType = theTagType; + myIsCrushable = CrushedTagManager.instance.isTagCrushable(myTag); + } + + public void addPub(Stream pubStream) + { + if (myIsCrushable) + { + byte[] newData = new byte[(int)(pubStream.Length - pubStream.Position)]; + pubStream.Read(newData, 0, newData.Length); + myData.Clear(); + myData.Add(newData); + myTotalSize = newData.Length; + } + else + { + byte[] newData = new byte[(int)(pubStream.Length - pubStream.Position)]; + pubStream.Read(newData, 0, newData.Length); + myData.Add(newData); + myTotalSize += newData.Length; + } + } + + public void writeTo(Stream s, string connection, string publishingApp) + { + new MetaData(new PubMetaData(myTagType, myTag, myTotalSize, connection, publishingApp)).save(s); + foreach (byte[] data in myData) + s.Write(data, 0, data.Length); + } + + public string tag { get { return myTag; } } + public string tagType { get { return myTagType; } } + public List data { get { return myData; } } + } + + public enum PublisherCoreMessageType + { + transactionBegin, + transactionCommit, + transactionAbort, + publish, + clear + } + + public class PublisherCoreMessage + { + public PublisherCoreMessageType msgType; + public string tagType, tag; + public Stream data; + + public PublisherCoreMessage(string tagToClear) + { + tag = tagToClear; + msgType = PublisherCoreMessageType.clear; + + } + + public PublisherCoreMessage(PublisherCoreMessageType theMsgType) + { + msgType = theMsgType; + } + + public PublisherCoreMessage(string theTagType, string theTag, Stream theData) + { + msgType = PublisherCoreMessageType.publish; + tagType = theTagType; + tag = theTag; + data = theData; + } + } + + // this class uses an internal queue + thread to serialize and process calls + // made through its public interface (IPublisherCore) + public class StaticPublisherCore : IPublisherCore + { + // used to allow for different transports + // to be used when communicating to a pubsubserver + protected PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(false); + + // internal queue used to serialize pubsub method calls + protected BlockingQueue myMainQueue = new BlockingQueue(); + + // all the data published through this object in snapshot form + // i.e. in a form that can be used repopulate a pubsub server + // in the event of a crash + protected AllPublishedTagState myState = new AllPublishedTagState(); + + // a different IPubSubServerConnection object is used to interact + // with each of the pubsub servers used by this publishercore + protected List mySenders = new List(); + + // a dictionary for fast retrieval of publishers + // tag -> IPubSubServerConnection + protected Dictionary myTagToSender = new Dictionary(); + + // data waiting for transaction to end or abort + // tag->TagWaitingInTransaction + protected Dictionary myWaitingInTransactionMessages = new Dictionary(); + + // router object, used to select which pubsub should be used for each tag + protected IPubSubRouter myRouter = null; + + // flag indicating wether we are in the middle of a transaction or not + protected bool myInTransaction = false; + + #region constructors + + public StaticPublisherCore() + // TODOds + : this(new PubSubRouter()) + { } + + public StaticPublisherCore(string serverURL) + : this(new PubSubRouter(new PubSubRoute(serverURL, ".*"))) + { } + + public StaticPublisherCore(IPubSubRouter r) + { + myRouter = r; + Thread pumpThread = new Thread(new ThreadStart(myMessagePump)); + pumpThread.IsBackground = true; + pumpThread.Start(); + } + + public void Dispose() + { } + #endregion + + public int getNumberOfWaitingMessages() { return myMainQueue.getCount(); } + + #region Main Thread - all library functions are called on the main thread and queued to be picked by the pump + + public void publish(string tag, Stream content) + { publish("", tag, content); } + + public void publish(string tagType, string tag, Stream content) + { + myMainQueue.enqueue(new PublisherCoreMessage(tagType, tag, content)); + } + + public void beginTransaction() + { + myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionBegin)); + } + + public void commitTransaction() + { + myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionCommit)); + } + + public void abortTransaction() + { + myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionAbort)); + } + + public bool inTransaction + { get { return myInTransaction; } } + + public void clearState(string tag) + { + myMainQueue.enqueue(new PublisherCoreMessage(tag)); + } + + #endregion + + #region Message Pump Thread - this is where messages are dequeued and the action performed + + private void myMessagePump() + { + while (true) + { + try + { + while (true) + { + object data = myMainQueue.dequeue(1000); + if (data != null) + { + int qc = myMainQueue.getCount(); + PublisherCoreMessage msg = data as PublisherCoreMessage; + if (msg.msgType == PublisherCoreMessageType.transactionBegin) + { + myBeginTransaction(); + } + else if (msg.msgType == PublisherCoreMessageType.transactionCommit) + { + myCommitTransaction(); + } + else if (msg.msgType == PublisherCoreMessageType.transactionAbort) + { + myAbortTransaction(); + } + else if (msg.msgType == PublisherCoreMessageType.publish) + { + myPublish(msg.tagType, msg.tag, msg.data); + } + else if (msg.msgType == PublisherCoreMessageType.clear) + { + myClearState(msg.tag); + } + } + + foreach (IPubSubServerConnection conn in mySenders) + { + conn.doWork(); + } + } + } + catch (Exception ex) + { + // BlueEventLog.logEvent(ex); + if (ex is OutOfMemoryException) + throw ex; + } + } + } + + private void myBeginTransaction() + { + // BlueEventLog.logAssert(!inTransaction); + //BlueEventLog.logEvent("StaticPublisherCore Starting Transaction.", BlueEventLevel.Information); + myInTransaction = true; + } + + private void myCommitTransaction() + { + myEndTransaction(true); + } + + private void myAbortTransaction() + { + myEndTransaction(false); + } + + + + //based on success, sends a TA or TE, and sets myInTransaction to false + //TODO: if a transaction is sending to multiple pubsubs (in order to + //keep it consistent with the way it was done earlier), this should + //simply publish it non transactionally (easy way out), or + //transactionally with respect to each pubsub + protected virtual void myEndTransaction(bool success) + { + lock (this) + { + // BlueEventLog.logAssert(inTransaction); + //BlueEventLog.logEvent("StaticPublisherCore Finishing Transaction.", BlueEventLevel.Information); + myInTransaction = false; + if (success) + { + // pubsub server -> tags with updates + Dictionary> routesToTags = new Dictionary>(); + + // group updates per pubsub server + foreach (string tag in myWaitingInTransactionMessages.Keys) + { + string thisURL = myRouter.getPubURLForTag(tag); + if (!routesToTags.ContainsKey(thisURL)) + routesToTags[thisURL] = new List(); + routesToTags[thisURL].Add(tag); + } + + // more than one server means we won't be able to honor the transactionality request + // log a warning if that is the case. + if (routesToTags.Keys.Count > 1) + { + //this is going to be a long error message, but it's an importat one to be able to debug. + StringBuilder errorMessageBuilder = new StringBuilder(); + errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n"); + errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n"); + + foreach (string route in routesToTags.Keys) + { + foreach (string tag in routesToTags[route]) + { + errorMessageBuilder.Append("\tTag "); + errorMessageBuilder.Append(tag); + errorMessageBuilder.Append(" is routed to "); + errorMessageBuilder.Append(route); + errorMessageBuilder.Append(".\n"); + } + + } + string errMsg = errorMessageBuilder.ToString(); + // BlueEventLog.logEvent(errMsg, BlueEventLevel.Diagnosis); + } + + // iterate through each server with pending messages + foreach (KeyValuePair> kvp in routesToTags) + { + IPubSubServerConnection transactionSender = null; + string publishingApp = ClientInfo.thisClient.processString; + string connection = kvp.Key; + Stream transactionData = new MemoryStream(); + + // iterate through each of the pending tag updates for one server + foreach (string tag in kvp.Value) + { + ///it would be the same transaction sender that would + ///return each time + transactionSender = myGetOrBuildSender(tag); + + + TagWaitingInTransaction waiting = myWaitingInTransactionMessages[tag]; + // iterate through each of the updates for a specific tag, specific server + foreach (byte[] data in waiting.data) + // update internal state + myState.publish(waiting.tag, waiting.tagType, data); + // add a pubsub header and copy each message to the transactionData stream + waiting.writeTo(transactionData, connection, publishingApp); + } + // now transactionData holds all the updates for all the tags + // that are to be sent to the PubSub represented by transactionSender + + // add a transaction header, indicating how many tags in the transaction + TransactionMetaData tMd = new TransactionMetaData(kvp.Value.Count); + + // rewind transaction data to the beggining + transactionData.Position = 0; + + // send the entire thing to the specific pubsub server. + transactionSender.send(new PubSubMessage(new MetaData(tMd), transactionData).toStream()); + } + } + myWaitingInTransactionMessages.Clear(); + } + } + + protected virtual void myPublish(string tagType, string tag, Stream content) + { + if (myInTransaction) + { + ///if its in transaction message is cached, and is sent + ///only when theres a commit or abort + if (!myWaitingInTransactionMessages.ContainsKey(tag)) + myWaitingInTransactionMessages.Add(tag, new TagWaitingInTransaction(tagType, tag)); + myWaitingInTransactionMessages[tag].addPub(content); + } + else + { + //if (CrushedTagManager.instance.isTagCrushable(tag)) + { + myState.publish(tag, tagType, content); + if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) + { + // BlueEventLog.logEvent("Sending crushable tag " + tag, BlueEventLevel.Diagnosis); + mySendData(tag, tagType, content); + } + else + { + // BlueEventLog.logEvent("Could not send tag " + tag + " since its waiting on an ack", BlueEventLevel.Diagnosis); + myCrushableTagsWaitingOnAck[tag].second++; + } + } + // else + // { + // // update snapshot state + // myState.publish(tag, tagType, content); + // // publish out the data + // mySendData(tag, tagType, content); + // } + } + } + + // add header, possibly compress the data, and sendto appropriate pubsub + private void mySendData(string tag, string tagType, Stream content) + { + long initPos = content.Position; + // locate right pubsub for this tag + IPubSubServerConnection serverConn = myGetOrBuildSender(tag); + Stream contentToPublish = content; + PubMetaData pubMd; + + if (PubSub3Config.instance.isTagCompressed(tag)) + { + contentToPublish = new MemoryStream(); + SharpZipCompressor Kompressor = new SharpZipCompressor(); + Kompressor.compress(content, contentToPublish); + contentToPublish.Position = 0; + pubMd = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString); + if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) + { + string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length); + // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio); + } + } + else + pubMd = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString); + + content.Position = initPos; + if (CrushedTagManager.instance.isTagCrushable(tag)) + { + string refId = String.Empty; + Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL); + serverConn.sendWithConfirmation((new PubSubMessage(new MetaData(pubMd), contentToPublish).toStream() as MemoryStream).ToArray(), ref refId); + myCrushableTagsWaitingOnAck.Add(tag, new Pair(refId, 0)); + } + else + { + Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL); + serverConn.send(new PubSubMessage(new MetaData(pubMd), contentToPublish).toStream()); + } + } + + private void myClearState(string tag) + { + if (myTagToSender.ContainsKey(tag)) + { + myTagToSender.Remove(tag); + } + myState.clear(tag); + + } + + protected virtual IPubSubServerConnection myGetOrBuildSender(string tag) + { + if (!myTagToSender.ContainsKey(tag)) + { + string route = myRouter.getPubURLForTag(tag); + IPubSubServerConnection sender = myConnectionFactory.getConnection(route); + myTagToSender.Add(tag, sender); + if (!mySenders.Contains(sender)) + { + mySenders.Add(sender); + sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot)); + sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm)); + } + + } + return myTagToSender[tag]; + } + + ///in case of the server coming up, it publishes all the stored state + ///for that server, back to it + protected void myHandleServerReboot(string serverURL, bool connection) + { + if (connection) + { + string thisPublisherInfo = ClientInfo.thisClient.processString; + foreach (KeyValuePair p in myTagToSender) + { + if (p.Value.serverURL == serverURL) + { + string tagType; + string tag = p.Key; + myCrushableTagsWaitingOnAck.Clear(); + byte[] data = myState.getContent(tag, out tagType); + mySendData(tag, tagType, new MemoryStream(data)); + + //Stream content = new MemoryStream(data); + //Stream contentToPublish = content; + //PubMetaData pubMd; + + //if (PubSub3Config.instance.isTagCompressed(tag)) + //{ + // contentToPublish = new MemoryStream(); + // SharpZipCompressor Kompressor = new SharpZipCompressor(); + // Kompressor.compress(content, contentToPublish); + // contentToPublish.Position = 0; + // pubMd = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, p.Value.serverURL, ClientInfo.thisClient.processString); + // //string msg = String.Format("Compressed tag {0} from {1} to {2} bytes.", tag, content.Length, contentToPublish.Length); + // //BlueEventLog.logEvent(msg, PS3LogLevels.info); + //} + //else + // pubMd = new PubMetaData(tagType, tag, (int)content.Length, p.Value.serverURL, ClientInfo.thisClient.processString); + + + //PubMetaData pMd = new PubMetaData(tagType, p.Key, data.Length, + // serverURL, thisPublisherInfo); + //p.Value.send(new PubSubMessage(new MetaData(pMd), new MemoryStream(data)).toStream()); + } + } + } + } + + #endregion + + /// + /// This is the state of the crushable tags. An entry here means + /// the tag is waiting for an ack. The value is a pair of conf id + /// and the number of packets that could not be sent because + /// of waiting for ack + /// + private Dictionary> myCrushableTagsWaitingOnAck = new Dictionary>(); + private void myOnSendConfirm(string messageId, bool confirmed) + { + foreach (string tag in new List(myCrushableTagsWaitingOnAck.Keys)) + { + Pair tagData = myCrushableTagsWaitingOnAck[tag]; + if (messageId == tagData.first) + { + myCrushableTagsWaitingOnAck.Remove(tag); + if (tagData.second > 0) + { + // BlueEventLog.logEvent("Missed sending " + tagData.second + " messages, sending latest message now", BlueEventLevel.Diagnosis); + /// this data was waiting on an + /// ack, need to resend it + string tagType; + byte[] data = myState.getContent(tag, out tagType); + myPublish(tagType, tag, new MemoryStream(data)); + } + } + } + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs b/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs new file mode 100644 index 000000000..b24b00829 --- /dev/null +++ b/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs @@ -0,0 +1,187 @@ +using System; +//using System.Messaging; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Text.RegularExpressions; + +using BCG; +using BCG.Utilities; +using BCG.Utilities.General; +using mbts; +//using BCG.Utilities.BlueConfiguration; + +namespace BCG +{ + + /// + /// This class serves as the subscrition interface to the pubsub. The only methods + /// it exposes are getUpdates and subscribe. Internally, it manages a SingleServerSubscriberCore + /// foreach server. So this class does the routing part, and each SingleServerSubscriberCore + /// handles dealing sending (sub/unsub) and receiving (data) messages from the transport. + /// There is no sharing with respect to this class. Ie, it creates new SingleServerSubscriberCore + /// to the servers even though there might be so created by other instances of the SubscriberCore + /// + /// + public class StaticSubscriberCore : ISubscriberCore + { + private Dictionary myServerURLToSubCore = new Dictionary(); + private IPubSubRouter myRouter; + private HighPerformanceTimer myTimer = new HighPerformanceTimer(); + private PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(true); + + public event PubSubConnectionChange onPubSubConnectionChange; + + + #region constructors + + public StaticSubscriberCore() + { + //this defaults to a basic pubsubrouter if the config key is not filled out + myInit(myGetRouter()); + } + public StaticSubscriberCore(string siteName) + { + myInit(new PubSubRouter(PubSub3Config.instance.getRouting(siteName))); + } + + private IPubSubRouter myGetRouter() + {return new PubSubRouter(PubSub3Config.instance.getRouting());} + + public StaticSubscriberCore(PubSubRoute[] routes) + { + myInit(new PubSubRouter(routes)); + } + + public StaticSubscriberCore(IPubSubRouter router) + { + myInit(router); + } + + #endregion + + private void myInit(IPubSubRouter router) + { + myRouter = router; + } + + public ArrayList getUpdates() + { + return getUpdates(10); + } + + public ArrayList getUpdates(TimeSpan timeout) + { + return getUpdates(Convert.ToInt32(timeout.TotalMilliseconds)); + } + + public ArrayList getUpdates(int timeout) + { + //BCG.BlueTrace.WriteTraceMethodEnterW("StaticSubscriberCore.getUpdates"); + lock (myServerURLToSubCore) + { + + if (myServerURLToSubCore.Count == 0) + { + return new ArrayList(); + } + else + { + ///gets a time share for every subscriber and + ///calls their updates + myTimer.start(); + int timeShare = timeout / myServerURLToSubCore.Count; + List retVal = new List(); + foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values) + retVal.AddRange(subCore.getUpdates(timeShare)); + myTimer.stop(); + //if (myTimer.lastIntervalTime > 10) + // BlueEventLog.logEvent("get updates taking " + myTimer.lastIntervalTime, BlueEventLevel.Error); + + //BCG.BlueTrace.WriteTraceMethodExitW("StaticSubscriberCore.getUpdates"); + return new ArrayList(retVal); + } + } + + } + + private Dictionary myBlueCommTags = new Dictionary (); + + bool myIsBlueCommTag(string tag) + { + return false; + } + + public void subscribe(params string[] tags) + { + lock (myServerURLToSubCore) + { + foreach (string tag in tags) + { + if (myIsBlueCommTag(tag)) + { + } + else + { + string serverURL = myRouter.getSubURLForTag(tag); + if (!myServerURLToSubCore.ContainsKey(serverURL)) + { + // BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information); + myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory)); + myServerURLToSubCore[serverURL].onPubSubConnectionChange += myHandleConnectionChange; + } + myServerURLToSubCore[serverURL].subscribe(tag); + } + + } + } + } + public void myHandleConnectionChange (bool isConnected, string connection) + { + if (onPubSubConnectionChange != null) + onPubSubConnectionChange(isConnected, connection); + + } + + public void unsubscribe(params string[] tags) + { + lock (myServerURLToSubCore) + { + + foreach (string tag in tags) + { + if (myIsBlueCommTag(tag)) + myServerURLToSubCore["bluecomm"].unsubscribe(tag); + else + { + string serverURL = myRouter.getSubURLForTag(tag); + + if (!myServerURLToSubCore.ContainsKey(serverURL)) + { + // BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information); + myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory)); + } + myServerURLToSubCore[serverURL].unsubscribe(tag); + } + } + } + } + public IPubSubRouter getRouter() + { + return myRouter; + } + + #region IDisposable Members + + public void Dispose() + { + + foreach (ISingleServerSubscriberCore sssc in myServerURLToSubCore.Values) + sssc.Dispose(); + myConnectionFactory.Dispose(); + } + + #endregion + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/StreamOpCode.cs b/src/DataStreamSuite/PubSub/StreamOpCode.cs new file mode 100644 index 000000000..df5481c76 --- /dev/null +++ b/src/DataStreamSuite/PubSub/StreamOpCode.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BCG +{ + public class StreamOpCode + { + public static readonly byte create = 0, //creates a table + add = 1, //adds a single row + remove = 2, //removes a single row + change = 3, //changes a single row + multipleRemoveAdd = 4,//removes and adds multiple rows + multipleChange = 5; //changes multiples rows + + } +} diff --git a/src/DataStreamSuite/PubSub/SubscriberCore.cs b/src/DataStreamSuite/PubSub/SubscriberCore.cs new file mode 100644 index 000000000..fbd893c28 --- /dev/null +++ b/src/DataStreamSuite/PubSub/SubscriberCore.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Collections; + +namespace BCG +{ + /* + * This class has been changed to be a thin pass-through shell around different subscriber core implementations. + * The specific implementation used is determined by the config. + * + * + * */ + public class SubscriberCore : ISubscriberCore + { + ISubscriberCore mySubscriberCore = null; + + public SubscriberCore() + { + // TODOds + mySubscriberCore = new StaticSubscriberCore();// PubSubComponentFactory.getSubscriberCore(); + mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange; + } + + public SubscriberCore(string siteName) + { + mySubscriberCore = new StaticSubscriberCore(siteName); + mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange; + } + public SubscriberCore(IPubSubRouter theRouter) + { + // TODOds + mySubscriberCore = new SubscriberCore(theRouter);// PubSubComponentFactory.getSubscriberCore(); + mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange; + } + + public SubscriberCore(PubSubRoute[] routes) + { + //this is always a static subscriber core + mySubscriberCore = new StaticSubscriberCore(routes); + mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange; + } + + + public IPubSubRouter getRouter() + { + return mySubscriberCore.getRouter(); + } + + public ArrayList getUpdates() + { + return mySubscriberCore.getUpdates(10); + } + + public ArrayList getUpdates(TimeSpan timeout) + { + return mySubscriberCore.getUpdates(Convert.ToInt32(timeout.TotalMilliseconds)); + } + + public ArrayList getUpdates(int timeout) + { + return mySubscriberCore.getUpdates(timeout); + } + + public void subscribe(params string[] tags) + { + mySubscriberCore.subscribe(tags); + } + + public void unsubscribe(params string[] tags) + { + mySubscriberCore.unsubscribe(tags); + } + + public void Dispose() + { + mySubscriberCore.Dispose(); + + } + private void myHandleConnectionChange(bool isConnected, string connection) + { + if (onPubSubConnectionChange != null) + onPubSubConnectionChange(isConnected, connection); + } + + public event PubSubConnectionChange onPubSubConnectionChange; + + } +} diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs index 015c77104..311681e5c 100644 --- a/src/DataStreamSuite/PubSubServer/Program.cs +++ b/src/DataStreamSuite/PubSubServer/Program.cs @@ -1,3 +1,52 @@ using System; +using System.Threading; +using System.Linq; -Console.WriteLine("Starting PubSubServer..."); +//using BCG.Utilities; +//using BCG.Utilities.General; + + +namespace BCG +{ + class PubSub2Main + { + + /// + /// The main entry point for the application. + /// + [STAThread] + public static void Main(string[] args) + { + if (args.Length > 0 && args[0] == "daemon") + args = args.Skip(1).ToArray(); + + //CoreServicesInit.init(); + PubSub3Config config = PubSub3Config.instance; + + long currentMemory = 0; + int defaultWaitTime = 100; + bool gced = false; + new Thread(() => + { + while (true) + { + gced = false; + currentMemory = GC.GetTotalMemory(false); + if (config.PS3_GCActivationThreshold > 0 && currentMemory > config.PS3_GCActivationThreshold) + { + string gcMsg = String.Format("GCing memory, mem before GC = {0}", currentMemory); + // BlueEventLog.logEvent(gcMsg, PS3LogLevels.info); + Console.WriteLine(gcMsg); + GC.Collect(); + gced = true; + } + Thread.Sleep(gced ? config.PS3_WaitPeriodAfterGC : defaultWaitTime); + } + }).Start(); + + PubSubMessageReactor pubSubServer = new PubSubMessageReactor(); + pubSubServer.start(); + + } + } +} diff --git a/src/DataStreamSuite/PubSubServer/PubSubServer.csproj b/src/DataStreamSuite/PubSubServer/PubSubServer.csproj index 8c8d40ddc..f2d3b0459 100644 --- a/src/DataStreamSuite/PubSubServer/PubSubServer.csproj +++ b/src/DataStreamSuite/PubSubServer/PubSubServer.csproj @@ -1,5 +1,11 @@ + + + + + + Exe net7.0