refactor: Move a few low-level DataStream types to DataStreamCore

So the lex/yacc generators can reference them
This commit is contained in:
John Colvin committed 2023-09-29 13:10:49 -04:00
1 parent d1f5ae86fd
commit ac1ede5986
10 files changed
+6 -30

No files matched your search

@@ -1,3 +1,4 @@
#nullable enable
using System;
using System.Collections;
@@ -21,18 +22,18 @@ namespace BCG
/// a sizeable time, eg for a table with 20000 rows and a schema of 20 columns, we will save about
/// half a million type checkings
/// </summary>
object this[int index] {get;}
object this[string colName] {get;}
object? this[int index] {get;}
object? this[string colName] {get;}
}
namespace Extensions
{
public static class IBlueRowExtensions
{
public static bool isDBNull(this object val)
public static bool isDBNull(this object? val)
{
return (val == null || val == DBNull.Value);
}
public static bool isNotDBNull(this object val)
public static bool isNotDBNull(this object? val)
{
return (val != null && val != DBNull.Value);
}
@@ -0,0 +1,111 @@
using System;
//using BCG.Utilities.BlueConfiguration;
namespace BCG
{
public class DataStreamLoadFileConfig //: ConfigurationRequest
{
private static DataStreamLoadFileConfig myInstance;
private string[] myAssemblies;
private string myIsTestMode = "false";
private bool myProfileStreams = false;
private double myProfileThreshold = 500;
private bool myUseNewJoin, myUseIncrementsForMultipleOps, myUseNewAppend, myUseNewFilter, myUseNewMap,
myUseUncopiedNewRowsInFireEvent, myUseModularizedBridge;
private bool myLogBlobPackets = false;
private bool myUseStaticParser = true;
//[Description("DataStream: list of assemblies providing the streams")]
public string [] loadedAssemblies
{
get { return myAssemblies; }
set { myAssemblies = value; }
}
public bool useStaticParser
{
get { return myUseStaticParser; }
set { myUseStaticParser = value; }
}
public bool logBlobPackets
{
get { return myLogBlobPackets; }
set { myLogBlobPackets = value; }
}
public string isTestMode
{
get {return myIsTestMode;}
set {myIsTestMode = value;}
}
public bool profileStreams
{
get { return myProfileStreams; }
set { myProfileStreams = Convert.ToBoolean(value); }
}
public double profileThreshold
{
get { return myProfileThreshold; }
set { myProfileThreshold = Convert.ToDouble(value); }
}
public bool isValid()
{
return true;
//return myVerify (myAssemblies);
}
public bool useNewJoin
{
get { return myUseNewJoin; }
set { myUseNewJoin = value; }
}
public bool useNewAppend
{
get { return myUseNewAppend; }
set { myUseNewAppend = value; }
}
public bool useNewFilter
{
get { return myUseNewFilter; }
set { myUseNewFilter = value; }
}
public bool useNewMap
{
get { return myUseNewMap; }
set { myUseNewMap = value; }
}
public bool useIncrementsForMultipleOps
{
get { return myUseIncrementsForMultipleOps; }
set { myUseIncrementsForMultipleOps = value; }
}
public bool useUncopiedNewRowsInFireEvent
{
get { return myUseUncopiedNewRowsInFireEvent; }
set { myUseUncopiedNewRowsInFireEvent = value; }
}
public bool useModularizedDataStreamBridge
{
get { return myUseModularizedBridge; }
set { myUseModularizedBridge = value; }
}
public static DataStreamLoadFileConfig instance =>
myInstance ??= new DataStreamLoadFileConfig();
// myInstance == null ?
// ConfigurationRequester.request (typeof (DataStreamLoadFileConfig)) as DataStreamLoadFileConfig
// as DataStreamLoadFileConfig :
// myInstance;
}
}
@@ -0,0 +1,181 @@
using System;
using System.Collections;
//using BCG.Utilities;
//using BCG.Utilities.General;
using mbts;
using mbts.spark;
namespace BCG
{
/// <summary>
/// Base class for all the sreams
/// </summary>
public class DataStream
{
internal static event EventHandler onFireStart;
protected bool myTestMode = Convert.ToBoolean (DataStreamLoadFileConfig.instance.isTestMode);
protected bool myFirstSubImplemented = true,
myLastSubImplemented = true;
protected IDictionary myDelMap = new Hashtable ();
private Guid myID = Guid.Empty;
public DateTime last_fired_time = DateTime.MinValue;
public int n_updates = 0;
public DataStream ()
:
this (null)
{}
public DataStream (DataStream realSource)
{
myRealSource = realSource;
myOnChangeDelegate = null;
myPendingAdds = new Stack();
}
public int numberOfSubscribers
{
get
{
return
myOnChangeDelegate != null?
myOnChangeDelegate.GetInvocationList().Length
:
0;
}
}
public delegate void ChangeDelegate(object newValue);
public event ChangeDelegate onChange
{
add
{
ChangeDelegate currValue = value;
if (myTestMode)
{
ChangeDelegateWrapper w = new ChangeDelegateWrapper (value);
currValue = new ChangeDelegate (w.wrapperFunction);
myDelMap.Add (value, currValue);
}
if (myRealSource != null)
myRealSource.onChange += currValue;
else
{
myOnChangeDelegate =
(ChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);
if (myOnChangeDelegate.GetInvocationList().Length == 1) // first subscriber
{
myPendingAdds.Push(new ArrayList());
myOnFirstSubscriber();
ArrayList calledDelegates = (ArrayList) myPendingAdds.Pop();
object v = myGetValueForInitialSubscription();
//BlueTrace.WriteViewStreamSubscribe(this.ID, v != null);
if (v != null && calledDelegates.IndexOf(currValue) == -1)
currValue (v);
}
else
{
object v = myGetValueForInitialSubscription();
// BlueTrace.WriteViewStreamSubscribe(this.ID, v != null);
if (v != null)
currValue (v);
}
}
}
remove
{
ChangeDelegate currValue = value;
// BlueTrace.WriteViewStreamUnsubscribe(this.ID);
if (myTestMode)
{
currValue = myDelMap[value] as ChangeDelegate;
myDelMap.Remove (value);
}
if (myRealSource != null)
myRealSource.onChange -= currValue;
else
{
myOnChangeDelegate =
(ChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);
// this is need to avoid a callback in a reentrant add.
if (myPendingAdds.Count > 0)
((ArrayList) (myPendingAdds.Peek())).Add(currValue);
if (myOnChangeDelegate == null) // last subscriber
{
myOnLastSubscriber();
myLastValue = null;
}
}
}
}
protected virtual object myGetValueForInitialSubscription()
{return myLastValue;}
protected void mySetRealSource (DataStream src)
{myRealSource = src;}
protected virtual void myOnFirstSubscriber() {myFirstSubImplemented = false;}
protected virtual void myOnLastSubscriber () {myLastSubImplemented = false;}
protected virtual void myFireEvent(object o)
{
last_fired_time = DateTime.Now;
n_updates++;
// BlueEventLog.logAssert (myRealSource == null);
myLastValue = o;
if (myOnChangeDelegate != null)
{
// this is need to avoid a callback in a reentrant add.
if (myPendingAdds.Count > 0)
foreach (Delegate d in myOnChangeDelegate.GetInvocationList())
((ArrayList) (myPendingAdds.Peek())).Add(d);
if (onFireStart != null)
onFireStart(this, EventArgs.Empty);
myOnChangeDelegate (o);
}
}
public Guid ID
{
get
{
if (this.myID == Guid.Empty) // && BlueTraceHelper.IsEnabled(BlueTraceLevel.Verbose, BlueTraceKeywords.Streams))
{
// BlueTraceHelper.FastInitGuid(ref this.myID);
// BlueTrace.WriteViewStreamCreate(this.GetType().ToString(), this.myID);
}
return this.myID;
}
}
// this was made internal for testing purposes only.
// don't mess with it.
internal ChangeDelegate myOnChangeDelegate;
protected object myLastValue;
protected DataStream myRealSource;
private Stack myPendingAdds;
}
internal class ChangeDelegateWrapper
{
private HighPerformanceTimer myTimer = new HighPerformanceTimer ();
private DataStream.ChangeDelegate myDel;
internal ChangeDelegateWrapper (DataStream.ChangeDelegate del) {myDel = del;}
internal void wrapperFunction (object o)
{
myTimer.start ();
myDel (o);
myTimer.stop ();
string minfo = "Type = " + myDel.Method.DeclaringType.FullName + " , method = " + myDel.Method.Name;
// BlueEventLog.logEvent (minfo + " , time = " + myTimer.lastIntervalTime.ToString (), BlueEventLevel.Performance);
}
}
}
@@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public delegate void ExportableChangeDelegate(object data_packet);
public interface IExportableDataStream
{
void begin_subscription();
event ExportableChangeDelegate on_change;
}
}
@@ -0,0 +1,51 @@
using System;
using System.Collections;
using System.Data;
using BCG.Utilities;
/* this class reprsents a table stream for use with CppDataStreams. It does not hold on to the last value fired.
* This way, the last value is purged from managed memory and only the slimmer, sexier Cpp implementation of the row remains.
*
*/
namespace BCG
{
public class LightTableStream : TableStream
{
public LightTableStream(string tableName)
:
base(tableName, null, null)
{
}
public LightTableStream(string tableName, ViewStreamDatabase db)
:
base(tableName, db, null)
{}
public LightTableStream(string tableName, string colExp)
:
base(tableName,
null,
colExp)
{}
public LightTableStream(string ttableName, ViewStreamDatabase db, string colExp) :
base(ttableName,db,colExp)
{
}
protected override void myHandleUpdates(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
{
base.myHandleUpdates(newRows, addedRows, deletedRows);
//once you've fired the rows, delete them becuase we don't want to hold onto them in memory
myLastValue = null;
}
}
}
@@ -0,0 +1,526 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Diagnostics;
using System.Reflection;
using BCG.Utilities;
using BCG.Utilities.General;
namespace BCG
{
/// Derive from this class to create a new proxy stream for a stream that
/// should be computed out of process. In the proxy class add constructors
/// that exactly mirror the ones you want to expose from the real stream.
/// Each of the proxy constructors should just call the base constructor
/// passing it a, possibly null VSDB, and the arguments it received in
/// exactly the same order. Only non nullparameters of types string, int, bool or
/// DateTime are supported. DateTime arguments will loose their time portion,
/// string arguments can only have digits or letters and ints have to be positive.
///
/// For example, if you real stream is:
///
/// class TradeLog : ViewStream
/// {
/// public TradeLog(string forAccount, string forTrader) {....}
/// }
///
/// Then your proxy stream would like:
///
/// class TradeLogProxy : ProxyStream
/// {
/// public TradeLogProxy(string forAccount, string forTrader) : base(null,forTrader, forAccount) {}
/// }
///
/// Unfortunatly, PubSub tags have a very limited alphabet, one that does not even support upper and lower
/// case dinstinctions, so some special care needs to be taken by the proxy or the real stream when dealing
/// with case sensitive arguments or with string arguments that can contain _ (the only non alpha numeric
/// character allowed in a string). The proxy tries to help by supporting the following conventions:
///
/// - When passing a symbol, mark the parameter with the kind of symbol you mean, option or
/// underlyier. Example:
/// public TheosProxy([Option] string symbol) : base(symbol) {} or
/// public FuturePositionProxy([Underlyer] string future) : base(symbol) {}
///
/// - When passing an exchange mark it with the Exchange attribute. Example:
/// public QuoteWidthsProxy([Symbol] string symbol,
/// [Exchange] string e) : base(e) {}
///
/// - When passing a group name mark it with the group. Since group names can have _ and presumably numbers
/// there can be only one group argument which has to be last one.
/// attribute. Ex:
/// public RiskTheosProxy([Group] string g) : base(g) {}
///
/// - When passing an instrument type mark it as InstrumentType. Example:
/// public SymbolsOfTypeProxy([InstrumentType] string it) : base(it) {}
///
/// - When passing a trader mark it as trader. Example:
/// public TraderTrades([trader] string joes) : base(joes) {}
///
/// - When passing a string that can contain _, the string argument must
/// be the last one in the constructor and should be marked as [LongString]
/// public WorkerHealth([LongString] string machineName) : base(machineName) {}
///
/// - When passing a symbol, instrument type combo combine both in one string
/// with the symbol:InstrumentType format and market it with BCGSym:
/// public PositionProxy([BCGSym] string bcgsym) : base(bcgsym) {}
/// More complex cases should be handled by the real stream.
public abstract class ProxyStream : ViewStream
{
public enum ProxyParameterType
{
clrType,
optionSymbol,
trader,
underlyerSymbol,
group,
exchange,
instrumentType,
bcgSym
}
public class ProxyParameter
{
public ProxyParameterType type;
public object value;
}
public class MarshallingException : SystemException
{
public MarshallingException (string format, params object [] args)
: base(String.Format(format,args) )
{}
}
[AttributeUsage(AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)]
protected class ProxyParameterAttribute : Attribute
{}
protected class OptionAttribute : ProxyParameterAttribute
{ }
protected class LongStringAttribute : ProxyParameterAttribute
{ }
protected class UnderlyierAttribute : ProxyParameterAttribute
{ }
protected class BCGSymAttribute : ProxyParameterAttribute
{ }
protected class GroupAttribute : ProxyParameterAttribute
{ }
protected class ExchangeAttribute : ProxyParameterAttribute
{ }
protected class TraderAttribute: ProxyParameterAttribute
{ }
protected class InstrumentTypeAttribute : ProxyParameterAttribute
{ }
public string getTag() { return myTag; }
private string myTag;
protected ProxyStream(params object[] args)
{myInit(null, args);}
protected ProxyStream(ViewStreamDatabase db, params object[] args)
{myInit(db, args);}
class MyException : System.Exception
{
public MyException (string msg) : base (msg)
{
}
}
private static void myThrowIfFalse(bool condition, string msg)
{
if (!condition)
throw new MyException(msg);
}
private void myInit(ViewStreamDatabase db, params object[] args)
{
try
{
ConstructorInfo ci = GetType().GetConstructor(args.Select(t => t.GetType()).ToArray());
foreach (object o in args)
myThrowIfFalse(o != null,
"Null arguments are not supported.");
myTag = myGetTag(ci, args, db != null);
mySetRealSource(new TableStream("prepub_" + myTag, db));
}
catch (MyException e)
{
// BlueEventLog2.instance().logEvent(e.Message, BlueEventLog2Level.Fatal);
mySetRealSource(new TableStream("error_tag"));
}
}
public static string getProxyName(string tag)
{ return tag.Split('_')[0]; }
public static List<ProxyParameter> unmarshal(Type proxyType, string tag)
{
myThrowIfFalse (proxyType.IsSubclassOf(typeof(ProxyStream)),
String.Format("{0} is not a subclass of ProxyStream", proxyType.Name));
myThrowIfFalse (tag.ToLower().StartsWith(proxyType.Name.ToLower()),
String.Format("Tag {0} does not match proxy type {1}.", tag, proxyType.Name));
string encodedParameters =
tag.Length == proxyType.Name.Length ? "" :
tag.Substring((proxyType.Name + "_").Length);
ConstructorInfo ci;
if (myConstructorsNeedToBeEnconded(proxyType))
{
List<Type> signature = myUnmarshalConstructorSignature(ref encodedParameters);
ci = proxyType.GetConstructor(signature.ToArray());
if (ci == null)
{
signature.Insert(0, typeof(ViewStreamDatabase));
ci = proxyType.GetConstructor(signature.ToArray());
if (ci == null)
throw new MarshallingException("Cannot find constructor for {0} matching signature {1}",
proxyType.Name, encodedParameters);
}
}
else
ci = proxyType.GetConstructors()[0];
ParameterInfo[] parameters = ci.GetParameters();
List<ProxyParameter> result = new List<ProxyParameter>();
ParameterInfo [] pars = ci.GetParameters();
// skip VSDB if first parameter
int i = pars.Length > 0 && pars[0].ParameterType.Equals(typeof(ViewStreamDatabase)) ? 1 : 0;
for (;i < pars.Length; i++)
result.Add(new ProxyParameter
{
type = myGetParameterType(pars[i]),
value = myGetParameterValue(pars[i], ref encodedParameters)
});
return result;
}
private static List<Type> myUnmarshalConstructorSignature(ref string encodedParameters)
{
List<Type> result = new List<Type> ();
for (int i = 0; i < encodedParameters.Length; i++)
if (encodedParameters[i] == 's')
result.Add(typeof(string));
else if (encodedParameters[i] == 'd')
result.Add(typeof(DateTime));
else if (encodedParameters[i] == 'i')
result.Add(typeof(Int32));
else if (encodedParameters[i] == 'b')
result.Add(typeof(bool));
else if (encodedParameters[i] == '_')
break;
else
throw new MarshallingException("'{0}' does not start with a valid constructor signature",
encodedParameters);
if (result.Count > 0)
encodedParameters = encodedParameters.Substring(result.Count + 1);
return result;
}
private static ProxyParameterType myGetParameterType(ParameterInfo pi)
{
if (pi.ParameterType == typeof(string))
{
ProxyParameterType result = ProxyParameterType.clrType;
object[] attr = pi.GetCustomAttributes(false);
if (attr.Length > 0)
{
foreach (Attribute a in attr)
if (a is OptionAttribute)
result = ProxyParameterType.optionSymbol;
else if (a is ExchangeAttribute)
result = ProxyParameterType.exchange;
else if (a is TraderAttribute)
result = ProxyParameterType.trader;
else if (a is GroupAttribute)
result = ProxyParameterType.group;
else if (a is InstrumentTypeAttribute)
result = ProxyParameterType.instrumentType;
else if (a is UnderlyierAttribute)
result = ProxyParameterType.underlyerSymbol;
else if (a is BCGSymAttribute)
result = ProxyParameterType.bcgSym;
else if (a is LongStringAttribute)
result = ProxyParameterType.clrType;
}
return result;
}
else if (pi.ParameterType == typeof(Int32) ||
pi.ParameterType == typeof(DateTime) ||
pi.ParameterType == typeof(bool))
return ProxyParameterType.clrType;
else
{
myThrowIfFalse (false,String.Format(
"{0} is not a supported proxy parameter type", pi.ParameterType.Name));
return ProxyParameterType.clrType;
}
}
private static object myGetParameterValue(ParameterInfo pi, ref string encodedParameters)
{
if (pi.ParameterType == typeof(DateTime))
{
string[] parts = encodedParameters.Split('_');
if (parts.Length < 3)
throw new MarshallingException("{0} does not start with a valid date", encodedParameters);
int year;
if (!Int32.TryParse(parts[0], out year))
throw new MarshallingException("{0} is not a valid year string", parts[0]);
int month;
if (!Int32.TryParse(parts[1], out month))
throw new MarshallingException("{0} is not a valid month string", parts[1]);
int day;
if (!Int32.TryParse(parts[2], out day))
throw new MarshallingException("{0} is not a valid day string", parts[2]);
DateTime result;
try {
result = new DateTime(year, month, day);
}
catch (ArgumentException)
{
throw new MarshallingException("{0}/{1}/{2} is not a valid date", year, month, day );
}
if (parts.Length > 3)
encodedParameters = encodedParameters.Substring(parts[0].Length + parts[1].Length + parts[2].Length + 3);
else
encodedParameters = "";
return
result;
}
else
{
if (pi.ParameterType == typeof(string) && myIsBCGSym(pi))
{
string[] parts = encodedParameters.Split('_');
if (parts.Length < 2)
throw new MarshallingException("{0} does not start with a valid BCGSym string", encodedParameters);
else
{
if (parts.Length > 2)
encodedParameters = encodedParameters.Substring(parts[0].Length + parts[1].Length + 2);
else
encodedParameters = "";
return parts[0] + ":" + parts[1];
}
}
else if (pi.ParameterType == typeof(string) && myIsLongString(pi))
{
string result = encodedParameters;
encodedParameters = "";
return result;
}
else
{
string[] parts = encodedParameters.Split('_');
object result = null;
if (pi.ParameterType == typeof(string))
result = parts[0];
else if (pi.ParameterType == typeof(Int32))
{
int n;
if (!Int32.TryParse(parts[0], out n))
throw new MarshallingException("{0} is not a valid int", parts[0]);
result = n;
}
else if (pi.ParameterType == typeof(Boolean))
{
bool b;
if (!bool.TryParse(parts[0], out b))
throw new MarshallingException("{0} is not a valid bool", parts[0]);
result = b;
}
else
myThrowIfFalse (false,
String.Format("{0} is not a supported proxy parameter type", pi.ParameterType.Name));
if (parts.Length > 1)
encodedParameters = encodedParameters.Substring(parts[0].Length + 1);
else
encodedParameters = "";
return result;
}
}
}
private bool myValidateString(string s, bool allowUnderscore)
{
foreach (char c in s)
if (!Char.IsDigit(c) && !Char.IsLetter(c) && !(allowUnderscore && c.Equals('_')))
return false;
return true;
}
private static bool myHasAttribute(ParameterInfo pi, Type attrType)
{
object[] attr = pi.GetCustomAttributes(false);
if (attr.Length > 0)
foreach (Attribute at in attr)
if (at.GetType().Equals(attrType))
return true;
return false;
}
private static bool myIsBCGSym(ParameterInfo pi)
{return myHasAttribute(pi, typeof(BCGSymAttribute));}
private static bool myIsLongString(ParameterInfo pi)
{ return myHasAttribute(pi, typeof(LongStringAttribute)) || myHasAttribute(pi, typeof(GroupAttribute)); }
private static bool myFirstParameterIsVSDB(ConstructorInfo ci)
{
return
ci.GetParameters().Length > 0 &&
ci.GetParameters()[0].ParameterType.Equals(typeof(ViewStreamDatabase));
}
private static bool myIsSameModuloVSDB(ConstructorInfo ci0, ConstructorInfo ci1)
{
if ((ci1.GetParameters().Length - ci0.GetParameters().Length) != 1)
return false;
if (!myFirstParameterIsVSDB(ci1))
return false;
for (int i = 0; i < ci0.GetParameters().Length; i++)
if (!ci1.GetParameters()[i + 1].ParameterType.Equals(
ci0.GetParameters()[i].ParameterType))
return false;
return true;
}
private static bool myConstructorsNeedToBeEnconded(Type t)
{
ConstructorInfo [] cis = t.GetConstructors();
if (cis.Length == 1)
return false;
else if (cis.Length > 2)
return true;
else
{
ConstructorInfo ci0 = cis[0];
ConstructorInfo ci1 = cis[1];
return !(myIsSameModuloVSDB(ci0, ci1) || myIsSameModuloVSDB(ci1, ci0));
}
}
private string myGetConstructorSignature(ConstructorInfo ci)
{
string result ="";
int i = myFirstParameterIsVSDB(ci) ? 1 : 0;
for (; i < ci.GetParameters().Length; i++)
{
Type t = ci.GetParameters()[i].ParameterType;
if (t.Equals(typeof(string)))
result += "s";
else if (t.Equals(typeof(DateTime)))
result += "d";
else if (t.Equals(typeof(bool)))
result += "b";
else if (t.Equals(typeof(Int32)))
result += "i";
else
myThrowIfFalse (false,
String.Format("{0} is not a supported proxy parameter type", t.Name));
}
return result;
}
private string myGetTag(ConstructorInfo ci, object[] args, bool firstArgIsVSDB)
{
Type t = ci.DeclaringType;
string tag = t.Name;
if (myConstructorsNeedToBeEnconded(t))
tag += "_" + myGetConstructorSignature(ci);
int i = myFirstParameterIsVSDB(ci) ? 1 : 0;
foreach (object a in args)
{
Debug.Assert(a != null);
if (a.GetType() == typeof(DateTime))
{
DateTime dt = (DateTime)a;
tag += String.Format("_{0}_{1}_{2}", dt.Year, dt.Month, dt.Day);
}
else if (a.GetType() == typeof(string))
{
if (myIsBCGSym(ci.GetParameters()[i]))
{
string[] parts = a.ToString().Split(':');
myThrowIfFalse (parts.Length == 2,
String.Format("Invalid BCGSym string argument {0}. Use sym:InstrumentType",
a));
if (!myValidateString(parts[0], false) || !myValidateString(parts[1], false))
{
// BlueEventLog2.instance().logEvent(
// String.Format(
// "Invalid string argument {0}. Currently, only letters and digits are supported in string arguments",
// a),BlueEventLog2Level.Error);
return "error_tag";
}
tag += String.Format("_{0}_{1}", parts[0], parts[1]);
}
else if (myIsLongString(ci.GetParameters()[i]))
{
// not a BCGSym
if (!myValidateString(a.ToString(), true))
{
// BlueEventLog2.instance().logEvent(
// String.Format("Invalid string argument {0}. Currently, only letters, digits and _'s are supported in long string arguments",
// a),BlueEventLog2Level.Error);
return "error_tag";
}
//can't do this-- ci.GetParameters() order is undefined
//myThrowIfFalse (i.Equals(ci.GetParameters().Length - 1),
// "Long string argument must be last");
tag += "_" + a.ToString();
}
else
{
// not a BCGSym
if (!myValidateString(a.ToString(), false))
{
// BlueEventLog2.instance().logEvent(String.Format(
// "Invalid string argument {0}. Currently, only letters and digits are supported in string arguments", a),
// BlueEventLog2Level.Error);
return "error_tag";
}
tag += "_" + a.ToString();
}
}
else if (a.GetType() == typeof(Int32))
{
myThrowIfFalse ((int) a >= 0,
String.Format("Invalid int argument {0}. Currently only positive integers are supported.",
a));
tag += "_" + a.ToString();
} else if (a.GetType() == typeof(bool))
tag += "_" + a.ToString();
else
myThrowIfFalse (false,
String.Format("{0} is not a supported proxy parameter type", a.GetType().Name));
i++;
}
return tag;
}
}
}
@@ -0,0 +1,184 @@
using System;
using System.Collections;
using System.Data;
using BCG.Utilities;
namespace BCG
{
public class TableStream : ViewStream
{
public string tableName;
private ITableAccumulator myTA;
private bool myGotData = false;
public TableStream(string tableName)
:
this(tableName, null, null)
{
}
public TableStream(string tableName, ViewStreamDatabase db)
:
this(tableName, db, null)
{}
public TableStream(string tableName, string colExp)
:
this(tableName,
null,
colExp)
{}
public TableStream(string ttableName, ViewStreamDatabase db, string colExp)
{
tableName = PubStreamPrefixer.prefix (ttableName);
// if (colExp != null)
// mySetRealSource (new ProjectStream (new TableStream (tableName, db), colExp));
// else
{
if (db == null)
myTA = ViewStreamDatabase.instance.getITableAccumulator (tableName);
else
myTA = db.getITableAccumulator (tableName);
}
}
protected override void myOnFirstSubscriber()
{
myTA.onUpdate += new TableUpdateDelegate (myHandleUpdates);
}
protected override void myOnLastSubscriber()
{
myTA.onUpdate -= new TableUpdateDelegate (myHandleUpdates);
}
protected virtual void myHandleUpdates (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
{
if (!myGotData)
{
myGotData = true;
// BlueEventLog.logEvent("Received data for " + tableName + " for the first time", BlueEventLevel.Information);
}
myFireEvent (new DataPacket (newRows, addedRows, deletedRows));
}
public override string ToString()
{
return GetType() + " table = " + tableName;
}
}
public class ThreadedTableStream : TableStream, IExportableDataStream
{
static private Lazy<ViewStreamDatabase> _vsd = new Lazy<ViewStreamDatabase>(() => new ViewStreamDatabase(), true);
public ThreadedTableStream(string table) : base(table, _vsd.Value)
{
}
public void begin_subscription()
{
this.onChange += _stream_onChange;
}
public event ExportableChangeDelegate on_change;
void _stream_onChange(object newValue)
{
if(on_change != null)
on_change(newValue);
}
}
// public class ExportableTableStream : IExportableDataStream
// {
// TableStream _stream;
// public ExportableTableStream(string table, IDataPump pump)
// {
// _stream = new TableStream(table, ((ExportableDataPump)pump).vsd.Value);
// }
// public void begin_subscription()
// {
// _stream.onChange += _stream_onChange;
// }
// public event ExportableChangeDelegate on_change;
// void _stream_onChange(object newValue)
// {
// if(on_change != null)
// on_change(newValue);
// }
// }
// public class ExportableQuery : IExportableDataStream
// {
// ViewStream _stream;
// public ExportableQuery(string query, BCG.IDataPump _)
// {
// // BCG.Evaluator.DataStreamExpression.syntax parser = new BCG.Evaluator.DataStreamExpression.syntax();
// // BCG.Evaluator.DataStreamExpression.DataStreamExpression exp =
// // (BCG.Evaluator.DataStreamExpression.DataStreamExpression)parser.Parse(query);
// _stream = exp.getView(new System.Collections.Hashtable());
// }
// public void begin_subscription()
// {
// _stream.onChange += _stream_onChange;
// }
// public event ExportableChangeDelegate on_change;
// void _stream_onChange(object newValue)
// {
// if(on_change != null)
// on_change(newValue);
// }
// }
// public class ExportableDataPump : IDataPump
// {
// public Lazy<ViewStreamDatabase> vsd = new Lazy<ViewStreamDatabase>(() => new ViewStreamDatabase(), true);
// System.Threading.Thread _thread;
// bool _continue = true;
// public ExportableDataPump(string pumpname)
// {
// tag = pumpname;
// _thread = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart((_) =>
// {
// while(_continue)
// {
// vsd.Value.doWork();
// System.Threading.Thread.Sleep(50);
// }
// }));
// _thread.Name = "DataPump " + pumpname;
// _thread.IsBackground = true;
// }
// public string tag
// {
// get;
// private set;
// }
// public void start()
// {
// try
// {
// if(_thread.ThreadState != System.Threading.ThreadState.Running)
// _thread.Start();
// }
// catch(System.Threading.ThreadStateException) { }
// }
// public void stop()
// {
// try
// {
// _continue = false;
// if(_thread.ThreadState == System.Threading.ThreadState.Running)
// _thread.Join(500);
// }
// catch(System.Threading.ThreadStateException) { }
// }
// }
}
@@ -0,0 +1,97 @@
using System;
//using BCG.Utilities.BlueConfiguration;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Marker abstract class
/// </summary>
public abstract class ViewStream : DataStream
{
private IBlueRowCollection myLastValueForTestMode;
protected bool myUseUncopiedNewRowsInFireEvent = DataStreamLoadFileConfig.instance.useUncopiedNewRowsInFireEvent;
public ViewStream (ViewStream sourceStream)
:
base(sourceStream)
{}
public ViewStream()
{}
public IBlueRowCollection rows
{
get
{
if (myRealSource != null)
{
// BlueEventLog.logAssert (myRealSource is ViewStream);
return (myRealSource as ViewStream).rows;
}
else
{
// used to force a computation of a stream which
// happens to have no subscribers
onChange += new DataStream.ChangeDelegate(myDummy);
try
{
if (myLastValue != null)
return ((DataPacket) myLastValue).newRows;
else
return null;
}
finally
{
onChange -= new DataStream.ChangeDelegate(myDummy);
}
}
}
}
protected override object myGetValueForInitialSubscription()
{
if (myLastValue == null)
return null;
else
{
return new DataPacket(
(myLastValue as DataPacket).newRows,
(myLastValue as DataPacket).newRows,
new BlueTable((myLastValue as DataPacket).newRows.schema));
}
}
protected override void myFireEvent(object o)
{
DataPacket dp = (DataPacket) o;
// BlueTrace.WriteViewStreamFireEvent(this.ID,
// dp.newRows == null ? 0 : dp.newRows.Count,
// dp.addedRows == null ? 0 : dp.addedRows.Count,
// dp.deletedRows == null ? 0 : dp.deletedRows.Count);
if (myTestMode)
{
// BlueEventLog.logAssert ((myFirstSubImplemented && myLastSubImplemented) | myRealSource != null,
// @"either myOnFirstSub/myOnLastSub should be implemented or mySetRealSource
// called with a non-null arg in " + this.GetType().FullName);
if (myLastValueForTestMode != null)
BlueDataUtility.checkConsistency(myLastValueForTestMode,
dp.newRows, dp.deletedRows, dp.addedRows);
myLastValueForTestMode = new ImmutableBlueTable(dp.newRows);
}
// the view already garantees that added & deleted are immutable.
// this garantess that new is also immutable
base.myFireEvent(myUseUncopiedNewRowsInFireEvent ? dp : dp.createImmutable());
}
private void myDummy (object o)
{}
}
}
@@ -0,0 +1,314 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using BCG.Utilities.General;
//using BCG.Utilities.BlueConfiguration;
using System.Collections.Generic;
using System.Threading;
using mbts;
namespace BCG
{
using HighPerformanceTimer = mbts.spark.HighPerformanceTimer;
public class ViewStreamDatabase : IWorker, IDisposable
{
internal delegate void TagFireDelegate(string tag);
public delegate void DoWorkDelegate ();
public event DoWorkDelegate onDoWorkStart,
onDoWorkEnd;
internal event TagFireDelegate onTagFireStart,
onTagFireEnd;
//currently onTagUpdate is usd by the DataStreamPerfRecorder to keep track of how often the updates for various streams come in
//and how big they are.
//public delegate void OnTagUpdateDelegate(PubMetaData metaData);
//public event OnTagUpdateDelegate onTagUpdate;
private IDictionary myITableAccumulatorMap = new Hashtable ();
private static ViewStreamDatabase mySelf;
private ISubscriberCore mySubscriberCore;
private ArrayList myWorkers = new ArrayList ();
private bool myIsDisposed = false;
private bool myIsInDoWork = false;
private static object myTimeInterval;
private HighPerformanceTimer myTimer = new HighPerformanceTimer();
private double myLogTimeThreshold = 10000;
private readonly double _dowork_threshold = 500;
private Thread myThreadOwner = null; // ViewStreamDatabase takes a thread affinity when created. If called from a different thread it will assert.
private static readonly object myInstanceLockObject = new object();
private readonly object myStateLock = new object();
public ViewStreamDatabase ()
: this (new SubscriberCore ())
{
}
public ViewStreamDatabase (ISubscriberCore subscriberCore)
{
mySubscriberCore = subscriberCore;
// if (DataStreamLoadFileConfig.instance.profileStreams)
// new DataStreamPerfRecorder(this);
}
private static int myGetTimeInterval()
{
if (myTimeInterval == null)
{
// TODO
string timeInterval = String.Empty; //ConfigurationRequester.request("viewStreamDatabaseTimeOut");
if (timeInterval == null || timeInterval == "")
myTimeInterval = 0;
else
myTimeInterval = Int32.Parse(timeInterval);
}
return (int) myTimeInterval;
}
public static int instance_hashcode = -1;
public static ViewStreamDatabase instance
{
get
{
if(mySelf == null || mySelf.myIsDisposed)
lock(myInstanceLockObject)
{
if (mySelf == null || mySelf.myIsDisposed)
{
mySelf = new ViewStreamDatabase();
instance_hashcode = mySelf.GetHashCode();
}
}
return mySelf;
}
set
{
lock(myInstanceLockObject)
{
if (mySelf != null)
mySelf.mySubscriberCore.Dispose ();
mySelf = value;
instance_hashcode = mySelf.GetHashCode();
}
}
}
public ICollection<string> getCachedTables()
{
List<string> result = new List<string>();
foreach (string s in myITableAccumulatorMap.Keys)
if (((ITableAccumulator)myITableAccumulatorMap[s]).isActive())
result.Add(s);
return result;
}
public ISubscriberCore subscriber{get{return mySubscriberCore;}}
public ITableAccumulator getITableAccumulator (string tableName)
{
// If theres a ITableAccumulator for the tableName its returned (to share subscriptions), otherwise
// a new one is obtained from the Subscriber factory
myITableAccumulatorMap[tableName] = myITableAccumulatorMap[tableName] == null ?
AccumulatorFactory.getAccumulator(tableName, mySubscriberCore) : myITableAccumulatorMap[tableName];
return myITableAccumulatorMap [tableName] as ITableAccumulator;
}
public void doWork ()
{
verifyAccess();
if (myIsSuspended)
return;
if (myIsInDoWork)
return;
else
{
myIsInDoWork = true; // to allow reentrancy
try
{
if (onDoWorkStart != null)
onDoWorkStart();
ArrayList updateList = mySubscriberCore.getUpdates(myGetTimeInterval());
IDictionary updateMap = myGetUpdateMap(updateList);
foreach (DictionaryEntry de in updateMap)
{
Stream[] allUpdates = (de.Value as ArrayList).ToArray(typeof(Stream)) as Stream[];
string tag = de.Key.ToString();
if (myITableAccumulatorMap[tag] == null){}
// BlueEventLog.logEvent("Received a SubscriberCore's callback for a tag that was never subscribed", BlueEventLevel.Warning);
else
{
myTimer.start();
if (onTagFireStart != null)
onTagFireStart(tag);
(myITableAccumulatorMap[tag] as ITableAccumulator).update(allUpdates);
if (onTagFireEnd != null)
onTagFireEnd(tag);
myTimer.stop();
if (myTimer.lastIntervalTime > myLogTimeThreshold)
{
// BlueEventLog.logEvent("callback on tag " + tag.ToUpper() + " took " +
// myTimer.lastIntervalTime + " msecs", BlueEventLevel.Warning);
}
}
}
// to allow for registering of workers on the worker callback
myIsCallingWorkers = true;
myPendingOps = new ArrayList();
foreach (IWorker w in myWorkers)
{
myTimer.start();
w.doWork();
myTimer.stop();
if (myTimer.lastIntervalTime > _dowork_threshold){}
// mbts.Logger.log (bts.log.Level.Warn, bts.log.Subsystem.gen, String.Format ("VSD.dowork on worker {0} took {1}", w.GetType().ToString (), myTimer.lastIntervalTime));
}
myIsCallingWorkers = false;
foreach (PendingAddRemoveWorker op in myPendingOps)
if (op.isAdd)
addWorker(op.w);
else
removeWorker(op.w);
}
finally
{
myPendingOps = null;
myIsInDoWork = false;
myIsCallingWorkers = false;
if (onDoWorkEnd != null)
onDoWorkEnd ();
}
}//end if we're not in do work
}
private bool myIsSuspended = false;
public void suspend() { myIsSuspended = true; }
public void resume() { myIsSuspended = false; }
public bool checkAccess()
{
if (this.myThreadOwner == null) this.myThreadOwner = Thread.CurrentThread;
return this.myThreadOwner == Thread.CurrentThread;
}
public void verifyAccess()
{
if (!checkAccess())
{
string errorMessage = string.Format("Invalid cross thread access on ViewStreamDatabase. Owning thread: {0}. Accessing thread: {1}", this.myThreadOwner.ManagedThreadId, Thread.CurrentThread.ManagedThreadId);
//#if DEBUG
// BlueEventLog2.instance().assert(false, errorMessage);
//#else
// BlueEventLog2.instance().logEvent(errorMessage, BlueEventLog2Level.Fatal);
//#endif
}
}
private IDictionary myGetUpdateMap (ArrayList updates)
{
IDictionary retVal = new Hashtable ();
foreach (PubSubUpdate update in updates)
{
if (retVal[update.tag] == null)
retVal[update.tag] = new ArrayList ();
(retVal[update.tag] as ArrayList).Add (update.body);
}
if (retVal.Count > 0)
{
string tagString = "";
foreach (string tag in retVal.Keys)
tagString += tag + " , ";
}
return retVal;
}
public bool isConsistent{get {return true;}}
public void addWorker(IWorker w)
{
verifyAccess();
if (myIsCallingWorkers)
myPendingOps.Add(new PendingAddRemoveWorker(true,w));
else
myWorkers.Add(w);
}
public void removeWorker(IWorker w)
{
verifyAccess();
if (myIsCallingWorkers)
myPendingOps.Add(new PendingAddRemoveWorker(false,w));
else
myWorkers.Remove(w);
}
#region IDisposable Members
public void Dispose()
{
if (!myIsDisposed)
{
myIsDisposed = true;
mySubscriberCore.Dispose ();
}
}
private ArrayList myPendingOps;
private bool myIsCallingWorkers = false;
class PendingAddRemoveWorker
{
public PendingAddRemoveWorker(bool isAdd, IWorker w)
{
this.isAdd = isAdd;
this.w = w;
}
public bool isAdd;
public IWorker w;
}
#endregion
}
/*
/// <summary>
/// Returns a subscriber based on the name of the tag. Hardcoded right now, will change to config
/// later
/// </summary>
public class SubscriberFactory
{
public static ITableAccumulator getITableAccumulator (string tag, SubscriberCore subCore)
{
string tableName = tag.Substring (3);
if (!DataStreamGatewaySettings.instance.isSerializableUnderNewScheme (tableName))
return new TableSubscriber (tag, subCore);
else
{
string subName = DataStreamGatewaySettings.instance.getSubscriber (tableName);
if (subName.Equals ("BCG.NonIncrementalTableSubscriber"))
return new NonIncrementalTableSubscriber (tag, subCore);
else
return new TableSubscriber (tag, subCore);
}
}
}*/
}