Files
spark/src/DataStreamSuite/PubSub/StaticPublisherCore.cs
T
2024-01-13 23:45:51 -05:00

505 lines
22 KiB
C#

using System;
//using System.Messaging;
using System.Xml.Serialization;
using System.Text;
using System.IO;
using System.Collections;
using System.Collections.Generic;
using System.Threading;
using BCG;
//using BCG.Utilities.BlueConfiguration;
using BCG.Utilities;
using mbts;
/* StaticPublisherCore.cs
* This file was originally called 'PublisherCore.cs'. It was renamed so that the publisher core could be replaced.
*
* */
namespace BCG {
// Provides a function to check if a tag is crushabel or not
public class CrushedTagManager {
private static CrushedTagManager _self;
private Dictionary<string, bool> _tag_map = new Dictionary<string, bool>();
public static CrushedTagManager instance {
get {
if (_self == null)
_self = new CrushedTagManager();
return _self;
}
}
public bool is_tag_crushable(string tag) {
tag = tag.ToLower();
bool rv = false;
lock (typeof(CrushedTagManager)) {
if (!_tag_map.TryGetValue(tag, out rv)) {
foreach (string r in PubSub3Config.instance.PS3_CrushableTags)
if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag)) {
_tag_map.Add(tag, true);
break;
}
_tag_map.Add(tag, false);
}
}
return rv;
}
}
/// <summary>
/// All the data for a single tag, thats published within
/// a transaction
/// </summary>
public class TagWaitingInTransaction {
private bool _is_crushable;
private List<byte[]> _buffered_data = new List<byte[]>();
// this is the size of all the buffered data
private int _total_size = 0;
private string _tag, _tag_type;
public TagWaitingInTransaction(string theTagType, string theTag) {
_tag = theTag;
_tag_type = theTagType;
_is_crushable = CrushedTagManager.instance.is_tag_crushable(_tag);
}
public void add_pub(Stream pub_stream) {
if (_is_crushable) {
// extract bytes, clear _buffered_data and queue to _buffered_data
byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
pub_stream.Read(new_data, 0, new_data.Length);
_buffered_data.Clear();
_buffered_data.Add(new_data);
_total_size = new_data.Length;
} else {
// extract btes and queue to _buffered_data
byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
pub_stream.Read(new_data, 0, new_data.Length);
_buffered_data.Add(new_data);
_total_size += new_data.Length;
}
}
public void write_to(Stream s, string connection, string publishing_app) {
new MetaData(new PubMetaData(_tag_type, _tag, _total_size, connection, publishing_app)).save(s);
foreach (byte[] data in _buffered_data)
s.Write(data, 0, data.Length);
}
public string tag { get { return _tag; } }
public string tagType { get { return _tag_type; } }
public List<byte[]> data { get { return _buffered_data; } }
}
public enum PublisherCoreMessageType {
transactionBegin,
transactionCommit,
transactionAbort,
publish,
clear
}
public class PublisherCoreMessage {
public PublisherCoreMessageType msgType;
public string tagType, tag;
public Stream data;
public PublisherCoreMessage(string tagToClear) {
tag = tagToClear;
msgType = PublisherCoreMessageType.clear;
}
public PublisherCoreMessage(PublisherCoreMessageType theMsgType) {
msgType = theMsgType;
}
public PublisherCoreMessage(string theTagType, string theTag, Stream theData) {
msgType = PublisherCoreMessageType.publish;
tagType = theTagType;
tag = theTag;
data = theData;
}
public override string ToString() {
return $"PublisherCoreMessage tag: {tag} data.Position: {data.Position} data.Length: {data.Length}";
}
}
// this class uses an internal queue + thread to serialize and process calls
// made through its public interface (IPublisherCore)
public class StaticPublisherCore : IPublisherCore {
// used to allow for different transports
// to be used when communicating to a pubsubserver
protected PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(false);
// internal queue used to serialize pubsub method calls
protected BlockingQueue myMainQueue = new BlockingQueue();
// all the data published through this object in snapshot form
// i.e. in a form that can be used repopulate a pubsub server
// in the event of a crash
protected AllPublishedTagState myState = new AllPublishedTagState();
// a different IPubSubServerConnection object is used to interact
// with each of the pubsub servers used by this publishercore
protected List<IPubSubServerConnection> mySenders = new List<IPubSubServerConnection>();
// a dictionary for fast retrieval of publishers
// tag -> IPubSubServerConnection
protected Dictionary<string, IPubSubServerConnection> myTagToSender = new Dictionary<string, IPubSubServerConnection>();
// data waiting for transaction to end or abort
// tag->TagWaitingInTransaction
protected Dictionary<string, TagWaitingInTransaction> myWaitingInTransactionMessages = new Dictionary<string, TagWaitingInTransaction>();
// router object, used to select which pubsub should be used for each tag
protected IPubSubRouter myRouter = null;
// flag indicating wether we are in the middle of a transaction or not
protected bool myInTransaction = false;
#region constructors
public StaticPublisherCore()
// TODOds
: this(new PubSubRouter()) { }
public StaticPublisherCore(string serverURL)
: this(new PubSubRouter(new PubSubRoute(serverURL, ".*"))) { }
public StaticPublisherCore(IPubSubRouter r) {
myRouter = r;
Thread pumpThread = new Thread(new ThreadStart(myMessagePump));
pumpThread.IsBackground = true;
pumpThread.Start();
}
public void Dispose() { }
#endregion
public int getNumberOfWaitingMessages() { return myMainQueue.getCount(); }
#region Main Thread - all library functions are called on the main thread and queued to be picked by the pump
public void publish(string tag, Stream content) { publish("", tag, content); }
public void publish(string tagType, string tag, Stream content) {
PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content);
bts.Log.Gen.info($"StaticPublisherCore.publish tag: {tag} msg: {msg} len: {content.Length - content.Position}");
myMainQueue.enqueue(msg);
}
public void beginTransaction() {
myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionBegin));
}
public void commitTransaction() {
myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionCommit));
}
public void abortTransaction() {
myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionAbort));
}
public bool inTransaction { get { return myInTransaction; } }
public void clearState(string tag) {
myMainQueue.enqueue(new PublisherCoreMessage(tag));
}
#endregion
#region Message Pump Thread - this is where messages are dequeued and the action performed
private void myMessagePump() {
while (true) {
try {
while (true) {
object data = myMainQueue.dequeue(1000);
if (data != null) {
int qc = myMainQueue.getCount();
PublisherCoreMessage msg = data as PublisherCoreMessage;
bts.Log.Gen.info($"StaticPublisherCore.myMessagePump msg: {msg}");
if (msg.msgType == PublisherCoreMessageType.transactionBegin) {
myBeginTransaction();
} else if (msg.msgType == PublisherCoreMessageType.transactionCommit) {
myCommitTransaction();
} else if (msg.msgType == PublisherCoreMessageType.transactionAbort) {
myAbortTransaction();
} else if (msg.msgType == PublisherCoreMessageType.publish) {
myPublish(msg.tagType, msg.tag, msg.data);
} else if (msg.msgType == PublisherCoreMessageType.clear) {
myClearState(msg.tag);
}
}
foreach (IPubSubServerConnection conn in mySenders) {
conn.doWork();
}
}
} catch (Exception ex) {
bts.Log.Gen.error($"StaticPublisherCore.myMessagePump exception ex: {ex}");
if (ex is OutOfMemoryException)
throw ex;
}
}
}
private void myBeginTransaction() {
// BlueEventLog.logAssert(!inTransaction);
//BlueEventLog.logEvent("StaticPublisherCore Starting Transaction.", BlueEventLevel.Information);
myInTransaction = true;
}
private void myCommitTransaction() {
myEndTransaction(true);
}
private void myAbortTransaction() {
myEndTransaction(false);
}
//based on success, sends a TA or TE, and sets myInTransaction to false
//TODO: if a transaction is sending to multiple pubsubs (in order to
//keep it consistent with the way it was done earlier), this should
//simply publish it non transactionally (easy way out), or
//transactionally with respect to each pubsub
protected virtual void myEndTransaction(bool success) {
lock (this) {
// BlueEventLog.logAssert(inTransaction);
//BlueEventLog.logEvent("StaticPublisherCore Finishing Transaction.", BlueEventLevel.Information);
myInTransaction = false;
if (success) {
// pubsub server -> tags with updates
Dictionary<string, List<string>> routesToTags = new Dictionary<string, List<string>>();
// group updates per pubsub server
foreach (string tag in myWaitingInTransactionMessages.Keys) {
string thisURL = myRouter.getPubURLForTag(tag);
if (!routesToTags.ContainsKey(thisURL))
routesToTags[thisURL] = new List<string>();
routesToTags[thisURL].Add(tag);
}
// more than one server means we won't be able to honor the transactionality request
// log a warning if that is the case.
if (routesToTags.Keys.Count > 1) {
//this is going to be a long error message, but it's an importat one to be able to debug.
StringBuilder errorMessageBuilder = new StringBuilder();
errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n");
errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n");
foreach (string route in routesToTags.Keys) {
foreach (string tag in routesToTags[route]) {
errorMessageBuilder.Append("\tTag ");
errorMessageBuilder.Append(tag);
errorMessageBuilder.Append(" is routed to ");
errorMessageBuilder.Append(route);
errorMessageBuilder.Append(".\n");
}
}
string errMsg = errorMessageBuilder.ToString();
// BlueEventLog.logEvent(errMsg, BlueEventLevel.Diagnosis);
}
// iterate through each server with pending messages
foreach (KeyValuePair<string, List<string>> kvp in routesToTags) {
IPubSubServerConnection transactionSender = null;
string publishingApp = ClientInfo.thisClient.processString;
string connection = kvp.Key;
Stream transactionData = new MemoryStream();
// iterate through each of the pending tag updates for one server
foreach (string tag in kvp.Value) {
///it would be the same transaction sender that would
///return each time
transactionSender = myGetOrBuildSender(tag);
TagWaitingInTransaction waiting = myWaitingInTransactionMessages[tag];
// iterate through each of the updates for a specific tag, specific server
foreach (byte[] data in waiting.data)
// update internal state
myState.publish(waiting.tag, waiting.tagType, data);
// add a pubsub header and copy each message to the transactionData stream
waiting.write_to(transactionData, connection, publishingApp);
}
// now transactionData holds all the updates for all the tags
// that are to be sent to the PubSub represented by transactionSender
// add a transaction header, indicating how many tags in the transaction
TransactionMetaData tMd = new TransactionMetaData(kvp.Value.Count);
// rewind transaction data to the beggining
transactionData.Position = 0;
// send the entire thing to the specific pubsub server.
// this is where the data is actually sent out !
var ps_msg = new PubSubMessage(new MetaData(tMd), transactionData);
Console.WriteLine($"serverConn.send metadata: {ps_msg.meta_data}");
bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
transactionSender.send(ps_msg);
}
}
myWaitingInTransactionMessages.Clear();
}
}
protected virtual void myPublish(string tagType, string tag, Stream content) {
bts.Log.Gen.info($"StaticPublisherCore.myPublish tag: {tag} content: {content.Position} / {content.Length} myInTransaction: {myInTransaction}");
if (myInTransaction) {
///if its in transaction message is cached, and is sent
///only when theres a commit or abort
if (!myWaitingInTransactionMessages.ContainsKey(tag))
myWaitingInTransactionMessages.Add(tag, new TagWaitingInTransaction(tagType, tag));
myWaitingInTransactionMessages[tag].add_pub(content);
} else {
{
myState.publish(tag, tagType, content);
if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) {
bts.Log.Gen.info("Sending crushable tag: {tag}");
mySendData(tag, tagType, content, "myPublish");
} else {
bts.Log.Gen.info("Could not send tag: {tag} since its waiting on an ack");
myCrushableTagsWaitingOnAck[tag].second++;
}
}
// else
// {
// // update snapshot state
// myState.publish(tag, tagType, content);
// // publish out the data
// mySendData(tag, tagType, content);
// }
}
}
// add header, possibly compress the data, and sendto appropriate pubsub
private void mySendData(string tag, string tagType, Stream content, string reason) {
bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType} reason: {reason} len: {content.Length - content.Position}");
long initPos = content.Position;
// locate right pubsub for this tag
IPubSubServerConnection serverConn = myGetOrBuildSender(tag);
Stream contentToPublish = content;
PubMetaData pub_md;
if (PubSub3Config.instance.isTagCompressed(tag)) {
contentToPublish = new MemoryStream();
SharpZipCompressor Kompressor = new SharpZipCompressor();
Kompressor.compress(content, contentToPublish);
contentToPublish.Position = 0;
pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
// BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
}
} else
pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
content.Position = initPos;
if (CrushedTagManager.instance.is_tag_crushable(tag)) {
string refId = String.Empty;
Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.server_url);
// this is where the data is actually sent out (there are multiple sends)
PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
//Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
bts.Log.Gen.info($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
serverConn.send_with_confirmation(ps_msg, ref refId);
myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0));
} else {
// this is where the data is actually sent out
var ps_msg = new PubSubMessage(pub_md, contentToPublish);
bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
serverConn.send(ps_msg);
}
}
private void myClearState(string tag) {
if (myTagToSender.ContainsKey(tag)) {
myTagToSender.Remove(tag);
}
myState.clear(tag);
}
protected virtual IPubSubServerConnection myGetOrBuildSender(string tag) {
if (!myTagToSender.ContainsKey(tag)) {
string route = myRouter.getPubURLForTag(tag);
IPubSubServerConnection sender = myConnectionFactory.getConnection(route);
myTagToSender.Add(tag, sender);
if (!mySenders.Contains(sender)) {
mySenders.Add(sender);
sender.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot));
sender.register_send_confirmation_callback(new SendConfirmationDelegate(myOnSendConfirm));
}
}
return myTagToSender[tag];
}
///in case of the server coming up, it publishes all the stored state
///for that server, back to it
protected void myHandleServerReboot(string serverURL, bool connection) {
bts.Log.Gen.info($"StaticPublisherCore.myHandleServerReboot server_url: {serverURL}");
if (connection) {
string thisPublisherInfo = ClientInfo.thisClient.processString;
foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
if (p.Value.server_url == serverURL) {
string tagType;
string tag = p.Key;
myCrushableTagsWaitingOnAck.Clear();
byte[] data = myState.getContent(tag, out tagType);
mySendData(tag, tagType, new MemoryStream(data), "myHandleServerReboot");
//Stream content = new MemoryStream(data);
//Stream contentToPublish = content;
//PubMetaData pubMd;
//if (PubSub3Config.instance.isTagCompressed(tag))
//{
// contentToPublish = new MemoryStream();
// SharpZipCompressor Kompressor = new SharpZipCompressor();
// Kompressor.compress(content, contentToPublish);
// contentToPublish.Position = 0;
// pubMd = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, p.Value.serverURL, ClientInfo.thisClient.processString);
// //string msg = String.Format("Compressed tag {0} from {1} to {2} bytes.", tag, content.Length, contentToPublish.Length);
// //BlueEventLog.logEvent(msg, PS3LogLevels.info);
//}
//else
// pubMd = new PubMetaData(tagType, tag, (int)content.Length, p.Value.serverURL, ClientInfo.thisClient.processString);
//PubMetaData pMd = new PubMetaData(tagType, p.Key, data.Length,
// serverURL, thisPublisherInfo);
//p.Value.send(new PubSubMessage(new MetaData(pMd), new MemoryStream(data)).toStream());
}
}
}
}
#endregion
/// <summary>
/// This is the state of the crushable tags. An entry here means
/// the tag is waiting for an ack. The value is a pair of conf id
/// and the number of packets that could not be sent because
/// of waiting for ack
/// </summary>
private Dictionary<string, Pair<string, int>> myCrushableTagsWaitingOnAck = new Dictionary<string, Pair<string, int>>();
private void myOnSendConfirm(string messageId, bool confirmed) {
foreach (string tag in new List<string>(myCrushableTagsWaitingOnAck.Keys)) {
Pair<string, int> tagData = myCrushableTagsWaitingOnAck[tag];
if (messageId == tagData.first) {
myCrushableTagsWaitingOnAck.Remove(tag);
if (tagData.second > 0) {
// BlueEventLog.logEvent("Missed sending " + tagData.second + " messages, sending latest message now", BlueEventLevel.Diagnosis);
/// this data was waiting on an
/// ack, need to resend it
string tagType;
byte[] data = myState.getContent(tag, out tagType);
myPublish(tagType, tag, new MemoryStream(data));
}
}
}
}
}
}