505 lines
24 KiB
C#
505 lines
24 KiB
C#
// using System;
|
|
// //using System.Messaging;
|
|
// using System.Xml.Serialization;
|
|
// using System.Text;
|
|
// using System.IO;
|
|
// using System.Collections;
|
|
// using System.Collections.Generic;
|
|
// using System.Threading;
|
|
|
|
// using BCG;
|
|
// //using BCG.Utilities.BlueConfiguration;
|
|
// using BCG.Utilities;
|
|
// using mbts;
|
|
// /* StaticPublisherCore.cs
|
|
// * This file was originally called 'PublisherCore.cs'. It was renamed so that the publisher core could be replaced.
|
|
// *
|
|
// * */
|
|
// namespace BCG {
|
|
// // Provides a function to check if a tag is crushabel or not
|
|
// public class CrushedTagManager {
|
|
// private static CrushedTagManager _self;
|
|
// private Dictionary<string, bool> _tag_map = new Dictionary<string, bool>();
|
|
// public static CrushedTagManager instance {
|
|
// get {
|
|
// if (_self == null)
|
|
// _self = new CrushedTagManager();
|
|
// return _self;
|
|
// }
|
|
// }
|
|
// public bool is_tag_crushable(string tag) {
|
|
// tag = tag.ToLower();
|
|
// bool rv = false;
|
|
// lock (typeof(CrushedTagManager)) {
|
|
// if (!_tag_map.TryGetValue(tag, out rv)) {
|
|
// foreach (string r in PubSub3Config.instance.PS3_CrushableTags)
|
|
// if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag)) {
|
|
// _tag_map.Add(tag, true);
|
|
// break;
|
|
// }
|
|
// _tag_map.Add(tag, false);
|
|
// }
|
|
// }
|
|
// return rv;
|
|
// }
|
|
// }
|
|
// /// <summary>
|
|
// /// All the data for a single tag, thats published within
|
|
// /// a transaction
|
|
// /// </summary>
|
|
// public class TagWaitingInTransaction {
|
|
// private bool _is_crushable;
|
|
// private List<byte[]> _buffered_data = new List<byte[]>();
|
|
// // this is the size of all the buffered data
|
|
// private int _total_size = 0;
|
|
// private string _tag, _tag_type;
|
|
// public TagWaitingInTransaction(string theTagType, string theTag) {
|
|
// _tag = theTag;
|
|
// _tag_type = theTagType;
|
|
// _is_crushable = CrushedTagManager.instance.is_tag_crushable(_tag);
|
|
// }
|
|
|
|
// public void add_pub(Stream pub_stream) {
|
|
// if (_is_crushable) {
|
|
// // extract bytes, clear _buffered_data and queue to _buffered_data
|
|
// byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
|
|
// pub_stream.Read(new_data, 0, new_data.Length);
|
|
// _buffered_data.Clear();
|
|
// _buffered_data.Add(new_data);
|
|
// _total_size = new_data.Length;
|
|
// } else {
|
|
// // extract btes and queue to _buffered_data
|
|
// byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
|
|
// pub_stream.Read(new_data, 0, new_data.Length);
|
|
// _buffered_data.Add(new_data);
|
|
// _total_size += new_data.Length;
|
|
// }
|
|
// }
|
|
|
|
// public void write_to(Stream s, string connection, string publishing_app) {
|
|
// new MetaData(new PubMetaData(_tag_type, _tag, _total_size, connection, publishing_app)).save(s);
|
|
// foreach (byte[] data in _buffered_data)
|
|
// s.Write(data, 0, data.Length);
|
|
// }
|
|
|
|
// public string tag { get { return _tag; } }
|
|
// public string tagType { get { return _tag_type; } }
|
|
// public List<byte[]> data { get { return _buffered_data; } }
|
|
// }
|
|
|
|
// public enum PublisherCoreMessageType {
|
|
// transactionBegin,
|
|
// transactionCommit,
|
|
// transactionAbort,
|
|
// publish,
|
|
// clear
|
|
// }
|
|
|
|
// public class PublisherCoreMessage {
|
|
// public PublisherCoreMessageType msgType;
|
|
// public string tagType, tag;
|
|
// public Stream data;
|
|
|
|
// public PublisherCoreMessage(string tagToClear) {
|
|
// tag = tagToClear;
|
|
// msgType = PublisherCoreMessageType.clear;
|
|
|
|
// }
|
|
|
|
// public PublisherCoreMessage(PublisherCoreMessageType theMsgType) {
|
|
// msgType = theMsgType;
|
|
// }
|
|
|
|
// public PublisherCoreMessage(string theTagType, string theTag, Stream theData) {
|
|
// msgType = PublisherCoreMessageType.publish;
|
|
// tagType = theTagType;
|
|
// tag = theTag;
|
|
// data = theData;
|
|
// }
|
|
// public override string ToString() {
|
|
// return $"PublisherCoreMessage tag: {tag} data.Position: {data.Position} data.Length: {data.Length}";
|
|
// }
|
|
// }
|
|
|
|
// // this class uses an internal queue + thread to serialize and process calls
|
|
// // made through its public interface (IPublisherCore)
|
|
// public class StaticPublisherCore : IPublisherCore {
|
|
// // used to allow for different transports
|
|
// // to be used when communicating to a pubsubserver
|
|
// protected PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(false);
|
|
|
|
// // internal queue used to serialize pubsub method calls
|
|
// protected BlockingQueue myMainQueue = new BlockingQueue();
|
|
|
|
// // all the data published through this object in snapshot form
|
|
// // i.e. in a form that can be used repopulate a pubsub server
|
|
// // in the event of a crash
|
|
// protected AllPublishedTagState myState = new AllPublishedTagState();
|
|
|
|
// // a different IPubSubServerConnection object is used to interact
|
|
// // with each of the pubsub servers used by this publishercore
|
|
// protected List<IPubSubServerConnection> mySenders = new List<IPubSubServerConnection>();
|
|
|
|
// // a dictionary for fast retrieval of publishers
|
|
// // tag -> IPubSubServerConnection
|
|
// protected Dictionary<string, IPubSubServerConnection> myTagToSender = new Dictionary<string, IPubSubServerConnection>();
|
|
|
|
// // data waiting for transaction to end or abort
|
|
// // tag->TagWaitingInTransaction
|
|
// protected Dictionary<string, TagWaitingInTransaction> myWaitingInTransactionMessages = new Dictionary<string, TagWaitingInTransaction>();
|
|
|
|
// // router object, used to select which pubsub should be used for each tag
|
|
// protected IPubSubRouter myRouter = null;
|
|
|
|
// // flag indicating wether we are in the middle of a transaction or not
|
|
// protected bool myInTransaction = false;
|
|
|
|
// #region constructors
|
|
|
|
// public StaticPublisherCore()
|
|
// // TODOds
|
|
// : this(new PubSubRouter()) { }
|
|
|
|
// public StaticPublisherCore(string serverURL)
|
|
// : this(new PubSubRouter(new PubSubRoute(serverURL, ".*"))) { }
|
|
|
|
// public StaticPublisherCore(IPubSubRouter r) {
|
|
// myRouter = r;
|
|
// Thread pumpThread = new Thread(new ThreadStart(myMessagePump));
|
|
// pumpThread.IsBackground = true;
|
|
// pumpThread.Start();
|
|
// }
|
|
|
|
// public void Dispose() { }
|
|
// #endregion
|
|
|
|
// public int getNumberOfWaitingMessages() { return myMainQueue.getCount(); }
|
|
|
|
// #region Main Thread - all library functions are called on the main thread and queued to be picked by the pump
|
|
|
|
// public void publish(string tag, Stream content) { publish("", tag, content); }
|
|
|
|
// public void publish(string tagType, string tag, Stream content) {
|
|
// PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content);
|
|
// bts.Log.Gen.info($"StaticPublisherCore.publish tag: {tag} msg: {msg} len: {content.Length - content.Position}");
|
|
// myMainQueue.enqueue(msg);
|
|
// }
|
|
|
|
// public void beginTransaction() {
|
|
// myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionBegin));
|
|
// }
|
|
|
|
// public void commitTransaction() {
|
|
// myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionCommit));
|
|
// }
|
|
|
|
// public void abortTransaction() {
|
|
// myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionAbort));
|
|
// }
|
|
|
|
// public bool inTransaction { get { return myInTransaction; } }
|
|
|
|
// public void clearState(string tag) {
|
|
// myMainQueue.enqueue(new PublisherCoreMessage(tag));
|
|
// }
|
|
|
|
// #endregion
|
|
|
|
// #region Message Pump Thread - this is where messages are dequeued and the action performed
|
|
|
|
// private void myMessagePump() {
|
|
// while (true) {
|
|
// try {
|
|
// while (true) {
|
|
// object data = myMainQueue.dequeue(1000);
|
|
// if (data != null) {
|
|
// int qc = myMainQueue.getCount();
|
|
// PublisherCoreMessage msg = data as PublisherCoreMessage;
|
|
// bts.Log.Gen.info($"StaticPublisherCore.myMessagePump msg: {msg}");
|
|
// if (msg.msgType == PublisherCoreMessageType.transactionBegin) {
|
|
// myBeginTransaction();
|
|
// } else if (msg.msgType == PublisherCoreMessageType.transactionCommit) {
|
|
// myCommitTransaction();
|
|
// } else if (msg.msgType == PublisherCoreMessageType.transactionAbort) {
|
|
// myAbortTransaction();
|
|
// } else if (msg.msgType == PublisherCoreMessageType.publish) {
|
|
// myPublish(msg.tagType, msg.tag, msg.data);
|
|
// } else if (msg.msgType == PublisherCoreMessageType.clear) {
|
|
// myClearState(msg.tag);
|
|
// }
|
|
// }
|
|
|
|
// foreach (IPubSubServerConnection conn in mySenders) {
|
|
// conn.doWork();
|
|
// }
|
|
// }
|
|
// } catch (Exception ex) {
|
|
// bts.Log.Gen.error($"StaticPublisherCore.myMessagePump exception ex: {ex}");
|
|
// if (ex is OutOfMemoryException)
|
|
// throw ex;
|
|
// }
|
|
// }
|
|
// }
|
|
|
|
// private void myBeginTransaction() {
|
|
// // BlueEventLog.logAssert(!inTransaction);
|
|
// //BlueEventLog.logEvent("StaticPublisherCore Starting Transaction.", BlueEventLevel.Information);
|
|
// myInTransaction = true;
|
|
// }
|
|
|
|
// private void myCommitTransaction() {
|
|
// myEndTransaction(true);
|
|
// }
|
|
|
|
// private void myAbortTransaction() {
|
|
// myEndTransaction(false);
|
|
// }
|
|
|
|
|
|
|
|
// //based on success, sends a TA or TE, and sets myInTransaction to false
|
|
// //TODO: if a transaction is sending to multiple pubsubs (in order to
|
|
// //keep it consistent with the way it was done earlier), this should
|
|
// //simply publish it non transactionally (easy way out), or
|
|
// //transactionally with respect to each pubsub
|
|
// protected virtual void myEndTransaction(bool success) {
|
|
// lock (this) {
|
|
// // BlueEventLog.logAssert(inTransaction);
|
|
// //BlueEventLog.logEvent("StaticPublisherCore Finishing Transaction.", BlueEventLevel.Information);
|
|
// myInTransaction = false;
|
|
// if (success) {
|
|
// // pubsub server -> tags with updates
|
|
// Dictionary<string, List<string>> routesToTags = new Dictionary<string, List<string>>();
|
|
|
|
// // group updates per pubsub server
|
|
// foreach (string tag in myWaitingInTransactionMessages.Keys) {
|
|
// string thisURL = myRouter.get_pub_url_for_tag(tag);
|
|
// if (!routesToTags.ContainsKey(thisURL))
|
|
// routesToTags[thisURL] = new List<string>();
|
|
// routesToTags[thisURL].Add(tag);
|
|
// }
|
|
|
|
// // more than one server means we won't be able to honor the transactionality request
|
|
// // log a warning if that is the case.
|
|
// if (routesToTags.Keys.Count > 1) {
|
|
// //this is going to be a long error message, but it's an importat one to be able to debug.
|
|
// StringBuilder errorMessageBuilder = new StringBuilder();
|
|
// errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n");
|
|
// errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n");
|
|
|
|
// foreach (string route in routesToTags.Keys) {
|
|
// foreach (string tag in routesToTags[route]) {
|
|
// errorMessageBuilder.Append("\tTag ");
|
|
// errorMessageBuilder.Append(tag);
|
|
// errorMessageBuilder.Append(" is routed to ");
|
|
// errorMessageBuilder.Append(route);
|
|
// errorMessageBuilder.Append(".\n");
|
|
// }
|
|
|
|
// }
|
|
// string errMsg = errorMessageBuilder.ToString();
|
|
// // BlueEventLog.logEvent(errMsg, BlueEventLevel.Diagnosis);
|
|
// }
|
|
|
|
// // iterate through each server with pending messages
|
|
// foreach (KeyValuePair<string, List<string>> kvp in routesToTags) {
|
|
// IPubSubServerConnection transactionSender = null;
|
|
// string publishingApp = ClientInfo.thisClient.processString;
|
|
// string connection = kvp.Key;
|
|
// Stream transactionData = new MemoryStream();
|
|
|
|
// // iterate through each of the pending tag updates for one server
|
|
// foreach (string tag in kvp.Value) {
|
|
// ///it would be the same transaction sender that would
|
|
// ///return each time
|
|
// transactionSender = myGetOrBuildSender(tag);
|
|
|
|
|
|
// TagWaitingInTransaction waiting = myWaitingInTransactionMessages[tag];
|
|
// // iterate through each of the updates for a specific tag, specific server
|
|
// foreach (byte[] data in waiting.data)
|
|
// // update internal state
|
|
// myState.publish(waiting.tag, waiting.tagType, data);
|
|
// // add a pubsub header and copy each message to the transactionData stream
|
|
// waiting.write_to(transactionData, connection, publishingApp);
|
|
// }
|
|
// // now transactionData holds all the updates for all the tags
|
|
// // that are to be sent to the PubSub represented by transactionSender
|
|
|
|
// // add a transaction header, indicating how many tags in the transaction
|
|
// TransactionMetaData tMd = new TransactionMetaData(kvp.Value.Count);
|
|
|
|
// // rewind transaction data to the beggining
|
|
// transactionData.Position = 0;
|
|
|
|
// // send the entire thing to the specific pubsub server.
|
|
// // this is where the data is actually sent out !
|
|
// var ps_msg = new PubSubMessage(new MetaData(tMd), transactionData);
|
|
// Console.WriteLine($"serverConn.send metadata: {ps_msg.meta_data}");
|
|
// bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
|
|
// transactionSender.send(ps_msg);
|
|
// }
|
|
// }
|
|
// myWaitingInTransactionMessages.Clear();
|
|
// }
|
|
// }
|
|
|
|
// protected virtual void myPublish(string tagType, string tag, Stream content) {
|
|
// bts.Log.Gen.info($"StaticPublisherCore.myPublish tag: {tag} content: {content.Position} / {content.Length} myInTransaction: {myInTransaction}");
|
|
// if (myInTransaction) {
|
|
// ///if its in transaction message is cached, and is sent
|
|
// ///only when theres a commit or abort
|
|
// if (!myWaitingInTransactionMessages.ContainsKey(tag))
|
|
// myWaitingInTransactionMessages.Add(tag, new TagWaitingInTransaction(tagType, tag));
|
|
// myWaitingInTransactionMessages[tag].add_pub(content);
|
|
// } else {
|
|
// {
|
|
// myState.publish(tag, tagType, content);
|
|
// if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) {
|
|
// bts.Log.Gen.info("Sending crushable tag: {tag}");
|
|
// mySendData(tag, tagType, content, "myPublish");
|
|
// } else {
|
|
// bts.Log.Gen.info("Could not send tag: {tag} since its waiting on an ack");
|
|
// myCrushableTagsWaitingOnAck[tag].second++;
|
|
// }
|
|
// }
|
|
// // else
|
|
// // {
|
|
// // // update snapshot state
|
|
// // myState.publish(tag, tagType, content);
|
|
// // // publish out the data
|
|
// // mySendData(tag, tagType, content);
|
|
// // }
|
|
// }
|
|
// }
|
|
|
|
// // add header, possibly compress the data, and sendto appropriate pubsub
|
|
// private void mySendData(string tag, string tagType, Stream content, string reason) {
|
|
// bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType} reason: {reason} len: {content.Length - content.Position}");
|
|
// long initPos = content.Position;
|
|
// // locate right pubsub for this tag
|
|
// IPubSubServerConnection serverConn = myGetOrBuildSender(tag);
|
|
// Stream contentToPublish = content;
|
|
// PubMetaData pub_md;
|
|
|
|
// if (PubSub3Config.instance.isTagCompressed(tag)) {
|
|
// contentToPublish = new MemoryStream();
|
|
// SharpZipCompressor Kompressor = new SharpZipCompressor();
|
|
// Kompressor.compress(content, contentToPublish);
|
|
// contentToPublish.Position = 0;
|
|
// pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
|
|
// if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
|
|
// string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
|
|
// // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
|
|
// }
|
|
// } else
|
|
// pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
|
|
|
|
// content.Position = initPos;
|
|
// if (CrushedTagManager.instance.is_tag_crushable(tag)) {
|
|
// string refId = String.Empty;
|
|
// Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.server_url);
|
|
// // this is where the data is actually sent out (there are multiple sends)
|
|
// PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
|
|
// //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
|
|
// bts.Log.Gen.info($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
|
|
// bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
|
|
// serverConn.send_with_confirmation(ps_msg, ref refId);
|
|
// myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0));
|
|
// } else {
|
|
// // this is where the data is actually sent out
|
|
// var ps_msg = new PubSubMessage(pub_md, contentToPublish);
|
|
// bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
|
|
// serverConn.send(ps_msg);
|
|
// }
|
|
// }
|
|
|
|
// private void myClearState(string tag) {
|
|
// if (myTagToSender.ContainsKey(tag)) {
|
|
// myTagToSender.Remove(tag);
|
|
// }
|
|
// myState.clear(tag);
|
|
// }
|
|
|
|
// protected virtual IPubSubServerConnection myGetOrBuildSender(string tag) {
|
|
// if (!myTagToSender.ContainsKey(tag)) {
|
|
// string route = myRouter.get_pub_url_for_tag(tag);
|
|
// IPubSubServerConnection sender = myConnectionFactory.getConnection(route);
|
|
// myTagToSender.Add(tag, sender);
|
|
// if (!mySenders.Contains(sender)) {
|
|
// mySenders.Add(sender);
|
|
// sender.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot));
|
|
// sender.register_send_confirmation_callback(new SendConfirmationDelegate(myOnSendConfirm));
|
|
// }
|
|
|
|
// }
|
|
// return myTagToSender[tag];
|
|
// }
|
|
|
|
// ///in case of the server coming up, it publishes all the stored state
|
|
// ///for that server, back to it
|
|
// protected void myHandleServerReboot(string serverURL, bool connection) {
|
|
// bts.Log.Gen.info($"StaticPublisherCore.myHandleServerReboot server_url: {serverURL}");
|
|
// if (connection) {
|
|
// string thisPublisherInfo = ClientInfo.thisClient.processString;
|
|
// foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
|
|
// if (p.Value.server_url == serverURL) {
|
|
// string tagType;
|
|
// string tag = p.Key;
|
|
// myCrushableTagsWaitingOnAck.Clear();
|
|
// byte[] data = myState.getContent(tag, out tagType);
|
|
// mySendData(tag, tagType, new MemoryStream(data), "myHandleServerReboot");
|
|
|
|
// //Stream content = new MemoryStream(data);
|
|
// //Stream contentToPublish = content;
|
|
// //PubMetaData pubMd;
|
|
|
|
// //if (PubSub3Config.instance.isTagCompressed(tag))
|
|
// //{
|
|
// // contentToPublish = new MemoryStream();
|
|
// // SharpZipCompressor Kompressor = new SharpZipCompressor();
|
|
// // Kompressor.compress(content, contentToPublish);
|
|
// // contentToPublish.Position = 0;
|
|
// // pubMd = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, p.Value.serverURL, ClientInfo.thisClient.processString);
|
|
// // //string msg = String.Format("Compressed tag {0} from {1} to {2} bytes.", tag, content.Length, contentToPublish.Length);
|
|
// // //BlueEventLog.logEvent(msg, PS3LogLevels.info);
|
|
// //}
|
|
// //else
|
|
// // pubMd = new PubMetaData(tagType, tag, (int)content.Length, p.Value.serverURL, ClientInfo.thisClient.processString);
|
|
|
|
|
|
// //PubMetaData pMd = new PubMetaData(tagType, p.Key, data.Length,
|
|
// // serverURL, thisPublisherInfo);
|
|
// //p.Value.send(new PubSubMessage(new MetaData(pMd), new MemoryStream(data)).toStream());
|
|
// }
|
|
// }
|
|
// }
|
|
// }
|
|
|
|
// #endregion
|
|
|
|
// /// <summary>
|
|
// /// This is the state of the crushable tags. An entry here means
|
|
// /// the tag is waiting for an ack. The value is a pair of conf id
|
|
// /// and the number of packets that could not be sent because
|
|
// /// of waiting for ack
|
|
// /// </summary>
|
|
// private Dictionary<string, Pair<string, int>> myCrushableTagsWaitingOnAck = new Dictionary<string, Pair<string, int>>();
|
|
// private void myOnSendConfirm(string messageId, bool confirmed) {
|
|
// foreach (string tag in new List<string>(myCrushableTagsWaitingOnAck.Keys)) {
|
|
// Pair<string, int> tagData = myCrushableTagsWaitingOnAck[tag];
|
|
// if (messageId == tagData.first) {
|
|
// myCrushableTagsWaitingOnAck.Remove(tag);
|
|
// if (tagData.second > 0) {
|
|
// // BlueEventLog.logEvent("Missed sending " + tagData.second + " messages, sending latest message now", BlueEventLevel.Diagnosis);
|
|
// /// this data was waiting on an
|
|
// /// ack, need to resend it
|
|
// string tagType;
|
|
// byte[] data = myState.getContent(tag, out tagType);
|
|
// myPublish(tagType, tag, new MemoryStream(data));
|
|
// }
|
|
// }
|
|
// }
|
|
// }
|
|
// }
|
|
// } |