// using System; // //using System.Messaging; // using System.Xml.Serialization; // using System.Text; // using System.IO; // using System.Collections; // using System.Collections.Generic; // using System.Threading; // using BCG; // //using BCG.Utilities.BlueConfiguration; // using BCG.Utilities; // using mbts; // /* StaticPublisherCore.cs // * This file was originally called 'PublisherCore.cs'. It was renamed so that the publisher core could be replaced. // * // * */ // namespace BCG { // // Provides a function to check if a tag is crushabel or not // public class CrushedTagManager { // private static CrushedTagManager _self; // private Dictionary _tag_map = new Dictionary(); // public static CrushedTagManager instance { // get { // if (_self == null) // _self = new CrushedTagManager(); // return _self; // } // } // public bool is_tag_crushable(string tag) { // tag = tag.ToLower(); // bool rv = false; // lock (typeof(CrushedTagManager)) { // if (!_tag_map.TryGetValue(tag, out rv)) { // foreach (string r in PubSub3Config.instance.PS3_CrushableTags) // if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag)) { // _tag_map.Add(tag, true); // break; // } // _tag_map.Add(tag, false); // } // } // return rv; // } // } // /// // /// All the data for a single tag, thats published within // /// a transaction // /// // public class TagWaitingInTransaction { // private bool _is_crushable; // private List _buffered_data = new List(); // // this is the size of all the buffered data // private int _total_size = 0; // private string _tag, _tag_type; // public TagWaitingInTransaction(string theTagType, string theTag) { // _tag = theTag; // _tag_type = theTagType; // _is_crushable = CrushedTagManager.instance.is_tag_crushable(_tag); // } // public void add_pub(Stream pub_stream) { // if (_is_crushable) { // // extract bytes, clear _buffered_data and queue to _buffered_data // byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)]; // pub_stream.Read(new_data, 0, new_data.Length); // _buffered_data.Clear(); // _buffered_data.Add(new_data); // _total_size = new_data.Length; // } else { // // extract btes and queue to _buffered_data // byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)]; // pub_stream.Read(new_data, 0, new_data.Length); // _buffered_data.Add(new_data); // _total_size += new_data.Length; // } // } // public void write_to(Stream s, string connection, string publishing_app) { // new MetaData(new PubMetaData(_tag_type, _tag, _total_size, connection, publishing_app)).save(s); // foreach (byte[] data in _buffered_data) // s.Write(data, 0, data.Length); // } // public string tag { get { return _tag; } } // public string tagType { get { return _tag_type; } } // public List data { get { return _buffered_data; } } // } // public enum PublisherCoreMessageType { // transactionBegin, // transactionCommit, // transactionAbort, // publish, // clear // } // public class PublisherCoreMessage { // public PublisherCoreMessageType msgType; // public string tagType, tag; // public Stream data; // public PublisherCoreMessage(string tagToClear) { // tag = tagToClear; // msgType = PublisherCoreMessageType.clear; // } // public PublisherCoreMessage(PublisherCoreMessageType theMsgType) { // msgType = theMsgType; // } // public PublisherCoreMessage(string theTagType, string theTag, Stream theData) { // msgType = PublisherCoreMessageType.publish; // tagType = theTagType; // tag = theTag; // data = theData; // } // public override string ToString() { // return $"PublisherCoreMessage tag: {tag} data.Position: {data.Position} data.Length: {data.Length}"; // } // } // // this class uses an internal queue + thread to serialize and process calls // // made through its public interface (IPublisherCore) // public class StaticPublisherCore : IPublisherCore { // // used to allow for different transports // // to be used when communicating to a pubsubserver // protected PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(false); // // internal queue used to serialize pubsub method calls // protected BlockingQueue myMainQueue = new BlockingQueue(); // // all the data published through this object in snapshot form // // i.e. in a form that can be used repopulate a pubsub server // // in the event of a crash // protected AllPublishedTagState myState = new AllPublishedTagState(); // // a different IPubSubServerConnection object is used to interact // // with each of the pubsub servers used by this publishercore // protected List mySenders = new List(); // // a dictionary for fast retrieval of publishers // // tag -> IPubSubServerConnection // protected Dictionary myTagToSender = new Dictionary(); // // data waiting for transaction to end or abort // // tag->TagWaitingInTransaction // protected Dictionary myWaitingInTransactionMessages = new Dictionary(); // // router object, used to select which pubsub should be used for each tag // protected IPubSubRouter myRouter = null; // // flag indicating wether we are in the middle of a transaction or not // protected bool myInTransaction = false; // #region constructors // public StaticPublisherCore() // // TODOds // : this(new PubSubRouter()) { } // public StaticPublisherCore(string serverURL) // : this(new PubSubRouter(new PubSubRoute(serverURL, ".*"))) { } // public StaticPublisherCore(IPubSubRouter r) { // myRouter = r; // Thread pumpThread = new Thread(new ThreadStart(myMessagePump)); // pumpThread.IsBackground = true; // pumpThread.Start(); // } // public void Dispose() { } // #endregion // public int getNumberOfWaitingMessages() { return myMainQueue.getCount(); } // #region Main Thread - all library functions are called on the main thread and queued to be picked by the pump // public void publish(string tag, Stream content) { publish("", tag, content); } // public void publish(string tagType, string tag, Stream content) { // PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content); // bts.Log.Gen.info($"StaticPublisherCore.publish tag: {tag} msg: {msg} len: {content.Length - content.Position}"); // myMainQueue.enqueue(msg); // } // public void beginTransaction() { // myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionBegin)); // } // public void commitTransaction() { // myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionCommit)); // } // public void abortTransaction() { // myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionAbort)); // } // public bool inTransaction { get { return myInTransaction; } } // public void clearState(string tag) { // myMainQueue.enqueue(new PublisherCoreMessage(tag)); // } // #endregion // #region Message Pump Thread - this is where messages are dequeued and the action performed // private void myMessagePump() { // while (true) { // try { // while (true) { // object data = myMainQueue.dequeue(1000); // if (data != null) { // int qc = myMainQueue.getCount(); // PublisherCoreMessage msg = data as PublisherCoreMessage; // bts.Log.Gen.info($"StaticPublisherCore.myMessagePump msg: {msg}"); // if (msg.msgType == PublisherCoreMessageType.transactionBegin) { // myBeginTransaction(); // } else if (msg.msgType == PublisherCoreMessageType.transactionCommit) { // myCommitTransaction(); // } else if (msg.msgType == PublisherCoreMessageType.transactionAbort) { // myAbortTransaction(); // } else if (msg.msgType == PublisherCoreMessageType.publish) { // myPublish(msg.tagType, msg.tag, msg.data); // } else if (msg.msgType == PublisherCoreMessageType.clear) { // myClearState(msg.tag); // } // } // foreach (IPubSubServerConnection conn in mySenders) { // conn.doWork(); // } // } // } catch (Exception ex) { // bts.Log.Gen.error($"StaticPublisherCore.myMessagePump exception ex: {ex}"); // if (ex is OutOfMemoryException) // throw ex; // } // } // } // private void myBeginTransaction() { // // BlueEventLog.logAssert(!inTransaction); // //BlueEventLog.logEvent("StaticPublisherCore Starting Transaction.", BlueEventLevel.Information); // myInTransaction = true; // } // private void myCommitTransaction() { // myEndTransaction(true); // } // private void myAbortTransaction() { // myEndTransaction(false); // } // //based on success, sends a TA or TE, and sets myInTransaction to false // //TODO: if a transaction is sending to multiple pubsubs (in order to // //keep it consistent with the way it was done earlier), this should // //simply publish it non transactionally (easy way out), or // //transactionally with respect to each pubsub // protected virtual void myEndTransaction(bool success) { // lock (this) { // // BlueEventLog.logAssert(inTransaction); // //BlueEventLog.logEvent("StaticPublisherCore Finishing Transaction.", BlueEventLevel.Information); // myInTransaction = false; // if (success) { // // pubsub server -> tags with updates // Dictionary> routesToTags = new Dictionary>(); // // group updates per pubsub server // foreach (string tag in myWaitingInTransactionMessages.Keys) { // string thisURL = myRouter.get_pub_url_for_tag(tag); // if (!routesToTags.ContainsKey(thisURL)) // routesToTags[thisURL] = new List(); // routesToTags[thisURL].Add(tag); // } // // more than one server means we won't be able to honor the transactionality request // // log a warning if that is the case. // if (routesToTags.Keys.Count > 1) { // //this is going to be a long error message, but it's an importat one to be able to debug. // StringBuilder errorMessageBuilder = new StringBuilder(); // errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n"); // errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n"); // foreach (string route in routesToTags.Keys) { // foreach (string tag in routesToTags[route]) { // errorMessageBuilder.Append("\tTag "); // errorMessageBuilder.Append(tag); // errorMessageBuilder.Append(" is routed to "); // errorMessageBuilder.Append(route); // errorMessageBuilder.Append(".\n"); // } // } // string errMsg = errorMessageBuilder.ToString(); // // BlueEventLog.logEvent(errMsg, BlueEventLevel.Diagnosis); // } // // iterate through each server with pending messages // foreach (KeyValuePair> kvp in routesToTags) { // IPubSubServerConnection transactionSender = null; // string publishingApp = ClientInfo.thisClient.processString; // string connection = kvp.Key; // Stream transactionData = new MemoryStream(); // // iterate through each of the pending tag updates for one server // foreach (string tag in kvp.Value) { // ///it would be the same transaction sender that would // ///return each time // transactionSender = myGetOrBuildSender(tag); // TagWaitingInTransaction waiting = myWaitingInTransactionMessages[tag]; // // iterate through each of the updates for a specific tag, specific server // foreach (byte[] data in waiting.data) // // update internal state // myState.publish(waiting.tag, waiting.tagType, data); // // add a pubsub header and copy each message to the transactionData stream // waiting.write_to(transactionData, connection, publishingApp); // } // // now transactionData holds all the updates for all the tags // // that are to be sent to the PubSub represented by transactionSender // // add a transaction header, indicating how many tags in the transaction // TransactionMetaData tMd = new TransactionMetaData(kvp.Value.Count); // // rewind transaction data to the beggining // transactionData.Position = 0; // // send the entire thing to the specific pubsub server. // // this is where the data is actually sent out ! // var ps_msg = new PubSubMessage(new MetaData(tMd), transactionData); // Console.WriteLine($"serverConn.send metadata: {ps_msg.meta_data}"); // bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}"); // transactionSender.send(ps_msg); // } // } // myWaitingInTransactionMessages.Clear(); // } // } // protected virtual void myPublish(string tagType, string tag, Stream content) { // bts.Log.Gen.info($"StaticPublisherCore.myPublish tag: {tag} content: {content.Position} / {content.Length} myInTransaction: {myInTransaction}"); // if (myInTransaction) { // ///if its in transaction message is cached, and is sent // ///only when theres a commit or abort // if (!myWaitingInTransactionMessages.ContainsKey(tag)) // myWaitingInTransactionMessages.Add(tag, new TagWaitingInTransaction(tagType, tag)); // myWaitingInTransactionMessages[tag].add_pub(content); // } else { // { // myState.publish(tag, tagType, content); // if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) { // bts.Log.Gen.info("Sending crushable tag: {tag}"); // mySendData(tag, tagType, content, "myPublish"); // } else { // bts.Log.Gen.info("Could not send tag: {tag} since its waiting on an ack"); // myCrushableTagsWaitingOnAck[tag].second++; // } // } // // else // // { // // // update snapshot state // // myState.publish(tag, tagType, content); // // // publish out the data // // mySendData(tag, tagType, content); // // } // } // } // // add header, possibly compress the data, and sendto appropriate pubsub // private void mySendData(string tag, string tagType, Stream content, string reason) { // bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType} reason: {reason} len: {content.Length - content.Position}"); // long initPos = content.Position; // // locate right pubsub for this tag // IPubSubServerConnection serverConn = myGetOrBuildSender(tag); // Stream contentToPublish = content; // PubMetaData pub_md; // if (PubSub3Config.instance.isTagCompressed(tag)) { // contentToPublish = new MemoryStream(); // SharpZipCompressor Kompressor = new SharpZipCompressor(); // Kompressor.compress(content, contentToPublish); // contentToPublish.Position = 0; // pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString); // if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) { // string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length); // // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio); // } // } else // pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString); // content.Position = initPos; // if (CrushedTagManager.instance.is_tag_crushable(tag)) { // string refId = String.Empty; // Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.server_url); // // this is where the data is actually sent out (there are multiple sends) // PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish); // //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}"); // bts.Log.Gen.info($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}"); // bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}"); // serverConn.send_with_confirmation(ps_msg, ref refId); // myCrushableTagsWaitingOnAck.Add(tag, new Pair(refId, 0)); // } else { // // this is where the data is actually sent out // var ps_msg = new PubSubMessage(pub_md, contentToPublish); // bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}"); // serverConn.send(ps_msg); // } // } // private void myClearState(string tag) { // if (myTagToSender.ContainsKey(tag)) { // myTagToSender.Remove(tag); // } // myState.clear(tag); // } // protected virtual IPubSubServerConnection myGetOrBuildSender(string tag) { // if (!myTagToSender.ContainsKey(tag)) { // string route = myRouter.get_pub_url_for_tag(tag); // IPubSubServerConnection sender = myConnectionFactory.getConnection(route); // myTagToSender.Add(tag, sender); // if (!mySenders.Contains(sender)) { // mySenders.Add(sender); // sender.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot)); // sender.register_send_confirmation_callback(new SendConfirmationDelegate(myOnSendConfirm)); // } // } // return myTagToSender[tag]; // } // ///in case of the server coming up, it publishes all the stored state // ///for that server, back to it // protected void myHandleServerReboot(string serverURL, bool connection) { // bts.Log.Gen.info($"StaticPublisherCore.myHandleServerReboot server_url: {serverURL}"); // if (connection) { // string thisPublisherInfo = ClientInfo.thisClient.processString; // foreach (KeyValuePair p in myTagToSender) { // if (p.Value.server_url == serverURL) { // string tagType; // string tag = p.Key; // myCrushableTagsWaitingOnAck.Clear(); // byte[] data = myState.getContent(tag, out tagType); // mySendData(tag, tagType, new MemoryStream(data), "myHandleServerReboot"); // //Stream content = new MemoryStream(data); // //Stream contentToPublish = content; // //PubMetaData pubMd; // //if (PubSub3Config.instance.isTagCompressed(tag)) // //{ // // contentToPublish = new MemoryStream(); // // SharpZipCompressor Kompressor = new SharpZipCompressor(); // // Kompressor.compress(content, contentToPublish); // // contentToPublish.Position = 0; // // pubMd = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, p.Value.serverURL, ClientInfo.thisClient.processString); // // //string msg = String.Format("Compressed tag {0} from {1} to {2} bytes.", tag, content.Length, contentToPublish.Length); // // //BlueEventLog.logEvent(msg, PS3LogLevels.info); // //} // //else // // pubMd = new PubMetaData(tagType, tag, (int)content.Length, p.Value.serverURL, ClientInfo.thisClient.processString); // //PubMetaData pMd = new PubMetaData(tagType, p.Key, data.Length, // // serverURL, thisPublisherInfo); // //p.Value.send(new PubSubMessage(new MetaData(pMd), new MemoryStream(data)).toStream()); // } // } // } // } // #endregion // /// // /// This is the state of the crushable tags. An entry here means // /// the tag is waiting for an ack. The value is a pair of conf id // /// and the number of packets that could not be sent because // /// of waiting for ack // /// // private Dictionary> myCrushableTagsWaitingOnAck = new Dictionary>(); // private void myOnSendConfirm(string messageId, bool confirmed) { // foreach (string tag in new List(myCrushableTagsWaitingOnAck.Keys)) { // Pair tagData = myCrushableTagsWaitingOnAck[tag]; // if (messageId == tagData.first) { // myCrushableTagsWaitingOnAck.Remove(tag); // if (tagData.second > 0) { // // BlueEventLog.logEvent("Missed sending " + tagData.second + " messages, sending latest message now", BlueEventLevel.Diagnosis); // /// this data was waiting on an // /// ack, need to resend it // string tagType; // byte[] data = myState.getContent(tag, out tagType); // myPublish(tagType, tag, new MemoryStream(data)); // } // } // } // } // } // }