From 2bb7a89777f608478c16c8b3fba8c378e9cd7ba2 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Fri, 5 Jul 2024 21:21:27 +0000 Subject: [PATCH] Merging --- .../Messaging/Publish/PubTableBroker.cs | 14 +- .../Subscription/TableAccumulator2.cs | 2 +- .../Serialization/StreamSerializationUtils.cs | 8 +- .../PubsubServerStreamTagHandler.cs | 4 +- .../Configurations/MarketDataPublisher.json | 6 +- .../Configurations/TestDataPublisher.json | 4 +- .../PubSub/Client/Routing/PubSubRouter.cs | 4 +- .../AsyncTCPConnector_AsyncWrapper.cs | 25 +-- .../PubSub/documentation/internal_readme.txt | 117 ++++++++++++++ .../PubSub/documentation/readme.txt | 151 ++++++------------ .../Business/MarketData/BookStream.cs | 2 +- 11 files changed, 188 insertions(+), 149 deletions(-) create mode 100644 src/DataStreamSuite/PubSub/documentation/internal_readme.txt diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTableBroker.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTableBroker.cs index e7d20c6ca..3119202ba 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTableBroker.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTableBroker.cs @@ -51,13 +51,13 @@ namespace BCG { public virtual void make_batch_changes(IBlueRowCollection new_rows, IBlueRowCollection del_rows, IBlueRowCollection added_rows) { - string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}"; - foreach (IBlueRow r in del_rows) - log_msg += $"id: {r["id"]} "; - log_msg += "addedRows: "; - foreach (IBlueRow r in added_rows) - log_msg += $"id: {r["id"]}"; - bts.Log.Gen.info(log_msg); + // string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}"; + // foreach (IBlueRow r in del_rows) + // log_msg += $"id: {r["id"]} "; + // log_msg += "addedRows: "; + // foreach (IBlueRow r in added_rows) + // log_msg += $"id: {r["id"]}"; + // bts.Log.Gen.info(log_msg); SinglePubTableBroker table_publisher = null; var name_key = new_rows.schema.tableName.ToLower(); if (!_publisher_map.TryGetValue(name_key, out table_publisher)) diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs index d54f012da..d27353e79 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs @@ -73,7 +73,7 @@ namespace BCG foreach (var msg in msgs) { bts.StreamIOMessage str_msg = _deser.deserialize(msg); - var state_txt = _table == null ? "NULL" : BlueDataUtility.id_string(_table); + var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table); Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}"); switch (str_msg.op) { diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs index d1d802112..8d4727f03 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs @@ -95,8 +95,8 @@ namespace bts { } public static IBlueTableSchema deserialize_schema(ref BinaryReader br) { - var hex_txt = to_hex(br.BaseStream); - bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}"); + // var hex_txt = to_hex(br.BaseStream); + // bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}"); string table_name = br.read_string_cpp(); int n_cols = br.ReadInt32(); // doing a br.Read was only advancing the data 1 byte!! List cis = new List(); @@ -137,8 +137,8 @@ namespace bts { public static ColumnInfo deserialize_column_info(ref BinaryReader br) { //col_no, name, type> - var hex_txt = to_hex(br.BaseStream); - bts.Log.Gen.info($"StreamSerializationUtils.deserialize_column_info bytes: {hex_txt}"); + // var hex_txt = to_hex(br.BaseStream); + // bts.Log.Gen.info($"StreamSerializationUtils.deserialize_column_info bytes: {hex_txt}"); int col_no = br.ReadInt32(); string col_name = br.read_string_cpp(); int tc_int = br.ReadInt32(); diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs index 279ed0bad..16873194d 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs @@ -44,7 +44,7 @@ namespace bts { public void update(string tag, byte[] msg) { StreamIOMessage str_msg = _deser.deserialize(msg); var state_txt = _state == null ? "NULL" : _state.Count.ToString(); - Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (pre.1) tag: {tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}"); + Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (pre.1) tag: {tag} sz: {msg.Length} _state: {state_txt}"); switch (str_msg.op) { case StreamIOMessage.StreamOp.CREATE: //Log.assert(_state == null); @@ -56,7 +56,7 @@ namespace bts { _state.addRows(str_msg.multiple_remove_add_message.added_rows); break; } - Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (post) tag: {tag} sz: {msg.Length} str_msg: {str_msg} state: {_state.Count}"); + Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (post) tag: {tag} sz: {msg.Length} state: {_state.Count}"); } public byte[] get_content(string tag) { diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json index 8c5196f98..100cb0e1d 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json @@ -1,6 +1,6 @@ { "user": "bts", - "sparkd": "http://192.168.1.80:8000/config", + "sparkd": "http://127.0.0.1:8000/config", "pubsub": { "milliseconds_between_sub_info_updates": 1000, "milliseconds_between_time_slice_updates": 5000, @@ -89,8 +89,8 @@ "name": "test_data", "streams": [ { - "name": "seconds_1_10", - "expression": "second(1,10)", + "name": "milliseconds_1000_100", + "expression": "milliseconds(1000,100)", "is_on_websocket": true }, { diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json index 04ebfef32..de5c21918 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json @@ -6,7 +6,7 @@ "streams": [ { "name": "seconds_1_10", - "expression": "second(1,10)", + "expression": "milliseconds(1000,10)", "is_on_websocket": true }, { @@ -22,4 +22,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs index fbd7062d8..1f0c34bf9 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs @@ -47,7 +47,7 @@ namespace BCG { if (!_pub_routes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) { throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag)); } - Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}"); + Log.Gen.info($"PubSubRouter.get_sub_url_for_tag tag: {tag} route: {route}"); return RemoteConnectionInfo.parse_url(route.server_address); } @@ -55,7 +55,7 @@ namespace BCG { if (!_sub_routes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) { throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag)); } - Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}"); + Log.Gen.info($"PubSubRouter.get_sub_url_for_tag tag: {tag} route: {route}"); return RemoteConnectionInfo.parse_url(route.server_address); } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs index d8ee8aa31..984cfebf1 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs @@ -21,7 +21,7 @@ namespace bts { ThreadPool.QueueUserWorkItem(_ => _keep_trying_blocking_function(connect_once, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5))); } - // this is the asctual fuction thta calls TCPCLient sync methof and establishes a connection + // this is the actual fuction thta calls TCPCLient sync methof and establishes a connection private static void _connect_to(RemoteConnectionInfo server_rci, ITCPConnectionHandler handler) { // TODO, random sleep bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server: {server_rci}"); @@ -41,6 +41,8 @@ namespace bts { return conn; } + // this is a ganeric function that calls a sync action function, and if unsuccessful timesout after "timeout" + // wais for wait_before_retrying before trying again public static void _keep_trying_blocking_function(Action action, TimeSpan timeout, TimeSpan wait_before_retrying) { Func try_once = () => { var cts = new CancellationTokenSource(); @@ -64,26 +66,5 @@ namespace bts { System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); } } - - // public static void connect_to(RemoteConnectionInfo server_rci, TimeSpan socket_timeout, TimeSpan retry_interval, ITCPConnectionHandler handler) { - // while (true) { - // try { - // var client = new TcpClient(); - // var result = client.BeginConnect(server_rci.ip, server_rci.port, null, null); - // var success = result.AsyncWaitHandle.WaitOne(socket_timeout, true); - // if (success && client.Connected) { - // client.EndConnect(result); - // handler.onConnection(new AsyncTCPConnection_AsyncWrapper(client)); - // } else { - // throw new SocketException(); - // } - // } catch (Exception ex) { - // Log.Gen.info($"Connection attempt failed: {ex.Message}"); - // Log.Gen.info($"Retrying in {retry_interval.TotalSeconds} seconds..."); - // Thread.Sleep(retry_interval); - // } - // } - // } - } } diff --git a/src/DataStreamSuite/PubSub/documentation/internal_readme.txt b/src/DataStreamSuite/PubSub/documentation/internal_readme.txt new file mode 100644 index 000000000..aaa5afda2 --- /dev/null +++ b/src/DataStreamSuite/PubSub/documentation/internal_readme.txt @@ -0,0 +1,117 @@ +Basic TCP: +ITCPConnection implemented by AsyncTCPConnection_APM is the innermost class resposinsble for TCP sends and received. Its used +on both client and server symmetrically. All the hard async send/receive logic is there. +AsyncTCPConnectionAcceptor_APM opens a port and listens and accepts to connections from the client. It creates the ITCPConnection +on the server. +AsyncTCPConnector_APM connects to the server (above) and creates an ITCPConnection on the client side. + +IPubSubServerConnection: +All communication to PubSubServer happens using this. It has 3 main types of calls: +1. send: to asyncronoulsy end data +2. get data: either through a polling call or callback +3. get connection info: through callbacks + +The only implementation of the above is PubSubServerManagedTCPConnection +(there used to be an MSMQ one and another wrapped around native code - but neither exists anymore) + +PubSubMessages: +PubSubClientMessage is the only messages thats sent from client to pubsubserver. Its serialized on the +client and sent over to the pubsubserver, where uints received ad deserialized + +SubscriberCore:ISubscriberCore (retired now) +| +V +SingleServerSubscriberCore:ISingleServerSubscriberCore + +PublisherCore:IPublisherCore (transactions not supported for now), keeps AllPublishedTagState to cache state on the publisher +| +V +IPubSubServerConnection + +PubSubServer: +PubSubMessageReactor (top level, called from main), contain PubSubState (which has all data and connection state) +ManagedTCPMessageHub : IPubsubMessageHub (this is what handles each ITCPConnection for each connection) + +PubSubState keeps the total pubsub data in 2 classes +Dictionary and +Dictionary +SingleTagData has the tag and IPubSubServerTagHandler (or byte[]) (IPubSubServerTagHandler holds the data state) +for streams the IPubSubServerTagHandler is the PubsubServerTagHandler, which is designed for multiple tags and has a IDictionary +TagStateBank containes the PubsubDeserial izationController, which knows how to deserialize the stream publishes and build state, ad simialirly produce the state on demand +PubsubTagHandlerFactory returns a IPubSubServerTagHandler for each tag, called by the server state handler, SingleTagData and publisher tag state, AllPublishedTagState + +SingleTagConnectionData contains connection vs sub, and connection vs pub +PubSubState calls +SystemSubscriptionMapPublisher.init(this); +SystemStatsPublisher.init(this); +SystemHeartbeatPublisher.init(); +--------------------------------------------------------------- +How internal tags are published? +There are 3 internal publishers (ie compute somerthing an dpublish tit PubSubServer, such that clients can subscribe to it): +SystemStatsPublisher, SystemSubscriptionMapPublisher and SystemHeartbeatPublisher +These are somewhat tricky because they are both listeners to PubUsbState event and intrnally publish, but are also IPubSubServerTagHandler +SystemSubscriptionMapPublisher: +just one instance, a static init is called by PubSubState and it passes itself. SystemSubscriptionMapPublisher subscribes to the event +gets the affected tags, marks them dirty, in doWork creates the state and calls the internal_publish. +Now, as the result of an internal_publish (which is pretty much like a rmote publish as far as the PubsubServer is concerned), SingleTagData +ends up calling SystemSubscriptionMapPublisher (since it also serves as the tagHandler for the tag it publishes via internal_publish) + + + +PubSubState has onPublishStats (tag, sz) that is fired on update_pub_data +onPublishStats is subscribed by SystemStatsPublisher +SystemStatsPublisher.onPublishStats updates the state and on the next doWork calls PubSubMessageReactor.internal_send +SystemStatsPublisher is also an IPubSubServerTagHandler so SingleTagData calls it getContent to get full state + +Streams: +PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges -> +PubTablePublisher:IPubTablePublisher per tag (addRow, deleteRow, changeRow, createTable, batchRemoveAdd) +PubTablePublisher has BlueDataSerializationController:IBlueDataSerializer to serialize the rows to byte[] +BlueDataSerializationController uses a IBlueDataSerializer, which is specialized for the tag (using UniversalSerializer for now) + +IBlueDataSerializer has just one function +void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData); +the second arg is IBlueRowCollection[] because in case of op = StreamOpCode.multipleRemoveAdd, it has multiple IBlueRowCollection for rem and add +(BlueDataSerializer.serialize's comment has the serialization layout) + +For each serializer we have 2 matching deserializers, IPubsubDeserializer (for PubSubservr) and IClientDeserializer (for clients subscribing to streams) +For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer + +------------------------------------------------------------------- +TCP publishing: +To run the base TCP layer run PubSubServer as an echo server, +(set run_echo_tcp_server_only in spark\src\DataStreamSuite\PubSubServer\Program.cs) +to run a client that sends a tcp message tot he echo server, run +SimpleTCPEchoStream + + +Pubsub publishing: +----------------- +PubSubTestingStream (bounces pubs across the pubsub server) +mutableAppend(pubSubTesting(100,10000000,100000,"test_1"), +pubSubTesting(250,10000000,100000,"test_2")) + + +Stream publishing: +------------------ +PublisherStream (publishes any stream to the pubsubserver) +r=second(100) +r=project(r,"id") +r=math(r,"x = id*id") +publisher(r, "test_x") + + +tracking message into the pubsubserver +-------------------------------------- +connection: +ServerToClientManagedTCPConnectionAcceptor._on_accept + var log_msg = $"ServerToClientManagedTCPConnectionAcceptor._on_accept called port: {_port} state: {res.AsyncState}"; + Log.Gen.info(log_msg); + +data: +ManagedTCPMessageHub.onData +bts.Log.Gen.info($"ManagedTCPMessgageHub.onData recv msg: {message}"); +PubSubMessageReactor._polling_thread_func +bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}"); +PubSubMessageReactor._handle_client_msg +bts.Log.Gen.info($"PubSubMessageReator._handle_client_msg ps_client_msg: {ps_client_msg}"); diff --git a/src/DataStreamSuite/PubSub/documentation/readme.txt b/src/DataStreamSuite/PubSub/documentation/readme.txt index aaa5afda2..64b024e3c 100644 --- a/src/DataStreamSuite/PubSub/documentation/readme.txt +++ b/src/DataStreamSuite/PubSub/documentation/readme.txt @@ -1,117 +1,58 @@ -Basic TCP: -ITCPConnection implemented by AsyncTCPConnection_APM is the innermost class resposinsble for TCP sends and received. Its used -on both client and server symmetrically. All the hard async send/receive logic is there. -AsyncTCPConnectionAcceptor_APM opens a port and listens and accepts to connections from the client. It creates the ITCPConnection -on the server. -AsyncTCPConnector_APM connects to the server (above) and creates an ITCPConnection on the client side. +If the DSQA comes up empty, we need to take it one step at a time and ensure it in the following order: -IPubSubServerConnection: -All communication to PubSubServer happens using this. It has 3 main types of calls: -1. send: to asyncronoulsy end data -2. get data: either through a polling call or callback -3. get connection info: through callbacks +-------------------- +PubsubServer routes: +-------------------- +the client (both publisher and subscriber) need to connect tot he rigth pubsubserver. There are 3 days to do this: +1. Bring up a DSQA and check the title bar, it should be tag:regExp: PubSunRoute +the ip and port matter, the prefix is just fo the human eye. +2. Run the stream pss_routes(). +It will give a row per tag reg exp vs the pss route (pss == pubsub server0) +3. Check the clirnt logs for: +PubSubRouter.get_sub_url_for_tag tag: and PubSubRouter.get_pub_url_for_tag tag: -The only implementation of the above is PubSubServerManagedTCPConnection -(there used to be an MSMQ one and another wrapped around native code - but neither exists anymore) +------------------- +Connection Attempt: +------------------- +Check the pss logs for: +"AsyncTCPConnectionAcceptor_AsyncWrapper.cons started on port:" +Make sure it is listening to the correct port -PubSubMessages: -PubSubClientMessage is the only messages thats sent from client to pubsubserver. Its serialized on the -client and sent over to the pubsubserver, where uints received ad deserialized - -SubscriberCore:ISubscriberCore (retired now) -| -V -SingleServerSubscriberCore:ISingleServerSubscriberCore - -PublisherCore:IPublisherCore (transactions not supported for now), keeps AllPublishedTagState to cache state on the publisher -| -V -IPubSubServerConnection - -PubSubServer: -PubSubMessageReactor (top level, called from main), contain PubSubState (which has all data and connection state) -ManagedTCPMessageHub : IPubsubMessageHub (this is what handles each ITCPConnection for each connection) - -PubSubState keeps the total pubsub data in 2 classes -Dictionary and -Dictionary -SingleTagData has the tag and IPubSubServerTagHandler (or byte[]) (IPubSubServerTagHandler holds the data state) -for streams the IPubSubServerTagHandler is the PubsubServerTagHandler, which is designed for multiple tags and has a IDictionary -TagStateBank containes the PubsubDeserial izationController, which knows how to deserialize the stream publishes and build state, ad simialirly produce the state on demand -PubsubTagHandlerFactory returns a IPubSubServerTagHandler for each tag, called by the server state handler, SingleTagData and publisher tag state, AllPublishedTagState - -SingleTagConnectionData contains connection vs sub, and connection vs pub -PubSubState calls -SystemSubscriptionMapPublisher.init(this); -SystemStatsPublisher.init(this); -SystemHeartbeatPublisher.init(); ---------------------------------------------------------------- -How internal tags are published? -There are 3 internal publishers (ie compute somerthing an dpublish tit PubSubServer, such that clients can subscribe to it): -SystemStatsPublisher, SystemSubscriptionMapPublisher and SystemHeartbeatPublisher -These are somewhat tricky because they are both listeners to PubUsbState event and intrnally publish, but are also IPubSubServerTagHandler -SystemSubscriptionMapPublisher: -just one instance, a static init is called by PubSubState and it passes itself. SystemSubscriptionMapPublisher subscribes to the event -gets the affected tags, marks them dirty, in doWork creates the state and calls the internal_publish. -Now, as the result of an internal_publish (which is pretty much like a rmote publish as far as the PubsubServer is concerned), SingleTagData -ends up calling SystemSubscriptionMapPublisher (since it also serves as the tagHandler for the tag it publishes via internal_publish) +On the client, make sure it is attemping to connect to the right pss +"AsyncTCPConnector_AsyncWrapper.connect_to server:" +------------------------------------ +Successful connection establishment: +------------------------------------ -PubSubState has onPublishStats (tag, sz) that is fired on update_pub_data -onPublishStats is subscribed by SystemStatsPublisher -SystemStatsPublisher.onPublishStats updates the state and on the next doWork calls PubSubMessageReactor.internal_send -SystemStatsPublisher is also an IPubSubServerTagHandler so SingleTagData calls it getContent to get full state +On pss, a successful connection should result in the log: +"calling onConnection now" +The whole log is +Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}, new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()} calling onConnection now"); -Streams: -PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges -> -PubTablePublisher:IPubTablePublisher per tag (addRow, deleteRow, changeRow, createTable, batchRemoveAdd) -PubTablePublisher has BlueDataSerializationController:IBlueDataSerializer to serialize the rows to byte[] -BlueDataSerializationController uses a IBlueDataSerializer, which is specialized for the tag (using UniversalSerializer for now) - -IBlueDataSerializer has just one function -void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData); -the second arg is IBlueRowCollection[] because in case of op = StreamOpCode.multipleRemoveAdd, it has multiple IBlueRowCollection for rem and add -(BlueDataSerializer.serialize's comment has the serialization layout) - -For each serializer we have 2 matching deserializers, IPubsubDeserializer (for PubSubservr) and IClientDeserializer (for clients subscribing to streams) -For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer - -------------------------------------------------------------------- -TCP publishing: -To run the base TCP layer run PubSubServer as an echo server, -(set run_echo_tcp_server_only in spark\src\DataStreamSuite\PubSubServer\Program.cs) -to run a client that sends a tcp message tot he echo server, run -SimpleTCPEchoStream +On the client, a successful connection should result in the log: +"AsyncTCPConnector_AsyncWrapper._connect_to new connection," -Pubsub publishing: ------------------ -PubSubTestingStream (bounces pubs across the pubsub server) -mutableAppend(pubSubTesting(100,10000000,100000,"test_1"), -pubSubTesting(250,10000000,100000,"test_2")) +------------------------------------ +Successful message sent from client: +------------------------------------ +"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent" - -Stream publishing: ------------------- -PublisherStream (publishes any stream to the pubsubserver) -r=second(100) -r=project(r,"id") -r=math(r,"x = id*id") -publisher(r, "test_x") - - -tracking message into the pubsubserver -------------------------------------- -connection: -ServerToClientManagedTCPConnectionAcceptor._on_accept - var log_msg = $"ServerToClientManagedTCPConnectionAcceptor._on_accept called port: {_port} state: {res.AsyncState}"; - Log.Gen.info(log_msg); +Successful message received on client: +-------------------------------------- +"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read" -data: -ManagedTCPMessageHub.onData -bts.Log.Gen.info($"ManagedTCPMessgageHub.onData recv msg: {message}"); -PubSubMessageReactor._polling_thread_func -bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}"); -PubSubMessageReactor._handle_client_msg -bts.Log.Gen.info($"PubSubMessageReator._handle_client_msg ps_client_msg: {ps_client_msg}"); + +pss's send/receive is the same log, since it uses the same class AsyncTCPConnection_AsyncWrapper +--------------------------------- +Successful message sent from pss: +--------------------------------- +"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent" + +----------------------------------- +Successful message received on pss: +----------------------------------- +"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read" \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs index 43caebf85..eb84462a0 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs @@ -62,7 +62,7 @@ namespace BCG { } private void _on_books(TableUpdate update) { - _log_updates(update); + //_log_updates(update); bts.Log.Gen.info($"BookStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}"); _cached_state = update.state; }