file restructuring for sdk

WARNING: This breaks the sparkd clang build (only works with gcc)

.devcontainer/Dockerfile

@@ -0,0 +1,122 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Note installations that take a while (more than 10s)
+
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
+
+# Python: ~30s
+RUN apt-get install -y \
+   python3-pip \
+   python3-venv \
+   && python3 -m pip install --upgrade pip
+
+# GCC
+ARG gcc_major=13
+RUN \
+   add-apt-repository ppa:ubuntu-toolchain-r/test \
+   && apt-get update \
+   && apt-get install -y g++-$gcc_major
+
+# Clang
+ARG clang_major=17
+RUN \
+   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+   | tee -a "/etc/apt/sources.list.d/llvm.list" \
+   && apt-get update \
+   && apt-get install -y \
+   clang-$clang_major \
+   clang-format-$clang_major \
+   clangd-$clang_major
+
+# CMake
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
+
+# Dotnet
+RUN apt-get update && apt-get install -y dotnet7
+
+# Node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+   | tee /etc/apt/sources.list.d/nodesource.list \
+   && apt-get update \
+   && apt-get install -y nodejs
+
+# Emumba (deps)
+RUN apt-get update && apt-get -y install rapidjson-dev
+# Emumba
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
+
+# Pcap
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+
+# Set up default executables (update-alternatives)
+RUN \
+   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+
+# Add compiler symlinks for ccache 
+RUN \
+   ln -s /usr/bin/ccache /usr/local/bin/clang; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+   ln -s /usr/bin/ccache /usr/local/bin/g++
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+   groupadd -g "$docker_uid" "$docker_user" \
+   && useradd \
+   -m \
+   -u "$docker_uid" \
+   -g "$docker_uid" \
+   -G sudo \
+   --password "$(openssl passwd -1 "$docker_pass")" \
+   "$docker_user"

.devcontainer/devcontainer.json

@@ -0,0 +1,48 @@
+{
+  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+  ],
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
+      "extensions": [
+        "eamodio.gitlens",
+        "llvm-vs-code-extensions.vscode-clangd",
+        "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools"
+      ]
+    }
+  }
+}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.gitignore

@@ -47,13 +47,13 @@ trace_data/
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
 /src/server/sparkd/cmds/spartan/analysis/data
-*.tmp/*
-/.venv*/*
+.tmp/
+.venv*/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-**/devcontainer.json
+.devcontainer/
 *.Generated.cs
 *.lst

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "externals"]
 	path = externals
 	url = git@bitbucket.org:bluetradesys/externals.git
+[submodule "vcpkg"]
+	path = vcpkg
+	url = https://github.com/microsoft/vcpkg.git

CMakePresets.json

@@ -26,6 +26,7 @@
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
@@ -112,8 +113,8 @@
       "description": "Debug build compiled w/ gcc",
       "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
-        "CMAKE_C_COMPILER": "gcc",
-        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_C_COMPILER": "gcc-12",
+        "CMAKE_CXX_COMPILER": "g++-12",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
       }
@@ -142,7 +143,7 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
       }
     },

docker/sparkd/dev/Dockerfile

@@ -48,9 +48,15 @@ RUN apt-get update && apt-get install -y \
 # cmake
 RUN \
     curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-    | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
-    && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
     && apt-get update \
+    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
+    && apt-get install -y kitware-archive-keyring \
+    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
+    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
 # node

docker/sparkd/dev/devcontainer.json

@@ -1,6 +1,7 @@
 {
   "image": "bluetradingsystems/sparkd-dev:latest",
-  "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
     "memory": "4gb",
     "storage": "6gb"
@@ -27,6 +28,6 @@
     "8003:8003"
   ],
   "mounts": [
-    "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
   ]
 }

docker/sparkd/dev/devcontainer_initialize.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnExpression">
+    <Scanner Include="ColumnExpressionTokens.lex" />
+    <Parser Include="ColumnExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -0,0 +1,62 @@
+namespace Bts.ColumnExpressions;
+
+public class ColumnExpression {
+   public ColumnExpression(string name, Type? type = null, object? value = null) {
+      Name = name;
+      Type = type
+         ?? value?.GetType()
+         ?? throw new ArgumentException(
+            "Type must be specified explicitly when value is null", nameof(type));
+      Value = value
+         ?? DBNull.Value;
+   }
+
+   public string Name { get; }
+
+   public Type Type { get; }
+
+   public object Value { get; }
+
+   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
+      // Try to convert value to 'type' to verify they are compatible
+      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
+      // account for implicit conversions (e.g., int -> long), just inheritance.
+      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
+         throw new ArgumentException(
+            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
+      }
+      return new(name, type ?? typeof(T), value);
+   }
+
+   internal static Type ParseType(string typeId) {
+      return Candidates().ExceptNull().FirstOrDefault() ??
+         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
+
+      IEnumerable<Type?> Candidates() {
+         if (typeId.Length == 0) {
+            yield break;
+         }
+         yield return Type.GetType(typeId);
+         yield return Type.GetType($"System.{typeId}");
+         if (typeId.Length > 1) {
+            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
+         }
+         yield return typeId.ToLower() switch {
+            "float64" or "double" or "real64" or "number" => typeof(double),
+            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
+            "int16" or "short" or "integer16" or "signed16" => typeof(short),
+            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
+            "int64" or "long" or "integer64" or "signed64" => typeof(long),
+            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
+            "uint32" or "uint" or "unsigned" => typeof(uint),
+            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
+            "char" => typeof(char),
+            "byte" or "uint8" => typeof(byte),
+            "bool" or "boolean" => typeof(bool),
+            "string" or "text" => typeof(string),
+            "date" or "datetime" => typeof(DateTime),
+            _ => null
+         };
+      }
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -0,0 +1,49 @@
+%namespace Bts.ColumnExpressions
+%partial
+%visibility internal
+%tokentype ColumnExpressionToken
+%parsertype ColumnExpressionParser
+
+%union {
+   public int Integer;
+   public double Real;
+   public bool Boolean;
+   public string String;
+   public string Identifier;
+   public string TypeId;
+   public DateTime DateTime;
+   public Tuple<string, Type> ColumnDefinition;
+   public ColumnExpression ColumnExpression;
+   public ImmutableList<ColumnExpression> ColumnExpressions;
+}
+
+%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
+%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
+
+%nonassoc EQUALS
+
+%type <ColumnDefinition> ColumnDefinition
+%type <ColumnExpression> ColumnExpression
+%type <ColumnExpressions> ColumnExpressions
+
+%%
+
+ColumnExpressions :
+   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
+   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
+   ;
+
+ColumnExpression :
+   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
+   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
+   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
+   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
+   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
+   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
+   ;
+
+ColumnDefinition :
+   IDENTIFIER { $$ = new($1.Identifier, null); }
+   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
+   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
+   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnExpressions;
+
+using System.IO;
+using System.Text;
+
+public class ColumnExpressionInterpreter {
+   public static IList<ColumnExpression> Evaluate(Stream stream) {
+      var parser = new ColumnExpressionParser();
+      var res = parser.Parse(stream);
+      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
+         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
+   }
+
+   public static IList<ColumnExpression> Evaluate(string str) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
+      return Evaluate(ms);
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnExpressions;
+
+using System.Text;
+
+partial class ColumnExpressionParser {
+   public ColumnExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -0,0 +1,9 @@
+namespace Bts.ColumnExpressions;
+
+partial class ColumnExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -0,0 +1,85 @@
+%namespace Bts.ColumnExpressions
+%scannertype ColumnExpressionScanner
+%visibility internal
+%tokentype ColumnExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+BooleanConstant          true|false
+StringConstant           '([^\'\\]*|\\.)*'
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+IntegerConstant          -?[0-9]+
+NullConstant             'null'
+RealConstant             -?[0-9]+\.[0-9]+
+
+/* Relational Operators */
+Equals                   =
+OfType                   /
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+NamespacePrefix          {Identifier}\.
+TypeId                   {NamespacePrefix}*{Identifier}
+
+/* Utility Tokens */
+OpenParenthesis          \(
+CloseParenthesis         \)
+Comma                    ,
+QuoteOpen                \[
+QuoteClose               \]
+
+/* Comments */
+LineComment \/\/[^\n\r]*(\n|\r|\r\n)
+
+%%
+
+{Equals} {
+   return (int)ColumnExpressionToken.EQUALS;
+}
+
+{OfType} {
+   return (int)ColumnExpressionToken.OF_TYPE;
+}
+
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
+   return (int)ColumnExpressionToken.STRING_CONSTANT;
+}
+
+{BooleanConstant} {
+   yylval.Boolean = yytext.ToLower() switch {
+      "true" => true,
+      "false" => false,
+      _ => throw new Exception(
+         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
+   };
+   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
+}
+
+{DateTimeConstant} {
+   yylval.DateTime = DateTime.Parse(yytext);
+   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
+}
+
+{IntegerConstant} {
+   yylval.Integer = int.Parse(yytext);
+   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
+}
+
+{RealConstant} {
+   yylval.Real = double.Parse(yytext);
+   return (int)ColumnExpressionToken.REAL_CONSTANT;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnExpressionToken.IDENTIFIER;
+}
+
+{TypeId} {
+   yylval.TypeId = yytext;   
+   return (int)ColumnExpressionToken.TYPE_ID;
+}
+
+%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnSortExpression">
+    <Scanner Include="ColumnSortExpressionTokens.lex" />
+    <Parser Include="ColumnSortExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -0,0 +1,11 @@
+namespace Bts.ColumnSortExpressions;
+
+public class ColumnSortExpression {
+   public ColumnSortExpression(string name, bool isAscending = true) {
+      Name = name;
+      IsAscending = isAscending;
+   }
+
+   public string Name { get; }
+   public bool IsAscending { get; }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -0,0 +1,29 @@
+%namespace Bts.ColumnSortExpressions
+%partial
+%visibility internal
+%tokentype ColumnSortExpressionToken
+%parsertype ColumnSortExpressionParser
+
+%union {
+   public string Identifier;
+   public ColumnSortExpression Column;
+   public ImmutableList<ColumnSortExpression> Columns;
+}
+
+%token ASCENDING, DESCENDING, IDENTIFIER
+
+%type <Column> Column
+%type <Columns> Columns
+
+%%
+
+Columns :
+   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
+   | Columns Column { $$ = $1.Add($2); }
+   ;
+
+Column :
+   IDENTIFIER { $$ = new($1.Identifier, true); }
+   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
+   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
+   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnSortExpressions;
+
+using System.IO;
+using System.Text;
+
+public class ColumnSortExpressionInterpreter {
+   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
+      var parser = new ColumnSortExpressionParser();
+      var res = parser.Parse(stream);
+      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
+         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
+   }
+
+   public static IList<ColumnSortExpression> Evaluate(string str) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
+      return Evaluate(ms);
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnSortExpressions;
+
+using System.Text;
+
+partial class ColumnSortExpressionParser {
+   public ColumnSortExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnSortExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -0,0 +1,9 @@
+namespace Bts.ColumnSortExpressions;
+
+partial class ColumnSortExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -0,0 +1,30 @@
+%namespace Bts.ColumnSortExpressions
+%scannertype ColumnSortExpressionScanner
+%visibility internal
+%tokentype ColumnSortExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+Ascending                +
+Descending               -
+
+%%
+
+{Ascending} {
+   return (int)ColumnSortExpressionToken.ASCENDING;
+}
+
+{Descending} {
+   return (int)ColumnSortExpressionToken.DESCENDING;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnSortExpressionToken.IDENTIFIER;
+}
+
+%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -9,9 +9,17 @@
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
+    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+  </ItemGroup>
+
 </Project>

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -0,0 +1,62 @@
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+//     public class PandasStream : ViewStream, IWorker {
+//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+//         }
+
+//         public PandasStream(ViewStream vs, string pandas_code) :
+//         this(vs, ViewStreamDatabase.instance, pandas_code) {
+//         }
+
+
+//         protected override void myOnFirstSubscriber() {
+
+//             // string python_code = @"
+//             //    import winsound
+//             //    frequency = 2500  # Set frequency (in Hz)
+//             //    duration = 1000   # Set duration (in milliseconds)
+//             //    winsound.Beep(frequency, duration)
+//             // ";
+//             string python_code = @"
+//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
+//             with open(file_path, \"w\") as f:
+//                 f.write(\"Hello, world!\n\")
+//             ";
+
+//             execute_python_code(python_code);
+//         }
+
+//         static void execute_python_code(string python_code) {
+//             ProcessStartInfo start = new ProcessStartInfo();
+//             start.FileName = "python"; // Python executable
+//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+//             start.UseShellExecute = false;
+//             start.RedirectStandardOutput = true;
+
+//             using (Process process = Process.Start(start)) {
+//                 using (System.IO.StreamReader reader = process.StandardOutput) {
+//                     string result = reader.ReadToEnd();
+//                     Console.Write(result);
+//                 }
+//             }
+//         }
+
+//         protected override void myOnLastSubscriber() {
+//         }
+
+//         public void doWork() {
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -0,0 +1,79 @@
+using BCG.Utilities.General;
+
+using System;
+
+namespace BCG {
+    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
+        class State {
+            public int n_sends = 0;
+            public int n_recvs = 0;
+        }
+        private readonly string _ip;
+        private readonly int _port;
+        private readonly ViewStreamDatabase _vsd;
+        State _state = new State();
+        private readonly int _msg_size;
+
+        private IBlueTableSchema _schema;
+        private TimeSpan _send_period;
+        private ITCPConnection _connection = null;
+        private byte[] _buff;
+
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
+        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
+            _ip = ip;
+            _port = port;
+            _msg_size = msg_size;
+            _vsd = vsd;
+            _schema = BlueDataUtility.form_schema("TCPClient",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("n_sends", TypeCode.UInt32),
+            Tuple.Create("n_recvs", TypeCode.UInt32));
+            _send_period = TimeSpan.FromSeconds(send_period_seconds);
+            _buff = new byte[msg_size];
+            for (int i = 0; i < _buff.Length; i++) {
+                _buff[i] = (byte)i;
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, _send_period);
+            var server_address = new BlutoSubAddress(_ip, _port);
+            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            if (_connection != null) {
+                _connection.send(_buff);
+                _state.n_sends++;
+            }
+            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
+            BlueTable rows = new BlueTable(r.schema);
+            rows.addRow(r);
+            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        public void onConnection(ITCPConnection connection) {
+            _connection = connection;
+            _connection.register(this);
+        }
+        public bool isDisposed { get; }
+
+        public void onData(RemoteMessage msg) {
+            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
+            _state.n_recvs++;
+        }
+
+        public void Dispose() {
+
+        }
+
+
+    }
+}

src/DataStreamSuite/DataStream/Misc/test.py

@@ -0,0 +1,3 @@
+file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
+with open(file_path, "w") as f:
+    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,13 +1,6 @@
-// using System;
-// using System.Data;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Utilities.General;
-// namespace BCG
-// {
-
-//    public class AddColumnsStream : MapStream
-//    {
-//       public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef))  {}    
-//    }
-// }
+namespace BCG;
+
+public class AddColumnsStream : MapStream {
+   public AddColumnsStream(ViewStream vs, string columnDef)
+      : base(vs, new ColumnAdder(columnDef)) { }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -0,0 +1,12 @@
+namespace bts {
+    using BCG;
+
+    using System;
+    using System.Collections;
+
+    public class AddConstantColumnRow {
+        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,73 +1,52 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Evaluator.AddColumnsExpression;
-
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Provides methods to add columns to the schema
-//    /// </summary>
-//    public class ColumnAdder : IStreamMapper
-//    {
-//       protected IDictionary myNewColumns;
-//       protected IBlueTableSchema mySchema;
-
-//       static syntax myParser =
-//          new syntax();
-
-//       /// <summary>
-//       /// string represting the columns to be added to the output table. 
-//       /// </summary>
-//       /// <param name="colAddExpression"></param>
-//       public ColumnAdder (string columnDef)
-//       {
-//          AddColumnsExpression ace;
-
-//          if (DataStreamLoadFileConfig.instance.useStaticParser)
-//          {
-//             lock (myParser)
-//             {
-//                ace =
-//                (AddColumnsExpression)myParser.Parse(columnDef);
-//             }
-//          }
-//          else
-//          {
-//             syntax parser = new syntax();
-//             ace =
-//             (AddColumnsExpression)parser.Parse(columnDef);
-//          }
-
-// 	 if (ace == null)
-// 	   throw new Exception ("Invalid columnn expression: " + columnDef);
-//          myNewColumns = ace.columns;
-//       }
-         
-//       public virtual IBlueRow map (IBlueRow row)
-//       {
-//          if (mySchema == null)
-//             myConstructSchema (row.schema);
-//          return new AddColumnsRow (mySchema, row);
-//       }
-
-//       public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
-//       {
-//          myConstructSchema (inputSchema);
-//          return mySchema;
-//       }
-
-//       private void myConstructSchema (IBlueTableSchema schema)
-//       {
-//          ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
-//          int ciIndex = 0;
-//          foreach (ColumnInfo ci in schema.columnInfo)
-//             cis[ciIndex++] = ci;
-//          foreach (DictionaryEntry de in myNewColumns)
-//             cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++, 
-//                ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
-//          mySchema = new BlueTableSchema (cis, schema.tableName);
-//       }
-
-//    }
-// }
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+using Bts.ColumnExpressions;
+
+namespace BCG;
+
+/// <summary>
+/// Provides methods to add columns to the schema
+/// </summary>
+/// <param name="columnsExpression">
+/// String represting the columns to add to the output table
+/// </param>
+public class ColumnAdder : IStreamMapper {
+   // Store tuples of (column, index) to preserve order
+   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
+   protected IBlueTableSchema? mySchema;
+
+   public ColumnAdder(string columnsExpression) =>
+      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
+         throw new Exception("Invalid columnn expression: " + columnsExpression))
+         .Select((c, i) => (c, i))
+         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
+
+   [MemberNotNull(nameof(mySchema))]
+   public virtual IBlueRow map(IBlueRow row) {
+      // Only rebuild schema if necessary (i.e., if it's null)
+      mySchema ??= buildSchema(row.schema, myNewColumns);
+      return new AddColumnsRow(mySchema, row);
+   }
+
+   [MemberNotNull(nameof(mySchema))]
+   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
+      // Always rebuild schema (even if it's already non-null)
+      mySchema = buildSchema(inputSchema, myNewColumns);
+
+   [MemberNotNull(nameof(mySchema))]
+   static BlueTableSchema buildSchema(
+      IBlueTableSchema schema,
+      IDictionary<string, (ColumnExpression column, int index)> newColumns
+   ) {
+      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
+      schema.columnInfo.CopyTo(cis, 0);
+      foreach (var (col, index) in newColumns.Values) {
+         var schemaIdx = schema.columnInfo.Length + index;
+         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
+      }
+      return new BlueTableSchema(cis, schema.tableName);
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -15,6 +15,11 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
+   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
+   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
+   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
+   // file is created. The file names are token_epoch_datetime
+   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -1,190 +1,190 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using System.Runtime.Serialization.Formatters.Binary;
-using BCG.Utilities.General;
-using bts.collections;
-using System.Reflection.Metadata;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
 
 
-namespace BCG {
+// namespace BCG {
 
-   public class PlaybackStream : ViewStream, IWorker {
-      private readonly PlaybackType _type;
-      private BlueTable _current_table;
-      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-      // these are the packets already downloaded from the current file
-      private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-      //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-      private IEnumerable<string> _file_names;
-      private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-      private DateTime _playback_t0, _playback_t1;
-      private bool _all_files_read = false;
-      SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
+//    public class PlaybackStream : ViewStream, IWorker {
+//       private readonly PlaybackType _type;
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       // these are the packets already downloaded from the current file
+//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+//       private IEnumerable<string> _file_names;
+//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+//       private DateTime _playback_t0, _playback_t1;
+//       private bool _all_files_read = false;
+//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
 
-      public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-         _type = PlaybackType.Streaming;
-         _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-         _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-         string token_directory_name = $"{directory_name}\\{token}";
-         bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-         List<string> files_in_range = new List<string>();
-         if (Directory.Exists(token_directory_name)) {
-            _file_names =
-         Directory.GetFiles(token_directory_name).Where(f => {
-            DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-            return file_time > _playback_t0 && file_time < _playback_t1;
-         });
-         }
-      }
+//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+//          _type = PlaybackType.Streaming;
+//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+//          List<string> files_in_range = new List<string>();
+//          if (Directory.Exists(token_directory_name)) {
+//             _file_names =
+//          Directory.GetFiles(token_directory_name).Where(f => {
+//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+//             return file_time > _playback_t0 && file_time < _playback_t1;
+//          });
+//          }
+//       }
 
-      protected override void myOnFirstSubscriber() {
-         FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-         _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-         if (snapshot_packet != null) {
-            foreach (PlaybackPacket p in _snapshot_data.packets) {
-               byte[] ops;
-               Array row_array;
-               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-               for (int i = 0; i < ops.Length; i++) {
-                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-                  byte op = ops[i];
-                  if (op == StreamOpCode.create) {
-                     IBlueRowCollection full_table = rows_args[0];
-                     _current_table = new BlueTable(full_table);
-                  } else if (op == StreamOpCode.multipleRemoveAdd) {
-                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
-                  } else {
-                     bts.Log.assert(false, $"Oprator: {op} is not expected");
-                  }
-                  if (p.idx == snapshot_packet.idx) {
-                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-                     break;
-                  }
-               }
-            }
-         }
-      }
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _snapshot_data.packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
 
 
-      private void _load_packets_from_current_file() {
-         int curr_idx = _current_fs.Item1;
-         FileStream curr_fs = _current_fs.Item2;
-         _packets = new List<PlaybackPacket>();
-         _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-      }
+//       private void _load_packets_from_current_file() {
+//          int curr_idx = _current_fs.Item1;
+//          FileStream curr_fs = _current_fs.Item2;
+//          _packets = new List<PlaybackPacket>();
+//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+//       }
 
-      private bool current_file_all_replayed {
-         get {
-            bool rv = true;
-            _packets.ForEach(p => rv = rv && p.replayed);
-            return rv;
-         }
-      }
+//       private bool current_file_all_replayed {
+//          get {
+//             bool rv = true;
+//             _packets.ForEach(p => rv = rv && p.replayed);
+//             return rv;
+//          }
+//       }
 
-      protected override void myOnLastSubscriber() {
-         var fs = _current_fs.Item2;
-         if (fs != null) {
-            fs.Close();
-         }
-         ViewStreamDatabase.instance.removeWorker(this);
-      }
+//       protected override void myOnLastSubscriber() {
+//          var fs = _current_fs.Item2;
+//          if (fs != null) {
+//             fs.Close();
+//          }
+//          ViewStreamDatabase.instance.removeWorker(this);
+//       }
 
-      // public event PlaybackStreamProgress onProgress;
-      // public event Action onFileExhausted;
-      private PlaybackPacket _locate_packet_to_replay() {
-         foreach (var p in _packets) {
-            // returns the first non replayed packet
-            if (!p.replayed)
-               return p;
-         }
-         return null;
-      }
+//       // public event PlaybackStreamProgress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          //else
+//          {
+//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
+//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
+//             if (onProgress != null)
+//                onProgress(myFileReader.Position, myFileReader.Length);
+//          }
+//       }
 
-      private PlaybackPacket _get_previous_packet(PlaybackPacket p) {
-         if (p.idx == 0)
-            return null;
-         return _packets[p.idx - 1];
-      }
+//       #region IWorker Members
 
-      #region IWorker Members
+//       public void doWork() {
+//          if (_all_files_read)
+//             return;
+//          if (current_file_all_replayed) {
+//             _load_next_file();
+//          }
+//          PlaybackPacket curr_packet = _locate_packet_to_replay();
+//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+//             byte[] ops;
+//             Array row_array;
+//             // what a stupid interface!
+//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//             _fire_packet(ops, row_array);
+//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//          } else {
+//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+//             // this is the time we need to fire (replay) this packet
+//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+//             long now_epoch = PersistorStream.epoch_now();
+//             // fire when we are past the 
+//             if (now_epoch >= time_to_fire) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//                _fire_packet(ops, row_array);
+//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//             }
+//          }
+//       }
 
-      public void doWork() {
-         if (_all_files_read)
-            return;
-         if (current_file_all_replayed) {
-            _load_next_file();
-         }
-         PlaybackPacket curr_packet = _locate_packet_to_replay();
-         if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-            byte[] ops;
-            Array row_array;
-            // what a stupid interface!
-            _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-            _fire_packet(ops, row_array);
-            curr_packet.replay_timestamp = PersistorStream.epoch_now();
-         } else {
-            PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-            // this is the time we need to fire (replay) this packet
-            long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-            long now_epoch = PersistorStream.epoch_now();
-            // fire when we are past the 
-            if (now_epoch >= time_to_fire) {
-               byte[] ops;
-               Array row_array;
-               _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-               _fire_packet(ops, row_array);
-               curr_packet.replay_timestamp = PersistorStream.epoch_now();
-            }
-         }
-      }
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
 
-      private void _fire_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-               //_current_table.addRows(full_table);
-               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-                                          (myLastValue as DataPacket).newRows));
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-               myFireEvent(new DataPacket(_current_table, adds, dels));
-            } else {
-               bts.Log.assert(false, $"Oprator: {op} is not expected");
-            }
+//          }
+//       }
 
-         }
-      }
+//       private void _load_next_file() {
+//          var new_idx = _current_fs.Item1 + 1;
+//          if (new_idx >= _file_names.Count()) {
+//             _all_files_read = true;
+//             bts.Log.Gen.info("All files read");
+//             return;
+//          }
+//          var prev_fs = _current_fs.Item2;
+//          if (prev_fs != null)
+//             prev_fs.Close();
+//          var new_filename = _file_names.ElementAt(new_idx);
+//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+//          _load_packets_from_current_file();
+//       }
 
-      private void _load_next_file() {
-         var new_idx = _current_fs.Item1 + 1;
-         if (new_idx >= _file_names.Count()) {
-            _all_files_read = true;
-            bts.Log.Gen.info("All files read");
-            return;
-         }
-         var prev_fs = _current_fs.Item2;
-         if (prev_fs != null)
-            prev_fs.Close();
-         var new_filename = _file_names.ElementAt(new_idx);
-         bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-         var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-         _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-         _load_packets_from_current_file();
-      }
-
-      #endregion
-   }
-}
+//       #endregion
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -11,13 +11,27 @@ using System.Reflection.Metadata;
 
 namespace BCG {
    public class SnapshotStream : ViewStream {
-
+      public enum SpecificSnapshot {
+         FirstOfDay,
+         LastOfDay
+      };
       private BlueTable _current_table;
       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
       public DateTime _snapshot_time;
       public string _snapshot_file;
       public IEnumerable<PlaybackPacket> _packets;
-
+      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+         DateTime dt;
+         var parsed = DateTime.TryParse(snapshot_date, out dt);
+         dt = dt.Date;
+         bts.Log.assert(parsed);
+         SpecificSnapshot specific_snapshot_enum;
+         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+         bts.Log.assert(parsed);
+         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+         mySetRealSource(rs);
+      }
 
       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -37,6 +37,20 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
+         DateTime max_dt = DateTime.MinValue;
+         string rv = String.Empty;
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            var dt = parse_datetime_from_filename(fn);
+            if (dt > max_dt) {
+               max_dt = dt;
+               rv = fn;
+            }
+         }
+         return rv;
+      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,47 +1,42 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
+using System;
+using System.Collections;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Computes the max for a given column
-// 	/// </summary>
-// 	public class MaxGroupByEvaluator : GroupByEvaluator
-// 	{
+using BCG.Utilities;
 
-//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the max for a given column
+    /// </summary>
+    public class MaxGroupByEvaluator : GroupByEvaluator {
 
-//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
-//             myOutputTable;
-//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
-//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
-//          if (maxRow != null)
-//          {
-//             BlueEventLog.logAssert (currMaxRow != null);
-//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
-//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
-//             {
-//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
-//                if (maxRow == null)
-//                   myOutputTable.removeRow (hashString);
-//                else
-//                   myOutputTable.addRow (maxRow, hashString);
-//             }
-//          }
-//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
-//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-//          // otherwise if maxRow > currMaxRow then its replaced
-//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
-//             myOutputTable.addRow (maxRow, hashString);
-//          return myProject (myOutputTable.getRow (hashString));
-//       }
-// 	}
-// }
+        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
+
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
+            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
+            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
+            if (maxRow != null) {
+                bts.Log.assert(currMaxRow != null);
+                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
+                    if (maxRow == null)
+                        myOutputTable.removeRow(hashString);
+                    else
+                        myOutputTable.addRow(maxRow, hashString);
+                }
+            }
+            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
+            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+            // otherwise if maxRow > currMaxRow then its replaced
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+                myOutputTable.addRow(maxRow, hashString);
+            return myProject(myOutputTable.getRow(hashString));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,20 +1,15 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-// 	public class MaxStream : ViewStream
-// 	{
-//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
-//       {
-//       }
-// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
-// 		{
-//          mySetRealSource (new GroupByStream (stream, 
-//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
-// 		}
+namespace BCG {
+    public class MaxStream : ViewStream {
+        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
+        }
+        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
+            mySetRealSource(new GroupByStream(stream,
+                                                new MaxGroupByEvaluator(outputCol, inputCols)));
+        }
 
-//       public MaxStream (ViewStream stream, string outputCol)
-//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-// 	}
-// }
+        public MaxStream(ViewStream stream, string outputCol)
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,40 +1,36 @@
-// using System;
-// using BCG.Utilities;
+using System;
 
-// namespace BCG
-// {
-//    public class MinGroupByEvaluator : MaxGroupByEvaluator
-//    {
-//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
-//          : base (outputCol, groupByCols)  {}
+using BCG.Utilities;
 
-//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
-//          base (colMapEpression, outputCol, groupByCols)  {}
+namespace BCG {
+    public class MinGroupByEvaluator : MaxGroupByEvaluator {
+        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
-//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
-//          if (minRow != null)
-//          {
-//             // BlueEventLog.logAssert (currMinRow != null);
-//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
-//             {
-//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
-//                if (minRow == null)
-//                   myOutputTable.removeRow (stringRep);
-//                else
-//                   myOutputTable.addRow (minRow, stringRep);
-//             }
-//          }
-//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
-//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
-//             myOutputTable.addRow (minRow, stringRep);
-//          return myOutputTable.getRow (stringRep);
-//       }
+        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
+           base(colMapEpression, outputCol, groupByCols) { }
 
-//    }
-// }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
+            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
+            if (minRow != null) {
+                // BlueEventLog.logAssert (currMinRow != null);
+                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
+                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
+                    if (minRow == null)
+                        myOutputTable.removeRow(stringRep);
+                    else
+                        myOutputTable.addRow(minRow, stringRep);
+                }
+            }
+            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
+                myOutputTable.addRow(minRow, stringRep);
+            return myOutputTable.getRow(stringRep);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,17 +1,13 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class MinStream : ViewStream
-//    {
-//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
-//       {
-//          mySetRealSource (new GroupByStream (stream, 
-//                            new MinGroupByEvaluator (outputCol, inputCol)));
-//       }
+namespace BCG {
+    public class MinStream : ViewStream {
+        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
+            mySetRealSource(new GroupByStream(stream,
+                              new MinGroupByEvaluator(outputCol, inputCol)));
+        }
 
-//       public MinStream (ViewStream stream, string outputCol)
-//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-//    }
-// }
+        public MinStream(ViewStream stream, string outputCol)
+                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,94 +1,82 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Computes the sum for a given column
-//    /// </summary>
-//    public class SumGroupByEvaluator : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the sum for a given column
+    /// </summary>
+    public class SumGroupByEvaluator : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-         
-//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+
+            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
 
-//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
-//       {
-//          myOutputCols = output_columns;
-//          myGroupByCols = groupby_columns;
-//       }
+        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
+            myOutputCols = output_columns;
+            myGroupByCols = groupby_columns;
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-//    public class SumGroupByEvaluator<T> : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-//          : base(outputCol, groupByCols)
-//       { }
+    public class SumGroupByEvaluator<T> : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-//          : base(colMapEpression, outputCol, groupByCols) { }
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate(DataPacket dp)
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
-//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-// }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,107 +1,89 @@
-// using System;
-// using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
-// namespace BCG
-// {
-//    public class NullMapper : IStreamMapper
-//    {
-//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+namespace BCG {
+    public class NullMapper : IStreamMapper {
+        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-//       public NullMapper(string[] valid_cols)
-//       {
-//          foreach (string c in valid_cols)
-//             _valid_cols[c] = true;
-//          _valid_cols["id"] = true;
-//       }
+        public NullMapper(string[] valid_cols) {
+            foreach (string c in valid_cols)
+                _valid_cols[c] = true;
+            _valid_cols["id"] = true;
+        }
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-//       public IBlueRow map(IBlueRow row)
-//       {
-//          MutableBlueRow out_row = new MutableBlueRow(row);
-//          foreach (string c in row.schema.columnNames)
-//             if (!_valid_cols.ContainsKey(c))
-//                out_row.setValue(c, DBNull.Value);
-//          return out_row;
-//       }
-//    }
+        public IBlueRow map(IBlueRow row) {
+            MutableBlueRow out_row = new MutableBlueRow(row);
+            foreach (string c in row.schema.columnNames)
+                if (!_valid_cols.ContainsKey(c))
+                    out_row.setValue(c, DBNull.Value);
+            return out_row;
+        }
+    }
 
-//    public class SumStream : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//       {
-//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//       }
+    public class SumStream : ViewStream {
+        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns)
-//       {
-//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-//          rs = new ProjectStream(rs, "!fake_col");
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns) {
+            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+            rs = new ProjectStream(rs, "!fake_col");
+            mySetRealSource(rs);
+        }
 
-//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
-//       {
-//          string[] output_columns = space_seperated_output_columns.Split(' ');
-//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
-//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
-//       }
+        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
+            string[] output_columns = space_seperated_output_columns.Split(' ');
+            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
+            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
-//       {
-//          ViewStream rs = null;
-//          groupby_columns = groupby_columns ?? new string[] {};
-//          if (groupby_columns.Length == 0)
-//          {
-//             rs = new AddColumnsStream(stream, "fake_col=0");
-//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
-//             rs = new ProjectStream(rs, "!fake_col");
-//          }
-//          else
-//          {
-//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-//          }
-//          if (force_null_into_other_columns)
-//          {
-//             List<string> valid_cols = new List<string>(output_columns);
-//             valid_cols.AddRange(groupby_columns);
-//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-//          }
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
+            ViewStream rs = null;
+            groupby_columns = groupby_columns ?? new string[] { };
+            if (groupby_columns.Length == 0) {
+                // rs = new AddColumnsStream(stream, "fake_col=0");
+                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
+                // rs = new ProjectStream(rs, "!fake_col");
+            } else {
+                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+            }
+            if (force_null_into_other_columns) {
+                List<string> valid_cols = new List<string>(output_columns);
+                valid_cols.AddRange(groupby_columns);
+                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+            }
+            mySetRealSource(rs);
+        }
 
-//    public SumStream (ViewStream stream, string outputCol)
-//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
-//    {
-//    }
-// }
+        public SumStream(ViewStream stream, string outputCol)
+        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
+        }
+    }
 
-// public class IntSumStream : ViewStream
-// {
-//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//    {
-//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//       mySetRealSource(new GroupByStream(stream,
-//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
-//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//    }
+    public class IntSumStream : ViewStream {
+        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-// }
+    }
 
 
-//    public class SumStream<T> : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
-//       {
-//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
-//       }
-//    }
-// }
+    public class SumStream<T> : ViewStream {
+        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
+            string[] inputCols = spaceSeparatedInputCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -0,0 +1,79 @@
+using BCG;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+using ZstdSharp.Unsafe;
+
+namespace bts {
+    public class RowDiffStream : ViewStream {
+        private ViewStream _vs;
+        private readonly string _order_col, _col_1, _col_2;
+        private readonly int _diff;
+        private IBlueTableSchema _out_schema;
+        private readonly string _diff_col_name;
+        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
+            _vs = vs;
+            _order_col = order_col;
+            _col_1 = col_1;
+            _col_2 = col_2;
+            _diff = diff;
+            _diff_col_name = diff_col_name;
+        }
+
+        private string _diff_col_name_perc {
+            get {
+                return $"{_diff_col_name}_percentage";
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _handle_vs;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _handle_vs;
+        }
+
+        private void _handle_vs(object dp_obj) {
+            var dp = dp_obj as DataPacket;
+            IBlueRowCollection nrows = dp.newRows;
+            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
+            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
+            List<IBlueRow> row_list = new List<IBlueRow>();
+            foreach (IBlueRow r in nrows)
+                row_list.Add(r);
+            //OrderBy sorts in ascedning order
+            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
+            var sorted_row_array = sorted_rows.ToArray();
+            // reverse it to make it descending    
+            sorted_rows.Reverse();
+            //sort rows so we can do the diff
+            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
+            int i = 0;
+            foreach (IBlueRow r in sorted_rows) {
+                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
+                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
+                // we assume x - y, and name variables like that
+                double x = r.get<double>(_col_1);
+                int y_row_idx = i + _diff;
+                double diff = System.Double.NaN;
+                double perc = System.Double.NaN;
+                if (y_row_idx < sorted_rows.Count()) {
+                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
+                    diff = x - y;
+                    perc = 100.0 * diff / y;
+                }
+                nr.setValue(_diff_col_name, diff);
+                nr.setValue(_diff_col_name_perc, perc);
+                rv.addRow(nr);
+                i++;
+            }
+
+            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -0,0 +1,31 @@
+namespace BCG {
+    using System.Collections.Generic;
+
+    public class KeyValueToRelationalStream : ViewStream {
+        private readonly ViewStream _kv_str;
+        private readonly string _doc_id_col, _key_col, _val_col;
+        public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
+            _kv_str = kv_str;
+            _doc_id_col = doc_id_col;
+            _key_col = key_col;
+            _val_col = val_col;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _kv_str.onChange += _on_str_update;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _kv_str.onChange -= _on_str_update;
+        }
+
+        private void _on_str_update(object dp_obj) {
+            DataPacket dp = dp_obj as DataPacket;
+            var rows = dp.newRows;
+            var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
+            var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -3,45 +3,37 @@ using Google.Protobuf.WellKnownTypes;
 using System;
 using BCG.Utilities.General;
 
-namespace BCG
-{
+namespace BCG {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 
-   public class TimeSeriesStream : ViewStream
-   {
+   public class TimeSeriesStream : ViewStream {
       private BlueTable _table;
       private ViewStream _stream;
       private int _id = 0;
 
-      public TimeSeriesStream(ViewStream stream)
-      {
+      public TimeSeriesStream(ViewStream stream) {
          _stream = stream;
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          _stream.onChange += new ChangeDelegate(_onstream_change);
       }
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
 
-      private void _onstream_change(object o)
-      {
+      private void _onstream_change(object o) {
          DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
-         if (_table == null)
-         {
+         if (_table == null) {
             IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
             schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
             _table = new BlueTable(schema);
          }
          BlueTable added_rows = new BlueTable(_table.schema);
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
             ar.setValue("timestamp", now);
             ar.setValue("id", _id++);
@@ -53,114 +45,6 @@ namespace BCG
       }
    }
 
-   public class TimeBucketStream : ViewStream, IStreamMapper
-   {
-      private ViewStream _src;
-      private readonly string _time_stamp_col;
-      private IBlueTableSchema _schema;
-      private int _bucket_size_seconds;
-
-      public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
-      {
-         _time_stamp_col = time_stamp_col;
-         _bucket_size_seconds = bucket_size_seconds;
-         var vs = new MapStream(input, this);
-         mySetRealSource(vs);
-      }
-
-      #region IStreamMapper Members
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
-      {
-         IBlueTableSchema rv = input_schema;
-         //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
-         rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
-         return rv;
-
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row)
-      {
-         DateTime ts = row.get<DateTime>(_time_stamp_col);
-         TimeSpan time_of_day = (ts - ts.Date);
-         int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
-         double buckets = seconds / _bucket_size_seconds;
-         int bucket_number = Convert.ToInt32(Math.Floor(buckets));
-         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue("bucket_number", bucket_number);
-         return rv;
-      }
-
-      #endregion
-   }
-
-   // public class ThrottleStream : ViewStream, IWorker
-   // {
-   //    private ViewStreamDatabase _vsd;
-   //    private ViewStream _stream;
-   //    private int _throttle_ms;
-   //    private TimeSpan _min_fire_period;
-   //    private DateTime _last_Fire_time = DateTime.MinValue;
-   //    private DataPacket _unfired_dp;
-   //    public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
-   //    {
-   //       _vsd = vsd;
-   //       _stream = base_stream;
-   //       _throttle_ms = throttle_ms;
-   //       _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
-   //    }
-
-   //    protected override void myOnFirstSubscriber()
-   //    {
-   //       _vsd.addWorker(this);
-   //       _stream.onChange += new ChangeDelegate(_onstream_change);
-   //    }
-   //    protected override void myOnLastSubscriber()
-   //    {
-   //       _vsd.removeWorker(this);
-   //       _stream.onChange -= new ChangeDelegate(_onstream_change);
-   //    }
-
-   //    private void _onstream_change(object obj)
-   //    {
-   //       DataPacket dp = obj as DataPacket;
-   //       if (DateTime.Now - _last_Fire_time >= _min_fire_period)
-   //       {
-   //          // ok to fire here
-   //       }
-   //       else
-   //       {
-   //          _last_cached_state = dp.newRows;
-   //       }
-   //       _possibly_fire(dp.newRows);
-   //    }
-
-   //    public void doWork()
-   //    {
-   //       if (_unfired_dp != null)
-   //       {
-
-   //       }
-   //    }
-
-   //    private void _possibly_fire()
-   //    {
-   //       if (_unfired_dp != null)
-   //       {
-   //          if (DateTime.Now - _last_fire_time > _min_fire_period)
-   //          {
-   //             DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
-   //             _unfired_dp = null;
-   //             _last_Fire_time = DateTime.Now;
-   //             myFireEvent(dp);
-   //          }
-   //       }
-   //    }
-
-   // }
 
 
 }

src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Security.Cryptography.X509Certificates;
+
+using BCG;
+using BCG.Utilities.General;
+
+using HtmlAgilityPack;
+
+namespace BCG {
+    class Program {
+        static void Main() {
+            string url = "http://quotes.toscrape.com";
+            List<string> quotes = ScrapeQuotes(url);
+
+            Console.WriteLine("Quotes from http://quotes.toscrape.com:");
+            foreach (var quote in quotes) {
+                Console.WriteLine(quote);
+            }
+        }
+
+        static List<string> ScrapeQuotes(string url) {
+            List<string> quotes = new List<string>();
+
+            // Create a new HtmlWeb instance
+            HtmlWeb web = new HtmlWeb();
+
+            try {
+                // Load the HTML document from the specified URL
+                HtmlDocument document = web.Load(url);
+
+                // Extract quotes from the HTML document
+                var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
+                if (quoteNodes != null) {
+                    foreach (var quoteNode in quoteNodes) {
+                        string quote = quoteNode.InnerText.Trim();
+                        quotes.Add(quote);
+                    }
+                }
+            } catch (Exception ex) {
+                Console.WriteLine($"Error: {ex.Message}");
+            }
+
+            return quotes;
+        }
+    }
+
+    public class WebScapperStream : ViewStream, IWorker {
+        private readonly string _url;
+        private readonly string _xpath;
+        private readonly ViewStreamDatabase _vsd;
+        private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
+        private readonly IBlueTableSchema _schema;
+
+        public WebScapperStream()
+        :
+        this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
+        public WebScapperStream(string url, string xpath)
+        : this(url, xpath, ViewStreamDatabase.instance) { }
+        public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
+            _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("url", TypeCode.UInt32),
+                                                Tuple.Create("line", TypeCode.String));
+            _url = url;
+            _xpath = xpath;
+            _vsd = vsd;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            var lines = scrape_lines(_url, _xpath);
+            Func<string, int> get_id = (line) => {
+                int rv = 0;
+                if (!_line_id_map.TryGetValue(line, out rv))
+                    _line_id_map.Add(line, _line_id_map.Count);
+                return rv;
+            };
+            BlueTable nrows = new BlueTable(_schema);
+            foreach (var l in lines) {
+                bts.Log.Gen.info($"line: {l}");
+                var id = get_id(l);
+                BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
+                nrows.addRow(nr);
+            }
+            var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        static List<string> scrape_lines(string url, string xpath) {
+            List<string> quotes = new List<string>();
+            // Create a new HtmlWeb instance
+            HtmlWeb web = new HtmlWeb();
+            try {
+                // Load the HTML document from the specified URL
+                HtmlDocument document = web.Load(url);
+                // Extract quotes from the HTML document
+                var quote_nodes = document.DocumentNode.SelectNodes(xpath);
+                if (quote_nodes != null) {
+                    foreach (var qn in quote_nodes) {
+                        string quote = qn.InnerText.Trim();
+                        quotes.Add(quote);
+                    }
+                }
+            } catch (Exception ex) {
+                Console.WriteLine($"Error: {ex.Message}");
+            }
+
+            return quotes;
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -0,0 +1,212 @@
+using System;
+using System.Collections.Generic;
+using MongoDB.Driver;
+using MongoDB.Bson;
+using BCG;
+using BCG.Utilities.General;
+using ZstdSharp.Unsafe;
+using System.Runtime.CompilerServices;
+using mbts.spark;
+
+
+namespace BCG {
+    [DataStreamUserRespresentation("mongodb_sql")]
+    public class MongoDBSQLStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
+
+        // Access the database
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
+
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+
+        }
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            _read_mogodb();
+        }
+
+        private void _read_mogodb() {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_row(document);
+            }
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+            timer.start();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
+
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+            foreach (var n in doc_row.Names) {
+                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+            }
+            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_row(BsonDocument doc) {
+            var key_val = doc["_id"];
+            var key = key_val.ToString();
+            int id;
+            if (!_id_map.TryGetValue(key, out id)) {
+                id = _id_map.Count;
+                _id_map.Add(key, id);
+            }
+            MutableBlueRow nr = new MutableBlueRow(_table.schema);
+            nr.setValue("id", id);
+            foreach (var n in doc.Names) {
+                var val = doc[n].ToString();
+                nr.setValue(n, val);
+            }
+            _table.addRow(nr);
+        }
+    }
+
+    [DataStreamUserRespresentation("mongodb")]
+    public class MongoDBStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
+
+        // Access the database
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
+
+        public MongoDBStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+
+        }
+        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            _read_mogodb();
+        }
+
+        private void _read_mogodb() {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_doc(document);
+            }
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+            timer.start();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
+
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("doc_id", TypeCode.String),
+            Tuple.Create("key", TypeCode.String),
+            Tuple.Create("value", TypeCode.String));
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_doc(BsonDocument doc) {
+            var doc_id_bson = doc["_id"];
+            var doc_id = doc_id_bson.ToString();
+            Func<string, string, int> get_id = (string doc_id, string key) => {
+                string hash_key = $"{doc_id}_{key}";
+                int id = 0;
+                if (!_id_map.TryGetValue(hash_key, out id)) {
+                    id = _id_map.Count;
+                    _id_map.Add(hash_key, id);
+                }
+                return id;
+            };
+            foreach (var n in doc.Names) {
+                string key = n;
+                var id = get_id(doc_id, key);
+                MutableBlueRow nr = new MutableBlueRow(_table.schema);
+                var val = doc[n].ToString();
+                nr.setValue("id", id);
+                nr.setValue("doc_id", doc_id);
+                nr.setValue("key", key);
+                nr.setValue("value", val);
+                _table.addRow(nr);
+            }
+        }
+    }
+
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -0,0 +1,34 @@
+using BCG;
+
+namespace bts {
+    public class SchemaGuesserStream : ViewStream {
+
+        private IBlueRowCollection _rows = null;
+        private ViewStream _vs;
+        public SchemaGuesserStream(ViewStream vs) {
+            _vs = vs;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _on_data;
+        }
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _on_data;
+        }
+
+        private void _on_data(object obj) {
+            DataPacket dp = obj as DataPacket;
+            if (_rows == null) {
+                _rows = BlueDataUtility.get_typed_table(dp.newRows);
+            }
+            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+
+
+        }
+
+
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -1,115 +1,107 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Evaluator.SortExpression;
-
-
-// namespace BCG
-// {
-//    public class SortRow : IComparable, IBlueRow
-//    {
-//       private IBlueRow myRow;
-//       private IEnumerable myColumns;
-//       private IComparer myComparer;
-
-//       public SortRow (IBlueRow row, SortExpression sortExpression)
-//       {
-//          myRow = row;
-//          myColumns = sortExpression.columns;
-//       }
-
-//       public SortRow (IBlueRow row, IEnumerable columns)
-//       {
-//          myRow = row;
-//          myColumns = columns;
-//       }
-
-//       public SortRow (IBlueRow row, IComparer comparer)
-//       {
-//          myRow = row;
-//          myComparer = comparer;
-//       }
-
-//       public IBlueRow internal_row { get { return myRow; } }
-
-//       #region IComparable method
-//       public int CompareTo (object obj)
-//       {  
-//          IBlueRow otherRow = (IBlueRow)obj;
-//          if (myComparer == null)
-//          {
-//             foreach (SortExpression.ColumnCriteria cc in myColumns)
-//             {
-//                int retVal = myIsNull (this[cc.columnName]) ?
-//                   myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
-//                   myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
-//                retVal *= cc.ascending ? 1 : -1;
-//                if (retVal != 0)
-//                   return retVal;
-//             }
-//             return 0;
-//          }
-//          else
-//             return myComparer.Compare (this, otherRow);
-//       }
-//       #endregion
-
-//       #region IBlueRow members
-//       public IBlueTableSchema schema{get{return myRow.schema;}}
-//       #endregion
-
-//       #region ICollection members
-//       public int Count {get{return myRow.Count;}}
-
-//       public bool IsSynchronized {get{return false;}}
-
-//       public object SyncRoot {get{return this;}}
-
-//       public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
-//       #endregion
-
-//       #region IDictionary members
-//       public bool IsFixedSize {get{return true;}}
-
-//       public bool IsReadOnly {get{return true;}}
-
-//       /// <summary>
-//       /// The key can either be the string corrsponding to the column name or
-//       /// the order of the column
-//       /// </summary>
-//       public object this [object key]
-//       {
-//          get
-//          {
-//             if (key is string)
-//                return this[(string)key];
-//             return this[(int)key];
-//          }
-//          set {throw new NotSupportedException ("The IBlueRow is readonly");}
-//       }
-
-//       public object this [int index]   {get{return myRow[index];}}
-
-//       public object this [string colName] {get{return myRow[colName];}}
-
-//       public ICollection Keys{get{return myRow.Keys;}}
-
-//       public ICollection Values{get{return myRow.Values;}}
-
-//       public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
-
-//       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
-
-//       public bool Contains (object key){return myRow.Contains (key);}
-
-//       IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
-
-//       IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
-
-//       public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
-//       #endregion
-
-//       private bool myIsNull (object o) {return o == null || o is DBNull;}
-
-//    }
-// }
+using Bts.ColumnSortExpressions;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace BCG;
+
+public class SortRow : IComparable, IBlueRow {
+   readonly IBlueRow row;
+   readonly IEnumerable<ColumnSortExpression>? columns;
+   readonly IComparer? comparer;
+
+   public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) {
+      this.row = row;
+      this.columns = columns;
+   }
+
+   public SortRow(IBlueRow row, IComparer comparer) {
+      this.row = row;
+      this.comparer = comparer;
+   }
+
+   public IBlueRow internal_row { get { return row; } }
+
+   #region IComparable method
+   public int CompareTo(object? obj) {
+      if (obj is not IBlueRow otherRow) {
+         throw new ArgumentException(
+            $"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})",
+            nameof(obj));
+      }
+      if (comparer is not null) {
+         return comparer.Compare(this, otherRow);
+      }
+      return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0);
+      int Compare(ColumnSortExpression col) {
+         var otherVal = otherRow[col.Name];
+         int rv = myIsNull(this[col.Name]) ?
+            (myIsNull(otherVal) ? 0 : -1) :
+            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
+         rv *= col.IsAscending ? 1 : -1;
+         return rv;
+      }
+   }
+   #endregion
+
+   #region IBlueRow members
+   public IBlueTableSchema schema { get { return row.schema; } }
+   #endregion
+
+   #region ICollection members
+   public int Count { get { return row.Count; } }
+
+   public bool IsSynchronized { get { return false; } }
+
+   public object SyncRoot { get { return this; } }
+
+   public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
+   #endregion
+
+   #region IDictionary members
+   public bool IsFixedSize { get { return true; } }
+
+   public bool IsReadOnly { get { return true; } }
+
+   /// <summary>
+   /// The key can either be the string corrsponding to the column name or
+   /// the order of the column
+   /// </summary>
+   public object this[object key] {
+      get {
+         if (key is string)
+            return this[(string)key];
+         return this[(int)key];
+      }
+      set { throw new NotSupportedException("The IBlueRow is readonly"); }
+   }
+
+   public object this[int index] => row[index];
+
+   public object this[string colName] => row[colName];
+
+   public ICollection Keys => row.Keys;
+
+   public ICollection Values => row.Values;
+
+   public bool Contains(object key) => row.Contains(key);
+
+   void IDictionary.Add(object key, object? value) =>
+      throw new NotSupportedException("The IBlueRow is readonly");
+
+   void IDictionary.Clear() =>
+      throw new NotSupportedException("The IBlueRow is readonly");
+
+   IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); }
+
+   IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); }
+
+   public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+   #endregion
+
+   bool myIsNull(object? o) =>
+      o is null || o == null || o is DBNull;
+
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,129 +1,99 @@
-// using System;
-// using System.Collections;
-// using BCG.Evaluator.SortExpression;
+using Bts.ColumnSortExpressions;
 
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-// 	/// to merge the additions to the accumulates table
-// 	/// </summary>
-// 	public class SortStream : ViewStream
-// 	{
-//       private ViewStream myStream;
-//       private IEnumerable mySortCols;
-//       private IComparer myComparer;
-//       private DataPacket myLastFiredPacket;
-//       private DataPacket myLastRecievedPacket;
+namespace BCG;
 
-//       public SortStream (ViewStream stream, string sortExpression)
-//       {
-//          myStream = stream;
-//          mySetSortExpression (sortExpression);
-//       }
+/// <summary>
+/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+/// to merge the additions to the accumulates table
+/// </summary>
+public class SortStream : ViewStream {
+   ViewStream stream;
+   IList<ColumnSortExpression> sortCols;
+   IComparer? comparer;
+   DataPacket? lastFiredPacket;
+   DataPacket? lastReceivedPacket;
 
-//       public SortStream (ViewStream vs, string colName, bool isAsc)
-//       {
-//          myStream = vs;
-//          string sortExpression = isAsc ? "+" + colName : "-" + colName;
-//          mySetSortExpression (sortExpression);
-//       }
+   public SortStream(ViewStream stream, string sortExpression) {
+      this.stream = stream;
+      SortExpression = sortExpression;
+   }
 
-//       public SortStream (ViewStream vs, IComparer comparer)
-//       {
-//          myStream = vs;
-//          myComparer = comparer;
-//       }
+   public SortStream(ViewStream vs, string colName, bool isAsc) {
+      stream = vs;
+      SortExpression = isAsc ? "+" + colName : "-" + colName;
+   }
 
-//       public string sortExpression
-//       {
-//          set
-//          {
-//             myComparer = null;
-//             myLastFiredPacket = null;
-//             mySetSortExpression (value);   
-//             myChangeHandler (myLastRecievedPacket);
-//          }
-//       }
+   public SortStream(ViewStream stream, IComparer comparer) {
+      this.stream = stream;
+      this.comparer = comparer;
+      sortCols = Array.Empty<ColumnSortExpression>();
+   }
 
-//       public IComparer comparer
-//       {
-//          set
-//          {
-//             myComparer = value;
-//             myLastFiredPacket = null;
-//             myChangeHandler(myLastRecievedPacket);
-//          }
-//       }
+   [MemberNotNull(nameof(sortCols))]
+   public string SortExpression {
+      set {
+         comparer = null;
+         lastFiredPacket = null;
+         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
 
-//       private void mySetSortExpression (string sortExpression)
-//       {
-//          syntax parser = 
-//             new syntax();
-         
-//          mySortCols = ((SortExpression)
-//             parser.Parse (sortExpression)).columns;
-//       }
+   public IComparer Comparer {
+      set {
+         comparer = value;
+         lastFiredPacket = null;
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
 
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); 
-//       }
+   public void myChangeHandlerTest(object o) =>
+      myChangeHandler(o);
 
-//       protected override void myOnLastSubscriber()
-//       {
-//          myInit ();
-//          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); 
-//       }
+   [MemberNotNull(nameof(lastFiredPacket))]
+   internal void myChangeHandler(object o) {
+      lastReceivedPacket = (DataPacket)o;
+      // First callback, so we need to sort the whole table, this is the
+      // only time we will do that for all the other sorting operations would
+      // be sorting the increments and inserting them into the whole list
+      if (lastFiredPacket is null) {
+         lastFiredPacket = new() {
+            newRows = comparer is null ?
+               new SortedTable(lastReceivedPacket.newRows, sortCols) :
+               new SortedTable(lastReceivedPacket.newRows, comparer)
+         };
+         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
+         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
+         myFireEvent(lastFiredPacket.Clone());
+         return;
+      }
+      // Deletes
+      lastFiredPacket.deletedRows =
+         comparer == null ?
+         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
+         new SortedTable(lastReceivedPacket.deletedRows, comparer);
+      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
+      // Adds
+      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
+      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
+      myFireEvent(lastFiredPacket.Clone());
+   }
 
-//       internal void myChangeHandler (object o)
-//       {
-//          myLastRecievedPacket = (DataPacket)o;
-//          //first callback, so we need to sort the whole table, this is the
-//          //only time we will do that for all the other sorting operations would
-//          //be sorting the increments and inserting them into the whole list
-//          if (myLastFiredPacket == null)
-//          {
-//             myLastFiredPacket = new DataPacket ();
-//             myLastFiredPacket.newRows = 
-//                myComparer == null ?
-//                new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
-//                new SortedTable (myLastRecievedPacket.newRows, myComparer);
-//             myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//             myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-//             myFireEvent (myLastFiredPacket.Clone());
-//             return;
-//          }
-//          myHandleDeletes (myLastRecievedPacket);
-//          myHandleAdds (myLastRecievedPacket);
-//          myFireEvent (myLastFiredPacket.Clone());
-//       }
+   protected override void myOnFirstSubscriber() =>
+      stream.onChange += myChangeHandler;
 
-//       public void myChangeHandlerTest(object o)
-//       {
-//           myChangeHandler(o);
-//       }
-
-//       private void myHandleDeletes (DataPacket dp)
-//       {
-//          myLastFiredPacket.deletedRows = 
-//             myComparer == null ?
-//             new SortedTable (dp.deletedRows, mySortCols) :
-//             new SortedTable (dp.deletedRows, myComparer);
-//          ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
-//       }
-
-//       private void myHandleAdds (DataPacket dp)
-//       {
-//          myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
-//          ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
-//       }
-
-//       private void myInit ()
-//       {
-//          myLastFiredPacket = null;
-//          myLastRecievedPacket = null;
-//       }
-// 	}
-// }
+   protected override void myOnLastSubscriber() {
+      lastFiredPacket = null;
+      lastReceivedPacket = null;
+      stream.onChange -= myChangeHandler;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs

@@ -1,136 +1,108 @@
-// using System;
-// using System.Collections;
-// using BCG.Evaluator.SortExpression;
-
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Represents an IBlueRowCollection with the rows sorted according to the
-//    /// 
-//    /// </summary>
-//    public class SortedTable : IBlueRowCollection 
-//    {
-//       private ArrayList myRows = new ArrayList ();
-//       private Hashtable myIndex = new Hashtable ();
-//       private IBlueTableSchema mySchema;
-//       private IEnumerable mySortCols;
-//       private IComparer myComparer;
-
-//       #region SortedTable method(s)
-//       public void addRows (IBlueRowCollection rows)
-//       {
-//          //requires merger of the addedRows to the already existing ones
-//          if (rows.Count == 0)
-//             //at times there are calls with 0 additions, in that case
-//             //we ofcourse need no processing
-//             return;
-//          SortedTable addedRows = 
-//             myComparer == null ? new SortedTable (rows, mySortCols) :
-//                new SortedTable (rows, myComparer);
-//          int addedRowsIndex = 0;
-//          SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
-//          for (int i = 0; i < myRows.Count; i ++)
-//             if (newRow.CompareTo(myRows[i]) <= 0)
-//             {
-//                myRows.Insert (i, newRow);
-//                myIndex[newRow["id"]] = newRow;
-//                if (++addedRowsIndex == addedRows.Count)
-//                   return;
-//                newRow = addedRows.getRowByIndex (addedRowsIndex);
-//             }
-//          if (addedRowsIndex < addedRows.Count)
-//             for (int i = addedRowsIndex; i < addedRows.Count; i++)
-//             {
-//                newRow = addedRows.getRowByIndex (i);
-//                myRows.Add (newRow);
-//                myIndex[newRow["id"]] = newRow;
-//             }
-//       }
-
-//       public void deleteRows (IBlueRowCollection deletedRows)
-//       {
-//          foreach (IBlueRow deletedRow in deletedRows)
-//          {
-//             IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
-//             myRows.Remove (row);
-//             myIndex.Remove (row["id"]);
-//          }
-//       }
-
-//       public SortRow getRowByIndex (int index)  {return (SortRow)myRows[index];}
-//       #endregion
-
-//       #region Constructor(s)
-//       /// <summary>
-//       /// Sorts the rows using sortExpression as the sorting criteria
-//       /// </summary>
-//       /// <param name="rows">
-//       /// Rows to be inserted into the table
-//       /// </param>
-//       /// <param name="sortExpression">
-//       /// Sorting criteria
-//       /// </param>
-//       public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
-//       {
-//          mySortCols = sortExpression.columns;
-//          myConstructorHelper (rows);
-//       }
-
-//       public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
-//       {
-//          mySortCols = sortCols;
-//          myConstructorHelper (rows);
-//       }
-
-//       public SortedTable (IBlueRowCollection rows, IComparer comparer)
-//       {
-//          myComparer = comparer;
-//          myConstructorHelper (rows);
-//       }
-
-//       private void myConstructorHelper (IBlueRowCollection rows)
-//       {
-//          mySchema = rows.schema;
-//          foreach (IBlueRow row in rows)
-//          {
-//             SortRow newRow = 
-//                myComparer == null ?
-//                new SortRow (row, mySortCols) : new SortRow (row, myComparer);
-//             myRows.Add (newRow);
-//             myIndex[row["id"]] = newRow;
-//          }
-//          myRows.Sort ();
-//       }
-
-//       #endregion
-
-//       #region IBlueRowCollections members
-//       public IBlueRow this[int id] 
-//       {
-//          get
-//          {
-//             if (myIndex[id] == null)
-//                throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
-//             return (IBlueRow)myIndex[id];
-//          }
-//       }
-//       public IBlueTableSchema schema {get{return mySchema;}}
-
-//       public bool containsID (int id)  {return myIndex[id] != null;}
-
-//       #endregion
-
-//       #region ICollection members
-//       public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
-
-//       public bool IsSynchronized {get{return false;}}
-
-//       public object SyncRoot {get{return this;}}
-
-//       public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
-
-//       public int Count{get{return myRows.Count;}}
-//          #endregion
-//    }
-
-// }
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+using Bts.ColumnSortExpressions;
+
+namespace BCG;
+
+/// <summary>
+/// Represents an IBlueRowCollection with the rows sorted according to the
+/// </summary>
+public class SortedTable : IBlueRowCollection {
+   readonly List<SortRow> rows = new(0);
+   readonly Dictionary<int, IBlueRow> rowIndex = new(0);
+   readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>();
+   readonly IComparer? comparer;
+
+   /// <summary>
+   /// Sorts the rows using sortExpression as the sorting criteria
+   /// </summary>
+   /// <param name="rows">Rows to be inserted into the table</param>
+   /// <param name="sortCols">Sorting criteria</param>
+   public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) {
+      this.sortCols = sortCols;
+      init(rows);
+   }
+
+   public SortedTable(IBlueRowCollection rows, IComparer comparer) {
+      this.comparer = comparer;
+      init(rows);
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; private set; }
+   public object SyncRoot => this;
+   public IBlueRow this[int id] => rowIndex[id]
+      ?? throw new BlueDataException(
+         $"ID '{id}' does not exist in table '{schema?.tableName}'");
+
+   public void addRows(IBlueRowCollection rows) {
+      // Requires merger of the addedRows to the already existing ones
+      if (rows.Count == 0) {
+         // At times there are calls with 0 additions, in that case
+         // we of course need no processing
+         return;
+      }
+      SortedTable addedRows =
+         comparer == null ? new SortedTable(rows, sortCols) :
+            new SortedTable(rows, comparer);
+      int addedRowsIndex = 0;
+      SortRow newRow = addedRows.getRowByIndex(addedRowsIndex);
+      for (int i = 0; i < this.rows.Count; i++) {
+         if (newRow.CompareTo(this.rows[i]) <= 0) {
+            this.rows.Insert(i, newRow);
+            rowIndex[(int)newRow["id"]] = newRow;
+            if (++addedRowsIndex == addedRows.Count) {
+               return;
+            }
+            newRow = addedRows.getRowByIndex(addedRowsIndex);
+         }
+      }
+      if (addedRowsIndex < addedRows.Count) {
+         for (int i = addedRowsIndex; i < addedRows.Count; i++) {
+            newRow = addedRows.getRowByIndex(i);
+            this.rows.Add(newRow);
+            rowIndex[(int)newRow["id"]] = newRow;
+         }
+      }
+   }
+
+   public void deleteRows(IBlueRowCollection deletedRows) {
+      foreach (IBlueRow deletedRow in deletedRows) {
+         var row = rowIndex[(int)deletedRow["id"]!] as SortRow
+            ?? throw new InvalidOperationException("Index did not return IBlueRow");
+         rows.Remove(row);
+         rowIndex.Remove((int)row["id"]!);
+      }
+   }
+
+   public SortRow getRowByIndex(int index) =>
+      rows[index]!;
+
+   public bool containsID(int id) =>
+      rowIndex[id] is not null;
+
+   public IEnumerator GetEnumerator() =>
+      rows.GetEnumerator();
+
+   public void CopyTo(Array array, int index) {
+      for (int i = 0; i < Count; i++) {
+         array.SetValue(rows[i], index + i);
+      }
+   }
+
+   [MemberNotNull(nameof(schema))]
+   void init(IBlueRowCollection rows) {
+      schema = rows.schema;
+      foreach (IBlueRow row in rows) {
+         SortRow newRow = comparer is null ?
+            new(row, sortCols) : new(row, comparer);
+         this.rows.Add(newRow);
+         rowIndex[(int)row["id"]!] = newRow;
+      }
+      this.rows.Sort();
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,49 +1,43 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class TopStream : MethodDrivenStream
-//    {
-//       private int myTopCount;
+namespace BCG {
+    public class TopStream : MethodDrivenStream {
+        private int myTopCount;
 
-//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
+        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
 
-//       public TopStream (ViewStream stream, 
-//          int topCount, 
-//          string sortExpression) 
-//          :
-//          this (sortExpression == null? 
-//          stream :
-//          new SortStream (stream, sortExpression), topCount)
-//       {}
+        public TopStream(ViewStream stream,
+           int topCount,
+           string sortExpression)
+           :
+           this(sortExpression == null ?
+           stream :
+           new SortStream(stream, sortExpression), topCount) { }
 
-//       protected override void myOnTableClear () 
-//       {
-//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//       }
+        protected override void myOnTableClear() {
+            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+        }
 
-//       protected override void myHandleFirstCall () 
-//       {
-//          myLastFiredPacket = new DataPacket ();
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
-//       }
+        protected override void myHandleFirstCall() {
+            myLastFiredPacket = new DataPacket();
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+        }
 
-//       protected override void myHandleIncrementalAdd ()     {}
+        protected override void myHandleIncrementalAdd() { }
 
-//       protected override void myHandleIncrementalDelete()   
-//       {
-//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
-//             oldRows,
-//             myLastFiredPacket.schema.columnNames,
-//             out myLastFiredPacket.deletedRows,
-//             out myLastFiredPacket.addedRows);
-//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-//       }
-//    }
-// }
+        protected override void myHandleIncrementalDelete() {
+            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+               oldRows,
+               myLastFiredPacket.schema.columnNames,
+               out myLastFiredPacket.deletedRows,
+               out myLastFiredPacket.addedRows);
+            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+        }
+    }
+}

src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs

@@ -11,17 +11,17 @@ namespace BCG
    public class CircularRowCollection : OrderedBlueTable
    {
       private int myBufferSize;
-      public CircularRowCollection (IBlueTableSchema schema, int bufferSize)
-         : base (schema)
-      {		
-         myBufferSize = bufferSize ;
+      public CircularRowCollection(IBlueTableSchema schema, int bufferSize)
+         : base(schema)
+      {
+         myBufferSize = bufferSize;
       }
 
-      public CircularRowCollection (IBlueRowCollection rows, int bufferSize)
-         : this (rows.schema, bufferSize)
+      public CircularRowCollection(IBlueTableSchema schema, IEnumerable<IBlueRow> rows, int bufferSize)
+         : this(schema, bufferSize)
       {
          foreach (IBlueRow r in rows)
-            addRow (r);
+            addRow(r);
       }
 
       public int bufferSize
@@ -39,30 +39,30 @@ namespace BCG
       /// </summary>
       /// <param name="r"></param>
       /// <returns></returns>
-      public IBlueRow addRowWithReturn (IBlueRow r)
+      public IBlueRow addRowWithReturn(IBlueRow r)
       {
-         base.addRow (r);
+         base.addRow(r);
          if (myOrderedRows.Count > myBufferSize)
          {
-            IBlueRow retVal = myOrderedRows [0] as IBlueRow;
-            deleteRow (retVal);
+            IBlueRow retVal = myOrderedRows[0] as IBlueRow;
+            deleteRow(retVal);
             return retVal;
          }
          return null;
       }
 
-      public override void addRow (IBlueRow row)
+      public override void addRow(IBlueRow row)
       {
-         base.addRow (row);
+         base.addRow(row);
          if (myOrderedRows.Count > myBufferSize)
-            deleteRow (myOrderedRows [0] as IBlueRow);
+            deleteRow(myOrderedRows[0] as IBlueRow);
       }
 
-      public override void addRow (DataRow row)
+      public override void addRow(DataRow row)
       {
-         base.addRow (row);
+         base.addRow(row);
          if (myOrderedRows.Count > myBufferSize)
-            deleteRow (myOrderedRows [0] as IBlueRow);
+            deleteRow(myOrderedRows[0] as IBlueRow);
       }
 
       public void add_rows(IEnumerable<IBlueRow> rows_to_add, out IBlueRowCollection added_rows, out IBlueRowCollection deleted_rows)

src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs

@@ -3,8 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BCG
-{
+namespace BCG {
    /*
       These fields are first going to be used by  BlueRay to help it determine what differnet tags mean, but eventually
     * the goal is that they will be used for all services in this system. 
@@ -12,11 +11,12 @@ namespace BCG
     * not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service
     * probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated
     */
-   public enum MetaDataField {   Protocol,  //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
-                                 Type,   //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
-                                 Symbol, 
-                                 Exchange,
-                                 Platform,
-                                 MarketDataType, // BAM (bid, ask, mid), last, etc
-                                 RealName }; //does this thing have a better name we could call it
+   public enum MetaDataField {
+      Protocol,       // is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
+      Type,           // if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
+      Symbol,
+      Exchange,
+      MarketDataType, // BAM (bid, ask, mid), last, etc
+      RealName
+   }; //does this thing have a better name we could call it
 }

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -74,7 +74,7 @@ namespace BCG {
                      break;
                   }
                if (isVisible) {
-                  Log.assert(myTypeLookupTable[userRep] == null);
+                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
                   myTypeLookupTable[userRep] = type;
                }
             }

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -6,16 +6,17 @@ using Bts.DataStreamExpressions;
 using System.Collections.Generic;
 
 using BCG;
+using System.IO;
+using BCG.Utilities.General;
+using bts;
+using System.Linq;
 
 
 bts.Log.Gen.info($"Starting DSPublisher");
 Console.WriteLine("Starting DSPublisher...");
-static void init(string[] args)
-{
-   Console.WriteLine("Hit key to continue");
-   Console.ReadLine();
-   Func<string, string, Task> login = async (l, p) =>
-   {
+bool run_echo_tcp_client_only = false;
+static void init(string[] args) {
+   Func<string, string, Task> login = async (l, p) => {
       var login = l;
       var passwd = p;
       var service_ctx = bts.api.core.ServiceContext.create();
@@ -25,60 +26,105 @@ static void init(string[] args)
    Configuration.initialize();
    Task.Run(async () => await login("bts", String.Empty));
 }
-
 init(args);
-_publish_streams();
+if (run_echo_tcp_client_only)
+   start_echo_client();
+else
+   _publish_streams();
 
-static ViewStream build_stream(string stream_expression)
-{
-   try
-   {
+static ViewStream build_stream(string stream_expression) {
+   try {
+      bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
       var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
       var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
       return rv;
 
-   }
-   catch (Exception ex)
-   {
+   } catch (Exception ex) {
       bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
       return null;
    }
 }
 
+static void start_echo_client() {
+   Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
+   Console.ReadLine();
+   SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345);
+   client.start();
+   while (true) {
+      System.Threading.Thread.Sleep(1000);
+   }
+   // await SimpleTCPClient.start("127.0.0.1", 12345);
+   // Task.Delay(1000000000).Wait();
+}
 
-void _publish_streams()
-{
+void _publish_streams() {
    PubTableBroker pb_broker = new PubTableBroker();
    CloudStreamManager csm = new CloudStreamManager();
+   string csv_folder = @"C:\Temp\csvs_topublish";
+   CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
    // get it from config
    Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
-        {"bbo","bbo()"},
-        {"instruments","instruments()"},
+        //{"bbo","bbo()"},
+        //{"instruments","instruments()"},
         //{"timesales","timesales(10000)"},
-        {"books","books()"},
-      // {"seconds_1_10","second(1,10)"},
+        //{"books","books()"},
+       {"seconds_1_10","second(1,10)"},
         {"literal_420","literal(\"x\", \"420.0\")"},
-      //  {"bbo_ts","timeSeries(bbo())"}
-   };
-   foreach (var kv in streams_to_publish)
-   {
+        {"tradebar_30","tradebars(30)"},
+        {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
+        };
+   //  {"bbo_ts","timeSeries(bbo())"}
+
+   foreach (var kv in streams_to_publish) {
       var name = kv.Key;
       var exp = kv.Value;
       BCG.ViewStream vs = build_stream(exp);
       ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
       vs.onChange += on_change;
    }
-   while (true)
-   {
+   while (true) {
       System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
       ViewStreamDatabase.instance.doWork();
       csm.doWork();
+      MasterWorker.getInstance(false).doWork();
    }
 }
 
 
 
-void on_change(object obj)
-{
+void on_change(object obj) {
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
+}
+
+public class CSVPublisher : IWorker {
+   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
+   // read from config
+   private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
+
+   private PubTableBroker _pub_table_broker;
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+      _csv_directory = csv_directory;
+      _pub_table_broker = pub_table_broker;
+      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+   }
+   public void doWork() {
+      var file_names = Directory.GetFiles(_csv_directory);
+      foreach (var fn in file_names) {
+         if (!_pubs.ContainsKey(fn)) {
+            _handle_new_file(fn);
+         }
+      }
+   }
+
+   private void _handle_new_file(string file_name) {
+      if (file_name.EndsWith(".csv")) {
+         ViewStream vs = new CSVReaderStream(file_name);
+         vs = new SchemaGuesserStream(vs);
+         var toks = file_name.Split('\\');
+         string table_name = toks.Last();
+         table_name = table_name.Remove(table_name.Length - ".csv".Length);
+         ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
+         _pubs.Add(file_name, vp);
+      }
+   }
 }

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -382,7 +382,7 @@ namespace BCG {
       private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
          myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
          myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
-         myQueryTextBox.SelectionColor = Color.Black;
+         myQueryTextBox.SelectionColor = Color.White;
          myQueryTextBox.SelectionLength = 0;
          myTryExpression(myQueryTextBox.Text);
       }

src/DataStreamSuite/DataStreamSuite.sln

@@ -1,4 +1,4 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.0.31903.59
 MinimumVisualStudioVersion = 10.0.40219.1
@@ -38,6 +38,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStre
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{134C3915-CB2B-494C-BD62-E0B18250CBF5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -119,5 +123,13 @@ Global
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU
+		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs

@@ -47,30 +47,30 @@ namespace BCG {
       /// of Int32.Max would return all the messages it has
       /// received between last getUpdates and this call.
       /// </summary
-      IList<byte[]> getUpdates(TimeSpan timeout, int maxElements);
+      IList<byte[]> get_updates(TimeSpan timeout, int maxElements);
       /// <summary>
       /// server URL to which the connection is made
       /// </summary>
-      string serverURL { get; }
+      string server_url { get; }
       /// <summary>
       /// client URL as seen by the server. if a connection has not
       /// been established (as can happen in TCP) an empty string 
       /// is returned
       /// </summary>
-      string clientURL { get; }
+      string client_url { get; }
 
       /// <summary>
       /// registers the callback to receive 
       /// connection notifications
       /// </summary>
-      void registerCallback(ConnectionStatusChange del);
+      void register_connection_callback(PubSubServerConnectionStatusChange del);
 
       /// <summary>
       /// registers callbacks to reveive data
       /// </summary>
       /// <param name="del"></param>
-      void registerDataCallback(DataDelegate del);
+      void register_data_callback(PubSubServerDataDelegate del);
 
-      void registerSendConfirmationCallback(SendConfirmationDelegate del);
+      void register_send_confirmation_callback(SendConfirmationDelegate del);
    }
 }

src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs

@@ -8,6 +8,7 @@ using System.IO;
 //using BCG.Utilities.General;
 using System.Threading;
 using mbts;
+using bts;
 
 namespace BCG {
    /// <summary>
@@ -20,26 +21,27 @@ namespace BCG {
    /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread
    /// 
    /// </summary>
-   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,
-                                                   ITCPMessageHandler,
-                                                   ITCPConnectionHandler {
+   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver   
+                                                   ITCPMessageHandler,     // to get messages from ITCPCOnnection
+                                                   ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM
       private ITCPConnection _connection, _new_connection;
       private readonly BlutoSubAddress _server_address;
       // if a message(with confirm) can not be sent , its id is queued here, 
       // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
       //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
       private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
-      // in onData, data is queued here
+      // in onData, data is queued here, for polling
       private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
       private DateTime _last_hb = DateTime.Now;
-      private ConnectionStatusChange _conn_callback;
-      private DataDelegate _data_callback;
-      private SendConfirmationDelegate _on_send_onfirm_callback;
+      private PubSubServerConnectionStatusChange _conn_callback;
+      private PubSubServerDataDelegate _data_callback;
+      private SendConfirmationDelegate _on_send_confirm_callback;
       private readonly TimeSpan _connection_drop_threshold =
-         new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
+         new TimeSpan(0, 0, 0, 0,
+                     PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
 
-      public PubSubServerManagedTCPConnection(string serverURL)
-         : this(new BlutoSubAddress(serverURL)) { }
+      public PubSubServerManagedTCPConnection(string server_url)
+         : this(new BlutoSubAddress(server_url)) { }
 
       public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
          _server_address = server_address;
@@ -48,17 +50,17 @@ namespace BCG {
 
       private void _connect_to_ps_server() {
          bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
-         ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
+         AsyncTCPConnector_APM.connect_to(_server_address, this);
       }
 
       #region IPubSubServerConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string conf_id) {
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
+      public void send_with_confirmation(byte[] data, ref string conf_id) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} _connection : {_connection != null}");
          if (_connection != null) {
             _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
-               if (_on_send_onfirm_callback != null)
-                  _on_send_onfirm_callback(msg_id, confirmed);
+               if (_on_send_confirm_callback != null)
+                  _on_send_confirm_callback(msg_id, confirmed);
             });
          } else {
             bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
@@ -86,18 +88,18 @@ namespace BCG {
 
       public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
          var bytes = msg.to_bytes();
-         sendWithConfirmation(bytes, ref conf_id);
+         send_with_confirmation(bytes, ref conf_id);
       }
 
-      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) {
+      public IList<byte[]> get_updates(TimeSpan timeout, int maxElements) {
          return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
       }
 
-      public string serverURL {
+      public string server_url {
          get { return _server_address.ToString(); }
       }
 
-      public string clientURL {
+      public string client_url {
          get {
             string rv = "";
             lock (this) {
@@ -118,16 +120,16 @@ namespace BCG {
          }
       }
 
-      public void registerCallback(ConnectionStatusChange del) {
+      public void register_connection_callback(PubSubServerConnectionStatusChange del) {
          _conn_callback = del;
       }
 
-      public void registerDataCallback(DataDelegate del) {
+      public void register_data_callback(PubSubServerDataDelegate del) {
          _data_callback = del;
       }
 
-      public void registerSendConfirmationCallback(SendConfirmationDelegate del) {
-         _on_send_onfirm_callback = del;
+      public void register_send_confirmation_callback(SendConfirmationDelegate del) {
+         _on_send_confirm_callback = del;
       }
 
       #endregion
@@ -185,18 +187,6 @@ namespace BCG {
                break;
 
          }
-         // this stupid shit, caused duplicating og messages and 2 days of wild goose chase
-         // if (message.messageType == RemoteMessageType.Data)
-         // {
-         //    if (_data_callback != null)
-         //       _data_callback(message.remoteConnectionInfo.ToString(), message.message);
-         //    else
-         //       _in_data_queue.enqueue(message.message);
-         // }
-         // else
-         // {
-         //    _last_hb = DateTime.Now;
-         // }
       }
 
       #endregion
@@ -204,8 +194,7 @@ namespace BCG {
       #region ITCPConnectionHandler Members
 
       /// <summary>
-      /// This function is called on the main thread (as all other functions
-      /// except onData)
+      ///  will be called on ana rbitrary .Net thread
       /// </summary>
       /// <param name="connection"></param>
       public void onConnection(ITCPConnection connection) {
@@ -246,10 +235,10 @@ namespace BCG {
             _connection.register(this);
             if (_conn_callback != null)
                _conn_callback(_server_address.ToString(), true);
-            if (_on_send_onfirm_callback != null) {
+            if (_on_send_confirm_callback != null) {
                var msg_ids = _negative_unsent_confs.dequeueAll();
                foreach (var id in msg_ids)
-                  _on_send_onfirm_callback(id, false);
+                  _on_send_confirm_callback(id, false);
             }
          }
       }
@@ -264,47 +253,6 @@ namespace BCG {
       public void doWork() {
          //_dispose_and_reconnect_if_missing_hb();
          _check_for_new_connection();
-         ///check for heartbeats and let clients know in 
-         ///case of missing hb (connection breaking)
-         // TimeSpan hb_interval = DateTime.Now - myLastHB;
-         // bool hb_missed = hb_interval > myConnectionDropThreshold;
-         // bool conn_disposed = myConnection != null || myConnection.isDisposed;
-         // if (conn_disposed || hb_missed)
-         // {
-         //    if (hb_missed)
-         //    {
-         //       // since conn breakage was diagnosed logically, lets physically close the connection
-         //       myConnection.Dispose();
-         //       string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
-         //       Console.WriteLine(dispose_msg);
-         //    }
-         //    myConnection = null; // this is used as a marker for a dead connection
-         //    if (myConnCallback != null)
-         //       myConnCallback(myServerAddress.ToString(), false);
-         //    string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
-         //    $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
-         //    ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
-         // }
-         // if (myTempNewConnection != null)
-         // {
-         //    if (myConnection != null)
-         //    {
-         //       // TODOds
-         //       // log error and close it
-         //       // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
-         //       //                                                 + " before a new connection is established");
-         //    }
-         //    Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
-         //    myConnection = myTempNewConnection;
-         //    myTempNewConnection = null;
-         //    myConnection.register(this);
-         //    myConfs.dequeueAll();
-         //    if (myConnCallback != null)
-         //       myConnCallback(myServerAddress.ToString(), true);
-         // }
-         // foreach (Pair<string, bool> p in myConfs.dequeueAll())
-         //    if (myOnSendConfirmCallback != null)
-         //       myOnSendConfirmCallback(p.first, p.second);
       }
 
       #endregion

src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs

@@ -16,6 +16,7 @@ namespace BCG {
    // SubMetaData
    // TransactionMetaData
    // UnSubMetaData
+   // 
    public class MetaData : ICloneable {
       private IDictionary _name_value_map = new Dictionary<string, string>();
       int mySize = -1;

src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs

@@ -1,10 +1,8 @@
 using System;
 
-namespace BCG
-{
-   [Serializable]
-   public class SubMetaData 
-   {
+namespace BCG {
+   //[Serializable]
+   public class SubMetaData {
       /// <summary>
       /// connection URL on which the connection will receive data
       /// </summary>
@@ -13,20 +11,18 @@ namespace BCG
       /// identification of the client and the connection on which to receive data
       /// machine; ip; port; clientURL
       /// </summary>
-      public string subscribingApp;
+      public readonly string subscribing_app = ClientInfo.thisClient.processString;
       /// <summary>
       /// tags that are to be subscribed
       /// </summary>
       public string[] tags;
 
-      public SubMetaData()
-      { }
+      public SubMetaData() { }
 
-      public SubMetaData(string theConnection, params string[] theTags)
-      {
-         connection = theConnection;
-         tags = theTags;
-         subscribingApp = ClientInfo.thisClient.processString;
+      public SubMetaData(string the_connection, params string[] the_tags) {
+         connection = the_connection;
+         tags = the_tags;
+         //subscribing_app = ClientInfo.thisClient.processString;
       }
    }
 }

src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs

@@ -1,21 +1,17 @@
 using System;
 
-namespace BCG
-{
-   [Serializable]
-   public class UnSubMetaData
-   {
+namespace BCG {
+   //[Serializable]
+   public class UnSubMetaData {
       public string connection;
-      public string[] tags;
-      public string unsubscribingApp = ClientInfo.thisClient.processString;
+      public readonly string[] tags;
+      public readonly string unsubscribing_app = ClientInfo.thisClient.processString;
 
-      public UnSubMetaData()
-      {}
+      public UnSubMetaData() { }
 
-      public UnSubMetaData(string theConnection, params string[] theTags)
-      {
-         connection = theConnection;
-         tags = theTags;
+      public UnSubMetaData(string the_connection, params string[] the_tags) {
+         connection = the_connection;
+         tags = the_tags;
       }
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection2.cs

@@ -1,296 +1,248 @@
-using BCG;
+// using BCG;
 
-using Google.Protobuf;
+// using Google.Protobuf;
 
-using System;
-using System.IO;
-using System.Linq;
-using System.Net;
-using System.Net.Sockets;
-using System.Runtime.CompilerServices;
-using System.Text;
-using System.Threading.Tasks;
-using BCG;
-using System.Xml.Serialization;
-using System.Reflection;
-using BCG.Utilities.General;
-using System.Net.WebSockets;
+// using System;
+// using System.IO;
+// using System.Linq;
+// using System.Net;
+// using System.Net.Sockets;
+// using System.Runtime.CompilerServices;
+// using System.Text;
+// using System.Threading.Tasks;
+// using BCG;
+// using System.Xml.Serialization;
+// using System.Reflection;
+// using BCG.Utilities.General;
+// using System.Net.WebSockets;
 
-namespace BCG
-{
+// namespace BCG {
 
-   public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable
-   {
-      //private Socket _socket;
-      // tied to the socket above
-      private NetworkStream _nw_stream;
-      private readonly ConnectionSide _side;
-      ITCPMessageHandler _message_handler;
-      private int _msg_id = 0;
-      RemoteConnectionInfo _peer_address;
-      RemoteConnectionInfo _self_address;
-      private TimeSpan _hb_period = TimeSpan.FromSeconds(1);
-      private DateTime _last_hb_sent = DateTime.MinValue;
-      private DateTime _last_hb_received = DateTime.MaxValue;
-      private static byte _msg_type_data = 1;
-      private static byte _msg_type_hb = 0;
-      private bool _is_disposed = false;
-      SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>();
+//    public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable {
+//       private Socket _socket;
+//       // tied to the socket above
+//       private NetworkStream _nw_stream;
+//       private readonly ConnectionSide _side;
+//       ITCPMessageHandler _message_handler;
+//       private int _msg_id = 0;
+//       RemoteConnectionInfo _peer_address;
+//       RemoteConnectionInfo _self_address;
+//       private TimeSpan _hb_period = TimeSpan.FromSeconds(1);
+//       private DateTime _last_hb_sent = DateTime.MinValue;
+//       private DateTime _last_hb_received = DateTime.MaxValue;
+//       private static byte _msg_type_data = 1;
+//       private static byte _msg_type_hb = 0;
+//       private bool _is_disposed = false;
+//       SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>();
 
-      public class QueuedMessage
-      {
-         public string id;
-         public SendConfirmationDelegate del;
-         public byte[] data;
+//       public class QueuedMessage {
+//          public string id;
+//          public SendConfirmationDelegate del;
+//          public byte[] data;
 
-         public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data)
-         {
-            id = id;
-            del = del;
-            data = data;
-         }
+//          public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data) {
+//             id = id;
+//             del = del;
+//             data = data;
+//          }
 
-         public QueuedMessage(byte[] data)
-         : this(String.Empty, null, data) { }
+//          public QueuedMessage(byte[] data)
+//          : this(String.Empty, null, data) { }
 
-      }
+//       }
 
-      public enum ConnectionSide
-      {
-         Client,
-         Server
-      }
+//       public enum ConnectionSide {
+//          Client,
+//          Server
+//       }
 
-      public AsyncTCPConnection2(Socket socket, ConnectionSide side)
-      {
-         //_socket = socket;
-         _nw_stream = new NetworkStream(socket);
-         _side = side;
-         IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint;
-         _peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port);
+//       public AsyncTCPConnection2(Socket socket, ConnectionSide side) {
+//          _socket = socket;
+//          _nw_stream = new NetworkStream(socket);
+//          _side = side;
+//          IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint;
+//          _peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port);
 
-         var loc_ep = socket.LocalEndPoint as IPEndPoint;
-         _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port);
-      }
+//          var loc_ep = socket.LocalEndPoint as IPEndPoint;
+//          _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port);
+//       }
 
-      public void register(ITCPMessageHandler message_handler)
-      {
-         _message_handler = message_handler;
-         Task.Run(async () => _keep_reading());
-         Task.Run(async () => await _keep_sending());
-         //_keep_reading();
-         // if add a _keep_sending threadis added then we can maek send a non async
-         // methid making it much easier to integrate with the rest
-      }
+//       public void register(ITCPMessageHandler message_handler) {
+//          _message_handler = message_handler;
+//          Task.Run(async () => _keep_reading());
+//          Task.Run(async () => await _keep_sending());
+//          //_keep_reading();
+//          // if add a _keep_sending threadis added then we can maek send a non async
+//          // methid making it much easier to integrate with the rest
+//       }
 
-      // this thread pulls 
-      private async Task _keep_sending()
-      {
-         bts.Log.Gen.info($"TCPConnection._keep_sending starting");
-         while (!_is_disposed)
-         {
-            try
-            {
-               QueuedMessage m = _data_send_queue.dequeue(100);
-               if (m == null)
-               {
-                  continue;
-               }
-               await send_async(m.data);
-               if (String.IsNullOrEmpty(m.id))
-               {
-                  m.del(m.id, true);
-               }
-            }
-            catch (Exception ex)
-            {
-               bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}");
-               await Task.Delay(100);
-               _handle_error(ex);
-            }
-         }
-      }
+//       // this thread pulls 
+//       private async Task _keep_sending() {
+//          bts.Log.Gen.info($"TCPConnection._keep_sending starting");
+//          while (!_is_disposed) {
+//             try {
+//                QueuedMessage m = _data_send_queue.dequeue(100);
+//                if (m == null) {
+//                   continue;
+//                }
+//                await send_async(m.data);
+//                if (String.IsNullOrEmpty(m.id)) {
+//                   m.del(m.id, true);
+//                }
+//             } catch (Exception ex) {
+//                bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}");
+//                await Task.Delay(100);
+//                _handle_error(ex);
+//             }
+//          }
+//       }
 
-      private async Task _keep_reading()
-      {
-         bts.Log.Gen.info($"TCPConnection._keep_reading starting");
-         while (!_is_disposed)
-         {
-            try
-            {
-               byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream);
-               bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}");
-               await _handle_msg(msg);
-            }
-            catch (Exception ex)
-            {
-               bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}");
-               await Task.Delay(100);
-               _handle_error(ex);
-            }
-         }
-      }
+//       private async Task _keep_reading() {
+//          bts.Log.Gen.info($"TCPConnection._keep_reading starting");
+//          while (!_is_disposed) {
+//             try {
+//                byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream);
+//                bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}");
+//                await _handle_msg(msg);
+//             } catch (Exception ex) {
+//                bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}");
+//                await Task.Delay(100);
+//                _handle_error(ex);
+//             }
+//          }
+//       }
 
-      private async Task _handle_msg(byte[] msg)
-      {
-         // read the first byte to read the msg type and deal with it accordingly
-         BinaryReader br = new BinaryReader(new MemoryStream(msg));
-         byte msg_type = br.ReadByte();
-         if (msg_type == _msg_type_data)
-         {
-            byte[] data = br.ReadBytes(msg.Length - 1);
-            RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address);
-            _message_handler.onData(m);
-         }
-         else
-         {
-            RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address);
-            _message_handler.onData(m);
-         }
-      }
+//       private async Task _handle_msg(byte[] msg) {
+//          // read the first byte to read the msg type and deal with it accordingly
+//          BinaryReader br = new BinaryReader(new MemoryStream(msg));
+//          byte msg_type = br.ReadByte();
+//          if (msg_type == _msg_type_data) {
+//             byte[] data = br.ReadBytes(msg.Length - 1);
+//             RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address);
+//             _message_handler.onData(m);
+//          } else {
+//             RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address);
+//             _message_handler.onData(m);
+//          }
+//       }
 
-      // not very efficient
-      private static byte[] _prefix_msg_type(byte[] data)
-      {
-         MemoryStream ms = new MemoryStream();
-         BinaryWriter bw = new BinaryWriter(ms);
-         bw.Write(_msg_type_data);
-         bw.Write(data);
-         ms.Position = 0;
-         return ms.ToArray();
-      }
+//       // not very efficient
+//       private static byte[] _prefix_msg_type(byte[] data) {
+//          MemoryStream ms = new MemoryStream();
+//          BinaryWriter bw = new BinaryWriter(ms);
+//          bw.Write(_msg_type_data);
+//          bw.Write(data);
+//          ms.Position = 0;
+//          return ms.ToArray();
+//       }
 
-      private async void _handle_error(Exception ex)
-      {
-         bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}");
-         try
-         {
-            var socket = _nw_stream.Socket;
-            socket.Close();
-            socket.Dispose();
-         }
-         catch (Exception ex2)
-         {
-            bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}");
-         }
-         _is_disposed = true;
-         RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address);
-         _message_handler.onData(rm);
-      }
+//       private async void _handle_error(Exception ex) {
+//          bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}");
+//          try {
+//             var socket = _nw_stream.Socket;
+//             socket.Close();
+//             socket.Dispose();
+//          } catch (Exception ex2) {
+//             bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}");
+//          }
+//          _is_disposed = true;
+//          RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address);
+//          _message_handler.onData(rm);
+//       }
 
-      public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { }
+//       public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { }
 
-      public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del)
-      {
-         try
-         {
-            conf_id = (++_msg_id).ToString();
-            await TCPSocketUtils.send_bytes_async(_nw_stream, data);
-            del(conf_id, true);
-         }
-         catch (Exception ex)
-         {
-            _handle_error(ex);
-         }
-      }
+//       public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del) {
+//          try {
+//             conf_id = (++_msg_id).ToString();
+//             await TCPSocketUtils.send_bytes_async(_nw_stream, data);
+//             del(conf_id, true);
+//          } catch (Exception ex) {
+//             _handle_error(ex);
+//          }
+//       }
 
-      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
-      {
-         conf_id = (++_msg_id).ToString();
-         send(data);
-         del(conf_id, true);
-      }
+//       public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
+//          conf_id = (++_msg_id).ToString();
+//          send(data);
+//          del(conf_id, true);
+//       }
 
-      public void send(byte[] data)
-      {
-         Task.Run(async () => await send_async(data));
-         bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully");
-         //_data_send_queue.enqueue(new QueuedMessage(data));
-      }
+//       public void send(byte[] data) {
+//          Task.Run(async () => await send_async(data));
+//          bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully");
+//          //_data_send_queue.enqueue(new QueuedMessage(data));
+//       }
 
-      public async Task send_async(byte[] data)
-      {
-         // prefix 4 bytes for type (data vs hb)
-         MemoryStream str = new MemoryStream();
-         BinaryWriter bw = new BinaryWriter(str);
-         bw.Write(_msg_type_data);
-         bw.Write(data);
-         try
-         {
-            await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray());
-         }
-         catch (Exception ex)
-         {
-            bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}");
-            _handle_error(ex);
-         }
-      }
+//       public async Task send_async(byte[] data) {
+//          // prefix 4 bytes for type (data vs hb)
+//          MemoryStream str = new MemoryStream();
+//          BinaryWriter bw = new BinaryWriter(str);
+//          bw.Write(_msg_type_data);
+//          bw.Write(data);
+//          try {
+//             await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray());
+//          } catch (Exception ex) {
+//             bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}");
+//             _handle_error(ex);
+//          }
+//       }
 
-      public void getPeerAddress(out string ip, out int port)
-      {
-         ip = _peer_address.ip;
-         port = _peer_address.port;
-         ///.Net has some problems getting the port, so using
-         ///native code for that
-         //sockaddr addr = new sockaddr();
-         //int namelen = 16;
-         //getsockname(mySock.Handle, ref addr, ref namelen);
-         //port = addr.sin_port;
-      }
+//       public void getPeerAddress(out string ip, out int port) {
+//          ip = _peer_address.ip;
+//          port = _peer_address.port;
+//          ///.Net has some problems getting the port, so using
+//          ///native code for that
+//          //sockaddr addr = new sockaddr();
+//          //int namelen = 16;
+//          //getsockname(mySock.Handle, ref addr, ref namelen);
+//          //port = addr.sin_port;
+//       }
 
-      public RemoteConnectionInfo get_peer_address()
-      {
-         string ip; int port;
-         getPeerAddress(out ip, out port);
-         return new RemoteConnectionInfo(ip, port);
-      }
+//       public RemoteConnectionInfo get_peer_address() {
+//          string ip; int port;
+//          getPeerAddress(out ip, out port);
+//          return new RemoteConnectionInfo(ip, port);
+//       }
 
-      public RemoteConnectionInfo get_self_address()
-      {
-         string ip; int port;
-         getSelfAddress(out ip, out port);
-         return new RemoteConnectionInfo(ip, port);
-      }
+//       public RemoteConnectionInfo get_self_address() {
+//          string ip; int port;
+//          getSelfAddress(out ip, out port);
+//          return new RemoteConnectionInfo(ip, port);
+//       }
 
 
-      public void getSelfAddress(out string ip, out int port)
-      {
-         ip = _self_address.ip;
-         port = _self_address.port;
-      }
+//       public void getSelfAddress(out string ip, out int port) {
+//          ip = _self_address.ip;
+//          port = _self_address.port;
+//       }
 
-      private async void _send_hb()
-      {
-         if (_last_hb_sent - DateTime.Now > _hb_period)
-         {
-            MemoryStream ms = new MemoryStream();
-            BinaryWriter bw = new BinaryWriter(ms);
-            bw.Write(_msg_type_hb);
-            ms.Position = 0;
-            byte[] hb_msg = ms.ToArray();
-            await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg);
-         }
-      }
+//       private async void _send_hb() {
+//          if (_last_hb_sent - DateTime.Now > _hb_period) {
+//             MemoryStream ms = new MemoryStream();
+//             BinaryWriter bw = new BinaryWriter(ms);
+//             bw.Write(_msg_type_hb);
+//             ms.Position = 0;
+//             byte[] hb_msg = ms.ToArray();
+//             await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg);
+//          }
+//       }
 
-      public void doWork()
-      {
-         _send_hb();
-      }
+//       public void doWork() {
+//          _send_hb();
+//       }
 
-      public void Dispose()
-      {
-         _nw_stream.Close();
-         _nw_stream.Dispose();
-         _is_disposed = true;
-      }
+//       public void Dispose() {
+//          _nw_stream.Close();
+//          _nw_stream.Dispose();
+//          _is_disposed = true;
+//       }
 
-      public bool isDisposed
-      {
-         get
-         {
-            return _is_disposed;
-         }
-      }
+//       public bool isDisposed {
+//          get {
+//             return _is_disposed;
+//          }
+//       }
 
-   }
-}
+//    }
+// }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs

@@ -0,0 +1,75 @@
+using System.Net.Sockets;
+using System.Net;
+
+using BCG;
+using System;
+using System.Collections.Generic;
+
+
+namespace bts {
+
+    // opens a TCP port and starts listening to connections
+    public class AsyncTCPConnectionAcceptor_APM {
+        private readonly int _port;
+        private Socket _accept_socket;
+        private ITCPConnectionHandler _connection_handler;
+        private HashSet<string> _key_map = new HashSet<string>();
+        public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) {
+            _port = port;
+            _connection_handler = connection_handler;
+        }
+
+        public void init() {
+            _init();
+            _begin_accepting();
+        }
+
+        private void _init() {
+            _accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+            IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port);
+            _accept_socket.Bind(endpoint);
+            _accept_socket.Listen(250);
+        }
+
+        private void _begin_accepting() {
+            var accept_key = Guid.NewGuid().ToString();
+            var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key);
+            if (res != null && res.IsCompleted && res.CompletedSynchronously)
+                _on_accept(res);
+        }
+
+        private void _on_accept(IAsyncResult res) {
+            bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}");
+            string key = res.AsyncState.ToString();
+            if (_key_map.Contains(key)) {
+                return;
+            }
+            _key_map.Add(key);
+            Socket new_sock;
+            try {
+                new_sock = _accept_socket.EndAccept(res);
+            } catch (Exception ex) {
+                bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept exception thrown by EndAccept ex: {ex}");
+                System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
+                _begin_accepting();
+                return;
+            }
+            new_sock.LingerState = new LingerOption(false, 0);
+            new_sock.Blocking = false;
+            ITCPConnection newConn = new AsyncTCPConnection_APM(new_sock, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server);
+            _connection_handler.onConnection(newConn);
+            // go back to accepting more conns
+            _begin_accepting();
+            string ip; int port;
+            newConn.getPeerAddress(out ip, out port);
+            string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, _port);
+            //         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
+            //Console.WriteLine(connMsg);       
+        }
+
+
+
+
+    }
+
+}

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -0,0 +1,584 @@
+//This implementation uses APM the beginFunc, endFunc pattern
+// https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/asynchronous-programming-model-apm#beginning-an-asynchronous-operation
+
+/// <summary>
+/// This class, mantains a connection to a peer socket and using the APM
+/// model to communicate. It sends heartbeats at regular
+/// intervals (config). It is in active state when messages
+/// are sent and received without an error state. It is assumed 
+/// dead state when either of the following happens
+/// BeginReceive returns a SocketError
+/// BeginSend returns a SocketError
+/// EndReceive returns a SocketError
+/// EndSend returns a SocketError
+/// As soon as it assumes dead state it stops sending or
+/// receiving any messages (including heartbeats).
+/// The user of this class can either detect that by
+/// missing heartbeats or checking the isConnected property.
+/// If the peer dies in a 'silent' way, where none of the above 
+/// methods indicate an error, missing heartbeats is guaranteed
+/// to detect it.
+/// </summary>
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Net;
+using System.Net.Sockets;
+using System.IO;
+using BCG.Utilities.General;
+using System.Runtime.InteropServices;
+using BCG;
+using System.Collections.ObjectModel;
+using System.Numerics;
+using System.Reflection.Metadata;
+using System.Threading.Tasks;
+using System.Linq;
+
+namespace bts {
+   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable {
+
+      // serves as the base class for both Send and ReeceiveMessage classes.
+      // since both TCP sends and receives happen in multiple transfers, this
+      // provides a wrapper around copying the partial transfers into the complete buffer
+      public class AsyncMessageBase {
+         readonly string msg_id = Guid.NewGuid().ToString();
+         // every BeginSend or BeginReceive has a unique key associated with it
+         public int key;
+         // total msg that needs to be transfered
+         public byte[] buffer;
+         // as data is transferred in chunks, transfer_ptr moves ahead by amount of data transferred.
+         // everything < transfer_ptr has been successfully transferred,
+         // ie this is pointing to where the next transfer will start from
+         public int transfer_ptr;
+         protected int _n_transfers = 0;
+         public AsyncMessageBase(byte[] the_buffer, int the_offset) {
+            buffer = the_buffer;
+            transfer_ptr = the_offset;
+         }
+         public int remaining_size {
+            get {
+               return buffer.Length - transfer_ptr;
+            }
+         }
+
+         public int original_size => buffer.Length;
+
+         public void advance_transfer_pointer(int advance_by) {
+            transfer_ptr += advance_by;
+            _n_transfers++;
+            if (transfer_ptr > original_size) {
+               bts.Log.Gen.error($"transfer went beyond initial size {ToString()}");
+            }
+            // TODO: log if it goes past the buffer
+         }
+
+         public bool fully_transfered() {
+            return transfer_ptr >= original_size;
+         }
+
+         public override string ToString() {
+            return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " +
+                   $"fully_transfered: {fully_transfered()}";
+         }
+      }
+      public class AsyncSendMessage : AsyncMessageBase {
+         public string message_id;
+         public SendConfirmationDelegate on_send_del;
+         //private int _n_transfers = 0;
+
+         public AsyncSendMessage(byte[] the_buffer, int the_offset)
+            : this(the_buffer, the_offset, String.Empty, null) { }
+
+         public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del)
+         : base(the_buffer, the_transfer_ptr) {
+            message_id = the_message_id;
+            on_send_del = the_on_send_del;
+         }
+         public override string ToString() {
+            return base.ToString();
+         }
+
+      }
+
+      public class AsyncReceiveMessage : AsyncMessageBase {
+         public enum ReadState {
+            reading_size,
+            reading_data
+         };
+         readonly ReadState read_state;
+
+         public AsyncReceiveMessage(int size, ReadState the_read_state) :
+         this(new byte[size], 0, the_read_state) { }
+         public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state)
+         : base(the_buffer, the_transfer_ptr) {
+            read_state = the_read_state;
+         }
+
+         public int read_int32(int start_idx = 0) {
+            return BitConverter.ToInt32(buffer, start_idx);
+         }
+
+         public byte[] read_bytes(int start_idx) {
+            byte[] rv = new byte[buffer.Length - start_idx];
+            Array.Copy(buffer, start_idx, rv, 0, rv.Length);
+            return rv;
+         }
+
+         public bool is_reading_size() => read_state == ReadState.reading_size;
+      }
+
+      public class ErrorMessage {
+         public readonly SocketError? socket_error = null;
+         public readonly Exception exception = null;
+         public readonly bool is_exception = false;
+         public readonly string context = String.Empty;
+         public ErrorMessage(SocketError socket_error_, string context_) {
+            socket_error = socket_error_;
+            context = context_;
+         }
+         public ErrorMessage(Exception exception_, string context_) {
+            is_exception = true;
+            exception = exception_;
+            context = context_;
+         }
+
+         public bool oom => exception != null && exception is OutOfMemoryException;
+
+         public override string ToString() {
+            if (is_exception)
+               return $"ErrorMessage exception: {exception} context: {context}";
+            return $"socket_error: {socket_error} context: {context}";
+         }
+
+
+      }
+
+      #region work around for .Net socket's LocalEndPoint bug
+
+      // [StructLayout(LayoutKind.Sequential)]
+      // private struct in_addr {
+      //    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)]
+      //    public byte[] sin_addr;
+      // }
+
+      // [StructLayout(LayoutKind.Sequential)]
+      // private struct sockaddr {
+      //    public short sin_family;
+      //    public ushort sin_port;
+      //    public in_addr sin_addr;
+      //    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
+      //    public byte[] sin_zero;
+      // }
+
+      // [DllImport("wsock32.dll")]
+      // private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen);
+
+      // [DllImport("wsock32.dll")]
+      // private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen);
+
+      #endregion
+
+      private const int DATA_MSG_SIGNAL = 0;
+      private const int HB_MSG_SIGNAL = 1;
+      private int _send_key = 0, _receive_key = 0;
+      private HashSet<int> _read_key_map = new HashSet<int>(),
+      _send_key_map = new HashSet<int>();
+      private readonly int _max_msg_size = PubSub3Config.instance.PS3_MaxPacketSize;
+      private readonly RemoteConnectionInfo _peer_conn_info, _self_conn_info;
+      private readonly ITCPConnection.ConnectionSide _connection_side;
+      private readonly Socket _sock;
+      private ITCPMessageHandler _message_handler = null;
+
+      private bool _is_alive = true;
+
+      private byte[] _cached_hb_message = null;
+      private readonly TimeSpan _hb_period = TimeSpan.FromSeconds(1);   // TODO get it from config
+      private MasterWorker _master_worker = MasterWorker.getInstance(true);
+      private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false;
+      private DateTime _last_hb_received_time = new DateTime(1900, 1, 1);
+
+      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) {
+         _sock = sock;
+         _hb_period = hb_period;
+         _connection_side = connection_side;
+         _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
+         string ip;
+         int port;
+         getPeerAddress(out ip, out port);
+         _peer_conn_info = new RemoteConnectionInfo(ip, port);
+         getSelfAddress(out ip, out port);
+         _self_conn_info = new RemoteConnectionInfo(ip, port);
+         bts.Log.Gen.info($"AsyncTCPConnection_APM created {ToString()}");
+      }
+
+      public override string ToString() {
+         return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}";
+      }
+
+      #region ITCPConnection Members
+
+      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
+         conf_id = Guid.NewGuid().ToString();
+         byte[] msg = _build_data_message(data);
+         _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del));
+      }
+
+      public void send(byte[] data) {
+         byte[] msg = _build_data_message(data);
+         _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
+      }
+
+      public void register(ITCPMessageHandler messageHandler) {
+         bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned");
+         bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered");
+         _message_handler = messageHandler;
+         _master_worker.addWorker(this, _hb_period);
+         // we send only when send is called but read continoulsy, starting here
+         _read_next_msg();
+      }
+
+      public void getPeerAddress(out string ip, out int port) {
+         ip = "";
+         port = 0;
+         if (_is_alive) {
+            IPEndPoint ipEp = _sock.RemoteEndPoint as IPEndPoint;
+            port = ipEp.Port;
+            ip = ipEp.Address.ToString();
+         }
+      }
+
+      public void getSelfAddress(out string ip, out int port) {
+         ip = "";
+         port = 0;
+         if (_is_alive) {
+            IPEndPoint ipEp = _sock.LocalEndPoint as IPEndPoint;
+            ip = ipEp.Address.ToString();
+            port = ipEp.Port;
+         }
+      }
+
+      public bool isDisposed {
+         get { return !_is_alive; }
+      }
+
+      #endregion
+
+      #region IWorker Members
+
+      bool _last_connected = false;
+      public void doWork() {
+         if (!_is_alive)
+            return;
+         return;
+         // disable hbs for now
+         if (!_hb_sent_once) {
+            bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time");
+            _hb_sent_once = true;
+         }
+         var msg = _build_hb_message();
+         _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
+      }
+
+      private static int _tid() {
+         var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
+         return tid;
+      }
+
+      private static string _cb_txt(IAsyncResult res) {
+         return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}";
+      }
+
+      #endregion
+
+      #region sending data
+
+      private void _begin_or_resume_sending(AsyncSendMessage message) {
+         bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}");
+         if (_is_alive) {
+            try {
+               SocketError begin_send_err;
+               // since we get message on on_send we dont need to keep the state around
+               message.key = _send_key++;
+               var res = _sock.BeginSend(message.buffer, message.transfer_ptr, message.remaining_size, SocketFlags.None,
+                                 out begin_send_err, new AsyncCallback(_on_send), message);
+               if (begin_send_err != SocketError.Success)
+                  _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending"));
+               else {
+                  // successfully sent, all we need to do here is make sure if it was sent asynchronously
+                  // if not call _on_send explicitely
+                  if (res != null && res.IsCompleted && res.CompletedSynchronously) {
+                     // for small messages on LANs this can happen, we will on_send to deal with it the same way                     
+                     Task.Run(() => {
+                        bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending calling _on_send explicitely");
+                        _on_send(res);
+                     });
+                  }
+               }
+            } catch (Exception ex) {
+               _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending"));
+            }
+         }
+      }
+
+      private void _on_send(IAsyncResult res) {
+         AsyncSendMessage message = res.AsyncState as AsyncSendMessage;
+         if (_send_key_map.Contains(message.key)) {
+            bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled");
+            return;
+         }
+         _send_key_map.Add(message.key);
+         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
+                                 $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
+         if (_is_alive) {
+            try {
+               SocketError send_err;
+               int bytes_sent = _sock.EndSend(res, out send_err);
+               //message.offset += bytesSent;
+               message.advance_transfer_pointer(bytes_sent);
+               if (bytes_sent == 0) {
+                  bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception");
+                  _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"),
+                        "_on_send"));
+                  return;
+               }
+               bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " +
+               $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive}");
+               if (send_err == SocketError.Success) {
+                  // successful send of some bytes
+                  //if (message.offset == message.buffer.Length) {
+                  if (message.fully_transfered()) {
+                     // fully sent, if confirmation needs to be sent out                     
+                     if (!String.IsNullOrEmpty(message.message_id)) {
+                        message.on_send_del(message.message_id, true);
+                     }
+                  } else {
+                     // there is more to send
+                     _begin_or_resume_sending(message);
+                  }
+               } else {
+                  _handle_error(new ErrorMessage(send_err, "_on_send"));
+               }
+            } catch (Exception ex) {
+               _handle_error(new ErrorMessage(ex, "_on_send"));
+            }
+         }
+      }
+
+      /// <summary>
+      /// Assembles the heartbeat message, and uses it for 
+      /// future references
+      /// </summary>
+      /// <returns></returns>
+      private byte[] _build_hb_message() {
+         if (_cached_hb_message == null) {
+            BinaryWriter bw = new BinaryWriter(new MemoryStream());
+            // the total msg =[msg_sz, msg_type] 
+            int msg_sz = 4;
+            bw.Write(msg_sz);
+            bw.Write(HB_MSG_SIGNAL);
+            _cached_hb_message = (bw.BaseStream as MemoryStream).ToArray();
+         }
+         return _cached_hb_message;
+      }
+
+      // writes size prefixed msg
+      private byte[] _build_data_message(byte[] data) {
+         // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter)
+         BinaryWriter bw = new BinaryWriter(new MemoryStream());
+         // the total msg =[msg_sz, msg_type, data] 
+         var msg_sz = data.Length + 4;
+         bw.Write(msg_sz); // +4 for the type of msg
+         // to indicate data vs hb
+         bw.Write(DATA_MSG_SIGNAL); // for data
+         // if we want to id every message, this is the place to do it
+         bw.Write(data);
+         return (bw.BaseStream as MemoryStream).ToArray();
+      }
+
+      #endregion
+
+      #region receiving data
+
+      private void _read_next_msg() {
+         _read_size();
+      }
+
+      private void _read_size() {
+         _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size));
+      }
+
+      private void _read_data(int data_size) {
+         // TODO, this could be a problem is the size is too big - maybe read it in chunks (later)
+         _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data));
+      }
+
+      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) {
+         bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}");
+         if (!_is_alive)
+            return;
+         try {
+            SocketError read_sock_err;
+            rcv_msg.key = _receive_key++;
+            var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr,
+                                 rcv_msg.remaining_size,
+                                 SocketFlags.None, out read_sock_err,
+                                 new AsyncCallback(_on_read),
+                                 rcv_msg);
+            if (read_sock_err != SocketError.Success) {
+               bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}");
+               _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading"));
+            } else {
+               // successfully started receiving
+               if (res != null && res.IsCompleted && res.CompletedSynchronously) {
+                  // make it follow the same code path
+                  Task.Run(() => {
+                     bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading calling _on_send explicitely");
+                     _on_read(res);
+                  }
+                     );
+               }
+            }
+         } catch (Exception ex) {
+            _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading"));
+         }
+      }
+
+      private void _on_read(IAsyncResult res) {
+         AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
+         if (_read_key_map.Contains(rcv_message.key)) {
+            bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled");
+            return;
+         }
+         _read_key_map.Add(rcv_message.key);
+         if (!_is_alive)
+            return;
+         try {
+            SocketError read_err;
+            int bytes_read = _sock.EndReceive(res, out read_err);
+            rcv_message.advance_transfer_pointer(bytes_read);
+            bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read: this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} bytes_read: {bytes_read} message: {res.AsyncState} _is_alive: {_is_alive} read_err: {read_err}");
+            if (bytes_read == 0) {
+               bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception");
+               _handle_error(new ErrorMessage(new ApplicationException("Read 0 bytes, peer socket is likely shutdown"), "_on_read"));
+               return;
+            }
+            //return;
+            if (read_err == SocketError.Success) {
+               if (rcv_message.fully_transfered()) {
+                  if (rcv_message.is_reading_size()) {
+                     var msg_size = _on_read_size_completely(rcv_message);
+                     if (msg_size != null && msg_size.HasValue) {
+                        _read_data(msg_size.Value);
+                     }
+                     return;
+                  } else {
+                     // data is read
+                     int msg_type = rcv_message.read_int32();
+                     bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL);
+                     if (msg_type == DATA_MSG_SIGNAL) {
+                        _on_read_data_completely(rcv_message);
+                        _read_next_msg();
+                        return;
+                        return;
+                     } else if (msg_type == HB_MSG_SIGNAL) {
+                        _on_read_hb_completely(rcv_message);
+                        _read_next_msg();
+                        return;
+                     }
+                     // go back to reading size for a new message
+                     //_read_size();
+                  }
+               } else {
+                  // msg has not been fully rd, we have already advanced the transfer ptr
+                  _begin_or_resume_reading(rcv_msg: rcv_message);
+               }
+            } else {
+               _handle_error(new ErrorMessage(read_err, "_on_read"));
+            }
+         } catch (Exception ex) {
+            _handle_error(new ErrorMessage(ex, "_on_read"));
+         }
+      }
+
+      #endregion
+
+      private int? _on_read_size_completely(AsyncReceiveMessage msg) {
+         bts.Log.assert(msg.is_reading_size());
+         bts.Log.assert(msg.fully_transfered());
+         bts.Log.assert(msg.original_size == 4);
+         var msg_sz = msg.read_int32();
+         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely this: {this} msg: {msg} msg_sz: {msg_sz}");
+         if (msg_sz > _max_msg_size) {
+            string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}";
+            _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read"));
+            return null;
+         } else {
+            // sz is determined, now we read that many bytes
+            //_read_data(msg_sz);
+            return msg_sz;
+         }
+      }
+
+      private void _on_read_hb_completely(AsyncReceiveMessage msg) {
+         bts.Log.assert(!msg.is_reading_size());
+         bts.Log.assert(msg.fully_transfered());
+         bts.Log.assert(msg.original_size == 4);
+         bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL);
+         if (!_hb_received_yet) {
+            _hb_received_yet = true;
+            bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}");
+         }
+         _last_hb_received_time = DateTime.Now;
+         _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
+      }
+      private void _on_read_data_completely(AsyncReceiveMessage msg) {
+         bts.Log.assert(!msg.is_reading_size());
+         bts.Log.assert(msg.fully_transfered());
+         var type = msg.read_int32(0);
+         bts.Log.assert(type == DATA_MSG_SIGNAL);
+         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_data_completely this: {this} msg: {msg} type: {type}");
+         byte[] data = msg.read_bytes(4);
+         ///callbacks can be made through thread pool to avoid this
+         ///thread from being blocked, by this callback
+         if (!_data_received_yet) {
+            _data_received_yet = true;
+            bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}");
+         }
+         // could be an index here
+         _message_handler.onData(new RemoteMessage(data, RemoteMessageType.Data, _peer_conn_info));
+      }
+
+      #region IDisposable Members
+
+      public void Dispose() {
+         // socket is closed and disposed here
+         lock (this) {
+            if (_is_alive) {
+               _master_worker.removeWorker(this);
+               _is_alive = false;
+               bts.Log.Gen.info($"AsycnTCPConnection.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
+               try {
+                  _sock.Shutdown(SocketShutdown.Both);
+               } catch (Exception ex) {
+                  String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
+                  if (ex is OutOfMemoryException)
+                     throw ex;
+               }
+               _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info));
+            }
+         }
+      }
+
+      #endregion
+
+      private void _handle_error(ErrorMessage err_msg) {
+         lock (this) {
+            Log.Gen.info($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
+            if (err_msg.oom)
+               throw err_msg.exception;
+            Dispose();
+         }
+      }
+   }
+}

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_SAEArgs.cs

@@ -0,0 +1,2 @@
+// this implementation uses SocketAsyncEventArgs
+// https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socketasynceventargs?view=net-8.0

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_TAP.cs

@@ -0,0 +1,2 @@
+//This implementation uses Task-based asynchronous pattern (TAP)
+//https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/task-based-asynchronous-pattern-tap

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs

@@ -0,0 +1,65 @@
+using System.Net.Sockets;
+using System.Net;
+
+using BCG;
+using System;
+using System.Collections.Generic;
+
+
+namespace bts {
+    // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
+    // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
+    // statefullness and hence non static
+    public static class AsyncTCPConnector_APM {
+
+        internal class SocketConnectionArgs {
+            public EndPoint ep;
+            public Socket sock;
+            public ITCPConnectionHandler handler;
+            public string server_url;
+
+            internal SocketConnectionArgs(EndPoint theEp, Socket the_sock, ITCPConnectionHandler the_handler,
+               string theServerURL) {
+                ep = theEp;
+                sock = the_sock;
+                handler = the_handler;
+                server_url = theServerURL;
+            }
+
+            public override string ToString() {
+                return $"SocketConnectionArgs: ep: {ep} server_url: {server_url}";
+            }
+        }
+
+        public static void connect_to(BlutoSubAddress server_address, ITCPConnectionHandler handler) {
+            bts.Log.Gen.info($"AsyncTCPConnector_APM.connectTo server: {server_address}");
+            // when connection is established, this sock passed to an ITCPCOnnection and sent back
+            Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+            EndPoint ep = new IPEndPoint(IPAddress.Parse(server_address.ip), server_address.port);
+            sock.BeginConnect(ep, new AsyncCallback(_on_connect),
+                            new SocketConnectionArgs(ep, sock, handler, server_address.ToString()));
+        }
+
+        private static void _on_connect(IAsyncResult res) {
+            bts.Log.Gen.info("AsyncTCPConnector_APM._on_connect " + res.ToString() + " " + res.AsyncState.ToString());
+            SocketConnectionArgs sc_args = res.AsyncState as SocketConnectionArgs;
+            try {
+                sc_args.sock.EndConnect(res);
+            } catch (SocketException socket_ex) {
+                bts.Log.Gen.info($"AsyncTCPConnector_APM._on_connect EndConnect failed, args: {sc_args} ex: {socket_ex}");
+                if (sc_args.handler.isDisposed) {
+                    return;
+                } else {
+                    System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(1000));
+                    bts.Log.Gen.info($"AsyncTCPConnector_APM retrying connectiont to server: {sc_args.server_url}");
+                    sc_args.sock.BeginConnect(sc_args.ep, new AsyncCallback(_on_connect), sc_args);
+                    return;
+                }
+            }
+            bts.Log.Gen.info($"AsyncTCPConnector_APM._on_connect EndConnect successful args: {sc_args}");
+            sc_args.handler.onConnection(new AsyncTCPConnection_APM(sc_args.sock, TimeSpan.FromSeconds(1),
+                                        ITCPConnection.ConnectionSide.Client));
+        }
+
+    }
+}

src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs

@@ -1,82 +1,82 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Net;
-using System.Net.Sockets;
-using System.Threading;
-using System.Threading.Tasks;
-//using BCG.Utilities;
-//using BCG.Utilities.General;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Net;
+// using System.Net.Sockets;
+// using System.Threading;
+// using System.Threading.Tasks;
+// //using BCG.Utilities;
+// //using BCG.Utilities.General;
 
-namespace BCG {
-   internal class SocketConnectionArgs {
-      public EndPoint ep;
-      public Socket sock;
-      public ITCPConnectionHandler handler;
-      public string serverURL;
+// namespace BCG {
+//    internal class SocketConnectionArgs {
+//       public EndPoint ep;
+//       public Socket sock;
+//       public ITCPConnectionHandler handler;
+//       public string serverURL;
 
-      internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
-         string theServerURL) {
-         ep = theEp;
-         sock = theSock;
-         handler = theHandler;
-         serverURL = theServerURL;
-      }
-   }
+//       internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
+//          string theServerURL) {
+//          ep = theEp;
+//          sock = theSock;
+//          handler = theHandler;
+//          serverURL = theServerURL;
+//       }
+//    }
 
-   public class ClientToServerManagedTCPConnector {
-      private static Random myRnd = new Random(DateTime.Now.Millisecond);
-      private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
-      private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
-                           myWaitTimeBeforeRetryingLowerLimit = 1000;
+//    public class ClientToServerManagedTCPConnector {
+//       private static Random myRnd = new Random(DateTime.Now.Millisecond);
+//       private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
+//       private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
+//                            myWaitTimeBeforeRetryingLowerLimit = 1000;
 
-      public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
-         Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-         Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-         EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
-         sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
-      }
+//       public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
+//          Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+//          EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
+//          sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
+//       }
 
-      public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
-         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-         await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
-      }
+//       public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+//          bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+//       }
 
-      public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
-         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-         Task.Run(async () => {
-            await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
-         });
-      }
+//       public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+//          bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          Task.Run(async () => {
+//             await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+//          });
+//       }
 
-      private static void myOnConnect(IAsyncResult res) {
-         Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
-         SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
-         try {
-            args.sock.EndConnect(res);
-         } catch (SocketException sex) {
-            string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
-            //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
-            //Console.WriteLine(errMsg);
-            if (args.handler.isDisposed) {
-               string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
-               //Console.WriteLine(doneMsg);
-               //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
-               return;
-            } else {
-               string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
-               //Console.WriteLine(retryMsg);
-               //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
+//       private static void myOnConnect(IAsyncResult res) {
+//          Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
+//          SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
+//          try {
+//             args.sock.EndConnect(res);
+//          } catch (SocketException sex) {
+//             string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
+//             //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
+//             //Console.WriteLine(errMsg);
+//             if (args.handler.isDisposed) {
+//                string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
+//                //Console.WriteLine(doneMsg);
+//                //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
+//                return;
+//             } else {
+//                string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
+//                //Console.WriteLine(retryMsg);
+//                //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
 
-               int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit);
-               Thread.Sleep(waitTime);
-               args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args);
-               return;
-            }
-         }
-         //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection);
-         args.handler.onConnection(new AsyncTCPConnection(args.sock, myHBTimeSpan));
-      }
+//                int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit);
+//                Thread.Sleep(waitTime);
+//                args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args);
+//                return;
+//             }
+//          }
+//          //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection);
+//          args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client));
+//       }
 
-   }
-}
+//    }
+// }

src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs

@@ -9,6 +9,10 @@ namespace BCG {
    /// on top of the sockets (used on both server and client)
    /// </summary>
    public interface ITCPConnection : IDisposable {
+      public enum ConnectionSide {
+         Client,
+         Server
+      }
       /// <summary>
       /// The delegate is fired signalling if the data send was
       /// successful or not

src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs

@@ -22,20 +22,20 @@ namespace BCG {
          return portNo;
       }
 
-      public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
-         lock (mySyncObject) {
-            if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
-               string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
-               // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
-               return port;
-            } else {
-               SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
-               int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan);
-               myPortToSingleServerAcceptorMap[assignedport] = theAcceptor;
-               return assignedport;
-            }
-         } //end lock
-      } //end activate code
+      // public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
+      //    lock (mySyncObject) {
+      //       if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
+      //          string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
+      //          // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
+      //          return port;
+      //       } else {
+      //          SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
+      //          int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan);
+      //          myPortToSingleServerAcceptorMap[assignedport] = theAcceptor;
+      //          return assignedport;
+      //       }
+      //    } //end lock
+      // } //end activate code
 
       public static int accept_connections(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
          lock (mySyncObject) {

src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs

@@ -89,7 +89,7 @@ namespace BCG {
          }
          newSock.LingerState = new LingerOption(false, 0);
          newSock.Blocking = false;
-         ITCPConnection newConn = new AsyncTCPConnection(newSock, myHBTimeSpan);
+         ITCPConnection newConn = new bts.AsyncTCPConnection_APM(newSock, myHBTimeSpan, ITCPConnection.ConnectionSide.Server);
          myHandler.onConnection(newConn);
          myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
          string ip; int port;

src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs

@@ -11,38 +11,34 @@ using System.Xml.Serialization;
 using System.Reflection;
 using BCG.Utilities.General;
 using System.Net.WebSockets;
+using bts;
+using System.Threading;
+using Google.Protobuf.WellKnownTypes;
 
-namespace BCG
-{
+namespace BCG {
    // utility functions for TCP socket async communications. Does the following
    // client to server connect
    // server to client connect accepting
    // send data
    // receive data
-   public static class TCPSocketUtils
-   {
+   public static class TCPSocketUtils {
       #region socket opening and connecting
       // keeps accepting connections forever
       public static async Task accept_connections_async(string ip, int port, ITCPConnectionHandler handler,
-                                                            TimeSpan wait_after_error)
-      {
+                                                            TimeSpan wait_after_error) {
          IPAddress ipAddress = String.IsNullOrEmpty(ip) ? IPAddress.Parse(ip) : IPAddress.Any;
          var ip_end_point = new IPEndPoint(ipAddress, port);
          Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
          listener.Bind(ip_end_point);
          listener.Listen(10);
          Socket client_socket = null;
-         while (true)
-         {
-            try
-            {
+         while (true) {
+            try {
                // keeps on accepting connections
                client_socket = await listener.AcceptAsync();
-               AsyncTCPConnection2 new_conn = new(client_socket, AsyncTCPConnection2.ConnectionSide.Server);
+               ITCPConnection new_conn = new bts.AsyncTCPConnection_APM(client_socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server);
                handler.onConnection(new_conn);
-            }
-            catch (Exception ex)
-            {
+            } catch (Exception ex) {
                bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again");
                Task.Delay(wait_after_error);
             }
@@ -52,25 +48,20 @@ namespace BCG
       // connects to the server and ona  successful connection, calls con__handler.on_connection_async with an ITCPConnection.
       // In case of failures to connect it waits for a given amount of time before retrying 
       public static async Task<Socket> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
-                                                TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024)
-      {
+                                                TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024) {
          Socket client_socket = null;
-         for (int i = 0; i < n_max_attempts; i++)
-         {
-            try
-            {
+         for (int i = 0; i < n_max_attempts; i++) {
+            try {
                string log_conn = $"{ip}:{server_port}";
                bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} attempt no: {i}");
                IPAddress server_ip_address = IPAddress.Parse(ip);
                client_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                await client_socket.ConnectAsync(server_ip_address, server_port);
                bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} successfully connected after {i} attempts");
-               ITCPConnection new_conn = new AsyncTCPConnection2(client_socket, AsyncTCPConnection2.ConnectionSide.Client);
+               ITCPConnection new_conn = new bts.AsyncTCPConnection_APM(client_socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client);
                conn_handler.onConnection(new_conn);
                break;
-            }
-            catch (Exception ex)
-            {
+            } catch (Exception ex) {
                bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now");
                Task.Delay(wait_after_failed_attempt);
             }
@@ -83,31 +74,116 @@ namespace BCG
 
       // prefixes the size of the data and sends it as 2 sends, for size and data
       // (can be done as one send perhaps)
-      public static async Task send_bytes_async(NetworkStream nw_stream, byte[] data_buff)
-      {
-         var data_sz = data_buff.Length;
+      public static async Task send_bytes_async(NetworkStream nw_stream, byte[] data_buff) {
+         Log.Gen.info($"send_bytes_async data_buff: {data_buff.Length}");
+         // var data_sz = data_buff.Length;
          byte[] sz_buff = new byte[4];
-         // aend the size first
+         // // send the size first
          BinaryWriter bw = new BinaryWriter(new MemoryStream(sz_buff));
-         bw.Write(data_sz);
-         await nw_stream.WriteAsync(sz_buff, 0, 4);
-         await nw_stream.WriteAsync(data_buff, 0, data_buff.Length);
+         bw.Write(data_buff.Length);
+         await SendMessageAsync(nw_stream.Socket, new Memory<byte>(sz_buff));
+         // await nw_stream.WriteAsync(sz_buff, 0, 4);
+         // await nw_stream.WriteAsync(data_buff, 0, data_buff.Length);
+         Memory<byte> mem = new Memory<byte>(data_buff);
+         await SendMessageAsync(nw_stream.Socket, mem);
       }
 
+      public static async Task SendMessageAsync(Socket sock, Memory<byte> mem_buff) {
+         // use ManualResetEvent to make it non async
+         Log.Gen.info($"SendMessageAsync mem: {mem_buff.Length}");
+         SocketAsyncEventArgs evargs = new SocketAsyncEventArgs();
+         evargs.SetBuffer(mem_buff);
+         var tcs = new TaskCompletionSource<bool>();
+         evargs.Completed += (s, e) => {
+            Log.Gen.info($"SendMessageAsync completed asynchronously e: {sock_event_args_txt(e)}");
+            tcs.SetResult(true);
+         };
+         bool completed = !sock.SendAsync(evargs);
+         if (completed) {
+            Log.Gen.info($"SendMessageAsync completed synchronously e: {sock_event_args_txt(evargs)}");
+            tcs.SetResult(true);
+         }
+         await tcs.Task;
+
+      }
       // reads the bytes on a connected socket, by doing the opposite of the above,
       // reads the size first and then the data
-      public static async Task<byte[]> read_bytes_async(NetworkStream nw_stream)
-      {
-         byte[] sz_buffer = new byte[4];
-         //Asynchronously reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read.
-         int bytesRead = await nw_stream.ReadAsync(sz_buffer, 0, sz_buffer.Length);
-         BinaryReader br = new BinaryReader(new MemoryStream(sz_buffer));
-         int msg_sz = br.ReadInt32();
-         byte[] data_buffer = new byte[msg_sz];
-         bytesRead = await nw_stream.ReadAsync(data_buffer, 0, data_buffer.Length);
-         //Console.WriteLine($"read_bytes sz: {data_buffer.Length}");
-         return data_buffer;
+      public static async Task<byte[]> read_bytes_async(NetworkStream nw_stream) {
+         // byte[] data_buffer;
+         // try {
+         //    byte[] sz_buffer = new byte[4];
+         //    //Asynchronously reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read.
+         //    int bytesRead = await nw_stream.ReadAsync(sz_buffer, 0, sz_buffer.Length);
+         //    BinaryReader br = new BinaryReader(new MemoryStream(sz_buffer));
+         //    int msg_sz = br.ReadInt32();
+         //    data_buffer = new byte[msg_sz];
+         //    bytesRead = await nw_stream.ReadAsync(data_buffer, 0, data_buffer.Length);
+         //    Log.Gen.info($"read_bytes_async intended: {msg_sz} received: {bytesRead}");
+         // } catch (Exception ex) {
+         //    Log.Gen.error($"read_bytes_async exception: {ex}");
+         //    return new byte[] { };
+         // }
+         // //Console.WriteLine($"read_bytes sz: {data_buffer.Length}");
+         // return data_buffer;
+         return ReceiveSizePrefixedMessageAsync(nw_stream.Socket);
+      }
+
+      static string sock_event_args_txt(SocketAsyncEventArgs seargs) {
+         return $"err: {seargs.SocketError} bytes: {seargs.BytesTransferred}";
+      }
+
+      // this is a limited sync wrapper ovr the async ReceiveAsync
+      static byte[] ReceiveSizePrefixedMessageAsync(Socket socket) {
+         const int buffer_size = 4; // Size of the buffer to read the message size
+
+         var size_buffer = new byte[buffer_size];
+         var size_event_args = new SocketAsyncEventArgs();
+         size_event_args.SetBuffer(size_buffer, 0, buffer_size);
+         var size_completed_event = new ManualResetEvent(false);
+
+         size_event_args.Completed += (sender, e) => {
+            Log.Gen.info($"ReceiveSizePrefixedMessageAsync async completed (msg len) e: {sock_event_args_txt(e)}");
+            size_completed_event.Set(); // Signal that the size receive operation is complete
+         };
+
+         if (!socket.ReceiveAsync(size_event_args)) {
+            Log.Gen.info($"ReceiveSizePrefixedMessageAsync msg size read completed synchronously e: {sock_event_args_txt(seargs: size_event_args)}");
+            // If ReceiveAsync completed synchronously, set the event manually
+            size_completed_event.Set();
+         }
+
+         size_completed_event.WaitOne(); // Wait for the size receive operation to complete
+
+         int message_size = BitConverter.ToInt32(size_buffer, 0);
+         //Log.assert_d(message_size > 0, $"message_size: {message_size}");
+         if (message_size <= 0)
+            throw new ApplicationException($"msg_size: {message_size} not allowed");
+         Log.Gen.info($"ReceiveSizePrefixedMessageAsync message_size: {message_size}");
+         // at this ppint message sz has been read, now read the data
+
+         var message_buffer = new byte[message_size];
+         var message_event_args = new SocketAsyncEventArgs();
+         message_event_args.SetBuffer(message_buffer, 0, message_size);
+         var message_completed_event = new ManualResetEvent(false);
+
+         message_event_args.Completed += (sender, e) => {
+            Log.Gen.info($"ReceiveSizePrefixedMessageAsync async completed (msg) e: {sock_event_args_txt(e)}");
+            message_completed_event.Set(); // Signal that the message receive operation is complete
+         };
+
+         if (!socket.ReceiveAsync(message_event_args)) {
+            Log.Gen.info($"ReceiveSizePrefixedMessageAsync msg read completed synchronously e: {sock_event_args_txt(seargs: message_event_args)}");
+            // If ReceiveAsync completed synchronously, set the event manually
+            message_completed_event.Set();
+         }
+
+         message_completed_event.WaitOne(); // Wait for the message receive operation to complete
+
+         return message_buffer;
       }
-      #endregion
    }
+
+
+
+   #endregion
 }

src/DataStreamSuite/PubSub/PubSubUpdate.cs

@@ -4,19 +4,22 @@ using System.IO;
 
 namespace BCG {
    /// <summary>
-   /// this is the data returned from the
-   /// pubsub server
+   /// this is the data returned from the pubsub server
    /// </summary>
    public class PubSubUpdate {
 
-      public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; }
+      public PubSubUpdate(string tag, string the_tag_type, Stream body) {
+         this.tag = tag;
+         this.body = body;
+         this.tag_type = the_tag_type;
+      }
 
       public string tag;
       public Stream body;
-      public string tagType;
+      public string tag_type;
 
       public override string ToString() {
-         return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}";
+         return $"PubSubUpdate tag: {tag} tag_type: {tag_type} len: {body.Position}";
       }
    };
 }

src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs

@@ -5,18 +5,16 @@ using System.IO;
 using System.Collections.Generic;
 using mbts;
 
-namespace BCG
-{
-   public delegate void ConnectionStatusChange(string peerURL, bool connection);
-   public delegate void DataDelegate(string sender, byte[] data);
+namespace BCG {
+   public delegate void PubSubServerConnectionStatusChange(string peerURL, bool connection);
+   public delegate void PubSubServerDataDelegate(string sender, byte[] data);
    /// <summary>
    /// Encapsulates PubsubServer's communication with the
    /// rest of the world. Has send and get methods, and also
    /// an event to tell if a client connects or disconnects.
    /// this event is prompted by the doWork.
    /// </summary>
-   public interface IPubsubMessageHub : IWorker
-   {
+   public interface IPubsubMessageHub : IWorker {
 
       /// <summary>
       /// The port that the message hub is listening on.  Return -1 if you're not listening on a port, i.e. for hte case of multicast
@@ -29,14 +27,14 @@ namespace BCG
       /// getUpdates. The delegate will be fired on an arbitrary thread.
       /// </summary>
       /// <param name="?"></param>
-      void registerDataCallback(DataDelegate del);
+      void registerDataCallback(PubSubServerDataDelegate del);
       /// <summary>
       /// registers a connection callback. this callback
       /// will be fired within the doWork call.
       /// </summary>
       /// <param name="del"></param>
       /// <param name="register"></param>
-      void registerConnectionCallback(ConnectionStatusChange del);
+      void registerConnectionCallback(PubSubServerConnectionStatusChange del);
       /// <summary>
       /// this is where this interface starts communication with the
       /// rest of the world. Obvioulsy, the resgiters should be

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -22,8 +22,8 @@ namespace BCG {
 
       }
       private bool myActivated = false;
-      private ConnectionStatusChange myConnCb;
-      private DataDelegate myDataCb;
+      private PubSubServerConnectionStatusChange myConnCb;
+      private PubSubServerDataDelegate myDataCb;
 
       /// <summary>
       /// client to connection
@@ -77,12 +77,12 @@ namespace BCG {
 
       #region IPubsubMessageHub Members
 
-      public void registerDataCallback(DataDelegate del) {
+      public void registerDataCallback(PubSubServerDataDelegate del) {
          // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate");
          myDataCb = del;
       }
 
-      public void registerConnectionCallback(ConnectionStatusChange del) {
+      public void registerConnectionCallback(PubSubServerConnectionStatusChange del) {
          //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate");
          myConnCb = del;
       }

src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs

@@ -4,25 +4,22 @@ using System.Threading;
 using System.IO;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Wraps around all the IPubsubMessageHub's that pubsub supports.
    /// ie MSMQ and TCP to begin with.
    /// </summary>
-   public class ServerMessageHub : IPubsubMessageHub
-   {
+   public class ServerMessageHub : IPubsubMessageHub {
       private static ServerMessageHub mySelf = null;
       private static object myInstanceLock = new object();
       private Dictionary<string, IPubsubMessageHub> myHubs = new Dictionary<string, IPubsubMessageHub>();
       private SyncQueue<Pair<string, byte[]>> myReceivingQueue = new SyncQueue<Pair<string, byte[]>>();
-      private ConnectionStatusChange myConnectionCb;
-      private DataDelegate myDataCb;
+      private PubSubServerConnectionStatusChange myConnectionCb;
+      private PubSubServerDataDelegate myDataCb;
       private MasterWorker myMasterWorker = MasterWorker.getInstance(false);
 
       protected ServerMessageHub() : this(PubSub3Config.instance.PS3_ServerPort) { }
-      protected ServerMessageHub(int portNumber)
-      {
+      protected ServerMessageHub(int portNumber) {
          // TODO: no more MSMQ
          // if (!PubSub3Config.instance.PS3_DisableMSMQ)
          //    myHubs.Add(MSMQSubAddress.prefix, new MSMQMessageHub());
@@ -35,19 +32,14 @@ namespace BCG
                                                       as IPubsubMessageHub);
       }
 
-      public int port
-      {
+      public int port {
          get { return myHubs[BlutoSubAddress.prefix].port; }
 
       }
-      public static ServerMessageHub instance
-      {
-         get
-         {
-            if (mySelf == null)
-            {
-               lock (myInstanceLock)
-               {
+      public static ServerMessageHub instance {
+         get {
+            if (mySelf == null) {
+               lock (myInstanceLock) {
                   if (mySelf == null)
                      mySelf = new ServerMessageHub();
                }
@@ -60,47 +52,39 @@ namespace BCG
 
       /* Builds the server message hub with a specified port number. Only works if the server message hub has not already been created. 
        * */
-      public static ServerMessageHub getInstance(int portNumber)
-      {
-         if (mySelf == null)
-         {
-            lock (myInstanceLock)
-            {
+      public static ServerMessageHub getInstance(int portNumber) {
+         if (mySelf == null) {
+            lock (myInstanceLock) {
                if (mySelf == null)
                   mySelf = new ServerMessageHub(portNumber);
             }
 
-         }
-         else
+         } else
             throw new ArgumentException("Error: Cannot specify a port number for the ServerMessageHub instance if it has already been created.");
          return mySelf;
       }
       #region IPubsubMessageHub Members
 
-      public void registerDataCallback(DataDelegate del) { myDataCb = del; }
+      public void registerDataCallback(PubSubServerDataDelegate del) { myDataCb = del; }
 
-      public void registerConnectionCallback(ConnectionStatusChange del) { myConnectionCb = del; }
+      public void registerConnectionCallback(PubSubServerConnectionStatusChange del) { myConnectionCb = del; }
 
-      public void activate()
-      {
+      public void activate() {
          string msg = "PubSubServer started .. initializing TCP transport components .. ";
          Console.WriteLine(msg);
-         foreach (IPubsubMessageHub hub in myHubs.Values)
-         {
-            hub.registerDataCallback(new DataDelegate(myOnData));
-            hub.registerConnectionCallback(new ConnectionStatusChange(myOnConnection));
+         foreach (IPubsubMessageHub hub in myHubs.Values) {
+            hub.registerDataCallback(new PubSubServerDataDelegate(myOnData));
+            hub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myOnConnection));
             hub.activate();
          }
          myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, 250));
       }
 
-      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements)
-      {
+      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements) {
          return myReceivingQueue.dequeueMultipleElements(timeout, maxElements);
       }
 
-      public bool send(string receiver, byte[] data)
-      {
+      public bool send(string receiver, byte[] data) {
          if (receiver.StartsWith(BlutoSubAddress.prefix))
             return myHubs[BlutoSubAddress.prefix].send(receiver, data);
          else if (receiver.StartsWith(MSMQSubAddress.prefix))
@@ -109,8 +93,7 @@ namespace BCG
             return false;
       }
 
-      public bool send(string receiver, Stream str)
-      {
+      public bool send(string receiver, Stream str) {
          if (receiver.StartsWith(BlutoSubAddress.prefix))
             return myHubs[BlutoSubAddress.prefix].send(receiver, str);
          else if (receiver.StartsWith(MSMQSubAddress.prefix))
@@ -126,8 +109,7 @@ namespace BCG
       /// <summary>
       /// connection callback will be called within this doWork
       /// </summary>
-      public void doWork()
-      {
+      public void doWork() {
          foreach (IPubsubMessageHub hub in myHubs.Values)
             hub.doWork();
       }
@@ -138,21 +120,18 @@ namespace BCG
       public static string internalPublisherConnection = "InternalPublisher";
 
 
-      public void internalSend(byte[] data)
-      {
+      public void internalSend(byte[] data) {
          myReceivingQueue.enqueue(new Pair<string, byte[]>(internalPublisherConnection, data));
       }
 
-      private void myOnData(string sender, byte[] msg)
-      {
+      private void myOnData(string sender, byte[] msg) {
          if (myDataCb != null)
             myOnData(sender, msg);
          else
             myReceivingQueue.enqueue(new Pair<string, byte[]>(sender, msg));
       }
 
-      private void myOnConnection(string clientURL, bool connected)
-      {
+      private void myOnConnection(string clientURL, bool connected) {
          if (myConnectionCb != null)
             myConnectionCb(clientURL, connected);
       }

src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs

@@ -58,7 +58,7 @@ namespace BCG {
 
          myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc));
          myMessageHub = ServerMessageHub.instance;
-         myMessageHub.registerConnectionCallback(new ConnectionStatusChange(myHandleConnections));
+         myMessageHub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myHandleConnections));
 
 
       }

src/DataStreamSuite/PubSub/Server/PubSubState.cs

@@ -55,7 +55,7 @@ namespace BCG {
             myTagToPubsubMap[tag].onSub(sub);
 
             if (onSingleTagChange != null)
-               onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribingApp));
+               onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
          }
          if (onTagMapChanged != null)
             onTagMapChanged(sub.tags);
@@ -68,10 +68,10 @@ namespace BCG {
                ///and publishers for the tag then remove it.
                myTagToPubsubMap[tag].onUnsub(unsub);
                if (onSingleTagChange != null)
-                  onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp));
+                  onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
             } else {
                string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
-                  " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp);
+                  " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
                //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             }
          }

src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs

@@ -1,19 +1,19 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+
 using BCG.Utilities;
+
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// SingleTagPubSubMap consists of all the state w.r.t to
    /// publishing and subscription of a single tag. this information
    /// consists of the publisher, and a map of the subscriber
    /// connection vs the apps they're coming from 
    /// </summary>
-   internal class SingleTagPubSubMap
-   {
+   internal class SingleTagPubSubMap {
       private string myTag;
       /// <summary>
       /// map of clientURL to client app (machine;app;pid;clientURL)
@@ -29,8 +29,7 @@ namespace BCG
       /// </summary>
       List<Pair<string, string>> myDeadPubConnToApp = new List<Pair<string, string>>();
 
-      public SingleTagPubSubMap(string tag)
-      {
+      public SingleTagPubSubMap(string tag) {
          myTag = tag;
       }
 
@@ -38,8 +37,7 @@ namespace BCG
       /// Returns a list all the active connections that 
       /// have subscribed to this tag
       /// </summary>
-      public ICollection<string> subscriberConnections
-      {
+      public ICollection<string> subscriberConnections {
          get { return mySubConnToApp.Keys; }
       }
 
@@ -47,8 +45,7 @@ namespace BCG
       /// Returns a list of all the applications that have
       /// subscribed to this tag
       /// </summary>
-      public ICollection<string> subscripingApps
-      {
+      public ICollection<string> subscripingApps {
          get { return mySubConnToApp.Values; }
       }
 
@@ -57,8 +54,7 @@ namespace BCG
       /// connections as key and the application
       /// as value
       /// </summary>
-      public Dictionary<string, string> subscribers
-      {
+      public Dictionary<string, string> subscribers {
          get { return mySubConnToApp; }
       }
 
@@ -66,8 +62,7 @@ namespace BCG
       /// Returns a dictionary for all active publishing
       /// connection as key and the application as value
       /// </summary>
-      public Dictionary<string, string> activePublishers
-      {
+      public Dictionary<string, string> activePublishers {
          get { return myActivePubConnToApp; }
       }
 
@@ -75,8 +70,7 @@ namespace BCG
       /// Returns a list of pair of dead connections and
       /// theur corresponding applications
       /// </summary>
-      public ICollection<Pair<string, string>> deadPublishers
-      {
+      public ICollection<Pair<string, string>> deadPublishers {
          get { return myDeadPubConnToApp; }
       }
 
@@ -87,18 +81,14 @@ namespace BCG
       /// </summary>
       /// <param name="sub"></param>
       /// <returns></returns>
-      public bool onSub(SubMetaData sub)
-      {
-         if (mySubConnToApp.ContainsKey(sub.connection))
-         {
+      public bool onSub(SubMetaData sub) {
+         if (mySubConnToApp.ContainsKey(sub.connection)) {
             string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}"
-               + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribingApp);
+               + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribing_app);
             //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             return false;
-         }
-         else
-         {
-            mySubConnToApp.Add(sub.connection, sub.subscribingApp);
+         } else {
+            mySubConnToApp.Add(sub.connection, sub.subscribing_app);
             return true;
          }
       }
@@ -110,17 +100,13 @@ namespace BCG
       /// </summary>
       /// <param name="unsub"></param>
       /// <returns></returns>
-      public bool onUnsub(UnSubMetaData unsub)
-      {
-         if (mySubConnToApp.ContainsKey(unsub.connection))
-         {
+      public bool onUnsub(UnSubMetaData unsub) {
+         if (mySubConnToApp.ContainsKey(unsub.connection)) {
             mySubConnToApp.Remove(unsub.connection);
             return true;
-         }
-         else
-         {
+         } else {
             string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}",
-               myTag, unsub.connection, unsub.unsubscribingApp);
+               myTag, unsub.connection, unsub.unsubscribing_app);
             //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             return false;
          }
@@ -133,25 +119,20 @@ namespace BCG
       /// </summary>
       /// <param name="subscriber"></param>
       /// <returns></returns>
-      public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber)
-      {
+      public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber) {
          bool rv = false;
-         if (mySubConnToApp.ContainsKey(connection))
-         {
+         if (mySubConnToApp.ContainsKey(connection)) {
             mySubConnToApp.Remove(connection);
             wasSubscriber = true;
             rv = true;
-         }
-         else
+         } else
             wasSubscriber = false;
-         if (myActivePubConnToApp.ContainsKey(connection))
-         {
+         if (myActivePubConnToApp.ContainsKey(connection)) {
             myDeadPubConnToApp.Add(new Pair<string, string>(connection, myActivePubConnToApp[connection]));
             myActivePubConnToApp.Remove(connection);
             rv = true;
             wasPublisher = true;
-         }
-         else
+         } else
             wasPublisher = false;
          return rv;
       }
@@ -162,14 +143,10 @@ namespace BCG
       /// </summary>
       /// <param name="pub"></param>
       /// <param name="publisherChanged"></param>
-      public void onPub(PubMetaData pub, out bool newPublisher)
-      {
-         if (myActivePubConnToApp.ContainsKey(pub.connection))
-         {
+      public void onPub(PubMetaData pub, out bool newPublisher) {
+         if (myActivePubConnToApp.ContainsKey(pub.connection)) {
             newPublisher = false;
-         }
-         else
-         {
+         } else {
             myActivePubConnToApp.Add(pub.connection, pub.publishingApp);
             newPublisher = true;
          }

src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs

@@ -6,92 +6,79 @@ using BCG.Utilities;
 using BCG.Utilities.General;
 using mbts;
 using mbts.spark;
+using System.Linq;
 
 namespace BCG {
    /// <summary>
-   /// Deals with the subscriptions (and getting data) from one
-   /// pubsub server. Once again, theres no sharing of conenctions
+   /// Deals with the subscriptions (and getting data) from one sole
+   /// pubsub server. Theres no sharing of conenctions
    /// from this point. Every new instance of SingleServerSubscriberCore
    /// will open its new dedicated connection to the serverURL that
    /// its concerned with (passed as a conctructor arg).
    /// </summary>
 
    internal interface ISingleServerSubscriberCore : IDisposable {
-      IList<PubSubUpdate> getUpdates(int timeout);
+      IList<PubSubUpdate> get_updates(int timeout);
       event PubSubConnectionChange onPubSubConnectionChange;
       void subscribe(params string[] tags);
       void unsubscribe(params string[] tags);
    }
+
    internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore {
-      private HighPerformanceTimer myTimer = new HighPerformanceTimer();
-      private PubSubServerConnectionFactory myConnectionFactory;
-      private IPubSubServerConnection myConnection;
-      private List<string> myTags = new List<string>();
+      private readonly IPubSubServerConnection _connection;
+      private HashSet<string> _tags = new HashSet<string>();
 
       public event PubSubConnectionChange onPubSubConnectionChange;
 
-      public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) {
-         myConnectionFactory = connectionFactory;
-         myConnection = myConnectionFactory.getConnection(serverURL);
-         myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
+      public SingleServerSubscriberCore(string server_url, PubSubServerConnectionFactory connection_factory) {
+         _connection = connection_factory.getConnection(server_url);
+         _connection.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot));
       }
 
-      public IList<PubSubUpdate> getUpdates() {
-         return getUpdates(1);
-      }
-
-      /// <summary>
-      /// gets as many data items that it can get in
-      /// timeout milliseconds
-      /// </summary>
-      /// <param name="timeout"></param>
-      /// <returns></returns>
-      public IList<PubSubUpdate> getUpdates(int timeout) {
-         myConnection.doWork();
-         List<PubSubUpdate> retVal = new List<PubSubUpdate>();
-         IList<byte[]> dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
-         foreach (byte[] data in dataItems) {
-            processMessage(data, retVal);
+      // gets as many data items that it can get in
+      // timeout milliseconds
+      public IList<PubSubUpdate> get_updates(int timeout) {
+         _connection.doWork();
+         List<PubSubUpdate> rv = new List<PubSubUpdate>();
+         IList<byte[]> data_items = _connection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
+         foreach (byte[] data in data_items) {
+            process_message(data, ref rv);
          }
-         return retVal;
+         return rv;
 
       }
 
-      public List<string> getSubscribedTags() {
-         return myTags;
-      }
-
       public void subscribe(params string[] tags) {
+         HashSet<string> this_call_tags = new HashSet<string>();
          foreach (string tag in tags) {
-            if (myTags.Contains(tag)) {
-               string errMsg = String.Format("tag {0} was already subscribed from server {1}",
-                  tag, myConnection.serverURL);
-               // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
+            if (_tags.Contains(tag)) {
+               bts.Log.Gen.warn($"SingleServerSubscriberCore.subscribe tag: {tag} has already " +
+                                 $"been subscribed to server: {_connection.server_url}");
             } else {
-               myTags.Add(tag);
+               _tags.Add(tag);
+               this_call_tags.Add(tag);
             }
          }
-         mySubscribe(tags);
+         _subscribe(this_call_tags.ToArray());
       }
 
       public void unsubscribe(params string[] tags) {
-         List<string> unsubTags = new List<string>();
+         List<string> unsub_tags = new List<string>();
          foreach (string tag in tags) {
-            if (myTags.Contains(tag)) {
-               myTags.Remove(tag);
-               unsubTags.Add(tag);
+            if (_tags.Contains(tag)) {
+               _tags.Remove(tag);
+               unsub_tags.Add(tag);
             } else {
-               string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " +
-                  "it was never subscribed", tag, myConnection.serverURL);
-               // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
+               bts.Log.Gen.warn($"tag {tag} is being unsubscribed from server {_connection.server_url}, eventhough " +
+                                 "it was never subscribed");
             }
          }
-         UnSubMetaData unsub = new UnSubMetaData(myConnection.clientURL, unsubTags.ToArray());
+         UnSubMetaData unsub = new UnSubMetaData(_connection.client_url, unsub_tags.ToArray());
          Stream str = new MemoryStream();
          new MetaData(unsub).save(str);
          str.Position = 0;
          // this is where data is actually sent out
-         myConnection.send(str);
+         _connection.send(str);
       }
 
       #region IDisposable Members
@@ -99,44 +86,40 @@ namespace BCG {
       public void Dispose() {
          ///remove the subscriber
          //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream());
-         myConnection.Dispose();
+         _connection.Dispose();
 
          //clear the list of tags you've subscribed to. This way, if you're ever used again, 
          //you have to send all the subscription metadata out again.
-         myTags.Clear();
+         _tags.Clear();
       }
 
       #endregion
 
-      private void mySubscribe(params string[] tags) {
-         SubMetaData sub = new SubMetaData(myConnection.clientURL, tags);
+      // The only place where a tag is subsribed from the Pubsub server. the subscriptions are wrapped into SubMetaData,
+      // which has a to_stream method
+      private void _subscribe(params string[] tags) {
+         SubMetaData sub = new SubMetaData(_connection.client_url, tags);
          Stream str = new MemoryStream();
          new MetaData(sub).save(str);
          str.Position = 0;
-         myConnection.send(str);
+         _connection.send(str);
       }
 
-      private void myHandleServerReboot(string serverURL, bool connection) {
+      private void _handle_server_reboot(string server_url, bool connection) {
+         bts.Log.Gen.info($"SingleServerSubscriberCore._handle_server_reboot connection: {connection} server_url: {server_url}");
          if (connection) {
-            //ie connection established again, so has to
-            //resubscribe to all the tags
-            // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags",
-            //    PS3LogLevels.disconnection);
-            mySubscribe(myTags.ToArray());
-
-
+            _subscribe(_tags.ToArray());
          } else {
-            // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...",
-            //    PS3LogLevels.disconnection);
+            bts.Log.Gen.warn($"SingleServerSubscriberCore._handle_server_reboot connection to {server_url}  broke....attempting to reconnect...");
          }
-
          //let anyone watching you know that your connection has changed. 
          if (onPubSubConnectionChange != null)
-            onPubSubConnectionChange(connection, serverURL);
+            onPubSubConnectionChange(connection, server_url);
       }
 
-      internal static void processMessage(byte[] data, List<PubSubUpdate> updates) {
-         bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}");
+      // processes one message
+      internal static void process_message(byte[] data, ref List<PubSubUpdate> updates) {
+         bts.Log.Gen.info($"SingleServerSubscriberCore.process_message data_len: {data.Length}");
          if (data != null && data.Length > 0) {
             MemoryStream str = new MemoryStream(data);
             MetaData md = new MetaData(str);
@@ -156,9 +139,8 @@ namespace BCG {
                if (p.isCompressed) {
                   long compressedSize = bodyStream.Length - bodyStream.Position;
                   if (compressedSize == 0) {
-                     string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
-                     Console.WriteLine(errorMsg);
-                     //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
+                     string error_msg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
+                     bts.Log.Gen.error(error_msg);
                      return;
                   }
                   realBody = StreamDecompressor.decompressStream(bodyStream);
@@ -174,8 +156,9 @@ namespace BCG {
                   bodyStream.Read(trConstituent, 0, trConstituent.Length);
                   updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent)));
                }
-            } else { }
-            //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
+            } else {
+               bts.Log.assert_d(false, $"Unknown message received: {md} only Pub and TransactionMetaData are legal");
+            }
          }
       }
    }

src/DataStreamSuite/PubSub/StaticPublisherCore.cs

@@ -387,18 +387,18 @@ namespace BCG {
             SharpZipCompressor Kompressor = new SharpZipCompressor();
             Kompressor.compress(content, contentToPublish);
             contentToPublish.Position = 0;
-            pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
+            pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
             if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
                string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
                // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
             }
          } else
-            pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
+            pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
 
          content.Position = initPos;
          if (CrushedTagManager.instance.is_tag_crushable(tag)) {
             string refId = String.Empty;
-            Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL);
+            Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.server_url);
             //  this is where the data is actually sent out (there are multiple sends)
             PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
             //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
@@ -428,8 +428,8 @@ namespace BCG {
             myTagToSender.Add(tag, sender);
             if (!mySenders.Contains(sender)) {
                mySenders.Add(sender);
-               sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
-               sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm));
+               sender.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot));
+               sender.register_send_confirmation_callback(new SendConfirmationDelegate(myOnSendConfirm));
             }
 
          }
@@ -443,7 +443,7 @@ namespace BCG {
          if (connection) {
             string thisPublisherInfo = ClientInfo.thisClient.processString;
             foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
-               if (p.Value.serverURL == serverURL) {
+               if (p.Value.server_url == serverURL) {
                   string tagType;
                   string tag = p.Key;
                   myCrushableTagsWaitingOnAck.Clear();

src/DataStreamSuite/PubSub/StaticSubscriberCore.cs

@@ -80,7 +80,7 @@ namespace BCG {
                int timeShare = timeout / myServerURLToSubCore.Count;
                List<PubSubUpdate> retVal = new List<PubSubUpdate>();
                foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
-                  retVal.AddRange(subCore.getUpdates(timeShare));
+                  retVal.AddRange(subCore.get_updates(timeShare));
                _log_n_updates(retVal);
                myTimer.stop();
                //if (myTimer.lastIntervalTime > 10)