Fixing AsyncTCPCOnnection_APM's double partial messages for messages > 100 MB

.devcontainer/Dockerfile

@@ -1,122 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Note installations that take a while (more than 10s)
-
-RUN apt-get update
-RUN apt-get install -y ca-certificates
-RUN apt-get install -y ccache
-RUN apt-get install -y curl
-RUN apt-get install -y fish
-RUN apt-get install -y gdb
-# git-all: ~250s
-RUN apt-get install -y git-all
-RUN apt-get install -y gnupg
-# liblttng-ust-dev: ~30s
-RUN apt-get install -y liblttng-ust-dev
-RUN apt-get install -y libssh2-1-dev
-RUN apt-get install -y locales
-RUN apt-get install -y ninja-build
-# openssh-server: ~40s
-RUN apt-get install -y openssh-server
-RUN apt-get install -y pkg-config
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y sudo
-RUN apt-get install -y tzdata
-RUN apt-get install -y vim
-RUN apt-get install -y zip
-
-# Python: ~30s
-RUN apt-get install -y \
-   python3-pip \
-   python3-venv \
-   && python3 -m pip install --upgrade pip
-
-# GCC
-ARG gcc_major=13
-RUN \
-   add-apt-repository ppa:ubuntu-toolchain-r/test \
-   && apt-get update \
-   && apt-get install -y g++-$gcc_major
-
-# Clang
-ARG clang_major=17
-RUN \
-   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
-   | tee -a "/etc/apt/sources.list.d/llvm.list" \
-   && apt-get update \
-   && apt-get install -y \
-   clang-$clang_major \
-   clang-format-$clang_major \
-   clangd-$clang_major
-
-# CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
-
-# Dotnet
-RUN apt-get update && apt-get install -y dotnet7
-
-# Node
-ARG node_major=18
-RUN mkdir -p /etc/apt/keyrings \
-   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
-   | tee /etc/apt/sources.list.d/nodesource.list \
-   && apt-get update \
-   && apt-get install -y nodejs
-
-# Emumba (deps)
-RUN apt-get update && apt-get -y install rapidjson-dev
-# Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
-
-# Pcap
-RUN apt-get update && apt-get install -y libpcap0.8-dev
-
-# Set up default executables (update-alternatives)
-RUN \
-   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
-   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
-   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
-   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
-
-# Add compiler symlinks for ccache 
-RUN \
-   ln -s /usr/bin/ccache /usr/local/bin/clang; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-   groupadd -g "$docker_uid" "$docker_user" \
-   && useradd \
-   -m \
-   -u "$docker_uid" \
-   -g "$docker_uid" \
-   -G sudo \
-   --password "$(openssl passwd -1 "$docker_pass")" \
-   "$docker_user"

.devcontainer/devcontainer.json

@@ -1,48 +0,0 @@
-{
-  "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
-  "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
-  },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
-  "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
-  "updateRemoteUserUID": true,
-  "shutdownAction": "none",
-  // Note: If running multiple containers on the same host, each must use a
-  // unique set of ports. Usually we just add 1000 to each port as needed. For
-  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
-  "runArgs": [
-    "--shm-size",
-    "2gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003"
-  ],
-  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
-  ],
-  "customizations": {
-    "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
-      "extensions": [
-        "eamodio.gitlens",
-        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
-      ]
-    }
-  }
-}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.gitignore

@@ -54,6 +54,6 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-.devcontainer/
+**/.devcontainer
 *.Generated.cs
 *.lst

CMakePresets.json

@@ -113,8 +113,8 @@
       "description": "Debug build compiled w/ gcc",
       "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
-        "CMAKE_C_COMPILER": "gcc-12",
-        "CMAKE_CXX_COMPILER": "g++-12",
+        "CMAKE_C_COMPILER": "gcc",
+        "CMAKE_CXX_COMPILER": "g++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
       }
@@ -143,7 +143,7 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
       }
     },

playground-1.mongodb.js

@@ -0,0 +1,45 @@
+/* global use, db */
+// MongoDB Playground
+// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
+// Make sure you are connected to enable completions and to be able to run a playground.
+// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
+// The result of the last command run in a playground is shown on the results panel.
+// By default the first 20 documents will be returned with a cursor.
+// Use 'console.log()' to print to the debug output.
+// For more documentation on playgrounds please refer to
+// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
+
+// Select the database to use.
+use('mongodbVSCodePlaygroundDB');
+
+// Insert a few documents into the sales collection.
+db.getCollection('sales').insertMany([
+    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
+    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
+    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
+    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
+    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
+    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
+    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
+    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
+]);
+
+// Run a find command to view items sold on April 4th, 2014.
+const salesOnApril4th = db.getCollection('sales').find({
+    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
+}).count();
+
+// Print a message to the output window.
+console.log(`${salesOnApril4th} sales occurred in 2014.`);
+
+// Here we run an aggregation and open a cursor to the results.
+// Use '.toArray()' to exhaust the cursor to return the whole result set.
+// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
+db.getCollection('sales').aggregate([
+    // Find all of the sales that occurred in 2014.
+    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
+    // Group the total sales for each product.
+    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
+]);
+
+

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnExpression">
-    <Scanner Include="ColumnExpressionTokens.lex" />
-    <Parser Include="ColumnExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -1,62 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-public class ColumnExpression {
-   public ColumnExpression(string name, Type? type = null, object? value = null) {
-      Name = name;
-      Type = type
-         ?? value?.GetType()
-         ?? throw new ArgumentException(
-            "Type must be specified explicitly when value is null", nameof(type));
-      Value = value
-         ?? DBNull.Value;
-   }
-
-   public string Name { get; }
-
-   public Type Type { get; }
-
-   public object Value { get; }
-
-   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
-      // Try to convert value to 'type' to verify they are compatible
-      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
-      // account for implicit conversions (e.g., int -> long), just inheritance.
-      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
-         throw new ArgumentException(
-            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
-      }
-      return new(name, type ?? typeof(T), value);
-   }
-
-   internal static Type ParseType(string typeId) {
-      return Candidates().ExceptNull().FirstOrDefault() ??
-         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
-
-      IEnumerable<Type?> Candidates() {
-         if (typeId.Length == 0) {
-            yield break;
-         }
-         yield return Type.GetType(typeId);
-         yield return Type.GetType($"System.{typeId}");
-         if (typeId.Length > 1) {
-            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
-         }
-         yield return typeId.ToLower() switch {
-            "float64" or "double" or "real64" or "number" => typeof(double),
-            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
-            "int16" or "short" or "integer16" or "signed16" => typeof(short),
-            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
-            "int64" or "long" or "integer64" or "signed64" => typeof(long),
-            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
-            "uint32" or "uint" or "unsigned" => typeof(uint),
-            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
-            "char" => typeof(char),
-            "byte" or "uint8" => typeof(byte),
-            "bool" or "boolean" => typeof(bool),
-            "string" or "text" => typeof(string),
-            "date" or "datetime" => typeof(DateTime),
-            _ => null
-         };
-      }
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -1,49 +0,0 @@
-%namespace Bts.ColumnExpressions
-%partial
-%visibility internal
-%tokentype ColumnExpressionToken
-%parsertype ColumnExpressionParser
-
-%union {
-   public int Integer;
-   public double Real;
-   public bool Boolean;
-   public string String;
-   public string Identifier;
-   public string TypeId;
-   public DateTime DateTime;
-   public Tuple<string, Type> ColumnDefinition;
-   public ColumnExpression ColumnExpression;
-   public ImmutableList<ColumnExpression> ColumnExpressions;
-}
-
-%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
-%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
-
-%nonassoc EQUALS
-
-%type <ColumnDefinition> ColumnDefinition
-%type <ColumnExpression> ColumnExpression
-%type <ColumnExpressions> ColumnExpressions
-
-%%
-
-ColumnExpressions :
-   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
-   ;
-
-ColumnExpression :
-   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
-   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
-   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
-   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
-   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
-   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
-   ;
-
-ColumnDefinition :
-   IDENTIFIER { $$ = new($1.Identifier, null); }
-   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
-   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
-   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnExpressionInterpreter {
-   public static IList<ColumnExpression> Evaluate(Stream stream) {
-      var parser = new ColumnExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
-         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.Text;
-
-partial class ColumnExpressionParser {
-   public ColumnExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-partial class ColumnExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -1,85 +0,0 @@
-%namespace Bts.ColumnExpressions
-%scannertype ColumnExpressionScanner
-%visibility internal
-%tokentype ColumnExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-BooleanConstant          true|false
-StringConstant           '([^\'\\]*|\\.)*'
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
-IntegerConstant          -?[0-9]+
-NullConstant             'null'
-RealConstant             -?[0-9]+\.[0-9]+
-
-/* Relational Operators */
-Equals                   =
-OfType                   /
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-NamespacePrefix          {Identifier}\.
-TypeId                   {NamespacePrefix}*{Identifier}
-
-/* Utility Tokens */
-OpenParenthesis          \(
-CloseParenthesis         \)
-Comma                    ,
-QuoteOpen                \[
-QuoteClose               \]
-
-/* Comments */
-LineComment \/\/[^\n\r]*(\n|\r|\r\n)
-
-%%
-
-{Equals} {
-   return (int)ColumnExpressionToken.EQUALS;
-}
-
-{OfType} {
-   return (int)ColumnExpressionToken.OF_TYPE;
-}
-
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
-   return (int)ColumnExpressionToken.STRING_CONSTANT;
-}
-
-{BooleanConstant} {
-   yylval.Boolean = yytext.ToLower() switch {
-      "true" => true,
-      "false" => false,
-      _ => throw new Exception(
-         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
-   };
-   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
-}
-
-{DateTimeConstant} {
-   yylval.DateTime = DateTime.Parse(yytext);
-   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
-}
-
-{IntegerConstant} {
-   yylval.Integer = int.Parse(yytext);
-   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
-}
-
-{RealConstant} {
-   yylval.Real = double.Parse(yytext);
-   return (int)ColumnExpressionToken.REAL_CONSTANT;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnExpressionToken.IDENTIFIER;
-}
-
-{TypeId} {
-   yylval.TypeId = yytext;   
-   return (int)ColumnExpressionToken.TYPE_ID;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnSortExpression">
-    <Scanner Include="ColumnSortExpressionTokens.lex" />
-    <Parser Include="ColumnSortExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -1,11 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-public class ColumnSortExpression {
-   public ColumnSortExpression(string name, bool isAscending = true) {
-      Name = name;
-      IsAscending = isAscending;
-   }
-
-   public string Name { get; }
-   public bool IsAscending { get; }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -1,29 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%partial
-%visibility internal
-%tokentype ColumnSortExpressionToken
-%parsertype ColumnSortExpressionParser
-
-%union {
-   public string Identifier;
-   public ColumnSortExpression Column;
-   public ImmutableList<ColumnSortExpression> Columns;
-}
-
-%token ASCENDING, DESCENDING, IDENTIFIER
-
-%type <Column> Column
-%type <Columns> Columns
-
-%%
-
-Columns :
-   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | Columns Column { $$ = $1.Add($2); }
-   ;
-
-Column :
-   IDENTIFIER { $$ = new($1.Identifier, true); }
-   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
-   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
-   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnSortExpressionInterpreter {
-   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
-      var parser = new ColumnSortExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
-         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnSortExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.Text;
-
-partial class ColumnSortExpressionParser {
-   public ColumnSortExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnSortExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-partial class ColumnSortExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -1,30 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%scannertype ColumnSortExpressionScanner
-%visibility internal
-%tokentype ColumnSortExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-Ascending                +
-Descending               -
-
-%%
-
-{Ascending} {
-   return (int)ColumnSortExpressionToken.ASCENDING;
-}
-
-{Descending} {
-   return (int)ColumnSortExpressionToken.DESCENDING;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnSortExpressionToken.IDENTIFIER;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -9,8 +9,6 @@
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,62 +0,0 @@
-// using System;
-// using System.Diagnostics;
-
-// using BCG;
-// using BCG.Utilities.General;
-
-
-// namespace bts {
-
-//     public class PandasStream : ViewStream, IWorker {
-//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
-
-//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
-
-//         }
-
-//         public PandasStream(ViewStream vs, string pandas_code) :
-//         this(vs, ViewStreamDatabase.instance, pandas_code) {
-//         }
-
-
-//         protected override void myOnFirstSubscriber() {
-
-//             // string python_code = @"
-//             //    import winsound
-//             //    frequency = 2500  # Set frequency (in Hz)
-//             //    duration = 1000   # Set duration (in milliseconds)
-//             //    winsound.Beep(frequency, duration)
-//             // ";
-//             string python_code = @"
-//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
-//             with open(file_path, \"w\") as f:
-//                 f.write(\"Hello, world!\n\")
-//             ";
-
-//             execute_python_code(python_code);
-//         }
-
-//         static void execute_python_code(string python_code) {
-//             ProcessStartInfo start = new ProcessStartInfo();
-//             start.FileName = "python"; // Python executable
-//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
-//             start.UseShellExecute = false;
-//             start.RedirectStandardOutput = true;
-
-//             using (Process process = Process.Start(start)) {
-//                 using (System.IO.StreamReader reader = process.StandardOutput) {
-//                     string result = reader.ReadToEnd();
-//                     Console.Write(result);
-//                 }
-//             }
-//         }
-
-//         protected override void myOnLastSubscriber() {
-//         }
-
-//         public void doWork() {
-
-//         }
-
-//     }
-// }

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -1,79 +0,0 @@
-using BCG.Utilities.General;
-
-using System;
-
-namespace BCG {
-    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
-        class State {
-            public int n_sends = 0;
-            public int n_recvs = 0;
-        }
-        private readonly string _ip;
-        private readonly int _port;
-        private readonly ViewStreamDatabase _vsd;
-        State _state = new State();
-        private readonly int _msg_size;
-
-        private IBlueTableSchema _schema;
-        private TimeSpan _send_period;
-        private ITCPConnection _connection = null;
-        private byte[] _buff;
-
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
-        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
-            _ip = ip;
-            _port = port;
-            _msg_size = msg_size;
-            _vsd = vsd;
-            _schema = BlueDataUtility.form_schema("TCPClient",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("n_sends", TypeCode.UInt32),
-            Tuple.Create("n_recvs", TypeCode.UInt32));
-            _send_period = TimeSpan.FromSeconds(send_period_seconds);
-            _buff = new byte[msg_size];
-            for (int i = 0; i < _buff.Length; i++) {
-                _buff[i] = (byte)i;
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_address = new BlutoSubAddress(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            if (_connection != null) {
-                _connection.send(_buff);
-                _state.n_sends++;
-            }
-            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
-            BlueTable rows = new BlueTable(r.schema);
-            rows.addRow(r);
-            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        public void onConnection(ITCPConnection connection) {
-            _connection = connection;
-            _connection.register(this);
-        }
-        public bool isDisposed { get; }
-
-        public void onData(RemoteMessage msg) {
-            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
-            _state.n_recvs++;
-        }
-
-        public void Dispose() {
-
-        }
-
-
-    }
-}

src/DataStreamSuite/DataStream/Misc/test.py

@@ -1,3 +0,0 @@
-file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
-with open(file_path, "w") as f:
-    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -1,32 +0,0 @@
-using BCG;
-
-namespace BCG
-{
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("add_to_cloud")]
-   public class AddToCloudStream : ViewStream
-   {
-      private readonly string _name, _stream_exp;
-      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
-      public AddToCloudStream(string name, params string[] lines)
-      : this(name, 0, lines)
-      {
-      }
-
-      public AddToCloudStream(string name, int persist, params string[] lines)
-      {
-         _name = name;
-         _stream_exp = getExpression(lines);
-         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
-         mySetRealSource(new TableStream("cloud_streams"));
-      }
-
-      static string getExpression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -9,23 +9,20 @@ using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
 using System.Collections.Generic;
 using BCG.Utilities.General;
+using bts;
 
-namespace BCG
-{
+namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class CloudStreamManager : IWorker
-   {
-      public class StreamPubUnit
-      {
+   public class RemoteQueryManager : IWorker {
+      public class StreamPubUnit {
          public string name;
          public string stream_expression;
          public ViewStream stream;
          public bool persist;
          public ViewPublisher view_publisher;
 
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
-         {
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
             name = name_;
             stream_expression = stream_expression_;
             stream = stream_;
@@ -33,17 +30,13 @@ namespace BCG
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream build_stream(string stream_expression)
-      {
-         try
-         {
+      static ViewStream build_stream(string stream_expression) {
+         try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
             var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
             bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
@@ -52,34 +45,34 @@ namespace BCG
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       private ViewStream _stream = new TableStream("cloud_streams");
       private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
-      public CloudStreamManager()
-      {
+
+      public RemoteQueryManager() {
          _stream.onChange += _on_str;
-
+         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
       }
-      private void _on_str(object obj)
-      {
+      // add a RemoteQuery sub, additionally and subscribe to it
+      private void _on_str(object obj) {
          DataPacket dp = obj as DataPacket;
-         foreach (IBlueRow r in dp.newRows)
-         {
+         foreach (IBlueRow r in dp.newRows) {
             string name = r.get<string>("name");
             bool persist = r.get<bool>("persist");
             string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name))
-            {
-               _assign(name, stream_exp, persist);
+            if (!_stream_pub_map.ContainsKey(name)) {
+               _spawn_remote_stream(name, stream_exp, persist);
             }
          }
       }
 
-      private void _assign(string name, string stream_expression, bool persist)
-      {
+      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
-         if (persist)
-         {
+         if (persist) {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -87,13 +80,11 @@ namespace BCG
          _stream_pub_map.Add(name, u);
       }
 
-      private void _expose_to_ws(string name, ViewStream stream)
-      {
+      private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
 
-      public void doWork()
-      {
+      public void doWork() {
 
       }
    }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -0,0 +1,48 @@
+using BCG;
+
+using bts;
+
+using System;
+
+namespace BCG {
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
+   public class RemoteQueryStream : ViewStream {
+      private readonly string _name, _stream_exp;
+      private PublisherCore _publisher_core = new PublisherCore();
+      private ViewStream _source_str;
+
+      public RemoteQueryStream(params string[] lines) {
+         _stream_exp = get_expression(lines);
+         _name = Guid.NewGuid().ToString();
+         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
+         //_name.Replace("-", "_");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
+         byte[] rem_req_bytes = rem_req.serialize();
+         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
+      }
+
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }
+
+      static string get_expression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+
+      private void _handle_data(object o) {
+         myFireEvent(o);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -0,0 +1,31 @@
+namespace bts {
+   using System.IO;
+   using BCG;
+
+   using System.IO;
+   using System.Threading.Tasks.Dataflow;
+
+   public class RemoteRequest {
+      public readonly string stream_expression;
+      public readonly string published_table_name;
+      public RemoteRequest(string stream_expression_, string published_table_name_) {
+         stream_expression = stream_expression_;
+         published_table_name = published_table_name_;
+      }
+      public byte[] serialize() {
+         var str = new MemoryStream();
+         var bw = new BinaryWriter(str);
+         bw.write_string_cpp(stream_expression);
+         bw.write_string_cpp(published_table_name);
+         return str.ToArray();
+      }
+
+      public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
+         BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
+         string stream_expression_ = br.read_string_cpp();
+         string published_table_name_ = br.read_string_cpp();
+         return new RemoteRequest(stream_expression_, published_table_name_);
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -1,59 +1,54 @@
-using BCG;
+// using BCG;
 
-using System.Linq;
+// using bts;
 
-namespace BCG
-{
-   public class StreamOffloadRequestor
-   {
-      public class OffloadOp
-      {
-         string name;
-         string stream_exp;
-      }
-      private IBlueTableSchema _schema;
-      private IBlueRowCollection _rows;
-      private PubTableBroker _broker = new PubTableBroker();
-      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-      public StreamOffloadRequestor()
-      {
-         _schema = new BlueTableSchema("cloud_streams",
-                                   new ColumnInfo("id", 0, typeof(int)),
-                                   new ColumnInfo("name", 1, typeof(string)),
-                                   new ColumnInfo("persist", 2, typeof(bool)),
-                                   new ColumnInfo("stream_exp", 3, typeof(string)));
-         _existing_offloads_vs.onChange += _on_str;
-      }
-      private void _on_str(object obj)
-      {
-         DataPacket dp = obj as DataPacket;
-         _rows = dp.newRows;
-         // this is needed so the broker knows the state of the table
-         _broker.createTable(_rows);
-      }
-      public void add_to_cloud(string name, string stream_exp, bool persist)
-      {
-         // check if it exists
-         if (_rows == null)
-         {
-            BlueTable table = new BlueTable(_schema);
-            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-            table.addRow(nr);
-            _broker.createTable(table);
-            // create table  
-         }
-         else
-         {
-            if (exists(name))
-               return;
-            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-            _broker.addRow(nr);
-         }
-      }
+// using System.Linq;
 
-      bool exists(string name)
-      {
-         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-      }
-   }
-}
+// namespace BCG {
+//    // used on the client
+//    public class StreamOffloadRequestor {
+//       public class OffloadOp {
+//          string name;
+//          string stream_exp;
+//       }
+//       private IBlueTableSchema _schema;
+//       private IBlueRowCollection _rows;
+//       private PubTableBroker _broker = new PubTableBroker();
+//       ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+//       public StreamOffloadRequestor() {
+//          _schema = new BlueTableSchema("cloud_streams",
+//                                    new ColumnInfo("id", 0, typeof(int)),
+//                                    new ColumnInfo("name", 1, typeof(string)),
+//                                    new ColumnInfo("persist", 2, typeof(bool)),
+//                                    new ColumnInfo("stream_exp", 3, typeof(string)));
+//          _existing_offloads_vs.onChange += _on_str;
+//       }
+//       private void _on_str(object obj) {
+//          DataPacket dp = obj as DataPacket;
+//          _rows = dp.newRows;
+//          // this is needed so the broker knows the state of the table
+//          _broker.create_table(_rows);
+//       }
+//       public void add_to_cloud(string name, string stream_exp, bool persist) {
+//          // check if it exists
+//          if (_rows == null) {
+//             BlueTable table = new BlueTable(_schema);
+//             IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+//             table.addRow(nr);
+//             _broker.create_table(table);
+//             // create table  
+//          } else {
+//             if (exists(name))
+//                return;
+//             IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+//             // TODO: use make_batch_changes instead
+//             Log.assert(false, "Fix addRow");
+//             //_broker.addRow(nr);
+//          }
+//       }
+
+//       bool exists(string name) {
+//          return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs

@@ -1,17 +1,14 @@
 using System;
 using System.Collections;
 
-namespace BCG
-{
-   internal class AddColumnsRow : IBlueRow
-   {
+namespace BCG {
+   internal class AddColumnsRow : IBlueRow {
       private IBlueTableSchema mySchema;
       private IBlueRow myRow;
       //private object[] myCachedValues;
 
       #region Constructors
-      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
-      {
+      public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
          mySchema = schema;
          myRow = row;
          //myCachedValues = new object[schema.columnNames.Length];
@@ -19,85 +16,78 @@ namespace BCG
       #endregion
 
       #region IBlueRow members
-      public IBlueTableSchema schema{get{return mySchema;}}
+      public IBlueTableSchema schema { get { return mySchema; } }
       #endregion
 
       #region ICollection members
-      public int Count {get{return mySchema.columnNames.Length;}}
+      public int Count { get { return mySchema.columnNames.Length; } }
 
-      public bool IsSynchronized {get{return false;}}
+      public bool IsSynchronized { get { return false; } }
 
-      public object SyncRoot {get{return this;}}
+      public object SyncRoot { get { return this; } }
 
-      public void CopyTo (Array array, int index)  
-      {
+      public void CopyTo(Array array, int index) {
          int i = index;
          foreach (object o in this as IEnumerable)
-            array.SetValue (o, i++);
+            array.SetValue(o, i++);
       }
       #endregion
 
       #region IDictionary members
-      public bool IsFixedSize {get{return true;}}
+      public bool IsFixedSize { get { return true; } }
 
-      public bool IsReadOnly {get{return true;}}
+      public bool IsReadOnly { get { return true; } }
 
       /// <summary>
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public object this [object key]
-      {
-         get
-         {
+      public object this[object key] {
+         get {
             if (key is string)
                return this[(string)key];
             return this[(int)key];
          }
-         set {throw new NotSupportedException ("The IBlueRow is readonly");}
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
       }
 
-      public object this[int index]
-      {
-         get
-         {
+      public object this[int index] {
+         get {
             //if (myCachedValues[index] == null)
             //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
             //return myCachedValues[index];
-            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+            return mySchema[index].isConstant ? DBNull.Value : myRow[index];
          }
       }
 
-      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}
+      public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
 
-      public ICollection Keys{get{return mySchema.columnNames;}}
+      public ICollection Keys { get { return mySchema.columnNames; } }
 
-      public ICollection Values
-      {
-         get
-         {
+      public ICollection Values {
+         get {
             object[] retVal = new object[mySchema.columnNames.Length];
-            CopyTo (retVal, 0);
+            CopyTo(retVal, 0);
             return retVal;
          }
       }
 
-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
 
-      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
 
-      public bool Contains (object key){return mySchema[(string)key] != null;}
+      public bool Contains(object key) { return mySchema[(string)key] != null; }
 
       /// <summary>
       /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
       /// iterate through the values, typecast this into an IEnumerable
       /// </summary>
       /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
-      
-      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
 
-      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
       #endregion
 
    }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,6 +1,38 @@
-namespace BCG;
-
-public class AddColumnsStream : MapStream {
-   public AddColumnsStream(ViewStream vs, string columnDef)
-      : base(vs, new ColumnAdder(columnDef)) { }
-}
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using BCG.Utilities.General;
+namespace BCG {
+
+   public class AddColumnsStream : ViewStream {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
+         string[] toks = exp.Split(':');
+         string col_name = toks[0];
+         string col_value = toks[1];
+         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
+         return Tuple.Create(col_name, tup.Item1, tup.Item2);
+      }
+
+      private Tuple<string, Type, object> _new_col;
+      private IBlueTableSchema _out_schema;
+
+      public AddColumnsStream(ViewStream vs, string col_def) {
+         _new_col = parse_exp(col_def);
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
+         mySetRealSource(rs);
+      }
+
+      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
+         return _out_schema;
+      }
+
+      private IBlueRow _map(IBlueRow r) {
+         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
+         rv.setValue(_new_col.Item1, _new_col.Item3);
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -1,12 +0,0 @@
-namespace bts {
-    using BCG;
-
-    using System;
-    using System.Collections;
-
-    public class AddConstantColumnRow {
-        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
-
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,52 +1,63 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-
-using Bts.ColumnExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Provides methods to add columns to the schema
-/// </summary>
-/// <param name="columnsExpression">
-/// String represting the columns to add to the output table
-/// </param>
-public class ColumnAdder : IStreamMapper {
-   // Store tuples of (column, index) to preserve order
-   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
-   protected IBlueTableSchema? mySchema;
-
-   public ColumnAdder(string columnsExpression) =>
-      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
-         throw new Exception("Invalid columnn expression: " + columnsExpression))
-         .Select((c, i) => (c, i))
-         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
-
-   [MemberNotNull(nameof(mySchema))]
-   public virtual IBlueRow map(IBlueRow row) {
-      // Only rebuild schema if necessary (i.e., if it's null)
-      mySchema ??= buildSchema(row.schema, myNewColumns);
-      return new AddColumnsRow(mySchema, row);
-   }
-
-   [MemberNotNull(nameof(mySchema))]
-   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
-      // Always rebuild schema (even if it's already non-null)
-      mySchema = buildSchema(inputSchema, myNewColumns);
-
-   [MemberNotNull(nameof(mySchema))]
-   static BlueTableSchema buildSchema(
-      IBlueTableSchema schema,
-      IDictionary<string, (ColumnExpression column, int index)> newColumns
-   ) {
-      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
-      schema.columnInfo.CopyTo(cis, 0);
-      foreach (var (col, index) in newColumns.Values) {
-         var schemaIdx = schema.columnInfo.Length + index;
-         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
-      }
-      return new BlueTableSchema(cis, schema.tableName);
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.AddColumnsExpression;
+
+// namespace BCG {
+//     /// <summary>
+//     /// Provides methods to add columns to the schema
+//     /// </summary>
+//     public class ColumnAdder : IStreamMapper {
+//         protected IDictionary myNewColumns;
+//         protected IBlueTableSchema mySchema;
+
+//         static syntax myParser =
+//            new syntax();
+
+//         /// <summary>
+//         /// string represting the columns to be added to the output table. 
+//         /// </summary>
+//         /// <param name="colAddExpression"></param>
+//         public ColumnAdder(string columnDef) {
+//             AddColumnsExpression ace;
+
+//             if (DataStreamLoadFileConfig.instance.useStaticParser) {
+//                 lock (myParser) {
+//                     ace =
+//                     (AddColumnsExpression)myParser.Parse(columnDef);
+//                 }
+//             } else {
+//                 syntax parser = new syntax();
+//                 ace =
+//                 (AddColumnsExpression)parser.Parse(columnDef);
+//             }
+
+//             if (ace == null)
+//                 throw new Exception("Invalid columnn expression: " + columnDef);
+//             myNewColumns = ace.columns;
+//         }
+
+//         public virtual IBlueRow map(IBlueRow row) {
+//             if (mySchema == null)
+//                 myConstructSchema(row.schema);
+//             return new AddColumnsRow(mySchema, row);
+//         }
+
+//         public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+//             myConstructSchema(inputSchema);
+//             return mySchema;
+//         }
+
+//         private void myConstructSchema(IBlueTableSchema schema) {
+//             ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
+//             int ciIndex = 0;
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 cis[ciIndex++] = ci;
+//             foreach (DictionaryEntry de in myNewColumns)
+//                 cis[ciIndex] = new ColumnInfo((string)de.Key, ciIndex++,
+//                    ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
+//             mySchema = new BlueTableSchema(cis, schema.tableName);
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,48 +1,24 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
+using BCG;
 
-using System.Runtime.Serialization.Formatters.Binary;
-
-namespace BCG
-{
+namespace bts {
    [DataStreamUserRespresentation("loader")]
-   public class LoaderStream : ViewStream, ISideEffectStream
-   {
-      public class Exception : System.Exception
-      {public Exception(string msg) : base(msg) { }}
+   public class LoaderStream : ViewStream {
 
-      public LoaderStream(string fileName, int secondsBetweenPolls)
-      {
-         mySetRealSource(
-            new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-               ViewStreamDatabase.instance));
+      private string _filename;
+      private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
+
+
+      public LoaderStream(string filename) {
+         _filename = filename;
       }
 
-      public LoaderStream(string fileName)
-      {
-        if (fileName == null)
-           throw new LoaderStream.Exception("File name cannot be null");
-
-        for (int i = 0; i < 5; i++)
-           try
-           {
-              using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-              {
-                  BinaryFormatter bf = new BinaryFormatter();
-#pragma warning disable SYSLIB0011
-                  BlueTable bt = (BlueTable)bf.Deserialize(fs);
-#pragma warning restore SYSLIB0011
-                  mySetRealSource(new BlueRowCollectionStaticStream(bt));
-              }
-              break;
-           }
-           catch (IOException)
-           {
-              System.Threading.Thread.Sleep(50);
-           }
-
+      protected override void myOnFirstSubscriber() {
+         byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
+         StreamIOMessage msg = _deser.deserialize(ser_msg);
+         Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
+         IBlueRowCollection rows = msg.create_message.rows;
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,6 +5,7 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
+using bts;
 
 namespace BCG {
    // Writes all the udpates to a stream in a file.
@@ -15,16 +16,11 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
-   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
-   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
-   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
-   // file is created. The file names are token_epoch_datetime
-   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;
       private FileStream _out_fs;
-      private IBlueDataSerializer _serializer;
+      private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
       private string _file_name = String.Empty;
       private long _rollover_size;
       private readonly string _directory_path;
@@ -94,13 +90,12 @@ namespace BCG {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
          bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
+         _serializer = _serializer ?? new UniversalStreamIOSerializer();
          if (_write_create) {
             // new files always starts with create
-            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
-            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
-
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
          BinaryWriter bw = new BinaryWriter(_out_fs);
          long utc_epoch = epoch_now();
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +1,68 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
 
-namespace BCG
-{
-   public class RecorderStream : ViewStream
-   {
-      private ViewStream mySource;
-      private DateTime myStartTime;
-      private FileStream myOutFile;
-      private string myFileName;
-      private IBlueDataSerializer mySerializer;
-            
-      public RecorderStream(ViewStream src, string recordFile)
-      {
-         mySource = src;
-         myFileName = recordFile;
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;
 
-         myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-      }
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;
 
-      protected override void myOnFirstSubscriber()
-      {
-         myStartTime = DateTime.Now;         
-         mySource.onChange += new ChangeDelegate(mySource_onChange);
-         myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-      }
-      
-      protected override void myOnLastSubscriber()
-      {
-         mySource.onChange -= mySource_onChange;
-         myOutFile.Close();
-      }
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }
 
-      void mySource_onChange(object newValue)
-      {
-         DataPacket dp = (DataPacket)newValue;         
-         
-         BlueTable toWrite = new BlueTable(dp.schema);
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }
 
-         if (mySerializer == null)
-         {
-            mySerializer = new UniversalSerializer(dp.schema);
-         }
-         foreach (IBlueRow r in dp.newRows)
-            toWrite.addRow(r);
-         
-         byte[] result;
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }
 
-         if (myLastValue == null)
-            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-         else
-            mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         
 
-         BinaryWriter bw = new BinaryWriter(myOutFile);
-         //write down what time this happened
-         bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-         bw.Write(result.Length);
-         bw.Write(result);         
-         myOutFile.Flush();
+//          BlueTable toWrite = new BlueTable(dp.schema);
 
-         myFireEvent(dp);
-      }
-   }
-}
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -1,64 +1,37 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using BCG;
 
-// using System.Runtime.Serialization.Formatters.Binary;
+namespace bts {
+   // this saves the total state in the filr provided in the cons
+   [DataStreamUserRespresentation("saver")]
+   public class SaverStream : ViewStream {
+      private ViewStream _stream;
+      private string _filename;
+      private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
 
-// namespace BCG
-// {
-//    [DataStreamUserRespresentation("saver")]
-//    public class SaverStream : ViewStream, ISideEffectStream
-//    {
-//       private ViewStream myStream;
-//       private string myFileName;
+      public SaverStream(ViewStream vs, string filename) {
+         _stream = vs;
+         _filename = filename;
+      }
 
-//       public class Exception : System.Exception
-//       {
-//          public Exception(string msg) : base(msg) { }
-//       }
+      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
 
-//       public SaverStream(ViewStream vs, string fileName)
-//       {
-//          if (fileName == null)
-//             throw new SaverStream.Exception("File name cannot be null");
-//          myStream = vs;
-//          myFileName = fileName;
+      void _handle_changes(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _save(dp);
+         myFireEvent(dp_obj);
+      }
 
-//       }
+      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
 
-//       protected override void myOnFirstSubscriber()
-//       { myStream.onChange += new ChangeDelegate(myHandleChanges); }
+      protected void _save(DataPacket dp) {
+         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
+         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
+         StreamSerializationUtils.write_to_file(str_bytes, _filename);
 
-//       void myHandleChanges(object newValue)
-//       {
-//          DataPacket dp = newValue as DataPacket;
-//          mySave(dp);
-//          myFireEvent(newValue);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
-
-//       protected void mySave(DataPacket dp)
-//       {
-//          for (int i = 0; i < 5 ; i++)
-//             try
-//             {
-//                using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
-//                {
-//                   BinaryFormatter bf = new BinaryFormatter();
-//                   BlueTable t = new BlueTable(dp.schema);
-//                   foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
-//                      t.addRow(new BlueRow(r));
-//                   bf.Serialize(fs, t);
-//                }
-//                break;
-//             }
-//             catch (IOException)
-//             {
-//                System.Threading.Thread.Sleep(5);
-//             }
-//       }
-//    }
-// }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -1,129 +1,115 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using System.Runtime.Serialization.Formatters.Binary;
-using BCG.Utilities.General;
-using bts.collections;
-using System.Reflection.Metadata;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
 
 
-namespace BCG {
-   public class SnapshotStream : ViewStream {
-      public enum SpecificSnapshot {
-         FirstOfDay,
-         LastOfDay
-      };
-      private BlueTable _current_table;
-      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-      public DateTime _snapshot_time;
-      public string _snapshot_file;
-      public IEnumerable<PlaybackPacket> _packets;
-      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-         DateTime dt;
-         var parsed = DateTime.TryParse(snapshot_date, out dt);
-         dt = dt.Date;
-         bts.Log.assert(parsed);
-         SpecificSnapshot specific_snapshot_enum;
-         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-         bts.Log.assert(parsed);
-         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-         mySetRealSource(rs);
-      }
+// namespace BCG {
+//    public class SnapshotStream : ViewStream {
 
-      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-         string token_directory_name = $"{directory_name}\\{token}";
-         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-      }
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       public DateTime _snapshot_time;
+//       public string _snapshot_file;
+//       public IEnumerable<PlaybackPacket> _packets;
 
-      protected override void myOnFirstSubscriber() {
-         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-         _packets = StreamPersistorUtils.load_packets_from_file(fs);
-         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-         if (snapshot_packet != null) {
-            foreach (PlaybackPacket p in _packets) {
-               byte[] ops;
-               Array row_array;
-               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-               for (int i = 0; i < ops.Length; i++) {
-                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-                  byte op = ops[i];
-                  if (op == StreamOpCode.create) {
-                     IBlueRowCollection full_table = rows_args[0];
-                     _current_table = new BlueTable(full_table);
-                  } else if (op == StreamOpCode.multipleRemoveAdd) {
-                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
-                  } else {
-                     bts.Log.assert(false, $"Oprator: {op} is not expected");
-                  }
-                  if (p.idx == snapshot_packet.idx) {
-                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-                     break;
-                  }
-               }
-            }
-         }
-      }
 
-      private void _absorb_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-            } else {
-               bts.Log.assert(false, $"Operator: {op} is not expected");
-            }
-         }
-      }
+//       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+//          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+//       }
 
-      protected override void myOnLastSubscriber() {
-      }
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+//          _packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
 
-      // public event SnapshotStream.Progress onProgress;
-      // public event Action onFileExhausted;
-      private PlaybackPacket _locate_packet_to_replay() {
-         foreach (var p in _packets) {
-            // returns the first non replayed packet
-            if (!p.replayed)
-               return p;
-         }
-         return null;
-      }
+//       private void _absorb_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//             } else {
+//                bts.Log.assert(false, $"Operator: {op} is not expected");
+//             }
+//          }
+//       }
 
-      private void _fire_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-               //_current_table.addRows(full_table);
-               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-                                          (myLastValue as DataPacket).newRows));
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-               myFireEvent(new DataPacket(_current_table, adds, dels));
-            } else {
-               bts.Log.assert(false, $"Oprator: {op} is not expected");
-            }
+//       protected override void myOnLastSubscriber() {
+//       }
 
-         }
-      }
-   }
-}
+//       // public event SnapshotStream.Progress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          return null;
+//       }
+
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
+
+//          }
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -37,20 +37,6 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
-      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
-         DateTime max_dt = DateTime.MinValue;
-         string rv = String.Empty;
-         string[] file_names = Directory.GetFiles(directory_name);
-         for (int i = 0; i < file_names.Count() - 1; i++) {
-            string fn = file_names[i];
-            var dt = parse_datetime_from_filename(fn);
-            if (dt > max_dt) {
-               max_dt = dt;
-               rv = fn;
-            }
-         }
-         return rv;
-      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -1,37 +1,66 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using System.Text.RegularExpressions;
 
 using BCG.Utilities.General;
 
-namespace BCG
-{
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
-   {
+namespace BCG {
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
+
+      public static TimeSpan parse_duration(string input) {
+         int days = 0, hours = 0, minutes = 0, seconds = 0;
+
+         // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
+         Regex regex = new Regex(@"(\d+)([dhms])");
+         MatchCollection matches = regex.Matches(input);
+
+         foreach (Match match in matches) {
+            int value = int.Parse(match.Groups[1].Value);
+            string unit = match.Groups[2].Value;
+
+            switch (unit) {
+               case "d":
+                  days = value;
+                  break;
+               case "h":
+                  hours = value;
+                  break;
+               case "m":
+                  minutes = value;
+                  break;
+               case "s":
+                  seconds = value;
+                  break;
+            }
+         }
+
+         return new TimeSpan(days, hours, minutes, seconds);
+      }
+
       #region constructors
+
+      public TimeBasedFilter(string col_name, string row_time_to_live)
+      : this(col_name, parse_duration(row_time_to_live)) {
+
+      }
+
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-      {
-         myVsd               = vsd;
-         myRowTimeColumnName = rowTimeColumnName;
-         myRowTimeToLive     = rowTimeToLive;
-         myFilterFrequency   = filterFrequency;
-         myNextFilter        = DateTime.MinValue;
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+         myVsd = vsd;
+         _time_col = rowTimeColumnName;
+         _ttl = rowTimeToLive;
+         myFilterFrequency = filterFrequency;
+         myNextFilter = DateTime.MinValue;
 
          // add worker
          myVsd.addWorker(this);
@@ -41,32 +70,25 @@ namespace BCG
       public event FilterChangeDelegate onFilterChange;
 
       #region IStreamFilter
-      public bool filter(IBlueRow row)
-      {
-         DateTime now       = DateTime.Now;
-         DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
+      public bool filter(IBlueRow row) {
+         DateTime now = DateTime.Now;
 
-         if (now > timeStamp)
-         {
-            TimeSpan span = now - timeStamp;
-
-            if (span >= myRowTimeToLive)
-            {
+         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
+         if (now > row_time) {
+            TimeSpan span = now - row_time;
+            if (span >= _ttl) {
                return (false);
             }
          }
-
          return (true);
       }
       #endregion
 
       #region IWorker
-      public void doWork()
-      {
+      public void doWork() {
          DateTime now = DateTime.Now;
 
-         if (now > myNextFilter)
-         {
+         if (now > myNextFilter) {
             // fire event
             onFilterChange();
 
@@ -76,8 +98,7 @@ namespace BCG
       #endregion
 
       #region IDisposable
-      public void Dispose()
-      {
+      public void Dispose() {
          // remove worker
          myVsd.removeWorker(this);
       }
@@ -85,10 +106,20 @@ namespace BCG
 
       #region state
       private ViewStreamDatabase myVsd;
-      private string             myRowTimeColumnName;
-      private TimeSpan           myRowTimeToLive;
-      private TimeSpan           myFilterFrequency;
-      private DateTime           myNextFilter;
+      private string _time_col;
+      private TimeSpan _ttl;
+      private TimeSpan myFilterFrequency;
+      private DateTime myNextFilter;
       #endregion
    }
+   [DataStreamUserRespresentation("timefilter")]
+   public class TimeBasedFilterStream : ViewStream {
+      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
+      :
+      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
+      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
+         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
+         mySetRealSource(new FilterStream(base_stream, filter));
+      }
+   }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -4,18 +4,14 @@ using BCG.Utilities;
 using System.Collections.Generic;
 using System.Linq;
 
-namespace BCG
-{
-   public class GroupByStream : MethodDrivenStream
-   {
-      private class MyFuncGroupByEvaluator : IGroupByEvaluator
-      {
+namespace BCG {
+   public class GroupByStream : MethodDrivenStream {
+      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         {
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
-            _groupby_func = groupby_func;            
+            _groupby_func = groupby_func;
          }
          public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
 
@@ -29,7 +25,7 @@ namespace BCG
       /// Serves as  a quick way to access all rows belonging to the same group
       /// </summary>
       private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable ;
+      private RowDictionaryTable myBookKeepingTable;
       private IGroupByEvaluator myGroupByEvaluator;
       private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
       //can map a schema to another
@@ -37,112 +33,98 @@ namespace BCG
       private int myIdGenerator = 0;
 
       public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
-      { }
+         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
-         : this (stream, groupByEvaluator, true)
-      {}
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
+         : this(stream, groupByEvaluator, true) { }
 
-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
-         : base (stream)
-      {
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
+         : base(stream) {
          myGroupByEvaluator = groupbyEvaluator;
          myIdGuard = idGuard;
       }
 
-      protected override void myHandleFirstCall ()
-      {
+      protected override void myHandleFirstCall() {
          mySchema = myLastArrivedPacket.schema;
-         myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
-         myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, 
+         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
+         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
             myGroupByEvaluator.groupByColumns);
-         GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
-         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
-         {
-            IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
-               rowGroup, new BlueTable (mySchema)));
+         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
+         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
+            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
+               rowGroup, new BlueTable(mySchema)));
             if (newRow != null)
-               temp.addRow (newRow, myGetGroupString (rowGroup));
+               temp.addRow(newRow, myGetGroupString(rowGroup));
          }
-         myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
+         myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
       }
 
       /// <summary>
       /// Adding need not to anything as deletion already takes care
       /// of addition
       /// </summary>
-      protected override void myHandleIncrementalAdd()   {}
-      
+      protected override void myHandleIncrementalAdd() { }
+
       /// <summary>
       /// Both deletes and adds are handled here to make the intermediate results not
       /// appear in the spitted fired DataPacket (note this is done only in GroupByStream 
       /// because of the nature of it)
       /// </summary>
-      protected override void myHandleIncrementalDelete()   
-      {
+      protected override void myHandleIncrementalDelete() {
          myFireSwitch = false;
          RowDictionaryTable delGroups, addGroups;
          //form groups for deleted and added rows
-         myInitIncrementalChangeData (out delGroups, out addGroups);
-         ArrayList handledAddGroups = new ArrayList ();
+         myInitIncrementalChangeData(out delGroups, out addGroups);
+         ArrayList handledAddGroups = new ArrayList();
          string groupRepString;
 
-         foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
             // BlueEventLog.logAssert (delGroup.Count > 0);
-            groupRepString = myGetGroupString (delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
-            handledAddGroups.Add (addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
-            IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
-            myFireSwitch |= myMergeRow (newRow, groupRepString);           
+            groupRepString = myGetGroupString(delGroup);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
+            handledAddGroups.Add(addGroup);
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
+            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            myFireSwitch |= myMergeRow(newRow, groupRepString);
          }
 
-         foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
             // BlueEventLog.logAssert (addGroup.Count > 0);
-            if (!handledAddGroups.Contains (addGroup))
-            {
+            if (!handledAddGroups.Contains(addGroup)) {
                //this is the case when there are rows added to a group and none
                //have been removed, hence it 'escapes' from being dealt with in 
                //the loop above
 
-               groupRepString = myGetGroupString (addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
+               groupRepString = myGetGroupString(addGroup);
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
                // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, 
-                  new BlueTable (aggGroup.schema)));
-               myFireSwitch |= myMergeRow (newRow, groupRepString);
+               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
+                  new BlueTable(aggGroup.schema)));
+               myFireSwitch |= myMergeRow(newRow, groupRepString);
             }
          }
       }
 
-      private bool myMergeRow (IBlueRow newRow, string groupRepString)
-      {
-         IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
+      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
+         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
          //BlueEventLog.logAssert (newRow != null || existingRow != null);
          if (existingRow == null && newRow == null)
             return false;
-         else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
+         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
             return false;
-         else if (existingRow == null)
-         {
+         else if (existingRow == null) {
             //row added for the first time
-            ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-            ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
             return true;
-         }
-         else
-         {
-            ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
-            ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
-            if (newRow != null)
-            {
-               ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-               ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+         } else {
+            ((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
+            ((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
+            if (newRow != null) {
+               ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+               ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
             }
             return true;
          }
@@ -154,77 +136,68 @@ namespace BCG
       /// </summary>
       /// <param name="delGroups"></param>
       /// <param name="addGroups"></param>
-      private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
-      {
+      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
          delGroups =
-            new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
          addGroups =
-            new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
          foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
-            myBookKeepingTable.removeRow (deletedRow);
+            myBookKeepingTable.removeRow(deletedRow);
          foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
-            myBookKeepingTable.addRow (addedRow);
-         myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
+            myBookKeepingTable.addRow(addedRow);
+         myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
       }
 
-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
          if (myBookKeepingTable.Count == 0)
             return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
-         myBookKeepingTable.clear ();
-         myGroupByEvaluator.clear ();
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
+         myBookKeepingTable.clear();
+         myGroupByEvaluator.clear();
       }
 
-      private IBlueRow myEvaluate (DataPacket dp)
-      {
-         IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
-         return evalRow == null ? null : 
+      private IBlueRow myEvaluate(DataPacket dp) {
+         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
+         return evalRow == null ? null :
             myIdGuard ?
-            BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
+            BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
       }
 
-      private string myGetGroupString (IBlueRowCollection group)
-      {
-         return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
+      private string myGetGroupString(IBlueRowCollection group) {
+         return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
       }
 
-      protected override  void myOnLastSubscriber () 
-      {
+      protected override void myOnLastSubscriber() {
          //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream 
          //gets resubscribed, we start the process of building data again, and treat the callback as the first 
          //callback.
-         myInit ();
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myInit();
+         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override void myInit ()
-      {
-         base.myInit ();
-         if (myBookKeepingTable != null)
-         {
+      protected override void myInit() {
+         base.myInit();
+         if (myBookKeepingTable != null) {
             //to avoid calling init before the first callback
-            myBookKeepingTable.clear ();
-            myGroupByEvaluator.clear ();
+            myBookKeepingTable.clear();
+            myGroupByEvaluator.clear();
          }
       }
 
-      public static string getRepString (DataPacket dp, string[] colNames)
-      {
-         return BlueDataUtility.rowToString (getRepRow (dp), colNames);
+      public static string getRepString(DataPacket dp, string[] colNames) {
+         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow (DataPacket dp)
-      {
+      public static IBlueRow getRepRow(DataPacket dp) {
          if (dp.newRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.newRows, 0);
+            return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.addedRows, 0);
+            return BlueDataUtility.extractRow(dp.addedRows, 0);
          else if (dp.deletedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.deletedRows, 0);
+            return BlueDataUtility.extractRow(dp.deletedRows, 0);
          // BlueEventLog.logAssert (false, "Illegal Data Packet");
          return null;
       }
@@ -237,22 +210,21 @@ namespace BCG
       /// <param name="newValue"></param>
       /// <param name="isAdd"></param>
       /// <returns></returns>
-      public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
-      {
-         if (lastValue == null || lastValue.Equals (DBNull.Value))
+      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
+         if (lastValue == null || lastValue.Equals(DBNull.Value))
             return newValue;
          // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
-            // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
-            // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
+         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
+         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
          Object retVal = null;
-         if (newValue.GetType().Equals (typeof(int)))
+         if (newValue.GetType().Equals(typeof(int)))
             retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
-         else if (newValue.GetType().Equals (typeof(Double)))
+         else if (newValue.GetType().Equals(typeof(Double)))
             retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
-         else if (lastValue.GetType().Equals (typeof(Decimal)))
+         else if (lastValue.GetType().Equals(typeof(Decimal)))
             retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
-         else{}
-            // BlueEventLog.logAssert (false, "Illegal type");
+         else { }
+         // BlueEventLog.logAssert (false, "Illegal type");
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,42 +1,47 @@
-using System;
-using System.Collections;
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
 
-using BCG.Utilities;
+// namespace BCG
+// {
+// 	/// <summary>
+// 	/// Computes the max for a given column
+// 	/// </summary>
+// 	public class MaxGroupByEvaluator : GroupByEvaluator
+// 	{
 
-namespace BCG {
-    /// <summary>
-    /// Computes the max for a given column
-    /// </summary>
-    public class MaxGroupByEvaluator : GroupByEvaluator {
+//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
+//       {}
 
-        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
+//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
+//          : base (colMapEpression, outputCol, groupByCols)   {}
 
-        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
-
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
-            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
-            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
-            if (maxRow != null) {
-                bts.Log.assert(currMaxRow != null);
-                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
-                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
-                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
-                    if (maxRow == null)
-                        myOutputTable.removeRow(hashString);
-                    else
-                        myOutputTable.addRow(maxRow, hashString);
-                }
-            }
-            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
-            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-            // otherwise if maxRow > currMaxRow then its replaced
-            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
-                myOutputTable.addRow(maxRow, hashString);
-            return myProject(myOutputTable.getRow(hashString));
-        }
-    }
-}
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
+//             myOutputTable;
+//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
+//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
+//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
+//          if (maxRow != null)
+//          {
+//             BlueEventLog.logAssert (currMaxRow != null);
+//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
+//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
+//             {
+//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
+//                if (maxRow == null)
+//                   myOutputTable.removeRow (hashString);
+//                else
+//                   myOutputTable.addRow (maxRow, hashString);
+//             }
+//          }
+//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
+//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+//          // otherwise if maxRow > currMaxRow then its replaced
+//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
+//             myOutputTable.addRow (maxRow, hashString);
+//          return myProject (myOutputTable.getRow (hashString));
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,15 +1,20 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class MaxStream : ViewStream {
-        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
-        }
-        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
-            mySetRealSource(new GroupByStream(stream,
-                                                new MaxGroupByEvaluator(outputCol, inputCols)));
-        }
+// namespace BCG
+// {
+// 	public class MaxStream : ViewStream
+// 	{
+//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
+//       {
+//       }
+// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
+// 		{
+//          mySetRealSource (new GroupByStream (stream, 
+//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
+// 		}
 
-        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
-    }
-}
+//       public MaxStream (ViewStream stream, string outputCol)
+//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
+//       {}
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,36 +1,40 @@
-using System;
+// using System;
+// using BCG.Utilities;
 
-using BCG.Utilities;
+// namespace BCG
+// {
+//    public class MinGroupByEvaluator : MaxGroupByEvaluator
+//    {
+//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
+//          : base (outputCol, groupByCols)  {}
 
-namespace BCG {
-    public class MinGroupByEvaluator : MaxGroupByEvaluator {
-        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
-           : base(outputCol, groupByCols) { }
+//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
+//          base (colMapEpression, outputCol, groupByCols)  {}
 
-        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
-           base(colMapEpression, outputCol, groupByCols) { }
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
+//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
+//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
+//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
+//          if (minRow != null)
+//          {
+//             // BlueEventLog.logAssert (currMinRow != null);
+//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
+//             {
+//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
+//                if (minRow == null)
+//                   myOutputTable.removeRow (stringRep);
+//                else
+//                   myOutputTable.addRow (minRow, stringRep);
+//             }
+//          }
+//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
+//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
+//             myOutputTable.addRow (minRow, stringRep);
+//          return myOutputTable.getRow (stringRep);
+//       }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
-            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
-            if (minRow != null) {
-                // BlueEventLog.logAssert (currMinRow != null);
-                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
-                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
-                    if (minRow == null)
-                        myOutputTable.removeRow(stringRep);
-                    else
-                        myOutputTable.addRow(minRow, stringRep);
-                }
-            }
-            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
-            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
-                myOutputTable.addRow(minRow, stringRep);
-            return myOutputTable.getRow(stringRep);
-        }
-
-    }
-}
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,13 +1,17 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class MinStream : ViewStream {
-        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
-            mySetRealSource(new GroupByStream(stream,
-                              new MinGroupByEvaluator(outputCol, inputCol)));
-        }
+// namespace BCG
+// {
+//    public class MinStream : ViewStream
+//    {
+//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
+//       {
+//          mySetRealSource (new GroupByStream (stream, 
+//                            new MinGroupByEvaluator (outputCol, inputCol)));
+//       }
 
-        public MinStream(ViewStream stream, string outputCol)
-                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
-    }
-}
+//       public MinStream (ViewStream stream, string outputCol)
+//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
+//       {}
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,82 +1,94 @@
-using System;
+// using System;
 
-namespace BCG {
-    /// <summary>
-    /// Computes the sum for a given column
-    /// </summary>
-    public class SumGroupByEvaluator : GroupByEvaluator {
-        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Computes the sum for a given column
+//    /// </summary>
+//    public class SumGroupByEvaluator : GroupByEvaluator
+//    {
+//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
+//       {}
 
-        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
+//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
+//          : base (colMapEpression, outputCol, groupByCols)   {}
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-
-            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-            return sumRow;
-        }
-    }
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
+         
+//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+//          return sumRow;
+//       }
+//    }
 
 
-    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
-        private string[] myOutputCols, myGroupByCols;
-        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
-            myOutputCols = spaceSeperatedOutputCols.Split(' ');
-            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-        }
+//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
+//    {
+//       private string[] myOutputCols, myGroupByCols;
+//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
+//       {
+//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
+//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+//       }
 
-        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
-            myOutputCols = output_columns;
-            myGroupByCols = groupby_columns;
-        }
+//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
+//       {
+//          myOutputCols = output_columns;
+//          myGroupByCols = groupby_columns;
+//       }
 
-        #region IGroupByEvaluator Members
+//       #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
+//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
 
-        public string[] groupByColumns { get { return myGroupByCols; } }
+//       public string[] groupByColumns { get { return myGroupByCols; } }
 
-        public void clear() {/*stateless*/}
+//       public void clear()  {/*stateless*/}
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
 
-        #endregion
-    }
+//       #endregion
+//    }
 
-    public class SumGroupByEvaluator<T> : GroupByEvaluator {
-        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-           : base(outputCol, groupByCols) { }
+//    public class SumGroupByEvaluator<T> : GroupByEvaluator
+//    {
+//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+//          : base(outputCol, groupByCols)
+//       { }
 
-        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
+//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+//          : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+//       public override IBlueRow evaluate(DataPacket dp)
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-            return sumRow;
-        }
-    }
+//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+//          return sumRow;
+//       }
+//    }
 
-    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
-        private string[] myOutputCols, myGroupByCols;
-        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
-            myOutputCols = spaceSeperatedOutputCols.Split(' ');
-            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-        }
+//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
+//    {
+//       private string[] myOutputCols, myGroupByCols;
+//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
+//       {
+//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
+//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+//       }
 
-        #region IGroupByEvaluator Members
+//       #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
+//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
 
-        public string[] groupByColumns { get { return myGroupByCols; } }
+//       public string[] groupByColumns { get { return myGroupByCols; } }
 
-        public void clear() {/*stateless*/}
+//       public void clear()  {/*stateless*/}
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
 
-        #endregion
-    }
+//       #endregion
+//    }
 
-}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,89 +1,107 @@
-using System;
-using System.Collections.Generic;
+// using System;
+// using System.Collections.Generic;
 
-namespace BCG {
-    public class NullMapper : IStreamMapper {
-        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+// namespace BCG
+// {
+//    public class NullMapper : IStreamMapper
+//    {
+//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-        public NullMapper(string[] valid_cols) {
-            foreach (string c in valid_cols)
-                _valid_cols[c] = true;
-            _valid_cols["id"] = true;
-        }
+//       public NullMapper(string[] valid_cols)
+//       {
+//          foreach (string c in valid_cols)
+//             _valid_cols[c] = true;
+//          _valid_cols["id"] = true;
+//       }
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-        public IBlueRow map(IBlueRow row) {
-            MutableBlueRow out_row = new MutableBlueRow(row);
-            foreach (string c in row.schema.columnNames)
-                if (!_valid_cols.ContainsKey(c))
-                    out_row.setValue(c, DBNull.Value);
-            return out_row;
-        }
-    }
+//       public IBlueRow map(IBlueRow row)
+//       {
+//          MutableBlueRow out_row = new MutableBlueRow(row);
+//          foreach (string c in row.schema.columnNames)
+//             if (!_valid_cols.ContainsKey(c))
+//                out_row.setValue(c, DBNull.Value);
+//          return out_row;
+//       }
+//    }
 
-    public class SumStream : ViewStream {
-        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
-            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-        }
+//    public class SumStream : ViewStream
+//    {
+//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
+//       {
+//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+//          mySetRealSource(new GroupByStream(stream,
+//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+//       }
 
-        public SumStream(ViewStream stream, string[] output_columns) {
-            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-            rs = new ProjectStream(rs, "!fake_col");
-            mySetRealSource(rs);
-        }
+//       public SumStream(ViewStream stream, string[] output_columns)
+//       {
+//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+//          rs = new ProjectStream(rs, "!fake_col");
+//          mySetRealSource(rs);
+//       }
 
-        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
-            string[] output_columns = space_seperated_output_columns.Split(' ');
-            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
-            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
-        }
+//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
+//       {
+//          string[] output_columns = space_seperated_output_columns.Split(' ');
+//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
+//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
+//       }
 
-        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
-            ViewStream rs = null;
-            groupby_columns = groupby_columns ?? new string[] { };
-            if (groupby_columns.Length == 0) {
-                // rs = new AddColumnsStream(stream, "fake_col=0");
-                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
-                // rs = new ProjectStream(rs, "!fake_col");
-            } else {
-                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-            }
-            if (force_null_into_other_columns) {
-                List<string> valid_cols = new List<string>(output_columns);
-                valid_cols.AddRange(groupby_columns);
-                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-            }
-            mySetRealSource(rs);
-        }
+//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
+//       {
+//          ViewStream rs = null;
+//          groupby_columns = groupby_columns ?? new string[] {};
+//          if (groupby_columns.Length == 0)
+//          {
+//             rs = new AddColumnsStream(stream, "fake_col=0");
+//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
+//             rs = new ProjectStream(rs, "!fake_col");
+//          }
+//          else
+//          {
+//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+//          }
+//          if (force_null_into_other_columns)
+//          {
+//             List<string> valid_cols = new List<string>(output_columns);
+//             valid_cols.AddRange(groupby_columns);
+//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+//          }
+//          mySetRealSource(rs);
+//       }
 
-        public SumStream(ViewStream stream, string outputCol)
-        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
-        }
-    }
+//    public SumStream (ViewStream stream, string outputCol)
+//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
+//    {
+//    }
+// }
 
-    public class IntSumStream : ViewStream {
-        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
-            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-        }
+// public class IntSumStream : ViewStream
+// {
+//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
+//    {
+//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+//       mySetRealSource(new GroupByStream(stream,
+//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
+//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+//    }
 
-    }
+// }
 
 
-    public class SumStream<T> : ViewStream {
-        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
-            string[] inputCols = spaceSeparatedInputCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
-        }
-    }
-}
+//    public class SumStream<T> : ViewStream
+//    {
+//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
+//       {
+//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
+//          mySetRealSource(new GroupByStream(stream,
+//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -1,79 +0,0 @@
-using BCG;
-
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-using ZstdSharp.Unsafe;
-
-namespace bts {
-    public class RowDiffStream : ViewStream {
-        private ViewStream _vs;
-        private readonly string _order_col, _col_1, _col_2;
-        private readonly int _diff;
-        private IBlueTableSchema _out_schema;
-        private readonly string _diff_col_name;
-        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
-            _vs = vs;
-            _order_col = order_col;
-            _col_1 = col_1;
-            _col_2 = col_2;
-            _diff = diff;
-            _diff_col_name = diff_col_name;
-        }
-
-        private string _diff_col_name_perc {
-            get {
-                return $"{_diff_col_name}_percentage";
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _vs.onChange += _handle_vs;
-        }
-
-        protected override void myOnLastSubscriber() {
-            _vs.onChange -= _handle_vs;
-        }
-
-        private void _handle_vs(object dp_obj) {
-            var dp = dp_obj as DataPacket;
-            IBlueRowCollection nrows = dp.newRows;
-            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
-            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
-            List<IBlueRow> row_list = new List<IBlueRow>();
-            foreach (IBlueRow r in nrows)
-                row_list.Add(r);
-            //OrderBy sorts in ascedning order
-            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
-            var sorted_row_array = sorted_rows.ToArray();
-            // reverse it to make it descending    
-            sorted_rows.Reverse();
-            //sort rows so we can do the diff
-            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
-            int i = 0;
-            foreach (IBlueRow r in sorted_rows) {
-                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
-                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
-                // we assume x - y, and name variables like that
-                double x = r.get<double>(_col_1);
-                int y_row_idx = i + _diff;
-                double diff = System.Double.NaN;
-                double perc = System.Double.NaN;
-                if (y_row_idx < sorted_rows.Count()) {
-                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
-                    diff = x - y;
-                    perc = 100.0 * diff / y;
-                }
-                nr.setValue(_diff_col_name, diff);
-                nr.setValue(_diff_col_name_perc, perc);
-                rv.addRow(nr);
-                i++;
-            }
-
-            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -1,51 +1,64 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;
+
 using BCG;
 using BCG.Utilities;
 
-namespace BCG
-{
-   public class LiteralStream : ViewStream
-   {
-      public LiteralStream(string tableName, string schema, string value)
-      {
-         string [] columns = schema.Split(' ');
+namespace BCG {
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');
 
 
-         ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id",0,typeof(int));
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
          int n = 1;
-         foreach (string c in columns)
-         {
-            string [] columnAndType = c.Split('/');
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
             if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c,n,typeof(double));
+               ci[n] = new ColumnInfo(c, n, typeof(double));
             else
-               ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
             n++;
          }
 
 
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
-         string [] rows = value.Split(';');
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
          n = 0;
-         foreach (string r in rows) 
-            try 
-            {
-               if(r == string.Empty)
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
                   continue;
                t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));   
-            }
-            catch (Exception e)
-            {
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
                throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
             }
 
          mySetRealSource(new BlueRowCollectionStaticStream(t));
       }
 
-      public LiteralStream(string schema, string value) : this ("literal",schema, value)
-      {}
+      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+
+      // public LiteralStream(string value) {
+
+      // }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs

@@ -1,24 +1,24 @@
-using System;
+// using System;
 
-namespace BCG
-{
-   public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
-   {
-      IBlueTableSchema mySchema;
+// namespace BCG
+// {
+//    public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
+//    {
+//       IBlueTableSchema mySchema;
 
-      public IBlueRow map (IBlueRow row)
-      {
-         if (mySchema == null)
-         {
-            ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
-            int i = 0;
-            foreach (ColumnInfo ci in row.schema.columnInfo)
-               cis[i++] = ci.colType == null ?
-                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
-                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
-            mySchema = new BlueTableSchema (row.schema.tableName, cis);
-         }
-         return new BlueRow (mySchema, row);
-      }
-   }
-}
+//       public IBlueRow map (IBlueRow row)
+//       {
+//          if (mySchema == null)
+//          {
+//             ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
+//             int i = 0;
+//             foreach (ColumnInfo ci in row.schema.columnInfo)
+//                cis[i++] = ci.colType == null ?
+//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
+//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
+//             mySchema = new BlueTableSchema (row.schema.tableName, cis);
+//          }
+//          return new BlueRow (mySchema, row);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -8,8 +8,7 @@ using mbts;
 using mbts.spark;
 using mbts.config;
 
-namespace BCG
-{
+namespace BCG {
 
 
    /// <summary>
@@ -18,27 +17,48 @@ namespace BCG
    /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
    /// is preserved across the rows, ie row["id"] = f(row)["id"]
    /// </summary>
-   [DataStreamUserRespresentation (false)]
-   public class MapStream : MethodDrivenStream
-   {
-      private HighPerformanceTimer t = new HighPerformanceTimer ();
+   [DataStreamUserRespresentation(false)]
+   public class MapStream : MethodDrivenStream {
+      private HighPerformanceTimer t = new HighPerformanceTimer();
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
 
-      public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
-      {
-         if (myUseNewMap)
-            mySetRealSource (new SuperMapStream (vs, mapper));
-         else
-            myMapper = mapper;
+      public class FuncWrapper : IStreamMapper {
+         private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func;
+         Func<IBlueRow, IBlueRow> _row_map_func;
+
+         public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
+         Func<IBlueRow, IBlueRow> row_map_func) {
+            _schema_map_func = schema_map_func;
+            _row_map_func = row_map_func;
+         }
+         public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+            return _schema_map_func(input_schema);
+         }
+         public IBlueRow map(IBlueRow row) {
+            return _row_map_func(row);
+         }
+
       }
-      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
-      {
+
+      public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
          if (myUseNewMap)
             mySetRealSource(new SuperMapStream(vs, mapper));
          else
-         {
+            myMapper = mapper;
+      }
+
+      public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
+                                       Func<IBlueRow, IBlueRow> row_map_func)
+                                       :
+                                       this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
+      }
+
+      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
+         if (myUseNewMap)
+            mySetRealSource(new SuperMapStream(vs, mapper));
+         else {
             mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
             myChangeableMapper = mapper;
          }
@@ -47,77 +67,68 @@ namespace BCG
       /// <summary>
       /// Gets or sets the value of the mapper. Setting the value will result in re
       /// </summary>
-      private void myMapperChanged () 
-      {
+      private void myMapperChanged() {
          if (myLastArrivedPacket != null)
-            myChangeHandler (new DataPacket (myLastArrivedPacket.newRows, 
+            myChangeHandler(new DataPacket(myLastArrivedPacket.newRows,
                                              myLastArrivedPacket.newRows,
                                              myLastArrivedPacket.newRows));
       }
 
 
-      protected override void myHandleFirstCall ()
-      {
-         
-         IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
-         myLastFiredPacket = new DataPacket ();
-         myLastFiredPacket.newRows = new BlueTable (outputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
+      protected override void myHandleFirstCall() {
+
+         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
+         myLastFiredPacket = new DataPacket();
+         myLastFiredPacket.newRows = new BlueTable(outputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(outputSchema);
          foreach (IBlueRow row in myLastArrivedPacket.newRows)
-            ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
+            ((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row));
          myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
       }
 
-      protected override void myHandleIncrementalAdd()
-      {
+      protected override void myHandleIncrementalAdd() {
          //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows)
-         {
-            IBlueRow newRow = myMapper.map (row);
-            (myLastFiredPacket.newRows as BlueTable).addRow (newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
+            IBlueRow newRow = myMapper.map(row);
+            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
+            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
          }
          //t.stop ();
          //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
          //System.Diagnostics.Trace.WriteLine (" ");
       }
 
-      protected override void myHandleIncrementalDelete()
-      {  
+      protected override void myHandleIncrementalDelete() {
          //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
-         {
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
             IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow (row);
+            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
+            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
          }
          //t.stop ();
          //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
       }
-     
 
-      protected override void myOnFirstSubscriber() 
-      {
-         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
+
+      protected override void myOnFirstSubscriber() {
+         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override  void myOnLastSubscriber () 
-      {
-         myInit ();
+      protected override void myOnLastSubscriber() {
+         myInit();
          if (myStream != null)
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-         myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
       }
 
-      
+
    }
 
    /*

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -1,12 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+
 using BCG.Utilities;
 using BCG.Utilities.General;
+
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// SuperMap gives the user the ability
    /// to map one row to one or more rows
@@ -17,13 +18,11 @@ namespace BCG
    /// can chage itself and raise an event causing
    /// a remapping to happen
    /// </summary>
-   public class SuperMapStream : ViewStream
-   {
+   public class SuperMapStream : ViewStream {
       #region super mapper that wraps around stream mapper
-      private class MyMappingWrapper : ISuperStreamMapper
-      {
+      private class MyMappingWrapper : ISuperStreamMapper {
          private IStreamMapper myMapper;
-         
+
          public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
 
          #region ISuperStreamMapper Members
@@ -32,8 +31,7 @@ namespace BCG
 
          public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
 
-         public event MapperChangeDelegate onMapperChange
-         {
+         public event MapperChangeDelegate onMapperChange {
             add { }
             remove { }
          }
@@ -42,21 +40,18 @@ namespace BCG
       }
       #endregion
 
-      private class MyFuncWrapper : ISuperStreamMapper
-      {
+      private class MyFuncWrapper : ISuperStreamMapper {
          private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
          private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
          public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
-                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         {
+                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) {
             _get_schema = get_schema;
             _map_rows = map_rows;
          }
 
          public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
 
-         public IBlueRow[] map(IBlueRow r)
-         {
+         public IBlueRow[] map(IBlueRow r) {
             IEnumerable<IBlueRow> rows = _map_rows(r);
             return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
          }
@@ -76,19 +71,15 @@ namespace BCG
 
       public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
                                             Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, new MyFuncWrapper(get_schema, map_rows))
-      { }
+         : this(vs, new MyFuncWrapper(get_schema, map_rows)) { }
 
       public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, s => out_schema, map_rows)
-      { }
+         : this(vs, s => out_schema, map_rows) { }
 
       public SuperMapStream(ViewStream vs, IStreamMapper mapper)
-         : this(vs, new MyMappingWrapper(mapper))
-      { }
+         : this(vs, new MyMappingWrapper(mapper)) { }
 
-      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
-      {
+      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) {
          myStream = vs;
          myMapper = mapper;
          myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
@@ -96,8 +87,7 @@ namespace BCG
 
       protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          myStream.onChange -= new ChangeDelegate(myHandleData);
          myRows = null;
          myBaseRows = null;
@@ -105,8 +95,7 @@ namespace BCG
          //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
       }
 
-      private void myHandleData(object newValue)
-      {
+      private void myHandleData(object newValue) {
          DataPacket idp = newValue as DataPacket;
          //HighPerformanceTimer timer = new HighPerformanceTimer();
          //timer.start();
@@ -117,24 +106,19 @@ namespace BCG
          BlueTable dels = new BlueTable(myRows.schema),
                      adds = new BlueTable(myRows.schema);
 
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
             List<IBlueRow> mappedRows = myMap[(int)r["id"]];
-            foreach (IBlueRow mr in mappedRows)
-            {
+            foreach (IBlueRow mr in mappedRows) {
                dels.addRow(mr);
                myRows.deleteRow(mr);
             }
             myMap.Remove((int)r["id"]);
          }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             IBlueRow[] mappedRows = myMapper.map(r);
-            foreach (IBlueRow mr in mappedRows)
-            {
+            foreach (IBlueRow mr in mappedRows) {
                IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"]))
-               {
+               if (!myMap.ContainsKey((int)r["id"])) {
                   myMap.Add((int)r["id"], new List<IBlueRow>());
                }
                myMap[(int)r["id"]].Add(idChangedMapRow);
@@ -145,13 +129,12 @@ namespace BCG
          //timer.stop();
          // if (timer.lastIntervalTime > 20)
          //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
-         // myFireEvent(new DataPacket(myRows, adds, dels));
+         myFireEvent(new DataPacket(myRows, adds, dels));
       }
 
-      private void myHandleMapChange()
-      {
+      private void myHandleMapChange() {
          if (myBaseRows != null)
-            myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
+            myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs

@@ -0,0 +1,142 @@
+using BCG;
+using System;
+using System.IO;
+using MongoDB.Bson;
+using BCG.Utilities.General;
+using System.Linq;
+using System.Collections.Generic;
+using System.Runtime.Intrinsics.X86;
+
+
+namespace bts {
+   // listens to the underlying stream, and writes each seriialized packet in a 
+   // seperate file
+   [DataStreamUserRespresentation("dp_recorder")]
+   public class DataPacketRecorderStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _folder;
+      StreamIOSerializationController _serializer = new StreamIOSerializationController();
+      private UInt32 _n_updates = 0;
+      public DataPacketRecorderStream(ViewStream base_stream, string folder) {
+         _base_stream = base_stream;
+         _folder = folder;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_update;
+      }
+
+      private void _handle_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         byte[] ser_data = null;
+         if (_n_updates == 0) {
+            StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
+            ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
+         } else {
+            StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
+            ser_data = _serializer.serialize(new StreamIOMessage(mra));
+         }
+         Log.assert(ser_data != null);
+         _n_updates++;
+         string file_name = $"{_folder}\\{_n_updates}.bin";
+         _write_to_file(ser_data, file_name);
+         myFireEvent(dp);
+      }
+
+      private static void _write_to_file(byte[] data, string file_name) {
+         using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
+            // Write the byte array to the file
+            fs.Write(data, 0, data.Length);
+         }
+      }
+   }
+   [DataStreamUserRespresentation("dp_replay")]
+   public class DataPacketReplayStream : ViewStream, IWorker {
+      private ViewStreamDatabase _vsd;
+      private readonly string _folder_path;
+      private readonly TimeSpan _fire_period;
+      IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
+      private int _file_ptr = 0;
+      private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
+      private BlueTable _state = null;
+
+      public DataPacketReplayStream(string folder_path, string fire_period)
+      :
+      this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
+      }
+
+      public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
+         _vsd = vsd;
+         _folder_path = folder_path;
+         _fire_period = fire_period;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         MasterWorker.getInstance(false).addWorker(this, _fire_period);
+         _load_all_files();
+      }
+
+      protected override void myOnLastSubscriber() {
+         MasterWorker.getInstance(false).removeWorker(this);
+      }
+
+      public void doWork() {
+         if (_file_ptr >= _file_data.Count())
+            // we can set _file_ptr to 0 and keep going as well
+            return;
+         var curr_d = _file_data[_file_ptr++];
+         var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
+         StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
+         Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
+         switch (msg.op) {
+            case StreamIOMessage.StreamOp.CREATE: {
+                  _state = new BlueTable(msg.create_message.rows);
+                  break;
+               }
+            case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
+                  var mra = msg.multiple_remove_add_message;
+                  foreach (IBlueRow r in mra.removed_rows)
+                     _state.deleteRow(r);
+                  foreach (IBlueRow r in mra.added_rows)
+                     _state.addRow(r);
+                  break;
+               }
+         }
+         var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
+         myFireEvent(dp);
+      }
+
+      private void _load_all_files() {
+         Func<string, string> extract_file_name = (string full_path) => {
+            string fn = full_path.Split("\\").Last();
+            fn = fn.Split(".").First();
+            return fn;
+
+         };
+         string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
+
+         // Iterate over the .bin files and perform an action
+         foreach (string file in bin_files) {
+            byte[] file_data = File.ReadAllBytes(file);
+            _file_data.Add(Tuple.Create(file, file_data));
+         }
+         _file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
+            string fn1 = extract_file_name(t1.Item1);
+            string fn2 = extract_file_name(t2.Item1);
+            int fn_i1 = Int32.Parse(fn1);
+            int fn_i2 = Int32.Parse(fn2);
+            if (fn_i1 < fn_i2)
+               return -1;
+            else if (fn_i1 == fn_i2)
+               return 0;
+            else if (fn_i1 > fn_i2)
+               return 1;
+            return 0;
+         });
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -0,0 +1,195 @@
+using BCG;
+using BCG.Utilities.General;
+using System.Collections.Generic;
+
+//using MongoDB.Driver;
+
+using System;
+using System.IO;
+using System.Linq;
+
+namespace bts {
+    public class PubSubTestingStream : ViewStream, IWorker {
+        public class PayloadMessage {
+            public int msg_id = 0;
+            public string guid;
+            public Int64 travel_epoch_ms = 0;
+            public byte[] payload = null;
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
+                msg_id = msg_id_args;
+                travel_epoch_ms = send_epoch_args;
+                payload = new byte[payload_size];
+                for (int i = 0; i < payload_size; i++) {
+                    payload[i] = payload_byte;
+                }
+            }
+            private PayloadMessage() { }
+            public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
+
+            public byte[] serialize() {
+                var str = new MemoryStream();
+                BinaryWriter bw = new BinaryWriter(str);
+                bw.Write(msg_id);
+                bw.Write(guid);
+                bw.Write(travel_epoch_ms);
+                bw.Write(payload.Length);
+                bw.Write(payload);
+                str.Position = 0;
+                var bytes = str.ToArray();
+                return bytes;
+            }
+
+            public static PayloadMessage deserialize(byte[] ser_bytes) {
+                var str = new MemoryStream(ser_bytes);
+                BinaryReader br = new BinaryReader(str);
+                int msg_id = br.ReadInt32();
+                string guid = br.ReadString();
+                Int64 epoch = br.ReadInt64();
+                int len = br.ReadInt32();
+                var payload = br.ReadBytes(len);
+                return new PayloadMessage() {
+                    msg_id = msg_id,
+                    guid = guid,
+                    travel_epoch_ms = epoch,
+                    payload = payload
+                };
+            }
+
+        }
+        //public void
+        private readonly int _msg_size;
+        private readonly TimeSpan _publish_period;
+        private DateTime _last_publish_time = DateTime.MinValue;
+        private PublisherCore _pub_core = new PublisherCore();
+        private BlueTable _table;
+        private int _n_pubs = 0;
+        private readonly int _max_pubs;
+        private int _n_updates = 0;
+        private bool _continue_sub_thread = true;
+        private ViewStreamDatabase _vsd;
+        private bool _fire_dirty = false;
+        private PayloadMessage _last_sent_msg, _last_recvd_msg;
+        private readonly int _payload_sz;
+        private const byte _payload_byte = (byte)'A';
+        private readonly string _tag_name;
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+            _vsd = vsd;
+            _payload_sz = payload_size;
+            var schema = BlueDataUtility.form_schema("PubSubTest",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("n_pubs", TypeCode.Int32),
+            Tuple.Create("n_recvs", TypeCode.Int32),
+            Tuple.Create("last_sent_id", TypeCode.Int32),
+            Tuple.Create("last_recv_id", TypeCode.Int32),
+            Tuple.Create("last_sent_guid", TypeCode.String),
+            Tuple.Create("last_recv_guid", TypeCode.String),
+            Tuple.Create("match", TypeCode.Boolean),
+            Tuple.Create("echo_time", TypeCode.Int32));
+            _table = new BlueTable(schema);
+            _publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
+            _msg_size = payload_size;
+            _max_pubs = max_pubs;
+            _tag_name = tag_name;
+        }
+
+        public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
+        : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
+
+        protected override void myOnFirstSubscriber() {
+            _vsd.addWorker(this);
+            System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
+            th.IsBackground = true;
+            th.Start();
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vsd.removeWorker(this);
+            _continue_sub_thread = false;
+        }
+
+        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
+            var idx = 0;
+            foreach (var msg in updates) {
+                bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
+                idx++;
+            }
+        }
+
+        private void _subscribe_thread_func() {
+            // subscription is done in another thread
+            SubscriberCore sub_core = new SubscriberCore();
+            sub_core.subscribe(_tag_name);
+            while (_continue_sub_thread) {
+                // TODO recv_id
+                var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
+                if (updates == null || updates.Count() == 0)
+                    continue;
+                _log_updates(updates);
+                _n_updates += updates.Count();
+                if (updates.Count() > 0) {
+                    var server_msg = updates.Last();
+                    if (server_msg.data != null && server_msg.data.Length > 0) {
+                        _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
+                        _last_recvd_msg.travel_epoch_ms = epoch_now();
+                    } else {
+                        // why are we getting empty or null data ? (server is sending data regardless?)
+                    }
+                    _fire_dirty = true;
+                }
+            }
+        }
+
+        private void _publish_next_packet() {
+            if (_last_sent_msg == null) {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
+                    guid = Guid.NewGuid().ToString()
+                };
+            } else {
+                _last_sent_msg.msg_id++;
+                _last_sent_msg.guid = Guid.NewGuid().ToString();
+                _last_sent_msg.travel_epoch_ms = epoch_now();
+            }
+            var ser_bytes = _last_sent_msg.serialize();
+            _n_pubs++;
+            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
+        }
+
+        private void _fire_dp() {
+            object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
+            object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
+            object echo_time = DBNull.Value;
+            object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
+            object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
+            if (_last_sent_msg != null && _last_recvd_msg != null) {
+                echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
+            }
+            var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
+            IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
+                                        last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
+            BlueTable table = new BlueTable(_table.schema);
+            table.addRow(nr);
+            var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        public void doWork() {
+            if (DateTime.Now - _last_publish_time > _publish_period)
+                if (_n_pubs <= _max_pubs) {
+                    _publish_next_packet();
+                    _last_publish_time = DateTime.Now;
+                }
+            // publish 
+            // TODO: send id
+            if (_fire_dirty) {
+                _fire_dirty = false;
+                _fire_dp();
+            }
+        }
+
+        public static long epoch_now() {
+            return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs

@@ -0,0 +1,55 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+
+   public class SimpleTCPEchoStream : ViewStream, IWorker {
+      private string _ip;
+      private int _port;
+      private SimpleTCPClient _tcp_client;
+      private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
+      Tuple.Create("id", TypeCode.Int32),
+      Tuple.Create("event", TypeCode.String));
+      private BlueTable _table;
+      private SyncQueue<string> _action_queue = new SyncQueue<string>();
+
+      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) {
+         _ip = ip;
+         _port = port;
+         _table = new BlueTable(_schema);
+         _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         ViewStreamDatabase.instance.addWorker(this);
+         _tcp_client.register_listener((msg) => { _on_msg(msg); });
+         _tcp_client.start();
+      }
+
+      private void _on_msg(string msg) {
+         _action_queue.enqueue(msg);
+      }
+
+      public void doWork() {
+         if (_action_queue.getCount() == 0)
+            return;
+         BlueTable added_rows = new BlueTable(_schema);
+         var sz = _action_queue.getCount();
+         for (int i = 0; i < sz; i++) {
+            string msg = _action_queue.dequeue();
+            if (msg == null)
+               break;
+            IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
+            _table.addRow(nr);
+            added_rows.addRow(nr);
+         }
+         if (added_rows.Count == 0)
+            return;
+         DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
+         myFireEvent(dp);
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs

@@ -0,0 +1,77 @@
+using System.Net.Sockets;
+using System.Net;
+
+using BCG;
+using System;
+using System.Collections.Generic;
+using MongoDB.Bson;
+using BCG.Utilities.General;
+using System.Linq;
+
+
+namespace bts {
+   // this class, a little specialized, takes the first snapshot thats fired 
+   // from the base stream sorts them by a given column, and fires one row at a time 
+   public class SnapshotReplayStream : ViewStream, IWorker {
+
+      private ViewStreamDatabase _vsd;
+      private ViewStream _snapshot_stream;
+      private readonly string _sort_col;
+      private readonly int _buffer_size;
+      private TimeSpan _fire_period;
+      private int _in_row_ptr = 0;
+      private List<IBlueRow> _in_rows = new List<IBlueRow>();
+      private CircularBufferTable _out_table;
+
+
+      public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period)
+      : this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { }
+
+      public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) {
+         _vsd = vsd;
+         _snapshot_stream = base_str;
+         _sort_col = sort_col;
+         _buffer_size = buffer_size;
+         _fire_period = fire_period;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _snapshot_stream.onChange += _handle_stream_update;
+         MasterWorker.getInstance(false).addWorker(this, _fire_period);
+      }
+
+      protected override void myOnLastSubscriber() {
+         base.myOnLastSubscriber();
+      }
+
+      private void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         if (_in_rows.Count > 0)
+            return;
+         _out_table = new CircularBufferTable(dp.schema, _buffer_size);
+         // _in_rows is filled up only once
+         foreach (IBlueRow r in dp.newRows) {
+            _in_rows.Add(r);
+         }
+         _in_rows.Sort((IBlueRow r1, IBlueRow r2) => {
+            return BlueDataUtility.compare_rows(r1, r2, _sort_col);
+         });
+      }
+
+      public void doWork() {
+         if (_in_row_ptr >= _in_rows.Count) {
+            // we can loop around or not here
+            return;
+         }
+         if (_in_rows.Count > 0) {
+            // fire row
+            IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
+            _out_table.add_row(curr_row);
+            DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
+               myLastValue == null ? null : myLastValue as DataPacket, false);
+            myFireEvent(dp);
+         }
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -0,0 +1,817 @@
+
+
+using Amazon.Runtime.Internal;
+
+using bts;
+
+using DnsClient.Protocol;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System;
+using Microsoft.VisualBasic;
+using Spark.Core.Collections;
+using System.Reflection;
+
+namespace BCG {
+
+    /// <summary>
+    /// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
+    /// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
+    /// </summary>
+    [DataStreamUserRespresentation("tests")]
+    public class TesStream : ViewStream {
+        static byte[] create_byte_array() {
+            Random random = new Random();
+            var sz = random.Next(1, 1000);// % 1000;
+
+            // Generate a random byte
+            byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
+            byte[] rv = new byte[sz];
+            for (int i = 0; i < sz; i++) {
+                rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
+            }
+            return rv;
+        }
+
+        private IBlueTableSchema _schema;
+        class TestResult {
+            public string test_name = MethodBase.GetCurrentMethod().Name;
+            public bool test_result;
+        }
+        private List<Func<TestResult>> _tests = new
+         List<Func<TestResult>>();
+
+        public TesStream() {
+            _schema = BlueDataUtility.form_schema("Tests",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("test", TypeCode.String),
+            Tuple.Create("result", TypeCode.Boolean),
+            Tuple.Create("exception", TypeCode.String)
+            );
+            // register tests here
+            // _tests.Add(new Func<TestResult>(_run_sub_msg_test));
+            // _tests.Add(new Func<TestResult>(_run_pub_msg_test));
+            // _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
+            // _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
+            // _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
+            // _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
+            // _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
+            //  _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
+            //_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
+            // _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
+            // _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
+            // _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
+            // _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
+            //_tests.Add(new Func<TestResult>(_cpp_data_test));
+            //_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
+            _tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
+
+        }
+
+        protected override void myOnFirstSubscriber() {
+            BlueTable rows = new BlueTable(_schema);
+            // u the tests here            
+            foreach (var test in _tests) {
+                string ex_msg = String.Empty;
+                TestResult tr = new TestResult() {
+                    test_name = test.Method.Name,
+                    test_result = false
+                };
+                try {
+                    tr = test();
+                } catch (Exception ex) {
+                    ex_msg = ex.Message + "\n" + ex.StackTrace;
+                }
+                IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
+                rows.addRow(r);
+            }
+            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+        protected override void myOnLastSubscriber() {
+        }
+
+        private TestResult _run_sub_msg_test() {
+            bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
+                connection = Guid.NewGuid().ToString(),
+                subscribing_app = Guid.NewGuid().ToString(),
+                tags = new string[] { "one", "two", "three" }
+            };
+            MemoryStream str = new MemoryStream();
+            sub_msg.serialize_into_str(str);
+            str.Position = 0;
+            bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
+            sub_msg_.consruct_deserialize(str);
+            bool res = sub_msg.Equals(sub_msg_);
+            return new TestResult {
+                test_name = "SubscribeMsg",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pub_msg_test() {
+            var pub_msg = new bts.PublishMsg() {
+                connection = Guid.NewGuid().ToString(),
+                publishing_app = Guid.NewGuid().ToString(),
+                tag = Guid.NewGuid().ToString(),
+                tag_type = "DS_",
+                is_compressed = false,
+                publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                data_to_publish = create_byte_array(),
+            };
+            MemoryStream str = new MemoryStream();
+            pub_msg.serialize_into_str(str);
+            str.Position = 0;
+            var pub_msg_ = new bts.PublishMsg();
+            pub_msg_.consruct_deserialize(str);
+            bool res = pub_msg.Equals(pub_msg_);
+            return new TestResult {
+                test_name = "PublishMsg",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pubsub_server_msg_test() {
+            var msg = new bts.PubSubServerMessage() {
+                tag = Guid.NewGuid().ToString(),
+                tag_type = Guid.NewGuid().ToString(),
+                data = create_byte_array()
+            };
+            MemoryStream str = new MemoryStream();
+            msg.serialize_into_str(str);
+            str.Position = 0;
+            var msg_ = new bts.PubSubServerMessage();
+            msg_.consruct_deserialize(str);
+            bool res = msg.Equals(msg_);
+            return new TestResult {
+                test_name = "PubSubServerMessage",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pubsub_client_msg_test() {
+            PubSubClientMessage msg = new PubSubClientMessage() {
+                msg_type = PubSubMessageType.SUBSCRIBE,
+                sub_msg = new bts.SubscribeMsg() {
+                    connection = Guid.NewGuid().ToString(),
+                    subscribing_app = Guid.NewGuid().ToString(),
+                    tags = new string[] { "one", "two", "three" }
+                }
+            };
+            var str = new MemoryStream();
+            msg.serialize_into_str(str);
+            str.Position = 0;
+            var msg_ = new PubSubClientMessage();
+            msg_.consruct_deserialize(str);
+            var res = msg.Equals(msg_);
+            return new TestResult {
+                test_name = "PubSubClientMessage",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_columninfo_serialization_test() {
+            Func<ColumnInfo, bool> test_ci = (ci) => {
+                MemoryStream str = new MemoryStream();
+                BinaryWriter bw = new BinaryWriter(str);
+                StreamSerializationUtils.serialize_column_info(ci, ref bw);
+                str.Position = 0;
+                //var ci_txt = BlueDataUtility.to_string(ci);
+                //var ci_ser = 
+                //bts.Log.Gen.info("ci: {}")
+                BinaryReader br = new BinaryReader(str);
+                var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
+                return BlueDataUtility.compute_equality(ci, ci_prime);
+            };
+            int i = 0;
+            foreach (Type t in BlueDataUtility.valid_column_types()) {
+                ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
+                i++, t);
+                var res = test_ci(ci);
+                if (!res) {
+                    return new TestResult {
+                        test_name = "ColumnInfoSerialization",
+                        test_result = false
+                    };
+                }
+            }
+            return new TestResult {
+                test_name = "ColumnInfoSerialization",
+                test_result = true
+            };
+        }
+
+
+        private TestResult _run_schema_serialization_test() {
+            var table_name = "test_table";
+            var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
+            IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            StreamSerializationUtils.serialize_schema(schema, ref bw);
+            _write_to_file(str, @"c:\Temp\tests\schema.bin");
+            var hex_txt = StreamSerializationUtils.to_hex(str);
+            bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
+
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
+            var res = BlueDataUtility.compute_equality(schema, schema_prime);
+
+            return new TestResult {
+                test_name = "SchemaSerialization",
+                test_result = res
+            };
+        }
+
+
+
+
+        private TestResult _run_datetime_seriliaztion_test() {
+            DateTime dt = DateTime.Now;
+            DateTimeOffset dt_off = new DateTimeOffset(dt);
+            long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
+            DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
+            DateTime dt_p = dt_off_pr.DateTime;
+            var passes = dt.Equals(dt_p);
+            return new TestResult {
+                test_name = "_run_datetime_seriliaztion_test",
+                test_result = false
+            };
+        }
+
+        private TestResult _run_full_stream_serialization_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
+            UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
+            ser.serialize(stream_io_msg, ref bw);
+            str.Position = 0;
+            BinaryReader br = new System.IO.BinaryReader(str);
+            UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
+            var stream_io_msg_prime = deser.deserialize(ref br);
+            bool passes = stream_io_msg.Equals(stream_io_msg_prime);
+            var n_dels = 50;
+            BlueTable del_rows = new BlueTable(cr_rows.schema);
+            foreach (IBlueRow r in cr_rows) {
+                del_rows.addRow(r);
+                if (del_rows.Count == n_dels)
+                    break;
+            }
+            IBlueRowCollection added_rows = del_rows;
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            str = new MemoryStream();
+            bw = new BinaryWriter(str);
+            stream_io_msg = new bts.StreamIOMessage(mra_msg);
+            ser.serialize(stream_io_msg, ref bw);
+            bw.BaseStream.Position = 0;
+            br = new BinaryReader(str);
+            //deser = new UniversalStreamIODeserializer();
+            stream_io_msg_prime = deser.deserialize(ref br);
+            passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
+            return new TestResult {
+                test_name = "UniversalSerialization",
+                test_result = passes
+            };
+        }
+
+        private TestResult _run_full_stream_serialization_with_controller_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
+            IStreamIOMsgSerializer ser = new StreamIOSerializationController();
+            var ser_bytes = ser.serialize(stream_io_msg);
+            IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
+            var stream_io_msg_prime = deser.deserialize(ser_bytes);
+            bool passes = stream_io_msg.Equals(stream_io_msg_prime);
+            //mult remove adds
+            // remove all rows and add 50 more
+
+            BlueTable del_rows = new BlueTable(cr_rows);
+            BlueTable added_rows = new BlueTable(cr_rows.schema);
+            var n_adds = 50;
+            for (int i = 0; i < n_adds; i++) {
+                var nr = generate_random_row(schema, i);
+                added_rows.addRow(nr);
+            }
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            stream_io_msg = new bts.StreamIOMessage(mra_msg);
+            ser_bytes = ser.serialize(stream_io_msg);
+            var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
+            Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
+            stream_io_msg_prime = deser.deserialize(ser_bytes);
+            passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
+            return new TestResult {
+                test_name = "UniversalSerializationController",
+                test_result = passes
+            };
+        }
+
+        private TestResult _row_serialization_tests_no_dbnulls() {
+            // IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+            //                                                     new Tuple<string, TypeCode>("id", TypeCode.Int32),
+            //                                                     new Tuple<string, TypeCode>("i1", TypeCode.Int32),
+            //                                                     new Tuple<string, TypeCode>("d1", TypeCode.Double),
+            //                                                     new Tuple<string, TypeCode>("string_1", TypeCode.String));
+            // bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            // IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
+            var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            MemoryStream str = new MemoryStream();
+            IBlueRow r = generate_random_row(schema, 0, false);
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            _write_to_file(str, @"C:\Temp\tests\row.bin");
+            str.Position = 0;
+            byte[] raw_byte_array = str.ToArray();
+            var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
+            Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            string row_str = r_prime.to_string();
+            Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_NonNull",
+                test_result = passes
+            };
+        }
+
+        private TestResult _deserialize_pub_msg_test() {
+            var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
+            PubSubClientMessage ps_client_msg = new PubSubClientMessage();
+            ps_client_msg.consruct_deserialize(pub_msg_ser);
+            bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
+            return new TestResult {
+                test_name = "_deserialize_pub_msg_test",
+                test_result = passes
+            };
+        }
+
+        private TestResult _cpp_data_test() {
+            //create and write schema
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d_1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String)
+                                                                );
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
+            StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+            StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+
+            IBlueTableSchema schema_p = null;
+            var schema_file = @"c:\Temp\tests\schema.bin";
+            var row_file = @"C:\Temp\tests\row.bin";
+            var create_op_file = @"C:\Temp\tests\create_op.bin";
+            var mra_op_file = @"C:\Temp\tests\mra_op.bin";
+            IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
+
+            //serialize schema, write it to a file, read it back, deserialize it and match schemas
+            byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
+            _write_to_file(ser_schema_bytes, schema_file);
+
+            byte[] schema_file_bytes = _read_from_file(schema_file);
+            schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
+            bool passes = BlueDataUtility.compute_equality(schema, schema_p);
+
+            //serialize row, write it to a file, read it back, deserialize it and match schemas    
+            byte[] ser_row_bytes = row_ser.serialize_row(row);
+            _write_to_file(ser_row_bytes, row_file);
+
+            byte[] row_file_bytes = _read_from_file(row_file);
+            var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
+            Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
+
+            IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
+            passes = passes && BlueDataUtility.compute_equality(row, row_p);
+
+            {
+                // serialize a create operation
+                int n_rows = 100;
+                BlueTable table = new BlueTable(schema);
+                for (int i = 0; i < n_rows; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
+                //StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
+                _write_to_file(create_ser_bytes, create_op_file);
+
+                // read it and make sure its right
+                byte[] create_op_file_bytes = _read_from_file(create_op_file);
+                //StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+                var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
+                _write_to_file(create_ser_bytes, mra_op_file);
+
+                passes = passes && cr_msg.Equals(cr_msg_p);
+
+            }
+            //multiple remove adds
+            {
+                // remove all rows and add them again
+                int n_rows = 100;
+                BlueTable table = new BlueTable(schema);
+                for (int i = 0; i < n_rows; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
+
+                //StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
+                _write_to_file(mra_ser_bytes, mra_op_file);
+
+                // reasd it and make sure it matches
+                byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
+                //StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+                StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
+                passes = passes && mra_msg.Equals(mra_msg_p);
+            }
+            return new TestResult {
+                test_name = "_cpp_data_test",
+                test_result = passes
+            };
+        }
+
+        private TestResult _write_stream_updates_to_files() {
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d_1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String)
+                                                                );
+            var packets_path = @"C:\Temp\tests\stream_packets\";
+            Func<long> now_epochs = () => {
+                DateTimeOffset now = DateTimeOffset.UtcNow;
+                // Convert to Unix time in milliseconds
+                long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
+                return unix_time_milliseconds;
+            };
+            Func<int, bool, string> curr_file_path = (id, create) => {
+                //var epoch_ms = now_epochs();
+                var file_path = $"{packets_path}{id}.bin";
+                if (create)
+                    file_path = $"{packets_path}create_{id}.bin";
+                return file_path;
+            };
+            // write 1000 stream updates
+            {
+                int init_sz = 100;
+                BlueTable table = new BlueTable(schema);
+                StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                // create table of 100 rows
+                for (int i = 0; i < init_sz; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                // write create msg
+                StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
+                byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
+                var create_file_path = curr_file_path(table.Count, true);
+                _write_to_file(create_ser_bytes, create_file_path);
+
+                // add 1000 rows and write an io msg for each update and write it to a file
+                int n_updates = 1000;
+                int period_ms = 50;
+
+                // write a create
+                for (int i = 0; i < n_updates; i++) {
+                    // addition
+                    BlueTable adds = new BlueTable(schema);
+                    int id = table.Count;
+                    IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
+                    adds.addRow(nr);
+                    BlueTable dels = new BlueTable(schema);
+                    table.addRow(nr);
+                    StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
+                    byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
+                    var mra_file_path = curr_file_path(id, false);
+                    _write_to_file(mra_ser_bytes, mra_file_path);
+
+                    System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
+                }
+                // now start writing increments and add 1 row at a time
+            }
+            return new TestResult {
+                test_name = "_write_updates_as_files",
+                test_result = true
+            };
+        }
+
+
+        private TestResult _row_serialization_tests_with_dbnulls() {
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String));
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_WithDBNulls",
+                test_result = passes
+            };
+        }
+
+        private TestResult _row_serialization_tests_all_types() {
+            var schema = create_schema_with_all_types();
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            IBlueRow r = generate_random_row(schema, 0, false);
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            _write_to_file(str, @"C:\Temp\tests\row.bin");
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_AllTypes",
+                test_result = passes
+            };
+        }
+
+        private static TestResult _run_pubsub_server_stream_tag_handler_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            var tag = "DS_" + schema.tableName;
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+
+            PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
+            IStreamIOMsgSerializer ser = new StreamIOSerializationController();
+            byte[] msg_bytes = ser.serialize(str_io_msg);
+            handler.update(tag, msg_bytes);
+
+            //now get it back for the handler
+            byte[] content = handler.get_content(tag);
+            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
+            bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
+            // test here
+            var passes = str_io_msg.Equals(str_io_msg_p);
+            // delete all rows
+            BlueTable del_rows = new BlueTable(cr_rows);
+            cr_rows.clear();
+            // added 50 new rows
+            BlueTable added_rows = new BlueTable(del_rows.schema);
+            int new_state_sz = 50;
+            for (int i = 0; i < new_state_sz; i++) {
+                var nr = generate_random_row(schema, i);
+                added_rows.addRow(nr);
+            }
+            cr_rows.addRows(added_rows);
+            // the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
+            // 1. generate op
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            str_io_msg = new bts.StreamIOMessage(mra);
+            // 2. serialize it into bytes
+            msg_bytes = ser.serialize(str_io_msg);
+            // 3. send it to handler
+            handler.update(tag, msg_bytes);
+            // 4. get the state from the handler
+            byte[] content_p = handler.get_content(tag);
+            // 5. deserialize the state into a stream op
+            str_io_msg_p = deser.deserialize(content_p);
+            // 6. check to make sure the state's rows are good
+            passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
+                    BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
+            return new TestResult {
+                test_result = passes,
+                test_name = "run_pubsub_server_stream_tag_handler_test"
+            };
+        }
+
+
+        private static TestResult _run_single_tag_data_test_clean() {
+            // create message
+            var test_name = MethodBase.GetCurrentMethod().Name;
+            var schema = create_schema_with_all_types();
+            var tag = "DS_" + schema.tableName;
+            BlueTable table = new BlueTable(schema);
+            IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
+            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
+            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
+            // create with 100 rows
+            int n_rows = 100;
+            bool is_succ = true;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                // perform the same action SingletagData, and verify the states match
+                PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
+                table.addRow(nr);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            // change all rows
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                var dr = table[i];
+                table.deleteRow(dr);
+                table.addRow(nr);
+                PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            // delete all rows
+            var aux_table = new BlueTable(table);
+            foreach (IBlueRow r in aux_table) {
+                table.deleteRow(r);
+                PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            return new TestResult() {
+                test_result = is_succ,
+                test_name = test_name
+            };
+        }
+
+
+
+        #region Helper functions
+
+        private static void _write_to_file(byte[] ser_bytes, string file_name) {
+            _write_to_file(new MemoryStream(ser_bytes), file_name);
+        }
+
+        private static void _write_to_file(MemoryStream str, string file_name) {
+            if (String.IsNullOrEmpty(file_name))
+                return;
+            byte[] str_bytes = str.ToArray();
+            using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
+                // Write the byte array to the file
+                fs.Write(str_bytes, 0, str_bytes.Length);
+            }
+        }
+
+        private static byte[] _read_from_file(string file_path) {
+            var rv = File.ReadAllBytes(file_path);
+            return rv;
+        }
+
+        public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
+            var content = st_data.get_current_content();
+            var msg_p = deser.deserialize(content);
+            Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
+            return msg_p.create_message.rows;
+
+        }
+
+        public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
+            ref IStreamIOMsgSerializer serializer) {
+            Func<IBlueTableSchema> get_schema = () => {
+                return (added_row != null ? added_row : removed_row).schema;
+            };
+            bts.StreamIOMessage str_io_msg = null;
+            if (is_create) {
+                Log.assert(added_row != null);
+                var aux_new = new BlueTable(added_row.schema);
+                aux_new.addRow(added_row);
+                var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
+                str_io_msg = new bts.StreamIOMessage(cr_msg);
+            } else {
+                var schema = get_schema();
+                BlueTable added_table = new BlueTable(schema);
+                if (added_row != null)
+                    added_table.addRow(added_row);
+                BlueTable rem_table = new BlueTable(schema);
+                if (removed_row != null)
+                    rem_table.addRow(removed_row);
+                bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
+                    new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
+                str_io_msg = new bts.StreamIOMessage(mra_msg);
+            }
+            byte[] msg_bytes = serializer.serialize(str_io_msg);
+            PublishMsg pub_msg = new PublishMsg() {
+                connection = "test_connection",
+                tag = $"DS_{get_schema()}",
+                tag_type = "DS",
+                data_to_publish = msg_bytes
+            };
+            return pub_msg;
+        }
+
+        public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
+            excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
+            var table_name = "test_table";
+            BlueTableSchema schema = new BlueTableSchema();
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            foreach (Type t in BlueDataUtility.valid_column_types()) {
+                if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
+                    continue;
+                var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
+                ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
+                cis.Add(ci);
+            }
+            cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
+            schema = new BlueTableSchema(table_name, cis.ToArray());
+            return schema;
+        }
+
+        public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
+            Random rnd = new Random();
+            object[] row_vals = new object[schema.columnInfo.Length];
+            foreach (ColumnInfo ci in schema.columnInfo) {
+                if (ci.colName == "id") {
+                    row_vals[ci.colNo] = id;
+                    continue;
+                }
+                object v = get_random_object(ci.type_code());
+                if (include_dbnulls && rnd.NextSingle() % 4 == 0)
+                    v = DBNull.Value;
+                row_vals[ci.colNo] = v;
+            }
+            return new BlueRow(schema, row_vals);
+        }
+
+        public static object get_random_object(TypeCode type_code) {
+            Random random = new Random();
+            switch (type_code) {
+                case TypeCode.Boolean:
+                    return random.Next(2) == 0; // Random boolean value (true or false)
+                case TypeCode.Byte:
+                    return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
+                case TypeCode.Char:
+                    return (char)random.Next(65, 65 + 27); // Random char value
+                case TypeCode.Decimal:
+                    Decimal dec = 100.123M;
+                    return dec;
+                case TypeCode.DateTime:
+                    return DateTime.Now;
+                case TypeCode.Double:
+                    return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
+                case TypeCode.Int16:
+                    return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
+                case TypeCode.Int32:
+                    return random.Next(int.MinValue, int.MaxValue); // Random int value
+                case TypeCode.Int64:
+                    return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
+                case TypeCode.SByte:
+                    return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
+                case TypeCode.Single:
+                    return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
+                case TypeCode.String:
+                    return Guid.NewGuid().ToString();
+                case TypeCode.UInt16:
+                    return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
+                case TypeCode.UInt32:
+                    return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
+                case TypeCode.UInt64:
+                    byte[] ulongBuffer = new byte[8];
+                    random.NextBytes(ulongBuffer);
+                    return BitConverter.ToUInt64(ulongBuffer, 0);
+                default:
+                    throw new ArgumentException($"Unsupported TypeCode: {type_code}");
+                    return null;
+            }
+        }
+        #endregion
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/BenzingaNewStream.cs

@@ -0,0 +1,66 @@
+namespace bts {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Net;
+   using System.Text.Json;
+   using System.Text.Json.Nodes;
+   [DataStreamUserRespresentation("benzingaNews")]
+   public class BenzingaNewStream : ViewStream {
+      private IBlueTableSchema _schema = null;
+      private int _id = 0;
+      // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
+      public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
+         // Hiding body (with project, columns can be hidden, or renamed)
+         ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
+         // add column news_source = 'Benzinga', so we can append other news sources
+         output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
+         // now SuperMapStream is used to map one row into multiple, one per symbol,
+         // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to 
+         // return an array of rows, per one row)
+         output_stream = new SuperMapStream(mongodb_raw_stream,
+
+        (IBlueTableSchema s) => {
+           // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
+           if (_schema == null) {
+              var out_schema = s;
+              out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
+              out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
+              out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
+              out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
+              out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
+              _schema = out_schema;
+           }
+           return _schema;
+        },
+        (IBlueRow r) => {
+           var json_objs = BlueDataUtility.json_to_json_node_list(r.get<string>("securities"));
+           List<IBlueRow> expanded_rows = new List<IBlueRow>();
+           if (json_objs.Count == 0) {
+              MutableBlueRow nr = new MutableBlueRow(_schema, r);
+              nr.setValue("id", _id++);
+              expanded_rows.Add(nr);
+           }
+           foreach (JsonNode jnode in json_objs) {
+              if (jnode is JsonObject json_object) {
+                 MutableBlueRow nr = new MutableBlueRow(_schema, r);
+                 nr.setValue("id", _id++);
+                 nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
+                 nr.setValue("symbol", jnode["symbol"]);
+                 nr.setValue("exchange", jnode["exchange"]);
+                 nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
+                 expanded_rows.Add(nr);
+              }
+           }
+           return expanded_rows;
+
+        });
+         // now we creaate a TimeFilter stream to keep the table as a circular buffer of the last N days
+         // passed A an argument in the constructor above
+         // time_filter is a string like 7d 1h 30 m 45s 
+         output_stream = new TimeBasedFilterStream(output_stream, "created_at", ttl);
+         mySetRealSource(output_stream);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -4,19 +4,20 @@ using MongoDB.Driver;
 using MongoDB.Bson;
 using BCG;
 using BCG.Utilities.General;
-using ZstdSharp.Unsafe;
-using System.Runtime.CompilerServices;
+using BCG;
 using mbts.spark;
 
 
-namespace BCG {
+namespace bts {
     [DataStreamUserRespresentation("mongodb_sql")]
     public class MongoDBSQLStream : ViewStream, IWorker {
         private ViewStreamDatabase _vsd;
         private readonly string _connection_string, _db_name, _collection_name;
         private MongoClient _client;
+        // make this a sorted table
         private BlueTable _table;
 
+
         // Access the database
         private IMongoDatabase _database;
         private IDictionary<string, int> _id_map = new Dictionary<string, int>();
@@ -38,7 +39,7 @@ namespace BCG {
         }
 
         protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
         }
 
         protected override void myOnLastSubscriber() {
@@ -50,6 +51,7 @@ namespace BCG {
         }
 
         private void _read_mogodb() {
+
             HighPerformanceTimer timer = new HighPerformanceTimer();
             timer.start();
             // Access the collection
@@ -71,7 +73,7 @@ namespace BCG {
             timer.start();
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
             timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            bts.Log.Gen.info($"MongoDBStream time to compare tables: {timer.lastIntervalTime} msecs");
             if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                 return;
             myFireEvent(dp);
@@ -134,7 +136,7 @@ namespace BCG {
         }
 
         protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
         }
 
         protected override void myOnLastSubscriber() {
@@ -156,7 +158,7 @@ namespace BCG {
             _table = _table == null ? null : new BlueTable(_table.schema);
             // Iterate through the documents and print values
             foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                //bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
                 if (_table == null) {
                     _init_table(document);
                 }
@@ -209,4 +211,46 @@ namespace BCG {
         }
     }
 
+    [DataStreamUserRespresentation("mongodb_flat")]
+    public class MongoDBFlatStream : ViewStream {
+        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+        :
+        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+
+        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+            mySetRealSource(rs);
+        }
+    }
+
+
+
+    [DataStreamUserRespresentation("newsreplay")]
+    public class NewsReplayStream : ViewStream {
+        public NewsReplayStream(string connection_string, string db_name, string collection_name,
+                                int buffer_sz, string fire_period) :
+        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+
+        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+                                int buffer_sz, TimeSpan fire_period) {
+            ViewStream r;
+
+            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+            r = new KeyValueToRelationalStream(r,
+                                                "doc_id",
+                                                "key",
+                                                "value");
+            r = new DateStandardizationStream(r,
+                                               "created_at",
+                                               "ISO8601");
+            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+
+            mySetRealSource(r);
+        }
+    }
+
+
 }

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -0,0 +1,172 @@
+namespace bts {
+   using BCG;
+
+   using ProtoBuf;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+
+   public class ProjectData {
+      // schema from the base stream
+      public readonly IBlueTableSchema in_schema;
+      // this is the projected schema
+      public readonly IBlueTableSchema out_schema;
+      // projected cols to the input cols, for cols who's names are changed
+      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
+      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
+                  IDictionary<string, string> mapped_to_orig_col_map_) {
+         in_schema = in_schema_;
+         out_schema = out_schema_;
+         mapped_to_orig_col_map = mapped_to_orig_col_map_;
+      }
+   }
+
+   public class NewProjectRow : IBlueRow {
+      #region IBlueRow members
+      private ProjectData _project_data;
+      private IBlueRow _row;
+      public NewProjectRow(ProjectData project_data, IBlueRow row) {
+         _project_data = project_data;
+         _row = row;
+      }
+      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
+      #endregion
+
+      #region ICollection members
+      public int Count { get { return schema.columnNames.Length; } }
+
+      public bool IsSynchronized { get { return false; } }
+
+      public object SyncRoot { get { return this; } }
+
+      public void CopyTo(Array array, int index) {
+         int i = index;
+         foreach (object o in this as IEnumerable)
+            array.SetValue(o, i++);
+      }
+      #endregion
+
+      #region IDictionary members
+      public bool IsFixedSize { get { return true; } }
+
+      public bool IsReadOnly { get { return true; } }
+
+      public object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            return this[(int)key];
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public object this[int index] {
+         get {
+            return this[schema.columnInfo[index].colName];
+         }
+      }
+
+      public object this[string col_name] {
+         get {
+            string mapped_to_col = null;
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+               return _row[mapped_to_col];
+            }
+            return _row[col_name];
+         }
+      }
+
+      public ICollection Keys { get { return schema.columnNames; } }
+
+      public ICollection Values {
+         get {
+            object[] retVal = new object[schema.columnNames.Length];
+            CopyTo(retVal, 0);
+            return retVal;
+         }
+      }
+
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public bool Contains(object key) { return schema[(string)key] != null; }
+
+      /// <summary>
+      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+      /// iterate through the values, typecast this into an IEnumerable
+      /// </summary>
+      /// <returns></returns>
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      #endregion
+
+   }
+
+   public class NewProjectStream : ViewStream, IStreamMapper {
+
+      private ViewStream _stream;
+      private string _project_expression;
+      private ProjectData _project_data = null;
+      //public ProjectData 
+      public NewProjectStream(ViewStream vs, string project_expression) {
+         _stream = vs;
+         _project_expression = project_expression;
+         _stream = new MapStream(vs, this);
+         mySetRealSource(_stream);
+      }
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+         _project_data = _create_project_data(input_schema, _project_expression);
+         return _project_data.out_schema;
+      }
+
+      public IBlueRow map(IBlueRow row) {
+         return new NewProjectRow(_project_data, row);
+      }
+
+      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
+                                                      string project_expression) {
+         project_expression = project_expression.Trim();
+         string[] project_toks = project_expression.Split(' ');
+         HashSet<string> removed_cols = new HashSet<string>();
+         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         foreach (string p_tok in project_toks) {
+            if (p_tok.StartsWith("!")) {
+               string removed_col = p_tok.Substring(1);
+               removed_cols.Add(removed_col);
+            } else if (p_tok.Contains(":")) {
+               string[] map_toks = p_tok.Split(":");
+               string orig_col = map_toks[0];
+               string mapped_col = map_toks[1];
+               mapped_to_orig_map.Add(mapped_col, orig_col);
+               orig_to_mapped.Add(orig_col, mapped_col);
+            } else {
+               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
+            }
+         }
+         // make the new schema
+         List<ColumnInfo> new_cols = new List<ColumnInfo>();
+         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (removed_cols.Contains(ci.colName)) {
+               continue;
+            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
+               string mapped_name = orig_to_mapped[ci.colName];
+               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            } else {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            }
+         }
+         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs

@@ -8,8 +8,8 @@ namespace BCG {
    using System.Collections.Generic;
    using System.Linq;
 
-   public class ProjectStream : ViewStream
-   {
+   [DataStreamUserRespresentation("oldproject")]
+   public class ProjectStream : ViewStream {
       private ProjectHelper myHelper;
       private string myColExp, myTableName;
       private ViewStream myVs;
@@ -19,8 +19,7 @@ namespace BCG {
       /// </summary>
       /// <param name="vs">stream to susbsribe to as the input stream</param>
       /// <param name="colExp">expression defining the column mapping</param>
-      public ProjectStream (ViewStream vs, string colExp)
-      {
+      public ProjectStream(ViewStream vs, string colExp) {
          myVs = vs;
          myColExp = colExp;
       }
@@ -30,12 +29,10 @@ namespace BCG {
       /// <param name="vs">stream to subscribe to as the input stream</param>
       /// <param name="colExp">expression defining the column mapping</param>
       /// <param name="tableName">name of the mapped table</param>
-      public ProjectStream (ViewStream vs, string colExp, string tableName)
-      {
+      public ProjectStream(ViewStream vs, string colExp, string tableName) {
          if (colExp == "!")
-            mySetRealSource (new ChangeTableNameStream (vs, tableName));
-         else
-         {
+            mySetRealSource(new ChangeTableNameStream(vs, tableName));
+         else {
             myVs = vs;
             myColExp = colExp;
             myTableName = tableName;
@@ -46,59 +43,51 @@ namespace BCG {
       /// </summary>
       /// <param name="vs">stream to subscribe to as the input stream</param>
       /// <param name="dic">dictionary mapping the aliased to the original</param>
-      public ProjectStream (ViewStream vs, IDictionary dic)
-         : this(vs, GenericDictionaryWrapper.Create<string, string>(dic)) {}
+      public ProjectStream(ViewStream vs, IDictionary dic)
+         : this(vs, GenericDictionaryWrapper.Create<string, string>(dic)) { }
 
-      public ProjectStream (ViewStream vs, IEnumerable<KeyValuePair<string, string>> dic)
-      {
+      public ProjectStream(ViewStream vs, IEnumerable<KeyValuePair<string, string>> dic) {
          myVs = vs;
          myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key);
       }
 
-      protected override void myOnFirstSubscriber()
-      {
-         myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
+      protected override void myOnFirstSubscriber() {
+         myVs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChange);
       }
 
-      protected override void myOnLastSubscriber()
-      {
-         myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
+      protected override void myOnLastSubscriber() {
+         myVs.onChange -= new BCG.DataStream.ChangeDelegate(myHandleChange); ;
       }
 
-      internal void myHandleChange (object newValue)
-      {
+      internal void myHandleChange(object newValue) {
          DataPacket dp = newValue as DataPacket;
-         if (myHelper != null)
-         {
+         if (myHelper != null) {
             _timer.start();
             IBlueRowCollection dels, adds;
-            BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
+            BlueDataUtility.getTrueDeletesAndAdds(dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
             _timer.stop();
             if (dels.Count + adds.Count > 0)
-               myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
-                                            new ProjectedTable (adds, myHelper),
-                                             new ProjectedTable (dels, myHelper)
+               myFireEvent(new DataPacket(new ProjectedTable(dp.newRows, myHelper),
+                                            new ProjectedTable(adds, myHelper),
+                                             new ProjectedTable(dels, myHelper)
                                           )
-                           );              
-         }
-         else
-         {  //first time a callback is received
+                           );
+         } else {  //first time a callback is received
             myHelper = myDic == null ?
-               new ProjectHelper (dp.schema, myColExp, myTableName) :
-               new ProjectHelper (dp.schema, myDic, myTableName);
+               new ProjectHelper(dp.schema, myColExp, myTableName) :
+               new ProjectHelper(dp.schema, myDic, myTableName);
 
             IBlueRowCollection temp;
-            myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper),
+            myFireEvent(new DataPacket(temp = new ProjectedTable(dp.newRows, myHelper),
                temp,
-               new BlueTable (temp.schema)
+               new BlueTable(temp.schema)
                )
                );
 
          }
       }
-      public void myHandleChangeTest(object newValue)
-      {
-          myHandleChange(newValue);
+      public void myHandleChangeTest(object newValue) {
+         myHandleChange(newValue);
       }
 
    }

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -1,34 +0,0 @@
-using BCG;
-
-namespace bts {
-    public class SchemaGuesserStream : ViewStream {
-
-        private IBlueRowCollection _rows = null;
-        private ViewStream _vs;
-        public SchemaGuesserStream(ViewStream vs) {
-            _vs = vs;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _vs.onChange += _on_data;
-        }
-        protected override void myOnLastSubscriber() {
-            _vs.onChange -= _on_data;
-        }
-
-        private void _on_data(object obj) {
-            DataPacket dp = obj as DataPacket;
-            if (_rows == null) {
-                _rows = BlueDataUtility.get_typed_table(dp.newRows);
-            }
-            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-
-
-        }
-
-
-
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -1,107 +1,115 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace BCG;
-
-public class SortRow : IComparable, IBlueRow {
-   readonly IBlueRow row;
-   readonly IEnumerable<ColumnSortExpression>? columns;
-   readonly IComparer? comparer;
-
-   public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) {
-      this.row = row;
-      this.columns = columns;
-   }
-
-   public SortRow(IBlueRow row, IComparer comparer) {
-      this.row = row;
-      this.comparer = comparer;
-   }
-
-   public IBlueRow internal_row { get { return row; } }
-
-   #region IComparable method
-   public int CompareTo(object? obj) {
-      if (obj is not IBlueRow otherRow) {
-         throw new ArgumentException(
-            $"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})",
-            nameof(obj));
-      }
-      if (comparer is not null) {
-         return comparer.Compare(this, otherRow);
-      }
-      return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0);
-      int Compare(ColumnSortExpression col) {
-         var otherVal = otherRow[col.Name];
-         int rv = myIsNull(this[col.Name]) ?
-            (myIsNull(otherVal) ? 0 : -1) :
-            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
-         rv *= col.IsAscending ? 1 : -1;
-         return rv;
-      }
-   }
-   #endregion
-
-   #region IBlueRow members
-   public IBlueTableSchema schema { get { return row.schema; } }
-   #endregion
-
-   #region ICollection members
-   public int Count { get { return row.Count; } }
-
-   public bool IsSynchronized { get { return false; } }
-
-   public object SyncRoot { get { return this; } }
-
-   public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
-   #endregion
-
-   #region IDictionary members
-   public bool IsFixedSize { get { return true; } }
-
-   public bool IsReadOnly { get { return true; } }
-
-   /// <summary>
-   /// The key can either be the string corrsponding to the column name or
-   /// the order of the column
-   /// </summary>
-   public object this[object key] {
-      get {
-         if (key is string)
-            return this[(string)key];
-         return this[(int)key];
-      }
-      set { throw new NotSupportedException("The IBlueRow is readonly"); }
-   }
-
-   public object this[int index] => row[index];
-
-   public object this[string colName] => row[colName];
-
-   public ICollection Keys => row.Keys;
-
-   public ICollection Values => row.Values;
-
-   public bool Contains(object key) => row.Contains(key);
-
-   void IDictionary.Add(object key, object? value) =>
-      throw new NotSupportedException("The IBlueRow is readonly");
-
-   void IDictionary.Clear() =>
-      throw new NotSupportedException("The IBlueRow is readonly");
-
-   IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); }
-
-   IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); }
-
-   public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-   #endregion
-
-   bool myIsNull(object? o) =>
-      o is null || o == null || o is DBNull;
-
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.SortExpression;
+
+
+// namespace BCG
+// {
+//    public class SortRow : IComparable, IBlueRow
+//    {
+//       private IBlueRow myRow;
+//       private IEnumerable myColumns;
+//       private IComparer myComparer;
+
+//       public SortRow (IBlueRow row, SortExpression sortExpression)
+//       {
+//          myRow = row;
+//          myColumns = sortExpression.columns;
+//       }
+
+//       public SortRow (IBlueRow row, IEnumerable columns)
+//       {
+//          myRow = row;
+//          myColumns = columns;
+//       }
+
+//       public SortRow (IBlueRow row, IComparer comparer)
+//       {
+//          myRow = row;
+//          myComparer = comparer;
+//       }
+
+//       public IBlueRow internal_row { get { return myRow; } }
+
+//       #region IComparable method
+//       public int CompareTo (object obj)
+//       {  
+//          IBlueRow otherRow = (IBlueRow)obj;
+//          if (myComparer == null)
+//          {
+//             foreach (SortExpression.ColumnCriteria cc in myColumns)
+//             {
+//                int retVal = myIsNull (this[cc.columnName]) ?
+//                   myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
+//                   myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
+//                retVal *= cc.ascending ? 1 : -1;
+//                if (retVal != 0)
+//                   return retVal;
+//             }
+//             return 0;
+//          }
+//          else
+//             return myComparer.Compare (this, otherRow);
+//       }
+//       #endregion
+
+//       #region IBlueRow members
+//       public IBlueTableSchema schema{get{return myRow.schema;}}
+//       #endregion
+
+//       #region ICollection members
+//       public int Count {get{return myRow.Count;}}
+
+//       public bool IsSynchronized {get{return false;}}
+
+//       public object SyncRoot {get{return this;}}
+
+//       public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
+//       #endregion
+
+//       #region IDictionary members
+//       public bool IsFixedSize {get{return true;}}
+
+//       public bool IsReadOnly {get{return true;}}
+
+//       /// <summary>
+//       /// The key can either be the string corrsponding to the column name or
+//       /// the order of the column
+//       /// </summary>
+//       public object this [object key]
+//       {
+//          get
+//          {
+//             if (key is string)
+//                return this[(string)key];
+//             return this[(int)key];
+//          }
+//          set {throw new NotSupportedException ("The IBlueRow is readonly");}
+//       }
+
+//       public object this [int index]   {get{return myRow[index];}}
+
+//       public object this [string colName] {get{return myRow[colName];}}
+
+//       public ICollection Keys{get{return myRow.Keys;}}
+
+//       public ICollection Values{get{return myRow.Values;}}
+
+//       public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+//       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+//       public bool Contains (object key){return myRow.Contains (key);}
+
+//       IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
+
+//       IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
+
+//       public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+//       #endregion
+
+//       private bool myIsNull (object o) {return o == null || o is DBNull;}
+
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,99 +1,129 @@
-using Bts.ColumnSortExpressions;
+// using System;
+// using System.Collections;
+// using BCG.Evaluator.SortExpression;
 
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 
-namespace BCG;
+// namespace BCG
+// {
+// 	/// <summary>
+// 	/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+// 	/// to merge the additions to the accumulates table
+// 	/// </summary>
+// 	public class SortStream : ViewStream
+// 	{
+//       private ViewStream myStream;
+//       private IEnumerable mySortCols;
+//       private IComparer myComparer;
+//       private DataPacket myLastFiredPacket;
+//       private DataPacket myLastRecievedPacket;
 
-/// <summary>
-/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-/// to merge the additions to the accumulates table
-/// </summary>
-public class SortStream : ViewStream {
-   ViewStream stream;
-   IList<ColumnSortExpression> sortCols;
-   IComparer? comparer;
-   DataPacket? lastFiredPacket;
-   DataPacket? lastReceivedPacket;
+//       public SortStream (ViewStream stream, string sortExpression)
+//       {
+//          myStream = stream;
+//          mySetSortExpression (sortExpression);
+//       }
 
-   public SortStream(ViewStream stream, string sortExpression) {
-      this.stream = stream;
-      SortExpression = sortExpression;
-   }
+//       public SortStream (ViewStream vs, string colName, bool isAsc)
+//       {
+//          myStream = vs;
+//          string sortExpression = isAsc ? "+" + colName : "-" + colName;
+//          mySetSortExpression (sortExpression);
+//       }
 
-   public SortStream(ViewStream vs, string colName, bool isAsc) {
-      stream = vs;
-      SortExpression = isAsc ? "+" + colName : "-" + colName;
-   }
+//       public SortStream (ViewStream vs, IComparer comparer)
+//       {
+//          myStream = vs;
+//          myComparer = comparer;
+//       }
 
-   public SortStream(ViewStream stream, IComparer comparer) {
-      this.stream = stream;
-      this.comparer = comparer;
-      sortCols = Array.Empty<ColumnSortExpression>();
-   }
+//       public string sortExpression
+//       {
+//          set
+//          {
+//             myComparer = null;
+//             myLastFiredPacket = null;
+//             mySetSortExpression (value);   
+//             myChangeHandler (myLastRecievedPacket);
+//          }
+//       }
 
-   [MemberNotNull(nameof(sortCols))]
-   public string SortExpression {
-      set {
-         comparer = null;
-         lastFiredPacket = null;
-         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
+//       public IComparer comparer
+//       {
+//          set
+//          {
+//             myComparer = value;
+//             myLastFiredPacket = null;
+//             myChangeHandler(myLastRecievedPacket);
+//          }
+//       }
 
-   public IComparer Comparer {
-      set {
-         comparer = value;
-         lastFiredPacket = null;
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
+//       private void mySetSortExpression (string sortExpression)
+//       {
+//          syntax parser = 
+//             new syntax();
+         
+//          mySortCols = ((SortExpression)
+//             parser.Parse (sortExpression)).columns;
+//       }
 
-   public void myChangeHandlerTest(object o) =>
-      myChangeHandler(o);
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); 
+//       }
 
-   [MemberNotNull(nameof(lastFiredPacket))]
-   internal void myChangeHandler(object o) {
-      lastReceivedPacket = (DataPacket)o;
-      // First callback, so we need to sort the whole table, this is the
-      // only time we will do that for all the other sorting operations would
-      // be sorting the increments and inserting them into the whole list
-      if (lastFiredPacket is null) {
-         lastFiredPacket = new() {
-            newRows = comparer is null ?
-               new SortedTable(lastReceivedPacket.newRows, sortCols) :
-               new SortedTable(lastReceivedPacket.newRows, comparer)
-         };
-         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-         myFireEvent(lastFiredPacket.Clone());
-         return;
-      }
-      // Deletes
-      lastFiredPacket.deletedRows =
-         comparer == null ?
-         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-         new SortedTable(lastReceivedPacket.deletedRows, comparer);
-      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-      // Adds
-      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-      myFireEvent(lastFiredPacket.Clone());
-   }
+//       protected override void myOnLastSubscriber()
+//       {
+//          myInit ();
+//          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); 
+//       }
 
-   protected override void myOnFirstSubscriber() =>
-      stream.onChange += myChangeHandler;
+//       internal void myChangeHandler (object o)
+//       {
+//          myLastRecievedPacket = (DataPacket)o;
+//          //first callback, so we need to sort the whole table, this is the
+//          //only time we will do that for all the other sorting operations would
+//          //be sorting the increments and inserting them into the whole list
+//          if (myLastFiredPacket == null)
+//          {
+//             myLastFiredPacket = new DataPacket ();
+//             myLastFiredPacket.newRows = 
+//                myComparer == null ?
+//                new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
+//                new SortedTable (myLastRecievedPacket.newRows, myComparer);
+//             myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+//             myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+//             myFireEvent (myLastFiredPacket.Clone());
+//             return;
+//          }
+//          myHandleDeletes (myLastRecievedPacket);
+//          myHandleAdds (myLastRecievedPacket);
+//          myFireEvent (myLastFiredPacket.Clone());
+//       }
 
-   protected override void myOnLastSubscriber() {
-      lastFiredPacket = null;
-      lastReceivedPacket = null;
-      stream.onChange -= myChangeHandler;
-   }
-}
+//       public void myChangeHandlerTest(object o)
+//       {
+//           myChangeHandler(o);
+//       }
+
+//       private void myHandleDeletes (DataPacket dp)
+//       {
+//          myLastFiredPacket.deletedRows = 
+//             myComparer == null ?
+//             new SortedTable (dp.deletedRows, mySortCols) :
+//             new SortedTable (dp.deletedRows, myComparer);
+//          ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
+//       }
+
+//       private void myHandleAdds (DataPacket dp)
+//       {
+//          myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
+//          ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
+//       }
+
+//       private void myInit ()
+//       {
+//          myLastFiredPacket = null;
+//          myLastRecievedPacket = null;
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs

@@ -1,108 +1,136 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-using Bts.ColumnSortExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Represents an IBlueRowCollection with the rows sorted according to the
-/// </summary>
-public class SortedTable : IBlueRowCollection {
-   readonly List<SortRow> rows = new(0);
-   readonly Dictionary<int, IBlueRow> rowIndex = new(0);
-   readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>();
-   readonly IComparer? comparer;
-
-   /// <summary>
-   /// Sorts the rows using sortExpression as the sorting criteria
-   /// </summary>
-   /// <param name="rows">Rows to be inserted into the table</param>
-   /// <param name="sortCols">Sorting criteria</param>
-   public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) {
-      this.sortCols = sortCols;
-      init(rows);
-   }
-
-   public SortedTable(IBlueRowCollection rows, IComparer comparer) {
-      this.comparer = comparer;
-      init(rows);
-   }
-
-   public int Count => rows.Count;
-   public bool IsSynchronized => false;
-   public IBlueTableSchema schema { get; private set; }
-   public object SyncRoot => this;
-   public IBlueRow this[int id] => rowIndex[id]
-      ?? throw new BlueDataException(
-         $"ID '{id}' does not exist in table '{schema?.tableName}'");
-
-   public void addRows(IBlueRowCollection rows) {
-      // Requires merger of the addedRows to the already existing ones
-      if (rows.Count == 0) {
-         // At times there are calls with 0 additions, in that case
-         // we of course need no processing
-         return;
-      }
-      SortedTable addedRows =
-         comparer == null ? new SortedTable(rows, sortCols) :
-            new SortedTable(rows, comparer);
-      int addedRowsIndex = 0;
-      SortRow newRow = addedRows.getRowByIndex(addedRowsIndex);
-      for (int i = 0; i < this.rows.Count; i++) {
-         if (newRow.CompareTo(this.rows[i]) <= 0) {
-            this.rows.Insert(i, newRow);
-            rowIndex[(int)newRow["id"]] = newRow;
-            if (++addedRowsIndex == addedRows.Count) {
-               return;
-            }
-            newRow = addedRows.getRowByIndex(addedRowsIndex);
-         }
-      }
-      if (addedRowsIndex < addedRows.Count) {
-         for (int i = addedRowsIndex; i < addedRows.Count; i++) {
-            newRow = addedRows.getRowByIndex(i);
-            this.rows.Add(newRow);
-            rowIndex[(int)newRow["id"]] = newRow;
-         }
-      }
-   }
-
-   public void deleteRows(IBlueRowCollection deletedRows) {
-      foreach (IBlueRow deletedRow in deletedRows) {
-         var row = rowIndex[(int)deletedRow["id"]!] as SortRow
-            ?? throw new InvalidOperationException("Index did not return IBlueRow");
-         rows.Remove(row);
-         rowIndex.Remove((int)row["id"]!);
-      }
-   }
-
-   public SortRow getRowByIndex(int index) =>
-      rows[index]!;
-
-   public bool containsID(int id) =>
-      rowIndex[id] is not null;
-
-   public IEnumerator GetEnumerator() =>
-      rows.GetEnumerator();
-
-   public void CopyTo(Array array, int index) {
-      for (int i = 0; i < Count; i++) {
-         array.SetValue(rows[i], index + i);
-      }
-   }
-
-   [MemberNotNull(nameof(schema))]
-   void init(IBlueRowCollection rows) {
-      schema = rows.schema;
-      foreach (IBlueRow row in rows) {
-         SortRow newRow = comparer is null ?
-            new(row, sortCols) : new(row, comparer);
-         this.rows.Add(newRow);
-         rowIndex[(int)row["id"]!] = newRow;
-      }
-      this.rows.Sort();
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Evaluator.SortExpression;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Represents an IBlueRowCollection with the rows sorted according to the
+//    /// 
+//    /// </summary>
+//    public class SortedTable : IBlueRowCollection 
+//    {
+//       private ArrayList myRows = new ArrayList ();
+//       private Hashtable myIndex = new Hashtable ();
+//       private IBlueTableSchema mySchema;
+//       private IEnumerable mySortCols;
+//       private IComparer myComparer;
+
+//       #region SortedTable method(s)
+//       public void addRows (IBlueRowCollection rows)
+//       {
+//          //requires merger of the addedRows to the already existing ones
+//          if (rows.Count == 0)
+//             //at times there are calls with 0 additions, in that case
+//             //we ofcourse need no processing
+//             return;
+//          SortedTable addedRows = 
+//             myComparer == null ? new SortedTable (rows, mySortCols) :
+//                new SortedTable (rows, myComparer);
+//          int addedRowsIndex = 0;
+//          SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
+//          for (int i = 0; i < myRows.Count; i ++)
+//             if (newRow.CompareTo(myRows[i]) <= 0)
+//             {
+//                myRows.Insert (i, newRow);
+//                myIndex[newRow["id"]] = newRow;
+//                if (++addedRowsIndex == addedRows.Count)
+//                   return;
+//                newRow = addedRows.getRowByIndex (addedRowsIndex);
+//             }
+//          if (addedRowsIndex < addedRows.Count)
+//             for (int i = addedRowsIndex; i < addedRows.Count; i++)
+//             {
+//                newRow = addedRows.getRowByIndex (i);
+//                myRows.Add (newRow);
+//                myIndex[newRow["id"]] = newRow;
+//             }
+//       }
+
+//       public void deleteRows (IBlueRowCollection deletedRows)
+//       {
+//          foreach (IBlueRow deletedRow in deletedRows)
+//          {
+//             IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
+//             myRows.Remove (row);
+//             myIndex.Remove (row["id"]);
+//          }
+//       }
+
+//       public SortRow getRowByIndex (int index)  {return (SortRow)myRows[index];}
+//       #endregion
+
+//       #region Constructor(s)
+//       /// <summary>
+//       /// Sorts the rows using sortExpression as the sorting criteria
+//       /// </summary>
+//       /// <param name="rows">
+//       /// Rows to be inserted into the table
+//       /// </param>
+//       /// <param name="sortExpression">
+//       /// Sorting criteria
+//       /// </param>
+//       public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
+//       {
+//          mySortCols = sortExpression.columns;
+//          myConstructorHelper (rows);
+//       }
+
+//       public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
+//       {
+//          mySortCols = sortCols;
+//          myConstructorHelper (rows);
+//       }
+
+//       public SortedTable (IBlueRowCollection rows, IComparer comparer)
+//       {
+//          myComparer = comparer;
+//          myConstructorHelper (rows);
+//       }
+
+//       private void myConstructorHelper (IBlueRowCollection rows)
+//       {
+//          mySchema = rows.schema;
+//          foreach (IBlueRow row in rows)
+//          {
+//             SortRow newRow = 
+//                myComparer == null ?
+//                new SortRow (row, mySortCols) : new SortRow (row, myComparer);
+//             myRows.Add (newRow);
+//             myIndex[row["id"]] = newRow;
+//          }
+//          myRows.Sort ();
+//       }
+
+//       #endregion
+
+//       #region IBlueRowCollections members
+//       public IBlueRow this[int id] 
+//       {
+//          get
+//          {
+//             if (myIndex[id] == null)
+//                throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
+//             return (IBlueRow)myIndex[id];
+//          }
+//       }
+//       public IBlueTableSchema schema {get{return mySchema;}}
+
+//       public bool containsID (int id)  {return myIndex[id] != null;}
+
+//       #endregion
+
+//       #region ICollection members
+//       public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
+
+//       public bool IsSynchronized {get{return false;}}
+
+//       public object SyncRoot {get{return this;}}
+
+//       public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
+
+//       public int Count{get{return myRows.Count;}}
+//          #endregion
+//    }
+
+// }

src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs

@@ -3,24 +3,23 @@ using System.Collections.Generic;
 using System.Text;
 
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
    /// </summary>
-   public enum DateConversion
-   {
+   public enum DateConversion {
       ConvertDoubleToDateTime = 0,
       ChopTime,
       UTCToCurrent,
       ConvertStringToDateTime,
       ConvertDateTimeToMilliseconds,
-      SecondsOfDay
+      SecondsOfDay,
+      ISO8601,
+      EpochToDatetime
    }
 
    [DataStreamUserRespresentation("dateStandardize")]
-   public class DateStandardizationStream : ViewStream, IStreamMapper
-   {
+   public class DateStandardizationStream : ViewStream, IStreamMapper {
       private IBlueTableSchema mySchema;
       private DateConversion myConversion;
       private string myConversionColName;
@@ -31,12 +30,13 @@ namespace BCG
          {DateConversion.UTCToCurrent                 , typeof(DateTime)},
          {DateConversion.ConvertStringToDateTime      , typeof(DateTime)},
          {DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
-         {DateConversion.SecondsOfDay                 , typeof(Double)}
+         {DateConversion.SecondsOfDay                 , typeof(Double)},
+         {DateConversion.ISO8601                      , typeof(DateTime)},
+         {DateConversion.EpochToDatetime              , typeof(DateTime)}
       };
 
 
-      private static DateConversion _parse(string s)
-      {
+      private static DateConversion _parse(string s) {
          DateConversion rv = DateConversion.ChopTime;
          Enum.TryParse<DateConversion>(s, false, out rv);
          return rv;
@@ -45,11 +45,9 @@ namespace BCG
       /// for DSQA
       /// </summary>
       public DateStandardizationStream(ViewStream vs, string colName, string conversion)
-         : this(vs, colName, _parse(conversion))
-      { }
+         : this(vs, colName, _parse(conversion)) { }
 
-      public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion)
-      {
+      public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion) {
          myConversionColName = colName;
          myConversion = conversion;
          mySetRealSource(new MapStream(vs, this));
@@ -57,24 +55,18 @@ namespace BCG
 
       #region IStreamMapper Members
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
             List<ColumnInfo> cis = new List<ColumnInfo>();
-            foreach (ColumnInfo c in inputSchema.columnInfo)
-            {
-               if (c.colName == myConversionColName)
-               {
+            foreach (ColumnInfo c in inputSchema.columnInfo) {
+               if (c.colName == myConversionColName) {
                   Type convType;
-                  if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType))
-                  {
+                  if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) {
                      //BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
                      convType = typeof(DateTime);
                   }
                   cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
-               }
-               else
+               } else
                   cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
             }
             mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
@@ -86,56 +78,59 @@ namespace BCG
 
       #region IDynamicStreamMapper Members
 
-      public IBlueRow map(IBlueRow row)
-      {
-         return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
+      public IBlueRow map(IBlueRow row) {
+         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
+         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
+         return rv;
+         //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
       }
 
       #endregion
 
-      public static object mapDateTime(object objIn, DateConversion conv)
-      {
-         if (objIn == DBNull.Value || objIn == null)
-         {
+      public static object mapDateTime(object obj_in, DateConversion conv) {
+         if (obj_in == DBNull.Value || obj_in == null) {
             return DBNull.Value;
-         }
-         else
-         {
-            if (conv == DateConversion.ChopTime)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.Date;
-            }
-            else if (conv == DateConversion.ConvertDoubleToDateTime)
-            {
-               double dtIn = (double)objIn;
-               return DateTime.FromOADate(dtIn);
-            }
-            else if (conv == DateConversion.UTCToCurrent)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.ToLocalTime();
-            }
-            else if (conv == DateConversion.ConvertStringToDateTime)
-            {
-               return DateTime.Parse((string)objIn);
-            }
-            else if (conv == DateConversion.ConvertDateTimeToMilliseconds)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.TimeOfDay.TotalMilliseconds;
-            }
-            else if (conv == DateConversion.SecondsOfDay)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               DateTime start_of_day = dtIn.Date;
-               TimeSpan day_period = dtIn - start_of_day;
-               return day_period.TotalSeconds;
-            }
-            else
-            {
-               bts.Log.Gen.info("incorrect type");
-               return null;
+         } else {
+            switch (conv) {
+               case DateConversion.ChopTime: {
+                     DateTime dtIn = (DateTime)obj_in;
+                     return dtIn.Date;
+                  }
+               case DateConversion.ConvertDoubleToDateTime: {
+                     double dtIn = (double)obj_in;
+                     return DateTime.FromOADate(dtIn);
+                  }
+               case DateConversion.UTCToCurrent: {
+                     DateTime dtIn = (DateTime)obj_in;
+                     return dtIn.ToLocalTime();
+                  }
+               case DateConversion.ConvertStringToDateTime: {
+                     return DateTime.Parse((string)obj_in);
+                  }
+               case DateConversion.ConvertDateTimeToMilliseconds: {
+                     DateTime dtIn = (DateTime)obj_in;
+                     return dtIn.TimeOfDay.TotalMilliseconds;
+                  }
+               case DateConversion.SecondsOfDay: {
+                     DateTime dtIn = (DateTime)obj_in;
+                     DateTime start_of_day = dtIn.Date;
+                     TimeSpan day_period = dtIn - start_of_day;
+                     return day_period.TotalSeconds;
+                  }
+               case DateConversion.ISO8601: {
+                     string iso8601Date = obj_in.ToString();
+                     return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind);
+                  }
+               case DateConversion.EpochToDatetime: {
+                     double d = Convert.ToDouble(obj_in);
+                     long unix_time_seconds = (long)d;
+                     DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+                     return epoch.AddSeconds(unix_time_seconds);
+                  }
+               default: {
+                     bts.Log.Gen.info("incorrect type");
+                     return DBNull.Value;
+                  }
             }
          }
       }

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,43 +1,49 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class TopStream : MethodDrivenStream {
-        private int myTopCount;
+// namespace BCG
+// {
+//    public class TopStream : MethodDrivenStream
+//    {
+//       private int myTopCount;
 
-        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
+//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
 
-        public TopStream(ViewStream stream,
-           int topCount,
-           string sortExpression)
-           :
-           this(sortExpression == null ?
-           stream :
-           new SortStream(stream, sortExpression), topCount) { }
+//       public TopStream (ViewStream stream, 
+//          int topCount, 
+//          string sortExpression) 
+//          :
+//          this (sortExpression == null? 
+//          stream :
+//          new SortStream (stream, sortExpression), topCount)
+//       {}
 
-        protected override void myOnTableClear() {
-            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-        }
+//       protected override void myOnTableClear () 
+//       {
+//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
+//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
+//       }
 
-        protected override void myHandleFirstCall() {
-            myLastFiredPacket = new DataPacket();
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
-        }
+//       protected override void myHandleFirstCall () 
+//       {
+//          myLastFiredPacket = new DataPacket ();
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
+//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
+//       }
 
-        protected override void myHandleIncrementalAdd() { }
+//       protected override void myHandleIncrementalAdd ()     {}
 
-        protected override void myHandleIncrementalDelete() {
-            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
-               oldRows,
-               myLastFiredPacket.schema.columnNames,
-               out myLastFiredPacket.deletedRows,
-               out myLastFiredPacket.addedRows);
-            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-        }
-    }
-}
+//       protected override void myHandleIncrementalDelete()   
+//       {
+//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
+//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
+//             oldRows,
+//             myLastFiredPacket.schema.columnNames,
+//             out myLastFiredPacket.deletedRows,
+//             out myLastFiredPacket.addedRows);
+//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs

@@ -8,38 +8,36 @@ namespace BCG {
       private ViewStream mySrc;
       private IBlueTableSchema mySchema = null;
 
-      public AlphabetizeColumnsStream (ViewStream src) {
+      public AlphabetizeColumnsStream(ViewStream src) {
          mySrc = src;
       }
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          mySrc.onChange += new ChangeDelegate(mySrc_onChange);
       }
-      protected override void myOnLastSubscriber () 
-      {
-         mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
+      protected override void myOnLastSubscriber() {
+         mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
       }
-      private void mySrc_onChange (object newValue) {
+      private void mySrc_onChange(object newValue) {
          DataPacket dp = (DataPacket)newValue;
          if (mySchema == null) {
-            SortedList alphabetical = new SortedList ();
+            SortedList alphabetical = new SortedList();
             foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo)
-               alphabetical.Add (cinfo.colName,cinfo);
+               alphabetical.Add(cinfo.colName, cinfo);
             ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count];
-            alphabetical.Values.CopyTo (cinfos,0);
+            alphabetical.Values.CopyTo(cinfos, 0);
             for (int i = 0; i < cinfos.Length; ++i)
-               cinfos[i] = new ColumnInfo (cinfos[i].colName,i,cinfos[i].colType,cinfos[i].colValue);
-            mySchema = new BlueTableSchema (cinfos,dp.schema.tableName);
+               cinfos[i] = new ColumnInfo(cinfos[i].colName, i, cinfos[i].colType);
+            mySchema = new BlueTableSchema(cinfos, dp.schema.tableName);
          }
          BlueTable
-            deleted = new BlueTable (mySchema),
-            added = new BlueTable (mySchema),
-            newrows = new BlueTable (mySchema);
-         deleted.addRows (dp.deletedRows);
-         added.addRows (dp.addedRows);
-         newrows.addRows (dp.newRows);
-      
-         myFireEvent (new DataPacket (newrows,added,deleted));
+            deleted = new BlueTable(mySchema),
+            added = new BlueTable(mySchema),
+            newrows = new BlueTable(mySchema);
+         deleted.addRows(dp.deletedRows);
+         added.addRows(dp.addedRows);
+         newrows.addRows(dp.newRows);
+
+         myFireEvent(new DataPacket(newrows, added, deleted));
       }
    }
 }

src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs

@@ -1,71 +1,61 @@
 using System;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Publishes the rows in the form of a circular buffer, ie if the buffer size is n
    /// then the n'th row being puclished results in the deletion of the 0th row
    /// </summary>
-	public class CircularPublisher
-	{
+	public class CircularPublisher {
       private CircularRowCollection myBuffer;
       private PubTableBroker myBroker;
       private int myBufferSize;
-		public CircularPublisher (int bufferSize)
-		{
+      public CircularPublisher(int bufferSize) {
          myBufferSize = bufferSize;
-		}
-
-      public CircularPublisher (int bufferSize, IBlueRowCollection rows) 
-         : this (bufferSize)
-      {
-         publishRows (rows);
       }
 
-      public void publishRow (IBlueRow nr)
-      {
-         if (myBuffer == null)
-         {           
-            myBuffer = new CircularRowCollection (
-               new BlueTableSchema (nr.schema.tableName, nr.schema.columnInfo), 
-               myBufferSize);
-            myBroker = new PubTableBroker ();
-            myBroker.createTable (myBuffer);
-         }
-         IBlueRow br = myBuffer.addRowWithReturn (nr);
-         myBroker.makeBatchChanges (myBuffer, 
-            myReturnCollection (br),
-            myReturnCollection (nr));
+      public CircularPublisher(int bufferSize, IBlueRowCollection rows)
+         : this(bufferSize) {
+         publishRows(rows);
       }
 
-      public void publishRows (IBlueRowCollection nrs)
-      {
-         if (myBuffer == null)
-         {           
-            myBuffer = new CircularRowCollection (
-               new BlueTableSchema (BlueDataUtility.extractRow (nrs, 0).schema.tableName, 
-               BlueDataUtility.extractRow (nrs, 0).schema.columnInfo),
+      public void publishRow(IBlueRow nr) {
+         if (myBuffer == null) {
+            myBuffer = new CircularRowCollection(
+               new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo),
                myBufferSize);
-            myBroker = new PubTableBroker ();
-            myBroker.createTable (myBuffer);
+            myBroker = new PubTableBroker();
+            myBroker.create_table(myBuffer);
          }
-         BlueTable deletes = new BlueTable (myBuffer.schema);
-         foreach (IBlueRow nr in nrs)
-         {
-            IBlueRow dr = myBuffer.addRowWithReturn (nr);
+         IBlueRow br = myBuffer.addRowWithReturn(nr);
+         myBroker.make_batch_changes(myBuffer,
+            myReturnCollection(br),
+            myReturnCollection(nr));
+      }
+
+      public void publishRows(IBlueRowCollection nrs) {
+         if (myBuffer == null) {
+            myBuffer = new CircularRowCollection(
+               new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName,
+               BlueDataUtility.extractRow(nrs, 0).schema.columnInfo),
+               myBufferSize);
+            myBroker = new PubTableBroker();
+            myBroker.create_table(myBuffer);
+         }
+         BlueTable deletes = new BlueTable(myBuffer.schema);
+         foreach (IBlueRow nr in nrs) {
+            IBlueRow dr = myBuffer.addRowWithReturn(nr);
             if (dr != null)
-               deletes.addRow (dr);
+               deletes.addRow(dr);
          }
-         myBroker.makeBatchChanges (myBuffer, deletes, nrs);
+         myBroker.make_batch_changes(myBuffer, deletes, nrs);
       }
 
-      private IBlueRowCollection myReturnCollection (IBlueRow r)
-      {
+      private IBlueRowCollection myReturnCollection(IBlueRow r) {
          if (r == null)
-            return new BlueTable (myBuffer.schema);
-         BlueTable retVal = new BlueTable (r.schema);
-         retVal.addRow (r);
+            return new BlueTable(myBuffer.schema);
+         BlueTable retVal = new BlueTable(r.schema);
+         retVal.addRow(r);
          return retVal;
       }
-	}
+   }
 }

src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs

@@ -1,184 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq.Expressions;
-using System.Xml.Serialization;
-
-using BCG.Utilities;
-
-namespace BCG {
-   /// <summary>
-   /// This class will be used by the clients to publish tabular data to the pubsub
-   /// </summary>
-   public class PubTableBroker {
-      protected PublisherCore _publisher_core;
-      protected IDictionary<string, IPubTablePublisher> _publisher_map = new Dictionary<string, IPubTablePublisher>();
-      protected Dictionary<string, IBlueTableSchema> _schema_map = new Dictionary<string, IBlueTableSchema>();
-
-      protected bool myWaitingForFirstOp = false;
-
-      public PubTableBroker()
-         : this(new PublisherCore()) { }
-
-      public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; }
-
-      public PubTableBroker(IBlueRowCollection[] tablesToPublish)
-         : this(new PublisherCore(), tablesToPublish) { }
-
-      public PubTableBroker(PublisherCore publisher, IBlueRowCollection[] tablesToPublish) {
-         _publisher_core = publisher;
-
-         _publisher_core.beginTransaction();
-         foreach (IBlueRowCollection table in tablesToPublish)
-            createTable(table);
-         _publisher_core.commitTransaction();
-      }
-
-      public void beginTransaction() {
-         myWaitingForFirstOp = true;
-      }
-
-      public void commitTransaction() {
-         bts.Log.Gen.info("PubTableBroker.commitTransaction");
-         if (myWaitingForFirstOp)
-            myWaitingForFirstOp = false;
-         else
-            _publisher_core.commitTransaction();
-      }
-
-      public void rollbackTransaction() {
-         bts.Log.Gen.info("PubTableBroker.rollbackTransaction");
-         if (myWaitingForFirstOp)
-            myWaitingForFirstOp = false;
-         else
-            _publisher_core.abortTransaction();
-      }
-
-      public virtual void createTable(IBlueRowCollection newTable) {
-         string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
-         foreach (IBlueRow r in newTable) {
-            log_msg += $"id: {r["id"]}";
-         }
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(newTable.schema);
-         myCheckTransaction();
-         IPubTablePublisher table_publisher = null;
-         var name_key = newTable.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
-            table_publisher = _create_pubtable_publisher(name_key);
-            _publisher_map.Add(name_key, table_publisher);
-            _schema_map.Add(name_key, newTable.schema);
-         }
-         table_publisher.createTable(newTable);
-      }
-
-      /// <summary>
-      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
-      /// will be thrown
-      /// </summary>
-      /// <param name="r"></param>
-      public virtual void addRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
-         myCheckTransaction();
-         IPubTablePublisher table_publisher = null;
-         var name_key = r.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
-            throw new ApplicationException("Row can not be added, as the table that has not been created yet");
-         else
-            table_publisher.addRow(r);
-      }
-
-      /// <summary>
-      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
-      /// will be thrown
-      /// </summary>
-      /// <param name="r"></param>
-      public virtual void deleteRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
-         myCheckTransaction();
-         IPubTablePublisher table_publisher = null;
-         var name_key = r.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
-            throw new ApplicationException("Row can not be removed, as the table has not been created yet");
-         else
-            table_publisher.deleteRow(r);
-      }
-
-      /// <summary>
-      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
-      /// will be thrown
-      /// </summary>
-      /// <param name="r"></param>
-      public virtual void changeRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
-         myCheckTransaction();
-         IPubTablePublisher table_publisher = null;
-         var name_key = r.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
-            throw new ApplicationException("Row can not be changed, as the table has not been created yet");
-         else
-            table_publisher.changeRow(r);
-      }
-
-      /// <summary>
-      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
-      /// will be thrown
-      /// </summary>
-      /// <param name="r"></param>
-      public virtual void makeBatchChanges(IBlueRowCollection newRows,
-                                             IBlueRowCollection deletedRows,
-                                             IBlueRowCollection addedRows) {
-         string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
-         foreach (IBlueRow r in deletedRows)
-            log_msg += $"id: {r["id"]} ";
-         log_msg += "addedRows: ";
-         foreach (IBlueRow r in addedRows)
-            log_msg += $"id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(newRows.schema);
-         _check_schema_consistency(deletedRows.schema);
-         _check_schema_consistency(addedRows.schema);
-         IPubTablePublisher table_publisher = null;
-         var name_key = newRows.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
-            throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
-         else
-            table_publisher.batchRemoveAdd(deletedRows, addedRows);
-      }
-
-      public virtual void deleteTableRows(string tableName) {
-         myCheckTransaction();
-         (_publisher_map[tableName.ToLower()] as IPubTablePublisher).createTable(
-            new BlueTable(_schema_map[tableName.ToLower()] as IBlueTableSchema));
-      }
-
-      public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
-
-      public void clearTable(string tableName) { _publisher_core.clearState(PubStreamPrefixer.prefix(tableName)); }
-
-      protected void myCheckTransaction() {
-         if (myWaitingForFirstOp)
-            _publisher_core.beginTransaction();
-         myWaitingForFirstOp = false;
-      }
-
-      protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
-         bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
-         return new PubTablePublisher(tag, _publisher_core);
-      }
-
-      private void _check_schema_consistency(IBlueTableSchema schema_arg) {
-         IBlueTableSchema schema;
-         if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
-            bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/Publish/PublisherStream.cs

@@ -0,0 +1,31 @@
+namespace BCG {
+   // this for testing purposes mainly, its wraps a ViewPublisher around a stream, 
+   // and hence epublishes it out to the PS
+   public class PublisherStream : ViewStream {
+      private readonly ViewStream _main_stream;
+      private readonly string _name;
+      private ViewPublisher _vs_publisher;
+      private readonly PubTableBroker _broker = new PubTableBroker();
+
+      public PublisherStream(ViewStream main_stream, string name) {
+         _main_stream = main_stream;
+         _name = name;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
+         _main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
+         _vs_publisher.Dispose();
+      }
+
+      private void _handle_update(object o) {
+         myFireEvent(o);
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs

@@ -7,48 +7,32 @@ using BCG.Utilities;
 namespace BCG {
    public class ViewPublisher : IDisposable {
       protected ViewPublisher() { }
+      private readonly PubTableBroker _broker;
+      protected readonly ViewStream _stream;
+      private int _n_updates = 0;
 
       /// If an name IsNullOrEmpty then it uses the name
       /// in the schema (be careful with that because this
       /// could lead to double publishes if the schema name
       /// is not uniquely chosen)
-      public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
-         myName = name;
-         myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
-         myBroker = br;
-         myFirstTime = true;
-         myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
+      public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) {
+         _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
+         _broker = broker;
+         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
       }
 
       public void Dispose() {
-         myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
-         if (!String.IsNullOrEmpty(myName))
-            myBroker.clearTable(myName);
+         _stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
       }
 
-      private void myChangeHandler(object o) {
-         DataPacket dp = (DataPacket)o;
-         if (myFirstTime) {
-            myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName;
-            myLastRowCount = dp.newRows.Count;
-            myBroker.createTable(dp.newRows);
-            myFirstTime = false;
+      private void _handle_update(object o) {
+         DataPacket dp = o as DataPacket;
+         if (_n_updates == 0) {
+            _broker.create_table(dp.newRows);
+            _n_updates++;
          } else {
-            if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) {
-               myLastRowCount = dp.newRows.Count;
-               myBroker.makeBatchChanges(dp.newRows, dp.deletedRows, dp.addedRows);
-            }
+            _broker.make_batch_changes(dp.newRows, dp.deletedRows, dp.addedRows);
          }
       }
-
-      public int getLastRowPublishedCount() { return myLastRowCount; }
-
-      public bool hasPublished() { return !myFirstTime; }
-
-      private int myLastRowCount = -1;
-      private string myName;
-      private PubTableBroker myBroker;
-      protected ViewStream myView;
-      private bool myFirstTime;
    }
 }

src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs

@@ -1,92 +0,0 @@
-using System;
-using BCG.Utilities;
-
-namespace BCG
-{
-	public class ViewStreamPublisher
-	{
-      private IDynamicStreamMapper myMapper;
-      private PubTableBroker myBroker;
-      private IBlueRowCollection myOutputTable;
-      private IBlueRowCollection myInputTable;
-
-      public ViewStreamPublisher (PubTableBroker broker)
-      {
-         myBroker = broker;
-      }
-
-      public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper)
-         : this (broker)
-      {
-         myMapper = mapper;
-      }
-
-		public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper, ViewStream vs)
-         : this (broker, mapper)
-		{
-         vs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChanges);
-      }
-
-      private void myHandleChanges (object newValue)
-      {
-         DataPacket dp = newValue as DataPacket;
-         if (myMapper == null)
-         {
-            if (myOutputTable == null)
-            {
-               myBroker.createTable (dp.newRows);
-            }
-            else
-            {
-               myBroker.makeBatchChanges (dp.newRows, dp.deletedRows, dp.addedRows);
-            }
-            myOutputTable = dp.newRows;
-         }
-         else
-         {
-            if (myOutputTable != null)
-            {
-               BlueTable dRows = new BlueTable (myOutputTable.schema),
-                  aRows = new BlueTable (myOutputTable.schema);
-               foreach (IBlueRow dRow in dp.deletedRows)
-               {
-                  (myOutputTable as BlueTable).deleteRow ((int)dRow["id"]);
-                  dRows.addRow (myMapper.map (dRow));
-               }
-               foreach (IBlueRow aRow in dp.addedRows)
-               {
-                  IBlueRow r = myMapper.map (aRow);
-                  (myOutputTable as BlueTable).addRow (r);
-                  aRows.addRow (r);
-               }
-               myBroker.makeBatchChanges (myOutputTable, dRows, aRows);
-            }
-            else
-            {
-               if (dp.newRows.Count > 0)
-               {
-                  myOutputTable = new BlueTable (
-                  myMapper.map (BlueDataUtility.extractRow (dp.newRows, 0)).schema);
-                  myHandleChanges (newValue);
-               }
-            }
-         }
-      }
-
-      public void addRows (IBlueRowCollection aRows)
-      {
-         if (myInputTable != null)
-            (myInputTable as BlueTable).addRows (aRows);
-         else
-            myInputTable = new BlueTable (aRows);
-         myHandleChanges (new DataPacket(myInputTable, aRows, new BlueTable (aRows.schema)));
-      }
-
-      public void deleteRows (IBlueRowCollection dRows)
-      {
-         // BlueEventLog.logAssert (myInputTable != null);
-         (myInputTable as BlueTable).deleteRows (dRows);
-         myHandleChanges (new DataPacket (myInputTable, new BlueTable (myInputTable.schema), dRows));
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -6,7 +6,6 @@ using System.Reflection;
 using System.IO;
 using System.Diagnostics;
 using System.Linq;
-using System.Globalization;
 
 //using BCG.Utilities;
 using BCG.Utilities.General;
@@ -14,7 +13,8 @@ using System.Linq.Expressions;
 //using mbts.spark;
 using mbts;
 using BCG.Extensions;
-using System.Runtime.CompilerServices;
+using System.Text.Json;
+using System.Text.Json.Nodes;
 
 namespace BCG {
    public static class BlueDataUtility {
@@ -49,12 +49,6 @@ namespace BCG {
             rv.Add(r.get<T>(col));
          return rv;
       }
-      public static IEnumerable<T> get_colvalues<T>(IBlueRowCollection rows, string col) {
-         List<T> rv = new List<T>();
-         foreach (IBlueRow r in rows)
-            rv.Add(r.get<T>(col));
-         return rv;
-      }
 
       public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
          Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -74,12 +68,17 @@ namespace BCG {
       // The values are returned as string values
       // doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
       public static IBlueRowCollection keyvalue_to_relational<T>(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
-         HashSet<string> unique_cols = new HashSet<string>();
+         HashSet<string> keys = new HashSet<string>();
          foreach (string col in get_column_values<string>(kv_rows, key_col)) {
-            unique_cols.Add(col);
+            keys.Add(col);
          }
-         //unique_cols.Add(doc_col);
-         var schema_cols = unique_cols.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
+         if (keys.Contains("id")) {
+            keys.Remove("id");
+         }
+         //unique_cols.Add(doc_col)
+         // all the keys become cols
+         var schema_cols = keys.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
+         //schema of the final table needs doc and id
          schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
          schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
          var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
@@ -91,6 +90,8 @@ namespace BCG {
             nr.setValue(doc_col, kv.Key);
             foreach (var r in kv.Value) {
                var col = r.get<string>(key_col);
+               if (!keys.Contains(col))
+                  continue;
                var val = r.get<string>(val_col);
                nr.setValue(col, val);
             }
@@ -118,6 +119,15 @@ namespace BCG {
          return rv;
       }
 
+      public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
+         Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
+         foreach (IBlueRow r in rows) {
+            var id = r.get<int>("id");
+            rv.Add(id, r);
+         }
+         return rv;
+      }
+
       public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
          List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
          foreach (Tuple<Type, string> c in cols)
@@ -179,6 +189,35 @@ namespace BCG {
 
       #endregion
 
+      #region json
+
+      public static List<JsonNode> json_to_json_node_list(string jsonString) {
+         var jsonNodeList = new List<JsonNode>();
+         bool noe = String.IsNullOrEmpty(jsonString);
+         if (noe)
+            return jsonNodeList;
+         try {
+            // Parse the JSON string
+            JsonArray jsonArray = JsonNode.Parse(jsonString) as JsonArray;
+
+            if (jsonArray == null) {
+               Console.WriteLine("The JSON string must represent a list.");
+               return jsonNodeList;
+            }
+
+            // Iterate through each JsonNode in the JsonArray
+            foreach (JsonNode jsonNode in jsonArray) {
+               jsonNodeList.Add(jsonNode);
+            }
+         } catch (JsonException ex) {
+            Console.WriteLine($"Invalid JSON: {ex.Message}");
+         }
+
+         return jsonNodeList;
+      }
+
+      #endregion
+
       public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width) {
          var rv = get_printed_rows(table, get_col_width);
          Console.WriteLine(rv);
@@ -208,37 +247,98 @@ namespace BCG {
          }
          return rv;
       }
+      #region Schema types
+      // these are the only types allowed in schemas
       private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
       {
-         { TypeCode.Boolean, typeof(bool) },
-         { TypeCode.Byte, typeof(byte) },
-         { TypeCode.Char, typeof(char) },
-         { TypeCode.DateTime, typeof(DateTime) },
-         { TypeCode.DBNull, typeof(DBNull) },
-         { TypeCode.Decimal, typeof(decimal) },
-         { TypeCode.Double, typeof(double) },
-         { TypeCode.Empty, null },
-         { TypeCode.Int16, typeof(short) },
-         { TypeCode.Int32, typeof(int) },
-         { TypeCode.Int64, typeof(long) },
-         { TypeCode.Object, typeof(object) },
-         { TypeCode.SByte, typeof(sbyte) },
-         { TypeCode.Single, typeof(Single) },
-         { TypeCode.String, typeof(string) },
-         { TypeCode.UInt16, typeof(UInt16) },
-         { TypeCode.UInt32, typeof(UInt32) },
-         { TypeCode.UInt64, typeof(UInt64) }
+         { TypeCode.Boolean, typeof(bool) }, //3
+         { TypeCode.Byte, typeof(byte) },    //6
+         { TypeCode.Char, typeof(char) },    //4
+         // TODO
+         { TypeCode.DateTime, typeof(DateTime) },//16
+         { TypeCode.Decimal, typeof(decimal) }, //15
+         { TypeCode.Double, typeof(double) },   //14
+         //{ TypeCode.Empty, null },
+         { TypeCode.Int16, typeof(short) },     //7
+         { TypeCode.Int32, typeof(int) },       //9
+         { TypeCode.Int64, typeof(long) },      //11
+         { TypeCode.SByte, typeof(sbyte) },     //5
+         { TypeCode.Single, typeof(Single) },   //13   
+         { TypeCode.String, typeof(string) },   //18
+         { TypeCode.UInt16, typeof(UInt16) },    //8
+         { TypeCode.UInt32, typeof(UInt32) },    //10
+         { TypeCode.UInt64, typeof(UInt64) }    //12
       };
 
+      public static IEnumerable<TypeCode> valid_column_typecodes() {
+         return _typecode_to_type.Keys;
+      }
+
+      public static IEnumerable<Type> valid_column_types() {
+         return _typecode_to_type.Values;
+      }
+
+      public static bool is_valid_col_type(Type t) {
+         TypeCode type_code = get_typecode(t);
+         return type_code != TypeCode.Empty;
+      }
+
       public static TypeCode get_typecode(Type t) {
          foreach (var kv in _typecode_to_type) {
             if (kv.Value == t)
                return kv.Key;
          }
-         bts.Log.assert_d(false, $"type: {t} not found");
+         //bts.Log.assert_d(false, $"type: {t} not found");
          return TypeCode.Empty;
       }
 
+      public static Type get_type(TypeCode type_code) {
+         Type rv = null;
+         if (_typecode_to_type.TryGetValue(type_code, out rv)) {
+            return rv;
+         }
+         return null;
+      }
+
+      public static TypeCode type_code(this ColumnInfo ci) {
+         return get_typecode(ci.colType);
+      }
+
+      #endregion
+
+      public static Tuple<Type, object> guess_type(string value) {
+         if (string.IsNullOrEmpty(value)) {
+            return Tuple.Create(typeof(string), (object)value);
+         }
+
+         int val = 0;
+         // Try to parse as integer
+         if (int.TryParse(value, out val)) {
+            return Tuple.Create(typeof(int), (object)val);
+         }
+
+         double dval = 0.0;
+         // Try to parse as double
+         if (double.TryParse(value, out dval)) {
+            return Tuple.Create(typeof(double), (object)dval);
+         }
+
+         DateTime dt;
+         // Try to parse as DateTime
+         if (DateTime.TryParse(value, out dt)) {
+            return Tuple.Create(typeof(DateTime), (object)dt);
+         }
+
+         bool bval;
+         // Try to parse as boolean
+         if (bool.TryParse(value, out bval)) {
+            return Tuple.Create(typeof(bool), (object)bval);
+         }
+
+         // If none of the above, return string type
+         return Tuple.Create(typeof(string), (object)value);
+      }
+
       public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
          List<ColumnInfo> cis = new List<ColumnInfo>();
          foreach (Tuple<string, TypeCode> c in cols)
@@ -248,165 +348,6 @@ namespace BCG {
          return new BlueTableSchema(table_name, cis.ToArray());
       }
 
-      public static TypeCode convert_column_type(IEnumerable<string> col_vals) {
-         // bool
-         bool success = true;
-         foreach (var val in col_vals) {
-            bool parsed_val;
-            success = success && Boolean.TryParse(val, out parsed_val);
-            if (!success)
-               break;
-         }
-         if (success)
-            return TypeCode.Boolean;
-         // int
-         success = true;
-         foreach (var val in col_vals) {
-            int parsed_val;
-            success = success && Int32.TryParse(val, out parsed_val);
-            if (!success)
-               break;
-         }
-         if (success)
-            return TypeCode.Int32;
-         // try double
-         success = true;
-         foreach (var val in col_vals) {
-            double parsed_val;
-            success = success && Double.TryParse(val, out parsed_val);
-            if (!success)
-               break;
-         }
-         if (success)
-            return TypeCode.Double;
-         // try DateTime
-         success = true;
-         foreach (var val in col_vals) {
-            DateTime parsed_val;
-            success = success && DateTime.TryParse(val, out parsed_val);
-            if (!success)
-               break;
-         }
-         if (success)
-            return TypeCode.DateTime;
-         bts.Log.assert(false);
-         return TypeCode.Empty;
-      }
-
-      // public static Type guess_type(string val) {
-      //    //DateTime
-      //    {
-      //       DateTime dt;
-      //       var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt);
-      //       if (success) {
-      //          return typeof(DateTime);
-      //       }
-      //       DateTimeOffset dtoff;
-      //       success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz",
-      //                                              null, System.Globalization.DateTimeStyles.None, out dtoff);
-
-      //       if (success) {
-      //          return typeof(DateTime);
-      //       }
-      //    }
-      //    //int
-      //    {
-      //       int i;
-      //       var success = Int32.TryParse(val, out i);
-      //       if (success) {
-      //          return typeof(Int32);
-      //       }
-      //    }
-      //    //double
-      //    {
-      //       double d;
-      //       var success = Double.TryParse(val, out d);
-      //       if (success) {
-      //          return typeof(Double);
-      //       }
-      //    }
-      //    //bool
-      //    {
-      //       bool b;
-      //       var success = Boolean.TryParse(val, out b);
-      //       if (success) {
-      //          return typeof(bool);
-      //       }
-      //    }
-      //    //string
-      //    return typeof(string);
-      // }
-
-      private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) };
-      public static Type guess_type(string val) {
-         foreach (var t in _parsable_types) {
-            var type_val = convert_to_type(t, val);
-            if (type_val != null)
-               return t;
-         }
-         return typeof(string);
-
-      }
-
-      public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
-         // just use one row to guess
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         foreach (IBlueRow r in rows) {
-            foreach (var ci in r.schema.columnInfo) {
-               if (ci.colType != typeof(System.String)) {
-                  cis.Add(ci);
-                  continue;
-               }
-               Type type = guess_type(r[ci.colName] as string);
-               cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
-            }
-            return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
-         }
-         return null;
-      }
-
-      static object convert_to_type(Type t, string value) {
-         // Attempt to convert the string to the specified type
-         try {
-            return Convert.ChangeType(value, t);
-         } catch (Exception) {
-            if (t == typeof(double)) {
-               // some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
-               string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
-               cleaned_val = cleaned_val.Replace('−', '-');
-               double rv_d;
-               if (Double.TryParse(cleaned_val, out rv_d))
-                  return rv_d;
-               else
-                  return null;
-            }
-            return null;
-            // Handle conversion failure gracefully
-            // throw new ArgumentException("Failed to convert the string to the specified type.");
-         }
-      }
-
-      // converts a row where all columns are string types to a schema where the best types are guesed
-      public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
-         Func<IBlueRow, IBlueTableSchema, IBlueRow> get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => {
-            MutableBlueRow rv = new MutableBlueRow(typed_schema);
-            foreach (ColumnInfo ci in typed_schema.columnInfo) {
-               var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString());
-               rv.setValue(ci.colName, typed_val);
-            }
-            return rv;
-         };
-         IBlueTableSchema typed_schema = get_typed_schema(string_rows);
-         var rv = new BlueTable(typed_schema);
-         foreach (IBlueRow r in string_rows) {
-            var typed_row = get_typed_row(r, typed_schema);
-            rv.addRow(typed_row);
-         }
-         return rv;
-      }
-
-
-
       /// <summary>
       /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
       /// be alphabetic or not
@@ -541,12 +482,12 @@ namespace BCG {
          foreach (PropertyInfo pi in t.GetProperties())
             if (myIsActiveProperty(pi))
                colInfos.Add(new ColumnInfo(pi.Name, i++,
-                  ColumnInfo.isAllowedType(pi.PropertyType) ?
+                  is_valid_col_type(pi.PropertyType) ?
                   pi.PropertyType : typeof(String)));
          foreach (FieldInfo fi in t.GetFields())
             if (myIsActiveField(fi))
                colInfos.Add(new ColumnInfo(fi.Name, i++,
-                  ColumnInfo.isAllowedType(fi.FieldType) ? fi.FieldType : typeof(String)));
+                  is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String)));
 
          if (createIdField)
             colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
@@ -571,12 +512,12 @@ namespace BCG {
          foreach (PropertyInfo pi in t.GetProperties())
             if (myIsActiveProperty(pi))
                rowArray[schema[pi.Name].colNo] =
-                  ColumnInfo.isAllowedType(pi.PropertyType) ?
+                  is_valid_col_type(pi.PropertyType) ?
                   pi.GetValue(o, null) : pi.GetValue(o, null).ToString();
          foreach (FieldInfo fi in t.GetFields())
             if (myIsActiveField(fi))
                rowArray[schema[fi.Name].colNo] =
-                  ColumnInfo.isAllowedType(fi.FieldType) ?
+                  is_valid_col_type(fi.FieldType) ?
                   fi.GetValue(o) : fi.GetValue(o).ToString();
          if (idVal != null)
             rowArray[schema["id"].colNo] = idVal;
@@ -590,12 +531,12 @@ namespace BCG {
          foreach (PropertyInfo pi in t.GetProperties())
             if (myIsActiveProperty(pi))
                rowArray[schema[pi.Name].colNo] =
-                  ColumnInfo.isAllowedType(pi.PropertyType) ?
+                  is_valid_col_type(pi.PropertyType) ?
                   pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null));
          foreach (FieldInfo fi in t.GetFields())
             if (myIsActiveField(fi))
                rowArray[schema[fi.Name].colNo] =
-                  ColumnInfo.isAllowedType(fi.FieldType) ?
+                  is_valid_col_type(fi.FieldType) ?
                   fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o));
          if (idVal != null)
             rowArray[schema["id"].colNo] = idVal;
@@ -784,15 +725,7 @@ namespace BCG {
       /// <param name="r1"></param>
       /// <param name="r2"></param>
       /// <returns></returns>
-      public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) {
-         return computeRowEquality(r1, r2, r1.schema.columnNames);
-      }
 
-      public static bool computeRowEquality(IBlueRow r1,
-         IBlueRow r2,
-         params string[] colNames) {
-         return compareRows(r1, r2, colNames) == 0 ? true : false;
-      }
 
       /// <summary>
       /// Computes a string equivelant for the row, using the columns provided
@@ -830,7 +763,7 @@ namespace BCG {
       /// <param name="r2"></param>
       /// <param name="colNames"></param>
       /// <returns></returns>
-      public static int compareRows(IBlueRow r1,
+      public static int compare_rows(IBlueRow r1,
          IBlueRow r2,
          params string[] colNames) {
          int retVal = 0;
@@ -981,7 +914,7 @@ namespace BCG {
       public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) {
          IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
          foreach (IBlueRow r in rows)
-            retVal = compareRows(r, retVal, cols) > 0 ? r : retVal;
+            retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal;
          return retVal;
       }
 
@@ -994,7 +927,7 @@ namespace BCG {
       public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) {
          IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
          foreach (IBlueRow r in rows)
-            retVal = compareRows(r, retVal, cols) < 0 ? r : retVal;
+            retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal;
          return retVal;
       }
 
@@ -1073,8 +1006,58 @@ namespace BCG {
             out aRows);
       }
 
+      public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) {
+         IBlueRowCollection d_rows, a_rows;
+         compute_differences_fast(rows2, rows1, out d_rows, out a_rows);
+         return d_rows.Count == 0 && a_rows.Count == 0;
+      }
+
+      public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) {
+         return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType);
+      }
+
+      public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) {
+         if (s1.tableName != s1.tableName)
+            return false;
+         if (s1.columnInfo.Length != s2.columnInfo.Length)
+            return false;
+         for (int i = 0; i < s1.columnInfo.Length; i++) {
+            ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i];
+            if (!compute_equality(ci1, ci2))
+               return false;
+         }
+         return true;
+      }
+
+      public static bool compute_equality(IBlueRow r1, IBlueRow r2) {
+         return compute_row_equality(r1, r2, r1.schema.columnNames);
+      }
+
+      public static bool compute_row_equality(IBlueRow r1,
+         IBlueRow r2,
+         params string[] col_names) {
+         return compare_rows(r1, r2, col_names) == 0 ? true : false;
+      }
+
+      public static string to_string(ColumnInfo ci) {
+         return ci.colNo + ";" + ci.colName + ";" + ci.colType;
+      }
+
+      public static string to_string(this IBlueRow r) {
+         string rv = String.Empty;
+         foreach (ColumnInfo c in r.schema.columnInfo) {
+            rv += $"{c.colName}: {r[c.colName]}\n";
+         }
+         return rv;
+      }
+
       public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                   out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
+         if (old_rows == null || old_rows.Count == 0) {
+            out_d_rows = new BlueTable(new_rows.schema);
+            out_a_rows = new BlueTable(new_rows);
+            return;
+         }
          int[] col_indices = new int[new_rows.schema.columnNames.Length];
          for (int i = 0; i < col_indices.Length; i++)
             col_indices[i] = i;
@@ -1083,6 +1066,11 @@ namespace BCG {
 
       public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                       out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
+         if (old_rows == null || old_rows.Count == 0) {
+            out_d_rows = new BlueTable(new_rows.schema);
+            out_a_rows = new BlueTable(new_rows);
+            return;
+         }
          int[] col_indices = new int[] { new_rows.schema["id"].colNo };
          compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
       }
@@ -1126,7 +1114,7 @@ namespace BCG {
          foreach (IBlueRow nr in c1)
             if (!BlueDataUtility.rowPresent(c2, nr))
                (aRows as BlueTable).addRow(nr);
-            else if (BlueDataUtility.compareRows(nr, c2[(int)
+            else if (BlueDataUtility.compare_rows(nr, c2[(int)
                nr["id"]], columns) != 0) {
                (aRows as BlueTable).addRow(nr);
                (dRows as BlueTable).addRow(c2[(int)nr["id"]]);
@@ -1206,7 +1194,7 @@ namespace BCG {
          truelyAdds = new BlueTable(adds);
          foreach (IBlueRow r in dels) {
             if (adds.containsID((int)r["id"]) &&
-               BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) {
+               BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) {
                (truelyDels as BlueTable).deleteRow((int)r["id"]);
                (truelyAdds as BlueTable).deleteRow((int)r["id"]);
             }
@@ -2210,33 +2198,5 @@ namespace BCG {
       }
 
       #endregion
-
-      public class RowComparer<T> : IComparer<IBlueRow> {
-         private readonly IEnumerable<Tuple<string, bool>> _sort_cols;
-         public RowComparer(IEnumerable<Tuple<string, bool>> sort_cols) {
-            _sort_cols = sort_cols;
-         }
-         public int Compare(IBlueRow? x, IBlueRow? y) {
-            Func<object, object, int> compare_cols = (o1, o2) => {
-               return (o1 as IComparable).CompareTo(o2);
-            };
-            foreach (var c in _sort_cols) {
-               int i = compare_cols(x[c.Item1], y[c.Item1]);
-
-            }
-            return 1;
-         }
-
-      }
-      public static IList<IBlueRow> get_sorted_row(IBlueRowCollection rows, IEnumerable<Tuple<string, bool>> sort_cols) {
-         List<IBlueRow> row_list = new List<IBlueRow>();
-         foreach (IBlueRow r in rows) {
-            row_list.Add(r);
-         }
-         var row_comparer = new RowComparer<IBlueRow>(sort_cols);
-         ParallelSort.QuicksortSequential<IBlueRow>(row_list, row_comparer);
-         return row_list;
-
-      }
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs

@@ -1,45 +1,42 @@
 using System;
 
-namespace BCG
-{
-	/// <summary>
-	/// Allowed types
-	/// </summary>
-	public class BlueDataValidTypes
-	{
-      public static readonly Type BOOL = typeof(bool);
-      public static readonly Type BYTE = typeof(byte);
-      public static readonly Type CHAR = typeof(char);
-      public static readonly Type DECIMAL = typeof(decimal);
-      public static readonly Type DOUBLE = typeof(double);
-      public static readonly Type SHORT = typeof(short);
-      public static readonly Type INT = typeof(int);
-      public static readonly Type LONG = typeof(long);
-      public static readonly Type SBYTE = typeof(sbyte);
-      //public static readonly Type FLOAT = typeof(float);
-      public static readonly Type STRING = typeof(string);
-      //public static readonly Type USHORT = typeof(ushort);
-      public static readonly Type UINT = typeof(uint);
-      public static readonly Type ULONG = typeof(ulong);
-      public static readonly Type DATETIME = typeof(DateTime);
-      public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
-      public static readonly Type SINGLE  = typeof(Single);
-      public static readonly Type BYTEARRAY = typeof(byte[]);
+namespace BCG {
+    /// <summary>
+    /// Allowed types
+    /// </summary>
+    public class BlueDataValidTypes {
+        public static readonly Type BOOL = typeof(bool);
+        public static readonly Type BYTE = typeof(byte);
+        public static readonly Type CHAR = typeof(char);
+        public static readonly Type DECIMAL = typeof(decimal);
+        public static readonly Type DOUBLE = typeof(double);
+        public static readonly Type SHORT = typeof(short);
+        public static readonly Type INT = typeof(int);
+        public static readonly Type LONG = typeof(long);
+        public static readonly Type SBYTE = typeof(sbyte);
+        //public static readonly Type FLOAT = typeof(float);
+        public static readonly Type STRING = typeof(string);
+        //public static readonly Type USHORT = typeof(ushort);
+        public static readonly Type UINT = typeof(uint);
+        public static readonly Type ULONG = typeof(ulong);
+        public static readonly Type DATETIME = typeof(DateTime);
+        public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
+        public static readonly Type SINGLE = typeof(Single);
+        public static readonly Type BYTEARRAY = typeof(byte[]);
 
-      public static int getValueTypeSize (Type t)
-      {
-          if (t == BOOL || t == BYTE  || t == SBYTE)
-              return 1;
-          else if (t == SHORT || t == CHAR)
-              return 2;
-          else if (t == INT || t == UINT || t == SINGLE)
-              return 4;
-          else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
-              return 8;
-          else if (t == DECIMAL)
-              return 16;
-          else
-              throw new ApplicationException("Unsupported Data Type");
-      }
-	}
+        public static int getValueTypeSize(Type t) {
+            if (t == BOOL || t == BYTE || t == SBYTE)
+                return 1;
+            else if (t == SHORT || t == CHAR)
+                return 2;
+            else if (t == INT || t == UINT || t == SINGLE)
+                return 4;
+            else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
+                return 8;
+            else if (t == DECIMAL)
+                return 16;
+            else
+                throw new ApplicationException("Unsupported Data Type");
+        }
+    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs

@@ -1,116 +1,117 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
 
-/*
- * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns. 
- * if groupNo is zero, then the group is not stord in a lookup table at all. 
- * 
- * 
- * 
- */
-namespace BCG
-{
-   public class BlueRowLookupGroup
-   {
-      BlueTableSchema myEntrySchema;
-      public IBlueTableSchema entrySchema
-      {
-         get { return myEntrySchema; }
-         //cant' set this value, have to build it from the row's schema
+//TODORemove
+// /*
+//  * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns. 
+//  * if groupNo is zero, then the group is not stord in a lookup table at all. 
+//  * 
+//  * 
+//  * 
+//  */
+// namespace BCG
+// {
+//    public class BlueRowLookupGroup
+//    {
+//       BlueTableSchema myEntrySchema;
+//       public IBlueTableSchema entrySchema
+//       {
+//          get { return myEntrySchema; }
+//          //cant' set this value, have to build it from the row's schema
 
-      }
+//       }
 
-      //for faster access and such
-      private int[] myGroupColNos;
-      public int[] groupColNos
-      {
-         get { return myGroupColNos; }
+//       //for faster access and such
+//       private int[] myGroupColNos;
+//       public int[] groupColNos
+//       {
+//          get { return myGroupColNos; }
 
-      }
-      private string[] myGroupColNames;
-      public string[] groupColNames
-      {
-         get { return myGroupColNames; }
-      }
+//       }
+//       private string[] myGroupColNames;
+//       public string[] groupColNames
+//       {
+//          get { return myGroupColNames; }
+//       }
 
-      private short myGroupNumber;
-      public short groupNumber
-      {
-         get { return myGroupNumber; }
+//       private short myGroupNumber;
+//       public short groupNumber
+//       {
+//          get { return myGroupNumber; }
 
-      }
-      
-      /* how many bits are the ids into the table?
-       * In c# everything is stored as integers, but when it goes across the wire
-       * the data can be represented as a byte, a short, or an int 
-       */
-      private short myTableSize;
-      public short tableSize
-      {
-         get { return myTableSize; }
+//       }
 
-      }
+//       /* how many bits are the ids into the table?
+//        * In c# everything is stored as integers, but when it goes across the wire
+//        * the data can be represented as a byte, a short, or an int 
+//        */
+//       private short myTableSize;
+//       public short tableSize
+//       {
+//          get { return myTableSize; }
 
-      //if the group number is 0, the group is not stored in tables for serialization
-      public bool isStoredInTable
-      {
-         get { return myGroupNumber != 0; }
+//       }
 
-      }
-      public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
-      {
+//       //if the group number is 0, the group is not stored in tables for serialization
+//       public bool isStoredInTable
+//       {
+//          get { return myGroupNumber != 0; }
 
-         myGroupNumber = num;
-         myTableSize = sizeInBits;
-         myGroupColNames = cols ;
-      }
+//       }
+//       public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
+//       {
 
-      /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
-       each of these entries in the lookup table.  This function should be called by the LookupTableSchema
-       during its constructor. 
-       
-       This function also caches the column numbers for this group, for quicker access*/
-      public void buildEntrySchema(LookupTableSchema theSchema)
-      {
-         ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
-         myGroupColNos = new int[myGroupColNames.Length];
-         //the first column in our table is always the ID
-         int idColumn = 0;
-         columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
-         //go through each of the columns
-         for (int col = 1; col < columns.Length; ++col)
-         {
-            ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
-            if (columnInInputTable.isConstant)
-               columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
-            else
-               columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
+//          myGroupNumber = num;
+//          myTableSize = sizeInBits;
+//          myGroupColNames = cols ;
+//       }
 
-            myGroupColNos[col - 1] = columnInInputTable.colNo;
-         }
+//       /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
+//        each of these entries in the lookup table.  This function should be called by the LookupTableSchema
+//        during its constructor. 
 
-         
+//        This function also caches the column numbers for this group, for quicker access*/
+//       public void buildEntrySchema(LookupTableSchema theSchema)
+//       {
+//          ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
+//          myGroupColNos = new int[myGroupColNames.Length];
+//          //the first column in our table is always the ID
+//          int idColumn = 0;
+//          columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
+//          //go through each of the columns
+//          for (int col = 1; col < columns.Length; ++col)
+//          {
+//             ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
+//             if (columnInInputTable.isConstant)
+//                columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
+//             else
+//                columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
 
-         myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
-
-      }
-
-      /*takes a row as input, and extracts the members of the group from the row */
-      public object[] extractGroupFromRow(IBlueRow input)
-      {
-         int numCols = myGroupColNames.Length;
-         object[] results = new object[numCols];
-
-         for (int col = 0; col < numCols; ++col)
-            results[col] = input[myGroupColNos[col]];
-         
-         return results;
-
-      }
+//             myGroupColNos[col - 1] = columnInInputTable.colNo;
+//          }
 
 
 
-   }
-}
+//          myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
+
+//       }
+
+//       /*takes a row as input, and extracts the members of the group from the row */
+//       public object[] extractGroupFromRow(IBlueRow input)
+//       {
+//          int numCols = myGroupColNames.Length;
+//          object[] results = new object[numCols];
+
+//          for (int col = 0; col < numCols; ++col)
+//             results[col] = input[myGroupColNos[col]];
+
+//          return results;
+
+//       }
+
+
+
+//    }
+// }

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs

@@ -8,8 +8,7 @@ using System.Linq;
 
 using BCG.Utilities.General;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
    /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
@@ -17,112 +16,105 @@ namespace BCG
    /// the respective data, sorted according to the ColumnOrder.
    /// </summary>
    [Serializable]
-   public class BlueTableSchema : IBlueTableSchema
-   {
-      public class BlueTableSchemaSerializationHelper
-      {
-         public static object parse(Type t, string data)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               return Boolean.Parse(data);
-            else if (t == BlueDataValidTypes.BYTE)
-               return Byte.Parse(data);
-            else if (t == BlueDataValidTypes.CHAR)
-               return char.Parse(data);
-            else if (t == BlueDataValidTypes.DATETIME)
-               return DateTime.Parse(data);
-            else if (t == BlueDataValidTypes.DOUBLE)
-               return Double.Parse(data);
-            else if (t == BlueDataValidTypes.DECIMAL)
-               return Decimal.Parse(data);
-            else if (t == BlueDataValidTypes.INT)
-               return int.Parse(data);
-            else if (t == BlueDataValidTypes.LONG)
-               return long.Parse(data);
-            else if (t == BlueDataValidTypes.SBYTE)
-               return SByte.Parse(data);
-            else if (t == BlueDataValidTypes.STRING)
-               return data;
-            else if (t == BlueDataValidTypes.UINT)
-               return uint.Parse(data);
-            else if (t == BlueDataValidTypes.ULONG)
-               return ulong.Parse(data);
-            else if (t == BlueDataValidTypes.SINGLE)
-               return Single.Parse(data);
-            else
-               throw new ApplicationException("Can not be parsed");
-         }
-         public static void serialize(BinaryWriter bw, Type t, object o)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               bw.Write((bool)o);
-            else if (t == BlueDataValidTypes.BYTE)
-               bw.Write((byte)o);
-            else if (t == BlueDataValidTypes.CHAR)
-               bw.Write((char)o);
-            else if (t == BlueDataValidTypes.DATETIME)
-               bw.Write(((DateTime)o).Ticks);
-            else if (t == BlueDataValidTypes.DOUBLE)
-               bw.Write((double)o);
-            else if (t == BlueDataValidTypes.DECIMAL)
-               bw.Write((decimal)o);
-            else if (t == BlueDataValidTypes.INT)
-               bw.Write((int)o);
-            else if (t == BlueDataValidTypes.LONG)
-               bw.Write((long)o);
-            else if (t == BlueDataValidTypes.SBYTE)
-               bw.Write((sbyte)o);
-            else if (t == BlueDataValidTypes.STRING)
-               bw.Write(o as string);
-            else if (t == BlueDataValidTypes.SINGLE)
-               bw.Write((Single)o);
-            else if (t == BlueDataValidTypes.BYTEARRAY)
-            {
-               byte[] bs = o as byte[];
-               bw.Write(bs.Length);
-               bw.Write(bs);
-            }
-            else if (t == BlueDataValidTypes.UINT)
-               bw.Write((uint)o);
-            else if (t == BlueDataValidTypes.ULONG)
-               bw.Write((ulong)o);
-            else
-               throw new ApplicationException("Unsupported type");
-         }
+   public class BlueTableSchema : IBlueTableSchema {
+      public class BlueTableSchemaSerializationHelper {
+         // public static object parse(Type t, string data) {
+         //    if (t == BlueDataValidTypes.BOOL)
+         //       return Boolean.Parse(data);
+         //    else if (t == BlueDataValidTypes.BYTE)
+         //       return Byte.Parse(data);
+         //    else if (t == BlueDataValidTypes.CHAR)
+         //       return char.Parse(data);
+         //    else if (t == BlueDataValidTypes.DATETIME)
+         //       return DateTime.Parse(data);
+         //    else if (t == BlueDataValidTypes.DOUBLE)
+         //       return Double.Parse(data);
+         //    else if (t == BlueDataValidTypes.DECIMAL)
+         //       return Decimal.Parse(data);
+         //    else if (t == BlueDataValidTypes.INT)
+         //       return int.Parse(data);
+         //    else if (t == BlueDataValidTypes.LONG)
+         //       return long.Parse(data);
+         //    else if (t == BlueDataValidTypes.SBYTE)
+         //       return SByte.Parse(data);
+         //    else if (t == BlueDataValidTypes.STRING)
+         //       return data;
+         //    else if (t == BlueDataValidTypes.UINT)
+         //       return uint.Parse(data);
+         //    else if (t == BlueDataValidTypes.ULONG)
+         //       return ulong.Parse(data);
+         //    else if (t == BlueDataValidTypes.SINGLE)
+         //       return Single.Parse(data);
+         //    else
+         //       throw new ApplicationException("Can not be parsed");
+         // }
+         // public static void serialize(BinaryWriter bw, Type t, object o) {
+         //    if (t == BlueDataValidTypes.BOOL)
+         //       bw.Write((bool)o);
+         //    else if (t == BlueDataValidTypes.BYTE)
+         //       bw.Write((byte)o);
+         //    else if (t == BlueDataValidTypes.CHAR)
+         //       bw.Write((char)o);
+         //    else if (t == BlueDataValidTypes.DATETIME)
+         //       bw.Write(((DateTime)o).Ticks);
+         //    else if (t == BlueDataValidTypes.DOUBLE)
+         //       bw.Write((double)o);
+         //    else if (t == BlueDataValidTypes.DECIMAL)
+         //       bw.Write((decimal)o);
+         //    else if (t == BlueDataValidTypes.INT)
+         //       bw.Write((int)o);
+         //    else if (t == BlueDataValidTypes.LONG)
+         //       bw.Write((long)o);
+         //    else if (t == BlueDataValidTypes.SBYTE)
+         //       bw.Write((sbyte)o);
+         //    else if (t == BlueDataValidTypes.STRING)
+         //       bw.Write(o as string);
+         //    else if (t == BlueDataValidTypes.SINGLE)
+         //       bw.Write((Single)o);
+         //    else if (t == BlueDataValidTypes.BYTEARRAY) {
+         //       byte[] bs = o as byte[];
+         //       bw.Write(bs.Length);
+         //       bw.Write(bs);
+         //    } else if (t == BlueDataValidTypes.UINT)
+         //       bw.Write((uint)o);
+         //    else if (t == BlueDataValidTypes.ULONG)
+         //       bw.Write((ulong)o);
+         //    else
+         //       throw new ApplicationException("Unsupported type");
+         // }
 
-         public static object deserialize(BinaryReader br, Type t)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               return br.ReadBoolean();
-            else if (t == BlueDataValidTypes.BYTE)
-               return br.ReadByte();
-            else if (t == BlueDataValidTypes.CHAR)
-               return br.ReadChar();
-            else if (t == BlueDataValidTypes.DATETIME)
-               return new DateTime(br.ReadInt64());
-            else if (t == BlueDataValidTypes.DOUBLE)
-               return br.ReadDouble();
-            else if (t == BlueDataValidTypes.DECIMAL)
-               return br.ReadDecimal();
-            else if (t == BlueDataValidTypes.INT)
-               return br.ReadInt32();
-            else if (t == BlueDataValidTypes.LONG)
-               return br.ReadInt64();
-            else if (t == BlueDataValidTypes.SBYTE)
-               return br.ReadSByte();
-            else if (t == BlueDataValidTypes.STRING)
-               return br.ReadString();
-            else if (t == BlueDataValidTypes.BYTEARRAY)
-               return br.ReadBytes(br.ReadInt32());
-            else if (t == BlueDataValidTypes.UINT)
-               return br.ReadUInt32();
-            else if (t == BlueDataValidTypes.ULONG)
-               return br.ReadUInt64();
-            else if (t == BlueDataValidTypes.SINGLE)
-               return br.ReadSingle();
-            else
-               throw new ApplicationException("Unsupported type");
-         }
+         // public static object deserialize(BinaryReader br, Type t) {
+         //    if (t == BlueDataValidTypes.BOOL)
+         //       return br.ReadBoolean();
+         //    else if (t == BlueDataValidTypes.BYTE)
+         //       return br.ReadByte();
+         //    else if (t == BlueDataValidTypes.CHAR)
+         //       return br.ReadChar();
+         //    else if (t == BlueDataValidTypes.DATETIME)
+         //       return new DateTime(br.ReadInt64());
+         //    else if (t == BlueDataValidTypes.DOUBLE)
+         //       return br.ReadDouble();
+         //    else if (t == BlueDataValidTypes.DECIMAL)
+         //       return br.ReadDecimal();
+         //    else if (t == BlueDataValidTypes.INT)
+         //       return br.ReadInt32();
+         //    else if (t == BlueDataValidTypes.LONG)
+         //       return br.ReadInt64();
+         //    else if (t == BlueDataValidTypes.SBYTE)
+         //       return br.ReadSByte();
+         //    else if (t == BlueDataValidTypes.STRING)
+         //       return br.ReadString();
+         //    else if (t == BlueDataValidTypes.BYTEARRAY)
+         //       return br.ReadBytes(br.ReadInt32());
+         //    else if (t == BlueDataValidTypes.UINT)
+         //       return br.ReadUInt32();
+         //    else if (t == BlueDataValidTypes.ULONG)
+         //       return br.ReadUInt64();
+         //    else if (t == BlueDataValidTypes.SINGLE)
+         //       return br.ReadSingle();
+         //    else
+         //       throw new ApplicationException("Unsupported type");
+         // }
       }
 
       protected Hashtable myIndexByColName = new Hashtable();
@@ -132,29 +124,24 @@ namespace BCG
 
       public BlueTableSchema() { }
 
-      public BlueTableSchema(DataTable table)
-      {
+      public BlueTableSchema(DataTable table) {
          myIndexByColOrder = new ColumnInfo[table.Columns.Count];
          myTableName = table.TableName;
-         foreach (DataColumn col in table.Columns)
-         {
+         foreach (DataColumn col in table.Columns) {
             ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
             myIndexByColName[ci.colName] = ci;
             myIndexByColOrder[ci.colNo] = ci;
          }
       }
 
-      public BlueTableSchema(ColumnInfo[] cis, string tableName)
-      {
+      public BlueTableSchema(ColumnInfo[] cis, string tableName) {
          myConstruct(cis, tableName);
       }
 
-      private void myConstruct(ColumnInfo[] cis, string tableName)
-      {
+      private void myConstruct(ColumnInfo[] cis, string tableName) {
          myIndexByColOrder = new ColumnInfo[cis.Length];
          myTableName = tableName;
-         foreach (ColumnInfo ci in cis)
-         {
+         foreach (ColumnInfo ci in cis) {
             myIndexByColOrder[ci.colNo] = ci;
             myIndexByColName[ci.colName] = ci;
          }
@@ -169,15 +156,13 @@ namespace BCG
       /// </summary>
       /// <param name="tableName">the table name</param>
       /// <param name="fromType">the plain old data type whose fields will be mirrored in the schema</param>
-      public BlueTableSchema(string tableName, Type fromType)
-      {
+      public BlueTableSchema(string tableName, Type fromType) {
          List<ColumnInfo> cis = new List<ColumnInfo>();
          int i = 0;
          // we sort the field name so that we have a deterministic order given 
          // the same type. 
          bool foundId = false;
-         foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; }))
-         {
+         foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; })) {
             if (fi.Name.ToLower().Equals("id"))
                foundId = true;
             cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType));
@@ -188,32 +173,25 @@ namespace BCG
          myConstruct(cis.ToArray(), tableName);
       }
 
-      public ColumnInfo this[string colName]
-      {
-         get
-         {
+      public ColumnInfo this[string colName] {
+         get {
             object retVal = myIndexByColName[colName];
             return retVal != null ? retVal as ColumnInfo : null;
          }
       }
 
-      public ColumnInfo this[int colNo]
-      {
+      public ColumnInfo this[int colNo] {
          get { return myIndexByColOrder[colNo]; }
       }
 
       /// <summary>
       /// The returned Array is sorted according to the Column No
       /// </summary>
-      public string[] columnNames
-      {
-         get
-         {
-            if (myTableColNames == null)
-            {
+      public string[] columnNames {
+         get {
+            if (myTableColNames == null) {
                myTableColNames = new string[myIndexByColName.Count];
-               foreach (DictionaryEntry de in myIndexByColName)
-               {
+               foreach (DictionaryEntry de in myIndexByColName) {
                   ColumnInfo temp = (ColumnInfo)de.Value;
                   myTableColNames[temp.colNo] = temp.colName;
                }
@@ -225,8 +203,7 @@ namespace BCG
       /// <summary>
       /// The returned Array is sorted according to the Column No
       /// </summary>
-      public ColumnInfo[] columnInfo
-      {
+      public ColumnInfo[] columnInfo {
          get { return myIndexByColOrder; }
       }
 
@@ -234,49 +211,30 @@ namespace BCG
 
 
       #region ISerializedType methods
-      public void serialize(BinaryWriter bw)
-      {
+      public void serialize(BinaryWriter bw) {
          bw.Write(columnNames.Length);
          bw.Write(tableName);
-         foreach (ColumnInfo ci in columnInfo)
-         {
+         foreach (ColumnInfo ci in columnInfo) {
             bw.Write(ci.colName);
             bw.Write(ci.colType.ToString());
-            if (ci.isConstant)
-            {
-               bw.Write(true);
-               BlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
-            }
-            else
             {
                bw.Write(false);
             }
          }
       }
 
-      public void deserialze(BinaryReader br)
-      {
+      public void deserialze(BinaryReader br) {
          ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
          string name = br.ReadString();
-         for (int i = 0; i < cis.Length; i++)
-         {
+         for (int i = 0; i < cis.Length; i++) {
             string colName = br.ReadString();
             Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
-            bool isConstant = br.ReadBoolean();
-            if (isConstant)
-            {
-               cis[i] = new ColumnInfo(colName, i, t, BlueTableSchemaSerializationHelper.deserialize(br, t));
-            }
-            else
-            {
-               cis[i] = new ColumnInfo(colName, i, t);
-            }
+            cis[i] = new ColumnInfo(colName, i, t);
          }
          myConstruct(cis, name);
       }
 
-      public int getSerializedSize()
-      {
+      public int getSerializedSize() {
          throw new NotImplementedException("Feature for provided constant values to" +
             "non-primitive types in schema, not provided as of now");
       }

src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+
+using BCG;
+
+namespace bts {
+   public class CircularBufferTable : IBlueRowCollection {
+      private BlueTable _table;
+      private List<IBlueRow> _ordered_list = new List<IBlueRow>();
+      private int _buffer_sz;
+      public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) {
+         _table = new BlueTable(schema);
+         _buffer_sz = buffer_sz;
+      }
+
+      // public OrderedBlueTable(List<IBlueRow> rows, int buffer_sz) {
+      //    _buffer_sz = buffer_sz;
+      //    foreach (IBlueRow r in rows)
+      //       add_row(r);
+      // }
+
+      // returns the row that might need to be deleted to keep
+      // the circular buffer intact
+      public IBlueRow add_row(IBlueRow row) {
+         _ordered_list.Add(row);
+         _table.addRow(row);
+         if (_ordered_list.Count > _buffer_sz) {
+            IBlueRow rv = _ordered_list.ElementAt(0);
+            _ordered_list.RemoveAt(0);
+            _table.deleteRow(rv);
+            return rv;
+         }
+         return null;
+      }
+
+      public IBlueRow this[int id] {
+         get {
+            return _table[id];
+         }
+      }
+      public IBlueTableSchema schema {
+         get {
+            return _table.schema;
+         }
+      }
+      public bool containsID(int id) {
+         return _table.containsID(id);
+      }
+
+      public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); }
+      public int Count { get { return _table.Count; } }
+      public bool IsSynchronized { get { return false; } }
+      public object SyncRoot { get { return this; } }
+      public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,56 +1,25 @@
 using System;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
    /// </summary>
    [Serializable]
-   public class ColumnInfo : IComparable
-   {
-      public static bool isAllowedType (Type t)
-      {
-         return 
-            t.Equals (typeof (System.Int32)) ||
-            t.Equals (typeof (System.Int64)) ||
-            t.Equals (typeof (System.Double)) ||
-            t.Equals (typeof (System.String)) ||
-            t.Equals (typeof (System.DateTime)) ? true :
-            false;
-      }
+   public class ColumnInfo : IComparable {
 
       //the default column info does not cach values in a lookup table, therefore it is in group 0
-      public ColumnInfo(string colName, int colNo, Type colType)
-         : this(colName, colNo, colType, 0)
-      {
-
-
-      }
-
-      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
-      {
+      public ColumnInfo(string colName, int colNo, Type colType) {
          myColName = colName;
          myColNo = colNo;
          myColType = colType;
-         myLookupTableGroup = colGroup;
-         myColValue = null;
+      }
+
+      public ColumnInfo(string col_name, int col_no, TypeCode type_code)
+         : this(col_name, col_no, BlueDataUtility.get_type(type_code)) {
       }
 
 
-      public ColumnInfo (string colName, int colNo, object colValue)
-         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
-
-      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
-         : this (colName, colNo, colType) 
-      {
-
-         myColValue = colValue == null ? DBNull.Value : colValue;
-      
-      }
-
-
-      public int CompareTo (object obj)
-      {
+      public int CompareTo(object obj) {
          int otherColNo = ((ColumnInfo)(obj)).colNo;
          if (myColNo < otherColNo)
             return -1;
@@ -59,37 +28,29 @@ namespace BCG
          return 0;
       }
 
-      //public bool isCalculated{get{return myExpression != null;}}
+      public override bool Equals(object obj) {
+         if (obj is ColumnInfo) {
+            ColumnInfo other = obj as ColumnInfo;
+            return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
+         }
+         return false;
+      }
 
-      public bool isConstant {get{return myColValue != null;}}
+      public override string ToString() {
+         return colName + " " + colNo + " " + colType;
+      }
+
+
+      // for backwards compatibility to allow build
+      public bool isConstant { get { return false; } }
 
       private int myColNo;
       private string myColName;
       private Type myColType;
-      //private string myExpression;
-      private object myColValue;
 
-      //no table will ever have more than 65,536 columns
-      //if it does, we are in serious trouble
-      private short myLookupTableGroup;
+      public int colNo { get { return myColNo; } }
+      public string colName { get { return myColName; } }
+      public Type colType { get { return myColType; } }
 
-      public int colNo{get{return myColNo;}}
-      public string colName {get{return myColName;}}
-      public Type colType {get{return myColType;}}
-
-
-      //this value corresponds to how the data in the column is serialized. 
-      //if the group number is 0, the values are not stored in a table
-      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
-      //are stored together in a lookup table
-
-      
-      public short lookupTableGroup
-      {
-         get { return myLookupTableGroup; }
-         set { myLookupTableGroup = value; }
-      }
-      //public string expression {get{return myExpression;}}
-      public object colValue {get{return myColValue;}}
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs

@@ -1,252 +1,252 @@
-using System;
-using System.Collections;
-//using BCG.Utilities;
+// using System;
+// using System.Collections;
+// //using BCG.Utilities;
 
-namespace BCG
-{
-   /// <summary>
-   /// Provides a multiple indexing mechanism, required to increase the peformance of the streams
-   /// </summary>
-   public class BlueTableIndexException : Exception
-   {
-      public BlueTableIndexException(string msg) : base(msg) { }
-   }
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Provides a multiple indexing mechanism, required to increase the peformance of the streams
+//    /// </summary>
+//    public class BlueTableIndexException : Exception
+//    {
+//       public BlueTableIndexException(string msg) : base(msg) { }
+//    }
 
-   public interface IBlueTableIndex
-   {
-      void addRow(IBlueRow row);
-      void deleteRowOnPrimaryKey(object pkValue);
-      void deleteRowOnPrimaryKey(IBlueRow row);
-      void deleteRowOnBothKeys(object pkValue, object skValue);
-      void deleteRowOnBothKeys(IBlueRow row);
-      ArrayList searchRowOnPrimaryKey(object pkValue);
-      ArrayList searchRowOnPrimaryKey(IBlueRow row);
-      ArrayList searchRowOnBothKeys(object pkValue, object skValue);
-      ArrayList searchRowOnBothKeys(IBlueRow row);
+//    public interface IBlueTableIndex
+//    {
+//       void addRow(IBlueRow row);
+//       void deleteRowOnPrimaryKey(object pkValue);
+//       void deleteRowOnPrimaryKey(IBlueRow row);
+//       void deleteRowOnBothKeys(object pkValue, object skValue);
+//       void deleteRowOnBothKeys(IBlueRow row);
+//       ArrayList searchRowOnPrimaryKey(object pkValue);
+//       ArrayList searchRowOnPrimaryKey(IBlueRow row);
+//       ArrayList searchRowOnBothKeys(object pkValue, object skValue);
+//       ArrayList searchRowOnBothKeys(IBlueRow row);
 
 
-   }
+//    }
 
-   /*
-   public class BlueTableDoubleIndex : IBlueTableIndex
-   {
-      private IBlueRowCollection myTable;         
-      private string myPrimaryIndex;
-      private string mySecondaryIndex;
-      private Hashtable myHt = new Hashtable ();
-      
-      /// <summary>
-      /// Builds an index with 2 columns of the table acting as primary index and secondary index
-      /// respectively
-      /// </summary>
-      /// <param name="table">
-      /// table to index
-      /// </param>
-      /// <param name="primaryIndex">
-      /// primary key to index upon
-      /// </param>
-      /// <param name="secondaryIndex">
-      /// column serving as the secondary key
-      /// </param>
-      public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
-      {
-         myTable = table;
-         myPrimaryIndex = primaryIndex;
-         mySecondaryIndex = secondaryIndex;
-         myInit ();
-      }
-      
-      private void myInit()
-      {
-         foreach (IBlueRow row in myTable)
-            addRow (row);
-      }
+//    /*
+//    public class BlueTableDoubleIndex : IBlueTableIndex
+//    {
+//       private IBlueRowCollection myTable;         
+//       private string myPrimaryIndex;
+//       private string mySecondaryIndex;
+//       private Hashtable myHt = new Hashtable ();
 
-      /// <summary>
-      /// To add a row to the index
-      /// </summary>
-      /// <param name="row">
-      /// row to be added
-      /// </param>
-      public void addRow (IBlueRow row)
-      {
-         if (myHt[row[myPrimaryIndex]] != null)
-            ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
-         else
-         {
-            TimeManager.start ();
-            Hashtable ht = new Hashtable();
-            TimeManager.stop ();
-            ht[row[mySecondaryIndex]] = row;
-            myHt[row[myPrimaryIndex]] = ht;
-         }
-      }
+//       /// <summary>
+//       /// Builds an index with 2 columns of the table acting as primary index and secondary index
+//       /// respectively
+//       /// </summary>
+//       /// <param name="table">
+//       /// table to index
+//       /// </param>
+//       /// <param name="primaryIndex">
+//       /// primary key to index upon
+//       /// </param>
+//       /// <param name="secondaryIndex">
+//       /// column serving as the secondary key
+//       /// </param>
+//       public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
+//       {
+//          myTable = table;
+//          myPrimaryIndex = primaryIndex;
+//          mySecondaryIndex = secondaryIndex;
+//          myInit ();
+//       }
 
-      /// <summary>
-      /// If index or the primary key has already been deleted appropriate exceptions are thrown
-      /// </summary>
-      /// <param name="row"></param>
-      public void deleteRow (IBlueRow row)
-      {
-         if (myHt[row[myPrimaryIndex]] == null)
-         {
-            throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
-         }
-         ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
-         if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
-            myHt.Remove(row[myPrimaryIndex]);
-      }
+//       private void myInit()
+//       {
+//          foreach (IBlueRow row in myTable)
+//             addRow (row);
+//       }
 
-      /// <summary>
-      /// Returns an array of DataRow that match the primary key being passed as an argument
-      /// </summary>
-      /// <param name="o">
-      /// Primary Key of the table
-      /// </param>
-      /// <returns>
-      /// Array of rows matching the criteria
-      /// </returns>
-      public IBlueRow[] findValueFromPrimaryKey (object o)
-      {
-         object oht = myHt[o];
-         if (oht == null)
-            return new IBlueRow[0];
-         Hashtable ht = (Hashtable)oht;
-         IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
-         ht.Values.CopyTo(rows, 0);
-         return rows;
-      }
+//       /// <summary>
+//       /// To add a row to the index
+//       /// </summary>
+//       /// <param name="row">
+//       /// row to be added
+//       /// </param>
+//       public void addRow (IBlueRow row)
+//       {
+//          if (myHt[row[myPrimaryIndex]] != null)
+//             ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
+//          else
+//          {
+//             TimeManager.start ();
+//             Hashtable ht = new Hashtable();
+//             TimeManager.stop ();
+//             ht[row[mySecondaryIndex]] = row;
+//             myHt[row[myPrimaryIndex]] = ht;
+//          }
+//       }
+
+//       /// <summary>
+//       /// If index or the primary key has already been deleted appropriate exceptions are thrown
+//       /// </summary>
+//       /// <param name="row"></param>
+//       public void deleteRow (IBlueRow row)
+//       {
+//          if (myHt[row[myPrimaryIndex]] == null)
+//          {
+//             throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
+//          }
+//          ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
+//          if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
+//             myHt.Remove(row[myPrimaryIndex]);
+//       }
+
+//       /// <summary>
+//       /// Returns an array of DataRow that match the primary key being passed as an argument
+//       /// </summary>
+//       /// <param name="o">
+//       /// Primary Key of the table
+//       /// </param>
+//       /// <returns>
+//       /// Array of rows matching the criteria
+//       /// </returns>
+//       public IBlueRow[] findValueFromPrimaryKey (object o)
+//       {
+//          object oht = myHt[o];
+//          if (oht == null)
+//             return new IBlueRow[0];
+//          Hashtable ht = (Hashtable)oht;
+//          IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
+//          ht.Values.CopyTo(rows, 0);
+//          return rows;
+//       }
 
 
-      /// <summary>
-      /// returns as array of the primary keys
-      /// </summary>
-      /// <returns>
-      /// object[] of primary keys
-      /// </returns>
-      public Array getPrimaryKeys()
-      {
-         object[] retval = new object [myHt.Keys.Count];
-         myHt.Keys.CopyTo (retval, 0);
-         return retval;
-      }
-   }
-   */
+//       /// <summary>
+//       /// returns as array of the primary keys
+//       /// </summary>
+//       /// <returns>
+//       /// object[] of primary keys
+//       /// </returns>
+//       public Array getPrimaryKeys()
+//       {
+//          object[] retval = new object [myHt.Keys.Count];
+//          myHt.Keys.CopyTo (retval, 0);
+//          return retval;
+//       }
+//    }
+//    */
 
-   /// <summary>
-   /// Provides a 1-Level indexing structure.
-   /// </summary>
-   public class BlueTableSingleIndex : IBlueTableIndex
-   {
-      string myPrimaryKey;
-      string mySecondaryKey;
-      Hashtable myHt = new Hashtable();
+//    /// <summary>
+//    /// Provides a 1-Level indexing structure.
+//    /// </summary>
+//    public class BlueTableSingleIndex : IBlueTableIndex
+//    {
+//       string myPrimaryKey;
+//       string mySecondaryKey;
+//       Hashtable myHt = new Hashtable();
 
-      public BlueTableSingleIndex(string primaryKey, string secondaryKey)
-      {
-         myPrimaryKey = primaryKey;
-         mySecondaryKey = secondaryKey;
-      }
+//       public BlueTableSingleIndex(string primaryKey, string secondaryKey)
+//       {
+//          myPrimaryKey = primaryKey;
+//          mySecondaryKey = secondaryKey;
+//       }
 
-      /// <summary>
-      /// The primary key is the indexing key, and the secondary key is used as the key for
-      /// searching and hence deletion
-      /// </summary>
-      /// <param name="table"></param>
-      /// <param name="primaryKey"></param>
-      /// <param name="secondaryKey"></param>
-      public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
-         : this(primaryKey, secondaryKey)
-      {
-         foreach (IBlueRow row in table)
-            addRow(row);
-      }
+//       /// <summary>
+//       /// The primary key is the indexing key, and the secondary key is used as the key for
+//       /// searching and hence deletion
+//       /// </summary>
+//       /// <param name="table"></param>
+//       /// <param name="primaryKey"></param>
+//       /// <param name="secondaryKey"></param>
+//       public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
+//          : this(primaryKey, secondaryKey)
+//       {
+//          foreach (IBlueRow row in table)
+//             addRow(row);
+//       }
 
 
 
-      /// <summary>
-      /// Adds row to the indexing new structure
-      /// </summary>
-      /// <param name="row"></param>
-      public void addRow(IBlueRow row)
-      {
-         if (myHt[row[myPrimaryKey]] == null)
-            myHt[row[myPrimaryKey]] = new ArrayList();
-         ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
-      }
+//       /// <summary>
+//       /// Adds row to the indexing new structure
+//       /// </summary>
+//       /// <param name="row"></param>
+//       public void addRow(IBlueRow row)
+//       {
+//          if (myHt[row[myPrimaryKey]] == null)
+//             myHt[row[myPrimaryKey]] = new ArrayList();
+//          ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
+//       }
 
-      /// <summary>
-      /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
-      /// </summary>
-      /// <param name="row"></param>
-      /// <returns></returns>
-      public ArrayList searchRowOnPrimaryKey(IBlueRow row)
-      {
-         return searchRowOnPrimaryKey(row[myPrimaryKey]);
-      }
+//       /// <summary>
+//       /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
+//       /// </summary>
+//       /// <param name="row"></param>
+//       /// <returns></returns>
+//       public ArrayList searchRowOnPrimaryKey(IBlueRow row)
+//       {
+//          return searchRowOnPrimaryKey(row[myPrimaryKey]);
+//       }
 
-      public ArrayList searchRowOnPrimaryKey(object pkValue)
-      {
-         if (myHt[pkValue] == null)
-            return new ArrayList();
-         return (ArrayList)myHt[pkValue];
-      }
+//       public ArrayList searchRowOnPrimaryKey(object pkValue)
+//       {
+//          if (myHt[pkValue] == null)
+//             return new ArrayList();
+//          return (ArrayList)myHt[pkValue];
+//       }
 
-      /// <summary>
-      /// Gets rows based on the primary AND the secondary key
-      /// </summary>
-      /// <param name="row"></param>
-      /// <returns></returns>
-      public ArrayList searchRowOnBothKeys(IBlueRow row)
-      {
-         return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
-      }
+//       /// <summary>
+//       /// Gets rows based on the primary AND the secondary key
+//       /// </summary>
+//       /// <param name="row"></param>
+//       /// <returns></returns>
+//       public ArrayList searchRowOnBothKeys(IBlueRow row)
+//       {
+//          return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
+//       }
 
-      public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
-      {
-         if (myHt[pkValue] == null)
-            return new ArrayList();
-         ArrayList matchingList = (ArrayList)myHt[pkValue];
-         ArrayList retVal = new ArrayList();
-         foreach (IBlueRow r in matchingList)
-            if (r[mySecondaryKey].Equals(skValue))
-               retVal.Add(r);
-         return retVal;
-      }
+//       public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
+//       {
+//          if (myHt[pkValue] == null)
+//             return new ArrayList();
+//          ArrayList matchingList = (ArrayList)myHt[pkValue];
+//          ArrayList retVal = new ArrayList();
+//          foreach (IBlueRow r in matchingList)
+//             if (r[mySecondaryKey].Equals(skValue))
+//                retVal.Add(r);
+//          return retVal;
+//       }
 
 
-      /// <summary>
-      /// Deletes rows based on the primary key only
-      /// </summary>
-      /// <param name="row"></param>
-      public void deleteRowOnPrimaryKey(IBlueRow row)
-      {
-         deleteRowOnPrimaryKey(row[myPrimaryKey]);
-      }
+//       /// <summary>
+//       /// Deletes rows based on the primary key only
+//       /// </summary>
+//       /// <param name="row"></param>
+//       public void deleteRowOnPrimaryKey(IBlueRow row)
+//       {
+//          deleteRowOnPrimaryKey(row[myPrimaryKey]);
+//       }
 
-      public void deleteRowOnPrimaryKey(object pkValue)
-      {
-         if (myHt[pkValue] == null)
-            throw new BlueTableIndexException("Key to be deleted was not found in the index");
-         myHt.Remove(pkValue);
-      }
+//       public void deleteRowOnPrimaryKey(object pkValue)
+//       {
+//          if (myHt[pkValue] == null)
+//             throw new BlueTableIndexException("Key to be deleted was not found in the index");
+//          myHt.Remove(pkValue);
+//       }
 
-      public void deleteRowOnBothKeys(IBlueRow row)
-      {
-         deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
-      }
+//       public void deleteRowOnBothKeys(IBlueRow row)
+//       {
+//          deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
+//       }
 
-      public void deleteRowOnBothKeys(object pkValue, object skValue)
-      {
-         if (myHt[pkValue] == null)
-            throw new BlueTableIndexException("Key to be deleted was not found in the index");
-         ArrayList matchingList = (ArrayList)myHt[pkValue];
-         ArrayList aux = new ArrayList();
-         foreach (IBlueRow r in matchingList)
-            if (r[mySecondaryKey].Equals(skValue))
-               aux.Add(r);
-         foreach (IBlueRow r in aux)
-            matchingList.Remove(r);
-      }
-   }
-}
+//       public void deleteRowOnBothKeys(object pkValue, object skValue)
+//       {
+//          if (myHt[pkValue] == null)
+//             throw new BlueTableIndexException("Key to be deleted was not found in the index");
+//          ArrayList matchingList = (ArrayList)myHt[pkValue];
+//          ArrayList aux = new ArrayList();
+//          foreach (IBlueRow r in matchingList)
+//             if (r[mySecondaryKey].Equals(skValue))
+//                aux.Add(r);
+//          foreach (IBlueRow r in aux)
+//             matchingList.Remove(r);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs

@@ -1,324 +1,324 @@
-using System;
-using System.Data;
-using System.Collections;
-using System.IO;
-using System.Reflection;
-using System.Collections.Generic;
-using System.Linq;
+// using System;
+// using System.Data;
+// using System.Collections;
+// using System.IO;
+// using System.Reflection;
+// using System.Collections.Generic;
+// using System.Linq;
 
-using BCG.Utilities.General;
+// using BCG.Utilities.General;
 
-namespace BCG
-{
-   /// <summary>
-   /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
-   /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
-   /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
-   /// the respective data, sorted according to the ColumnOrder.
-   /// </summary>
-   [Serializable]
-   public class LookupTableSchema : IBlueTableSchema
-   {
-      public class LookupTableSchemaSerializationHelper
-      {
-         public static object parse(Type t, string data)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               return Boolean.Parse(data);
-            else if (t == BlueDataValidTypes.BYTE)
-               return Byte.Parse(data);
-            else if (t == BlueDataValidTypes.CHAR)
-               return char.Parse(data);
-            else if (t == BlueDataValidTypes.DATETIME)
-               return DateTime.Parse(data);
-            else if (t == BlueDataValidTypes.DOUBLE)
-               return Double.Parse(data);
-            else if (t == BlueDataValidTypes.DECIMAL)
-               return Decimal.Parse(data);
-            else if (t == BlueDataValidTypes.INT)
-               return int.Parse(data);
-            else if (t == BlueDataValidTypes.LONG)
-               return long.Parse(data);
-            else if (t == BlueDataValidTypes.SBYTE)
-               return SByte.Parse(data);
-            else if (t == BlueDataValidTypes.STRING)
-               return data;
-            else if (t == BlueDataValidTypes.UINT)
-               return uint.Parse(data);
-            else if (t == BlueDataValidTypes.ULONG)
-               return ulong.Parse(data);
-            else if (t == BlueDataValidTypes.SINGLE)
-               return Single.Parse(data);
-            else
-               throw new ApplicationException("Can not be parsed");
-         }
-         public static void serialize(BinaryWriter bw, Type t, object o)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               bw.Write((bool)o);
-            else if (t == BlueDataValidTypes.BYTE)
-               bw.Write((byte)o);
-            else if (t == BlueDataValidTypes.CHAR)
-               bw.Write((char)o);
-            else if (t == BlueDataValidTypes.DATETIME)
-               bw.Write(((DateTime)o).Ticks);
-            else if (t == BlueDataValidTypes.DOUBLE)
-               bw.Write((double)o);
-            else if (t == BlueDataValidTypes.DECIMAL)
-               bw.Write((decimal)o);
-            else if (t == BlueDataValidTypes.INT)
-               bw.Write((int)o);
-            else if (t == BlueDataValidTypes.LONG)
-               bw.Write((long)o);
-            else if (t == BlueDataValidTypes.SBYTE)
-               bw.Write((sbyte)o);
-            else if (t == BlueDataValidTypes.STRING)
-               bw.Write(o as string);
-            else if (t == BlueDataValidTypes.SINGLE)
-               bw.Write((Single)o);
-            else if (t == BlueDataValidTypes.BYTEARRAY)
-            {
-               byte[] bs = o as byte[];
-               bw.Write(bs.Length);
-               bw.Write(bs);
-            }
-            else if (t == BlueDataValidTypes.UINT)
-               bw.Write((uint)o);
-            else if (t == BlueDataValidTypes.ULONG)
-               bw.Write((ulong)o);
-            else
-               throw new ApplicationException("Unsupported type");
-         }
+// namespace BCG
+// {
+//    /// <summary>
+//    /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
+//    /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
+//    /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
+//    /// the respective data, sorted according to the ColumnOrder.
+//    /// </summary>
+//    [Serializable]
+//    public class LookupTableSchema : IBlueTableSchema
+//    {
+//       public class LookupTableSchemaSerializationHelper
+//       {
+//          public static object parse(Type t, string data)
+//          {
+//             if (t == BlueDataValidTypes.BOOL)
+//                return Boolean.Parse(data);
+//             else if (t == BlueDataValidTypes.BYTE)
+//                return Byte.Parse(data);
+//             else if (t == BlueDataValidTypes.CHAR)
+//                return char.Parse(data);
+//             else if (t == BlueDataValidTypes.DATETIME)
+//                return DateTime.Parse(data);
+//             else if (t == BlueDataValidTypes.DOUBLE)
+//                return Double.Parse(data);
+//             else if (t == BlueDataValidTypes.DECIMAL)
+//                return Decimal.Parse(data);
+//             else if (t == BlueDataValidTypes.INT)
+//                return int.Parse(data);
+//             else if (t == BlueDataValidTypes.LONG)
+//                return long.Parse(data);
+//             else if (t == BlueDataValidTypes.SBYTE)
+//                return SByte.Parse(data);
+//             else if (t == BlueDataValidTypes.STRING)
+//                return data;
+//             else if (t == BlueDataValidTypes.UINT)
+//                return uint.Parse(data);
+//             else if (t == BlueDataValidTypes.ULONG)
+//                return ulong.Parse(data);
+//             else if (t == BlueDataValidTypes.SINGLE)
+//                return Single.Parse(data);
+//             else
+//                throw new ApplicationException("Can not be parsed");
+//          }
+//          public static void serialize(BinaryWriter bw, Type t, object o)
+//          {
+//             if (t == BlueDataValidTypes.BOOL)
+//                bw.Write((bool)o);
+//             else if (t == BlueDataValidTypes.BYTE)
+//                bw.Write((byte)o);
+//             else if (t == BlueDataValidTypes.CHAR)
+//                bw.Write((char)o);
+//             else if (t == BlueDataValidTypes.DATETIME)
+//                bw.Write(((DateTime)o).Ticks);
+//             else if (t == BlueDataValidTypes.DOUBLE)
+//                bw.Write((double)o);
+//             else if (t == BlueDataValidTypes.DECIMAL)
+//                bw.Write((decimal)o);
+//             else if (t == BlueDataValidTypes.INT)
+//                bw.Write((int)o);
+//             else if (t == BlueDataValidTypes.LONG)
+//                bw.Write((long)o);
+//             else if (t == BlueDataValidTypes.SBYTE)
+//                bw.Write((sbyte)o);
+//             else if (t == BlueDataValidTypes.STRING)
+//                bw.Write(o as string);
+//             else if (t == BlueDataValidTypes.SINGLE)
+//                bw.Write((Single)o);
+//             else if (t == BlueDataValidTypes.BYTEARRAY)
+//             {
+//                byte[] bs = o as byte[];
+//                bw.Write(bs.Length);
+//                bw.Write(bs);
+//             }
+//             else if (t == BlueDataValidTypes.UINT)
+//                bw.Write((uint)o);
+//             else if (t == BlueDataValidTypes.ULONG)
+//                bw.Write((ulong)o);
+//             else
+//                throw new ApplicationException("Unsupported type");
+//          }
 
-         public static object deserialize(BinaryReader br, Type t)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               return br.ReadBoolean();
-            else if (t == BlueDataValidTypes.BYTE)
-               return br.ReadByte();
-            else if (t == BlueDataValidTypes.CHAR)
-               return br.ReadChar();
-            else if (t == BlueDataValidTypes.DATETIME)
-               return new DateTime(br.ReadInt64());
-            else if (t == BlueDataValidTypes.DOUBLE)
-               return br.ReadDouble();
-            else if (t == BlueDataValidTypes.DECIMAL)
-               return br.ReadDecimal();
-            else if (t == BlueDataValidTypes.INT)
-               return br.ReadInt32();
-            else if (t == BlueDataValidTypes.LONG)
-               return br.ReadInt64();
-            else if (t == BlueDataValidTypes.SBYTE)
-               return br.ReadSByte();
-            else if (t == BlueDataValidTypes.STRING)
-               return br.ReadString();
-            else if (t == BlueDataValidTypes.BYTEARRAY)
-               return br.ReadBytes(br.ReadInt32());
-            else if (t == BlueDataValidTypes.UINT)
-               return br.ReadUInt32();
-            else if (t == BlueDataValidTypes.ULONG)
-               return br.ReadUInt64();
-            else if (t == BlueDataValidTypes.SINGLE)
-               return br.ReadSingle();
-            else
-               throw new ApplicationException("Unsupported type");
-         }
-      }
+//          public static object deserialize(BinaryReader br, Type t)
+//          {
+//             if (t == BlueDataValidTypes.BOOL)
+//                return br.ReadBoolean();
+//             else if (t == BlueDataValidTypes.BYTE)
+//                return br.ReadByte();
+//             else if (t == BlueDataValidTypes.CHAR)
+//                return br.ReadChar();
+//             else if (t == BlueDataValidTypes.DATETIME)
+//                return new DateTime(br.ReadInt64());
+//             else if (t == BlueDataValidTypes.DOUBLE)
+//                return br.ReadDouble();
+//             else if (t == BlueDataValidTypes.DECIMAL)
+//                return br.ReadDecimal();
+//             else if (t == BlueDataValidTypes.INT)
+//                return br.ReadInt32();
+//             else if (t == BlueDataValidTypes.LONG)
+//                return br.ReadInt64();
+//             else if (t == BlueDataValidTypes.SBYTE)
+//                return br.ReadSByte();
+//             else if (t == BlueDataValidTypes.STRING)
+//                return br.ReadString();
+//             else if (t == BlueDataValidTypes.BYTEARRAY)
+//                return br.ReadBytes(br.ReadInt32());
+//             else if (t == BlueDataValidTypes.UINT)
+//                return br.ReadUInt32();
+//             else if (t == BlueDataValidTypes.ULONG)
+//                return br.ReadUInt64();
+//             else if (t == BlueDataValidTypes.SINGLE)
+//                return br.ReadSingle();
+//             else
+//                throw new ApplicationException("Unsupported type");
+//          }
+//       }
 
-      protected Hashtable myIndexByColName = new Hashtable();
-      protected ColumnInfo[] myIndexByColOrder;
-      private string[] myTableColNames;
-      private BlueRowLookupGroup[] myLookupGroups;
-      protected string myTableName;
+//       protected Hashtable myIndexByColName = new Hashtable();
+//       protected ColumnInfo[] myIndexByColOrder;
+//       private string[] myTableColNames;
+//       private BlueRowLookupGroup[] myLookupGroups;
+//       protected string myTableName;
 
-      public LookupTableSchema() { }
+//       public LookupTableSchema() { }
 
 
-      public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
-      {
-         myConstruct(tableName, cis, groups);
-      }
+//       public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
+//       {
+//          myConstruct(tableName, cis, groups);
+//       }
 
-      private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
-      {
-         myIndexByColOrder = new ColumnInfo[cis.Length];
-         myTableName = tableName;
-         foreach (ColumnInfo ci in cis)
-         {
-            myIndexByColOrder[ci.colNo] = ci;
-            myIndexByColName[ci.colName] = ci;
-         }
-         myLookupGroups = groups;
+//       private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
+//       {
+//          myIndexByColOrder = new ColumnInfo[cis.Length];
+//          myTableName = tableName;
+//          foreach (ColumnInfo ci in cis)
+//          {
+//             myIndexByColOrder[ci.colNo] = ci;
+//             myIndexByColName[ci.colName] = ci;
+//          }
+//          myLookupGroups = groups;
 
-         foreach (BlueRowLookupGroup group in groups)
-         {
-            group.buildEntrySchema(this);
+//          foreach (BlueRowLookupGroup group in groups)
+//          {
+//             group.buildEntrySchema(this);
 
 
-         }
-      }
+//          }
+//       }
 
 
 
 
-      public ColumnInfo this[string colName]
-      {
-         get
-         {
-            object retVal = myIndexByColName[colName];
-            return retVal != null ? retVal as ColumnInfo : null;
-         }
-      }
+//       public ColumnInfo this[string colName]
+//       {
+//          get
+//          {
+//             object retVal = myIndexByColName[colName];
+//             return retVal != null ? retVal as ColumnInfo : null;
+//          }
+//       }
 
-      public BlueRowLookupGroup[] lookupGroups
-      {
-         get { return myLookupGroups; }
+//       public BlueRowLookupGroup[] lookupGroups
+//       {
+//          get { return myLookupGroups; }
 
-      }
-      public ColumnInfo this[int colNo]
-      {
-         get { return myIndexByColOrder[colNo]; }
-      }
+//       }
+//       public ColumnInfo this[int colNo]
+//       {
+//          get { return myIndexByColOrder[colNo]; }
+//       }
 
-      /// <summary>
-      /// The returned Array is sorted according to the Column No
-      /// </summary>
-      public string[] columnNames
-      {
-         get
-         {
-            if (myTableColNames == null)
-            {
-               myTableColNames = new string[myIndexByColName.Count];
-               foreach (DictionaryEntry de in myIndexByColName)
-               {
-                  ColumnInfo temp = (ColumnInfo)de.Value;
-                  myTableColNames[temp.colNo] = temp.colName;
-               }
-            }
-            return myTableColNames;
-         }
-      }
+//       /// <summary>
+//       /// The returned Array is sorted according to the Column No
+//       /// </summary>
+//       public string[] columnNames
+//       {
+//          get
+//          {
+//             if (myTableColNames == null)
+//             {
+//                myTableColNames = new string[myIndexByColName.Count];
+//                foreach (DictionaryEntry de in myIndexByColName)
+//                {
+//                   ColumnInfo temp = (ColumnInfo)de.Value;
+//                   myTableColNames[temp.colNo] = temp.colName;
+//                }
+//             }
+//             return myTableColNames;
+//          }
+//       }
 
-      /// <summary>
-      /// The returned Array is sorted according to the Column No
-      /// </summary>
-      public ColumnInfo[] columnInfo
-      {
-         get { return myIndexByColOrder; }
-      }
+//       /// <summary>
+//       /// The returned Array is sorted according to the Column No
+//       /// </summary>
+//       public ColumnInfo[] columnInfo
+//       {
+//          get { return myIndexByColOrder; }
+//       }
 
-      public string tableName { get { return myTableName; } }
+//       public string tableName { get { return myTableName; } }
 
 
-      #region ISerializedType methods
-      public void serialize(BinaryWriter bw)
-      {
-         bw.Write(columnNames.Length);
-         bw.Write(tableName);
+//       #region ISerializedType methods
+//       public void serialize(BinaryWriter bw)
+//       {
+//          bw.Write(columnNames.Length);
+//          bw.Write(tableName);
 
-         //first, write the column names
-         foreach (ColumnInfo ci in columnInfo)
-         {
-            bw.Write(ci.colName);
-            bw.Write(ci.colType.ToString());
-            if (ci.isConstant)
-            {
-               bw.Write(true);
-               LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
-            }
-            else
-            {
-               bw.Write(false);
-               bw.Write(ci.lookupTableGroup);
-            }
+//          //first, write the column names
+//          foreach (ColumnInfo ci in columnInfo)
+//          {
+//             bw.Write(ci.colName);
+//             bw.Write(ci.colType.ToString());
+//             if (ci.isConstant)
+//             {
+//                bw.Write(true);
+//                LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
+//             }
+//             else
+//             {
+//                bw.Write(false);
+//                bw.Write(ci.lookupTableGroup);
+//             }
 
-         }
+//          }
 
-         //this is here for error checking
-         bw.Write(myLookupGroups.Length);
-         //now, write the size of each lookup table group
-         foreach (BlueRowLookupGroup group in myLookupGroups)
-         {
-            bw.Write(group.groupNumber);
-            bw.Write(group.tableSize);
-            bw.Write(group.groupColNames.Length);
-            foreach (string colName in group.groupColNames)
-            {
-               //write the index of this column in the table
-               bw.Write(this[colName].colNo);
+//          //this is here for error checking
+//          bw.Write(myLookupGroups.Length);
+//          //now, write the size of each lookup table group
+//          foreach (BlueRowLookupGroup group in myLookupGroups)
+//          {
+//             bw.Write(group.groupNumber);
+//             bw.Write(group.tableSize);
+//             bw.Write(group.groupColNames.Length);
+//             foreach (string colName in group.groupColNames)
+//             {
+//                //write the index of this column in the table
+//                bw.Write(this[colName].colNo);
 
-            }
+//             }
 
 
-         }
+//          }
 
 
 
-      }
+//       }
 
-      public void deserialze(BinaryReader br)
-      {
-         int numCols = br.ReadInt32();
-         //these are the final output columns that will be created
-         ColumnInfo[] columnInfos = new ColumnInfo[numCols];
+//       public void deserialze(BinaryReader br)
+//       {
+//          int numCols = br.ReadInt32();
+//          //these are the final output columns that will be created
+//          ColumnInfo[] columnInfos = new ColumnInfo[numCols];
 
-         Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
+//          Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
 
 
-         string tableName = br.ReadString();
-         for (int colNo = 0; colNo < numCols; ++colNo)
-         {
-            string colName = br.ReadString();
-            Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
-            bool colIsConst = br.ReadBoolean();
-            if (colIsConst)
-            {
-               columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
+//          string tableName = br.ReadString();
+//          for (int colNo = 0; colNo < numCols; ++colNo)
+//          {
+//             string colName = br.ReadString();
+//             Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
+//             bool colIsConst = br.ReadBoolean();
+//             if (colIsConst)
+//             {
+//                columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
 
-            }
-            else
-            {
-               short thisColGroup = br.ReadInt16();
-               columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
-            }
-         }
+//             }
+//             else
+//             {
+//                short thisColGroup = br.ReadInt16();
+//                columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
+//             }
+//          }
 
-         int numGroups = br.ReadInt32();
-         BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
+//          int numGroups = br.ReadInt32();
+//          BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
 
-         for (int group = 0; group < numGroups; ++group)
-         {
-            short groupNo = br.ReadInt16();
-            short groupSizeInBits = br.ReadInt16();
-            int numColsInGroup = br.ReadInt32();
-            int[] groupCols = new int[numColsInGroup];
-            string[] groupColNames = new string[numColsInGroup];
-            for (int col = 0; col < numColsInGroup; ++col)
-            {
-               groupCols[col] = br.ReadInt32();
-               groupColNames[col] = columnInfos[groupCols[col]].colName;
-            }
+//          for (int group = 0; group < numGroups; ++group)
+//          {
+//             short groupNo = br.ReadInt16();
+//             short groupSizeInBits = br.ReadInt16();
+//             int numColsInGroup = br.ReadInt32();
+//             int[] groupCols = new int[numColsInGroup];
+//             string[] groupColNames = new string[numColsInGroup];
+//             for (int col = 0; col < numColsInGroup; ++col)
+//             {
+//                groupCols[col] = br.ReadInt32();
+//                groupColNames[col] = columnInfos[groupCols[col]].colName;
+//             }
 
-            //create the group
-            theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
+//             //create the group
+//             theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
 
 
-         }
+//          }
 
 
-         myConstruct(tableName, columnInfos, theGroups);
+//          myConstruct(tableName, columnInfos, theGroups);
 
 
 
-      }
+//       }
 
-      public int getSerializedSize()
-      {
-         throw new NotImplementedException("Feature for provided constant values to" +
-            "non-primitive types in schema, not provided as of now");
-      }
+//       public int getSerializedSize()
+//       {
+//          throw new NotImplementedException("Feature for provided constant values to" +
+//             "non-primitive types in schema, not provided as of now");
+//       }
 
-      #endregion
-   }
-}
+//       #endregion
+//    }
+// }

src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs

@@ -5,8 +5,7 @@ using System.IO;
 using BCG.Utilities.General;
 //using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
     /// <summary>
     /// The SerializedBlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
     /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
@@ -17,12 +16,9 @@ namespace BCG
     /// memory streams, and uses this schema to unpack the values when they are needed. 
     /// </summary>
     [Serializable]
-    public class SerializedBlueTableSchema : IBlueTableSchema
-    {
-        public class SerializedBlueTableSchemaSerializationHelper
-        {
-            public static object parse(Type t, string data)
-            {
+    public class SerializedBlueTableSchema : IBlueTableSchema {
+        public class SerializedBlueTableSchemaSerializationHelper {
+            public static object parse(Type t, string data) {
                 if (t == BlueDataValidTypes.BOOL)
                     return Boolean.Parse(data);
                 else if (t == BlueDataValidTypes.BYTE)
@@ -52,8 +48,7 @@ namespace BCG
                 else
                     throw new ApplicationException("Can not be parsed");
             }
-            public static void serialize(BinaryWriter bw, Type t, object o)
-            {
+            public static void serialize(BinaryWriter bw, Type t, object o) {
                 if (t == BlueDataValidTypes.BOOL)
                     bw.Write((bool)o);
                 else if (t == BlueDataValidTypes.BYTE)
@@ -76,13 +71,11 @@ namespace BCG
                     bw.Write(o as string);
                 else if (t == BlueDataValidTypes.SINGLE)
                     bw.Write((Single)o);
-                else if (t == BlueDataValidTypes.BYTEARRAY)
-                {
+                else if (t == BlueDataValidTypes.BYTEARRAY) {
                     byte[] bs = o as byte[];
                     bw.Write(bs.Length);
                     bw.Write(bs);
-                }
-                else if (t == BlueDataValidTypes.UINT)
+                } else if (t == BlueDataValidTypes.UINT)
                     bw.Write((uint)o);
                 else if (t == BlueDataValidTypes.ULONG)
                     bw.Write((ulong)o);
@@ -90,8 +83,7 @@ namespace BCG
                     throw new ApplicationException("Unsupported type");
             }
 
-            public static object deserialize(BinaryReader br, Type t)
-            {
+            public static object deserialize(BinaryReader br, Type t) {
                 if (t == BlueDataValidTypes.BOOL)
                     return br.ReadBoolean();
                 else if (t == BlueDataValidTypes.BYTE)
@@ -132,45 +124,37 @@ namespace BCG
         public SerializedBlueTableSchema() { }
 
         //copies the schema from an existing schema, building the dispatch table
-        public SerializedBlueTableSchema(IBlueTableSchema toCopy)
-        {
+        public SerializedBlueTableSchema(IBlueTableSchema toCopy) {
             int numCols = toCopy.columnInfo.Length;
             ColumnInfo[] theColumns = new ColumnInfo[toCopy.columnInfo.Length];
-            foreach (ColumnInfo toCopyCI in toCopy.columnInfo)
-            {
+            foreach (ColumnInfo toCopyCI in toCopy.columnInfo) {
                 theColumns[toCopyCI.colNo] = toCopyCI;
             }
             myConstruct(theColumns, toCopy.tableName);
         }
 
-        public SerializedBlueTableSchema(DataTable table)
-        {
+        public SerializedBlueTableSchema(DataTable table) {
             ColumnInfo[] theseCols = new ColumnInfo[table.Columns.Count];
             myTableName = table.TableName;
-            foreach (DataColumn col in table.Columns)
-            {
+            foreach (DataColumn col in table.Columns) {
                 theseCols[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
             }
             myConstruct(theseCols, myTableName);
         }
 
-        public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName)
-        {
+        public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName) {
             myConstruct(cis, tableName);
         }
 
-        public void serializeObject(object data, BinaryWriter bw, int colNo)
-        {
+        public void serializeObject(object data, BinaryWriter bw, int colNo) {
             mySerializeDispatchTable[colNo](data, bw);
 
         }
 
-        private void myConstruct(ColumnInfo[] cis, string tableName)
-        {
+        private void myConstruct(ColumnInfo[] cis, string tableName) {
             myIndexByColOrder = new ColumnInfo[cis.Length];
             myTableName = tableName;
-            foreach (ColumnInfo ci in cis)
-            {
+            foreach (ColumnInfo ci in cis) {
                 myIndexByColOrder[ci.colNo] = ci;
                 myIndexByColName[ci.colName] = ci;
             }
@@ -179,32 +163,25 @@ namespace BCG
 
         public SerializedBlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { }
 
-        public ColumnInfo this[string colName]
-        {
-            get
-            {
+        public ColumnInfo this[string colName] {
+            get {
                 object retVal = myIndexByColName[colName];
                 return retVal != null ? retVal as ColumnInfo : null;
             }
         }
 
-        public ColumnInfo this[int colNo]
-        {
+        public ColumnInfo this[int colNo] {
             get { return myIndexByColOrder[colNo]; }
         }
 
         /// <summary>
         /// The returned Array is sorted according to the Column No
         /// </summary>
-        public string[] columnNames
-        {
-            get
-            {
-                if (myTableColNames == null)
-                {
+        public string[] columnNames {
+            get {
+                if (myTableColNames == null) {
                     myTableColNames = new string[myIndexByColName.Count];
-                    foreach (DictionaryEntry de in myIndexByColName)
-                    {
+                    foreach (DictionaryEntry de in myIndexByColName) {
                         ColumnInfo temp = (ColumnInfo)de.Value;
                         myTableColNames[temp.colNo] = temp.colName;
                     }
@@ -216,64 +193,46 @@ namespace BCG
         /// <summary>
         /// The returned Array is sorted according to the Column No
         /// </summary>
-        public ColumnInfo[] columnInfo
-        {
+        public ColumnInfo[] columnInfo {
             get { return myIndexByColOrder; }
         }
 
         public string tableName { get { return myTableName; } }
 
 
-        public int IdColumn
-        {
+        public int IdColumn {
             get { return myIdCol; }
         }
 
 
         #region ISerializedType methods
-        public void serialize(BinaryWriter bw)
-        {
+        public void serialize(BinaryWriter bw) {
             bw.Write(columnNames.Length);
             bw.Write(tableName);
-            foreach (ColumnInfo ci in columnInfo)
-            {
+            foreach (ColumnInfo ci in columnInfo) {
                 bw.Write(ci.colName);
                 bw.Write(ci.colType.ToString());
-                if (ci.isConstant)
-                {
-                    bw.Write(true);
-                    SerializedBlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
-                }
-                else
-                {
-                    bw.Write(false);
-                }
+                bw.Write(false);
             }
         }
 
-        public void deserialze(BinaryReader br)
-        {
+        public void deserialze(BinaryReader br) {
             ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
             string name = br.ReadString();
-            for (int i = 0; i < cis.Length; i++)
-            {
+            for (int i = 0; i < cis.Length; i++) {
                 string colName = br.ReadString();
                 Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
                 bool isConstant = br.ReadBoolean();
-                if (isConstant)
-                {
-                    cis[i] = new ColumnInfo(colName, i, t, SerializedBlueTableSchemaSerializationHelper.deserialize(br, t));
-                }
-                else
-                {
+                if (isConstant) {
+                    cis[i] = new ColumnInfo(colName, i, t);
+                } else {
                     cis[i] = new ColumnInfo(colName, i, t);
                 }
             }
             myConstruct(cis, name);
         }
 
-        public int getSerializedSize()
-        {
+        public int getSerializedSize() {
             throw new NotImplementedException("Feature for provided constant values to" +
                "non-primitive types in schema, not provided as of now");
         }
@@ -282,8 +241,7 @@ namespace BCG
 
 
         //builds a dispatch table used to pull values out of the memory streams in the rows. 
-        private void myBuildDispatchTable()
-        {
+        private void myBuildDispatchTable() {
             int numCols = myIndexByColOrder.Length;
             mySerializeDispatchTable = new SerializeDelegate[numCols];
             myPullFromByteBufferDispatchTable = new PullFromByteBufferDelegate[numCols];
@@ -292,18 +250,14 @@ namespace BCG
 
             myIdCol = idColumn.colNo;
 
-            for (int col = 0; col < numCols; ++col)
-            {
+            for (int col = 0; col < numCols; ++col) {
                 Type thisColType = myIndexByColOrder[col].colType;
                 //we don't store serializers for the id column, or string columns.
-                if (col == myIdCol || thisColType == typeof(string))
-                {
+                if (col == myIdCol || thisColType == typeof(string)) {
                     mySerializeDispatchTable[col] = null;
                     myColumnSizesTable[col] = 0;
                     myPullFromByteBufferDispatchTable[col] = null;
-                }
-                else
-                {
+                } else {
                     mySerializeDispatchTable[col] = myGetDispatchMethodSerialize(thisColType);
                     myPullFromByteBufferDispatchTable[col] = myGetDispatchMethodPullFromBuffer(thisColType);
                     myColumnSizesTable[col] = BlueDataValidTypes.getValueTypeSize(thisColType);
@@ -315,16 +269,14 @@ namespace BCG
         }
 
 
-        public int getNullMapSize()
-        {
+        public int getNullMapSize() {
             int numCols = myIndexByColOrder.Length;
             return Convert.ToInt32(Math.Ceiling((Convert.ToDouble(numCols) / 8.0)));
         }
 
         //this function reads the null array from inside the byte array.
         //it has to copy the bytes over. 
-        public BitArray getNullMap(byte[] theBytes)
-        {
+        public BitArray getNullMap(byte[] theBytes) {
             int nullMapSize = getNullMapSize();
             //the null map starts after the id, so that we have instant access to the id. 
             int byteCursor = BlueDataValidTypes.getValueTypeSize(typeof(Int32));
@@ -336,8 +288,7 @@ namespace BCG
         }
         //gets the value either from the bytes stream but only if the value is not a string. 
         //doesn't work if you're getting the id.
-        public unsafe object getValueNonString(byte[] theBytes, int colNo)
-        {
+        public unsafe object getValueNonString(byte[] theBytes, int colNo) {
             if (colNo == myIdCol)
                 throw new BlueDataException("Error: do not use getValueNonString to try to fetch the id of a row");
             BitArray nullMap = getNullMap(theBytes);
@@ -345,8 +296,7 @@ namespace BCG
                 return DBNull.Value;
             //the rest of the row starts after the id and the null map
             int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
-            for (int currentColumn = 0; currentColumn < colNo; ++currentColumn)
-            {
+            for (int currentColumn = 0; currentColumn < colNo; ++currentColumn) {
                 //pull the data from the binary reader, but only if this type is not null or a string
                 //if the type wer'e looking for is a string, it would have been returned already by the accesor
                 //in the bluerow. 
@@ -357,16 +307,14 @@ namespace BCG
 
                     thisOffset += myColumnSizesTable[currentColumn];
             }
-            fixed (byte* thisBytePtr = &theBytes[thisOffset])
-            {
+            fixed (byte* thisBytePtr = &theBytes[thisOffset]) {
                 return myPullFromByteBufferDispatchTable[colNo](thisBytePtr);
             }
 
 
         }
 
-        public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray)
-        {
+        public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray) {
             int numCols = myColumnSizesTable.Length;
             object[] theObjs = new Object[numCols];
 
@@ -374,8 +322,7 @@ namespace BCG
 
             int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
             int thisStringNo = 0;
-            for (int currentColumn = 0; currentColumn < numCols; ++currentColumn)
-            {
+            for (int currentColumn = 0; currentColumn < numCols; ++currentColumn) {
                 //pull the data from the binary reader, but only if this type is not null or a string
                 //if the type wer'e looking for is a string, it would have been returned already by the accesor
                 //in the bluerow. 
@@ -384,10 +331,8 @@ namespace BCG
                     theObjs[currentColumn] = DBNull.Value;
                 else if (myIndexByColOrder[currentColumn].colType == typeof(string))
                     theObjs[currentColumn] = stringArray[thisStringNo++];
-                else if (currentColumn != myIdCol)
-                {
-                    fixed (byte* thisGuy = &theBytes[thisOffset])
-                    {
+                else if (currentColumn != myIdCol) {
+                    fixed (byte* thisGuy = &theBytes[thisOffset]) {
                         theObjs[currentColumn] = myPullFromByteBufferDispatchTable[currentColumn](thisGuy);
                     }
                     thisOffset += myColumnSizesTable[currentColumn];
@@ -416,8 +361,7 @@ namespace BCG
         private void mySerializeDateTime(object data, BinaryWriter bw) { bw.Write(((DateTime)data).ToOADate()); }
         private void mySerializeSingle(object data, BinaryWriter bw) { bw.Write((Single)data); }
 
-        private SerializeDelegate myGetDispatchMethodSerialize(Type t)
-        {
+        private SerializeDelegate myGetDispatchMethodSerialize(Type t) {
             if (t.Equals(BlueDataValidTypes.INT)) return new SerializeDelegate(mySerializeInt32);
             else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new SerializeDelegate(mySerializeDouble);
             else if (t.Equals(BlueDataValidTypes.CHAR)) return new SerializeDelegate(mySerializeChar);
@@ -448,16 +392,14 @@ namespace BCG
         private unsafe object myPullDecimal(byte* bytePtr) { return *((decimal*)bytePtr); }
         private unsafe object myPullUInt(byte* bytePtr) { return *((UInt32*)bytePtr); }
         private unsafe object myPullULong(byte* bytePtr) { return *((UInt64*)bytePtr); }
-        private unsafe object myPullDateTime(byte* bytePtr)
-        {
+        private unsafe object myPullDateTime(byte* bytePtr) {
             double thisDateTime = *((double*)bytePtr);
             return DateTime.FromOADate(thisDateTime);
         }
         private unsafe object myPullSingle(byte* bytePtr) { return *((Single*)bytePtr); ; }
 
 
-        private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t)
-        {
+        private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t) {
             if (t.Equals(BlueDataValidTypes.INT)) return new PullFromByteBufferDelegate(myPullInt32);
             else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new PullFromByteBufferDelegate(myPullDouble);
             else if (t.Equals(BlueDataValidTypes.CHAR)) return new PullFromByteBufferDelegate(myPullChar);

src/DataStreamSuite/DataStreamCore/LightTableStream.cs

@@ -1,51 +1,44 @@
-using System;
-using System.Collections;
-using System.Data;
+// using System;
+// using System.Collections;
+// using System.Data;
 
-using BCG.Utilities;
+// using BCG.Utilities;
 
 
-/* this class reprsents a table stream for use with CppDataStreams. It does not hold on to the last value fired.
- * This way, the last value is purged from managed memory and only the slimmer, sexier Cpp implementation of the row remains. 
- * 
- */
-namespace BCG
-{
-    public class LightTableStream : TableStream
-    {
-        public LightTableStream(string tableName)
-         :
-         base(tableName, null, null)
-      {
-      }
+// /* this class reprsents a table stream for use with CppDataStreams. It does not hold on to the last value fired.
+//  * This way, the last value is purged from managed memory and only the slimmer, sexier Cpp implementation of the row remains. 
+//  * 
+//  */
+// namespace BCG {
+//     public class LightTableStream : TableStream {
+//         public LightTableStream(string tableName)
+//          :
+//          base(tableName, null, null) {
+//         }
 
-      public LightTableStream(string tableName, ViewStreamDatabase db)
-         :
-         base(tableName, db, null)
-      {}
+//         public LightTableStream(string tableName, ViewStreamDatabase db)
+//            :
+//            base(tableName, db, null) { }
 
-      public LightTableStream(string tableName, string colExp)
-         :
-         base(tableName, 
-         null, 
-         colExp)
-      {}
-
-      
-      public LightTableStream(string ttableName, ViewStreamDatabase db, string colExp) :
-          base(ttableName,db,colExp)
-      {
-       }
+//         public LightTableStream(string tableName, string colExp)
+//            :
+//            base(tableName,
+//            null,
+//            colExp) { }
 
 
-        protected override void myHandleUpdates(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
-        {
+//         public LightTableStream(string ttableName, ViewStreamDatabase db, string colExp) :
+//             base(ttableName, db, colExp) {
+//         }
 
-            base.myHandleUpdates(newRows, addedRows, deletedRows);
-            //once you've fired the rows, delete them becuase we don't want to hold onto them in memory
-            myLastValue = null;
-            
-        }
 
-    }
-}
+//         protected override void _handle_updates(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) {
+
+//             base._handle_updates(newRows, addedRows, deletedRows);
+//             //once you've fired the rows, delete them becuase we don't want to hold onto them in memory
+//             myLastValue = null;
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs

@@ -1,351 +0,0 @@
-using System;
-using System.IO;
-//using BCG.Utilities;
-using mbts;
-using mbts.spark;
-
-namespace BCG
-{   
-	public class TableAccumulator : ITableAccumulator
-	{
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();
-      private IClientDeserializer myDeserializer = new ClientDeserializationController ();
-      private BlueTable myStateTable;
-      private ISubscriberCore mySubscriptionCore;
-      private TableUpdateDelegate myOnUpdateDelegate;
-      private string myTag;
-      private bool myUseIncrementsForMultipleOps = DataStreamGatewaySettings.instance.useIncrementsForMultipleOps;
-
-      public TableAccumulator (string tag, ISubscriberCore subCore) 
-      {
-         myTag = tag;
-         mySubscriptionCore = subCore;
-      }
-
-      #region ITableAccumulator Members
-
-      public bool isActive()
-      {return myOnUpdateDelegate != null;}
-
-      public event TableUpdateDelegate onUpdate
-      {
-         add
-         {
-            // BlueEventLog.logEvent ("TableSubscriber being Subscribed for tag :" + myTag, BlueEventLevel.Information);
-            if (myOnUpdateDelegate == null)
-            {
-               //ie first subscription
-               // BlueEventLog.logEvent ("Subscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
-               mySubscriptionCore.subscribe (myTag);
-            }
-            myOnUpdateDelegate = 
-               Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate;
-            if (myStateTable != null)
-               value (myStateTable, myStateTable, new BlueTable (myStateTable.schema));
-      }
-         remove
-         {
-            // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
-            if (myOnUpdateDelegate != null)
-            {
-               myOnUpdateDelegate = 
-                  Delegate.Remove (myOnUpdateDelegate, value) as TableUpdateDelegate;
-               if (myOnUpdateDelegate == null)
-               { 
-                  //all subscribers gone
-                  // BlueEventLog.logEvent ("Unubscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
-                  mySubscriptionCore.unsubscribe (myTag);
-                  myStateTable = null;                  
-               }
-            }
-
-         }
-      }
-
-      public void update(Stream[] streams)
-      {
-         bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
-         //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
-         //data element for that tag in the ViewStreamDatabase
-         if (myOnUpdateDelegate != null)
-         {
-            byte[] ops;
-            Array rows;
-            _timer.start();
-            myDeserializer.deserialize (streams, out ops, out rows);
-            _timer.stop();
-            // BlueEventLog.logEvent(String.Format("deserializing {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
-            _timer.start();
-            myAccumulateAndFire (ops, rows);
-            _timer.stop();
-            // BlueEventLog.logEvent(String.Format("myAccumulateAndFire for {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
-         }
-      }
-
-      #endregion
-
-      private void myAccumulateAndFire (byte[] ops, Array rows)
-      {
-         //if a non create comes before the create, the deserializer returns 0 sized arrays
-         if (ops.Length == 0 || rows.Length == 0)
-         {
-            //ie if one is zero sized so should be the other one
-            // BlueEventLog.logAssert (ops.Length + rows.Length == 0);
-         }
-         else
-         {
-            if (ops.Length > 1)
-            {
-               int index = 0;
-               if (myStateTable == null)
-               {
-                  for(index=0; index<ops.Length; index++)
-                     if (ops[index] == StreamOpCode.create)
-                        break;
-                  if (index == ops.Length)
-                     return;
-               }
-               if (myUseIncrementsForMultipleOps)
-                  myHandleMultipleOpsIncrementally(ops, rows, index);
-               else
-                  myHandleMultipleOps(ops, rows, index);
-            }
-            else
-            {
-               if (myStateTable != null || ops[0] == StreamOpCode.create)
-                  myHandleSingleOp (ops, rows);
-            }
-         }
-      }
-
-      private void myHandleMultipleOps (byte[] ops, Array rows, int offset)
-      {
-         HighPerformanceTimer timer = new HighPerformanceTimer();
-         timer.start();
-         IBlueRowCollection stateAtStart = myStateTable == null ? null : new BlueTable (myStateTable);
-         for (int i = offset; i < ops.Length; i++)
-         {
-            byte op = ops[i];
-            IBlueRowCollection[] tableArray = rows.GetValue (i) as IBlueRowCollection[];
-            if (op == StreamOpCode.add)
-               myStateTable.addRows (tableArray[0]);
-            else if (op == StreamOpCode.remove)
-               myStateTable.deleteRows (tableArray[0]);
-            else if (op == StreamOpCode.change)
-               myStateTable.changeRows (tableArray[0]);
-            else if (op == StreamOpCode.create)
-               myStateTable = new BlueTable (tableArray[0]);
-            else if (op == StreamOpCode.multipleChange)
-               myStateTable.changeRows (tableArray[0]);
-            else if (op == StreamOpCode.multipleRemoveAdd)
-            {
-               myStateTable.deleteRows (tableArray[0]);
-               myStateTable.addRows (tableArray[1]);
-            }
-         }
-         IBlueRowCollection dRows, aRows;
-         if (stateAtStart == null)
-         {
-            dRows = new BlueTable (myStateTable.schema);
-            aRows = myStateTable;
-         }
-         else
-         {
-            BlueDataUtility.computeDifferences (myStateTable, stateAtStart, out dRows, out aRows);
-         }
-         timer.stop();
-         //Console.WriteLine("multiple ops for table " + myTag + " took " + timer.lastIntervalTime + " msecs");
-         myOnUpdateDelegate (myStateTable, aRows, dRows);
-
-      }
-
-      #region handling mutliple ops in single update call, incrementally
-
-      private void myHandleMultipleOpsIncrementally(byte[] ops, Array rows, int offset)
-      {
-         BlueTable dels = null, adds = null;
-         if (myStateTable != null)
-         {
-            dels = new BlueTable(myStateTable.schema);
-            adds = new BlueTable(myStateTable.schema);
-         }
-         for (int i = offset; i < ops.Length; i++)
-         {
-            byte op = ops[i];
-            IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[];
-            if (op == StreamOpCode.add)
-            {
-               myHandleAdd(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
-            }
-            else if (op == StreamOpCode.remove)
-            {
-               myHandleRemove(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
-            }
-            else if (op == StreamOpCode.change)
-            {
-               myHandleChange(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
-            }
-            else if (op == StreamOpCode.create)
-            {
-               myHandleCreate(tableArray[0], ref dels, ref adds);
-            }
-            else if (op == StreamOpCode.multipleChange)
-            {
-               myHandleMultipleChanges(tableArray[0], dels, adds);
-            }
-            else if (op == StreamOpCode.multipleRemoveAdd)
-            {
-               myHandleMultipleRemovesAdds(tableArray[0], tableArray[1], dels, adds);
-            }
-         }
-         myOnUpdateDelegate(myStateTable, adds, dels);
-      }
-
-      private void myHandleAdd(IBlueRow r, BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-         {
-            myStateTable.addRow(r);
-            adds.addRow(r);
-         }
-      }
-
-      private void myHandleRemove(IBlueRow r, BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-         {
-            myStateTable.deleteRow(r);
-            if (adds.containsID((int)r["id"]))
-               adds.deleteRow(r);
-            else
-               dels.addRow(r);
-         }
-      }
-
-      private void myHandleChange(IBlueRow r, BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-         {
-            IBlueRow er = myStateTable[(int)r["id"]];
-            myHandleRemove(er, dels, adds);
-            myHandleAdd(r, dels, adds);
-         }
-      }
-
-      private void myHandleCreate(IBlueRowCollection newTable, ref BlueTable dels, ref BlueTable adds)
-      {
-         if (dels == null)
-         {
-            // BlueEventLog.logAssert(adds == null);
-            dels = new BlueTable(newTable.schema);
-            adds = new BlueTable(newTable);
-            myStateTable = new BlueTable(newTable);
-         }
-         else
-         {
-            dels = new BlueTable(myStateTable);
-            myStateTable = new BlueTable(newTable);
-            adds = new BlueTable(newTable);
-         }
-      }
-
-      private void myHandleMultipleRemovesAdds(IBlueRowCollection deletedRows, IBlueRowCollection addedRows,
-                                             BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-         {
-            foreach (IBlueRow r in deletedRows)
-               myHandleRemove(r, dels, adds);
-            foreach (IBlueRow r in addedRows)
-               myHandleAdd(r, dels, adds);
-         }
-      }
-
-      private void myHandleMultipleChanges(IBlueRowCollection changedRows, BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-            foreach (IBlueRow r in changedRows)
-               myHandleChange(r, dels, adds);
-      }
-
-      #endregion
-
-      private void myHandleSingleOp(byte[] ops, Array rows)
-      {
-         byte op = ops[0];
-         IBlueRowCollection[] tableArray = rows.GetValue (0) as IBlueRowCollection[];
-         IBlueRowCollection dRows, aRows;
-         if (op == StreamOpCode.create)
-         {
-            if (myStateTable == null)
-            {
-               aRows = myStateTable = new BlueTable(tableArray[0]);
-               dRows = new BlueTable(myStateTable.schema);
-               myOnUpdateDelegate(myStateTable, aRows, dRows);
-            }
-            else
-            {
-               IBlueRowCollection newState = new BlueTable(tableArray[0]);
-               BlueDataUtility.computeDifferences(newState, myStateTable, out dRows, out aRows);
-               if (aRows.Count > 0 || dRows.Count > 0)
-               {
-                  myStateTable = new BlueTable(newState);
-                  myOnUpdateDelegate(myStateTable, aRows, dRows);
-               }
-            }
-         }
-         else
-         {
-            if (op == StreamOpCode.add)
-            {
-               myStateTable.addRows(aRows = tableArray[0]);
-               dRows = new BlueTable(myStateTable.schema);
-            }
-            else if (op == StreamOpCode.remove)
-            {
-               myStateTable.deleteRows(dRows = tableArray[0]);
-               aRows = new BlueTable(myStateTable.schema);
-            }
-            else if (op == StreamOpCode.change)
-            {
-               IBlueRow cRow = BlueDataUtility.extractRow(tableArray[0], 0);
-               aRows = new BlueTable(myStateTable.schema);
-               dRows = new BlueTable(myStateTable.schema);
-               (aRows as BlueTable).addRow(cRow);
-               (dRows as BlueTable).addRow(myStateTable[(int)cRow["id"]]);
-               myStateTable.changeRow(cRow);
-            }
-            else if (op == StreamOpCode.multipleChange)
-            {
-               if (myStateTable.Count == tableArray[0].Count)
-               {
-                  //all rows are changed, so we can optimize this
-                  dRows = myStateTable;
-                  aRows = myStateTable = new BlueTable(tableArray[0]);
-               }
-               else
-               {
-                  dRows = new BlueTable(myStateTable.schema);
-                  aRows = tableArray[0];
-                  foreach (IBlueRow r in tableArray[0])
-                  {
-                     (dRows as BlueTable).addRow(myStateTable[(int)r["id"]]);
-                     myStateTable.changeRow(r);
-                  }
-               }
-            }
-            else if (op == StreamOpCode.multipleRemoveAdd)
-            {
-               BlueDataUtility.absorbChanges(myStateTable, dRows = tableArray[0], aRows = tableArray[1]);
-            }
-            else
-            {
-               aRows = dRows = null;
-               // BlueEventLog.logAssert(false, "unknown opcode");
-            }
-            myOnUpdateDelegate(myStateTable, aRows, dRows);
-         }
-      }
-   }
-   
-}