Fixing mangled merge

.devcontainer/Dockerfile

@@ -1,122 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Note installations that take a while (more than 10s)
-
-RUN apt-get update
-RUN apt-get install -y ca-certificates
-RUN apt-get install -y ccache
-RUN apt-get install -y curl
-RUN apt-get install -y fish
-RUN apt-get install -y gdb
-# git-all: ~250s
-RUN apt-get install -y git-all
-RUN apt-get install -y gnupg
-# liblttng-ust-dev: ~30s
-RUN apt-get install -y liblttng-ust-dev
-RUN apt-get install -y libssh2-1-dev
-RUN apt-get install -y locales
-RUN apt-get install -y ninja-build
-# openssh-server: ~40s
-RUN apt-get install -y openssh-server
-RUN apt-get install -y pkg-config
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y sudo
-RUN apt-get install -y tzdata
-RUN apt-get install -y vim
-RUN apt-get install -y zip
-
-# Python: ~30s
-RUN apt-get install -y \
-   python3-pip \
-   python3-venv \
-   && python3 -m pip install --upgrade pip
-
-# GCC
-ARG gcc_major=13
-RUN \
-   add-apt-repository ppa:ubuntu-toolchain-r/test \
-   && apt-get update \
-   && apt-get install -y g++-$gcc_major
-
-# Clang
-ARG clang_major=17
-RUN \
-   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
-   | tee -a "/etc/apt/sources.list.d/llvm.list" \
-   && apt-get update \
-   && apt-get install -y \
-   clang-$clang_major \
-   clang-format-$clang_major \
-   clangd-$clang_major
-
-# CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
-
-# Dotnet
-RUN apt-get update && apt-get install -y dotnet7
-
-# Node
-ARG node_major=18
-RUN mkdir -p /etc/apt/keyrings \
-   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
-   | tee /etc/apt/sources.list.d/nodesource.list \
-   && apt-get update \
-   && apt-get install -y nodejs
-
-# Emumba (deps)
-RUN apt-get update && apt-get -y install rapidjson-dev
-# Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
-
-# Pcap
-RUN apt-get update && apt-get install -y libpcap0.8-dev
-
-# Set up default executables (update-alternatives)
-RUN \
-   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
-   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
-   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
-   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
-
-# Add compiler symlinks for ccache 
-RUN \
-   ln -s /usr/bin/ccache /usr/local/bin/clang; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-   groupadd -g "$docker_uid" "$docker_user" \
-   && useradd \
-   -m \
-   -u "$docker_uid" \
-   -g "$docker_uid" \
-   -G sudo \
-   --password "$(openssl passwd -1 "$docker_pass")" \
-   "$docker_user"

.devcontainer/devcontainer.json

@@ -1,48 +0,0 @@
-{
-  "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
-  "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
-  },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
-  "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
-  "updateRemoteUserUID": true,
-  "shutdownAction": "none",
-  // Note: If running multiple containers on the same host, each must use a
-  // unique set of ports. Usually we just add 1000 to each port as needed. For
-  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
-  "runArgs": [
-    "--shm-size",
-    "2gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003"
-  ],
-  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
-  ],
-  "customizations": {
-    "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
-      "extensions": [
-        "eamodio.gitlens",
-        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
-      ]
-    }
-  }
-}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.gitignore

@@ -47,13 +47,13 @@ trace_data/
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
 /src/server/sparkd/cmds/spartan/analysis/data
-.tmp/
-.venv*/
+*.tmp/*
+/.venv*/*
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-.devcontainer/
+**/devcontainer.json
 *.Generated.cs
 *.lst

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "externals"]
 	path = externals
 	url = git@bitbucket.org:bluetradesys/externals.git
-[submodule "vcpkg"]
-	path = vcpkg
-	url = https://github.com/microsoft/vcpkg.git

CMakePresets.json

@@ -26,7 +26,6 @@
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
@@ -143,7 +142,7 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
       }
     },

docker/sparkd/dev/Dockerfile

@@ -48,15 +48,9 @@ RUN apt-get update && apt-get install -y \
 # cmake
 RUN \
     curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-    | gpg --dearmor - \
-    | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-    | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+    | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
+    && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
     && apt-get update \
-    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
-    && apt-get install -y kitware-archive-keyring \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
-    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
 # node

docker/sparkd/dev/devcontainer.json

@@ -1,7 +1,6 @@
 {
   "image": "bluetradingsystems/sparkd-dev:latest",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
+  "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
   "hostRequirements": {
     "memory": "4gb",
     "storage": "6gb"
@@ -28,6 +27,6 @@
     "8003:8003"
   ],
   "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+    "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
   ]
 }

docker/sparkd/dev/devcontainer_initialize.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionTokens.lex

@@ -14,7 +14,7 @@ ParenthesisClose   \)
 Add                \+
 Subtract           \-
 Multiply           \*
-Divide             /
+Divide             \\
 Equals             =
 Comma              ,
 

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -2,8 +2,6 @@ namespace Bts.BooleanExpressions.Arguments;
 
 using bts;
 
-using Spark.Core.Collections;
-
 static class ArgumentComparer {
    public static int Compare(
       IArgument value1, IArgument value2, IDictionary<string,object?> env) {
@@ -16,10 +14,7 @@ static class ArgumentComparer {
       var cmp1 = value1.Cast<IComparable>();
       var cmp2 = value2.Cast<IComparable>();
       try {
-         var v1 = cmp1.GetValue(env);
-         var v2 = cmp2.GetValue(env);
-         return v1.GetType().IsNumeric() ?
-            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
+         return cmp1.GetValue(env).CompareTo(cmp2.GetValue(env));
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;

src/DataStreamSuite/BooleanExpressions/BooleanExpressionGrammar.y

@@ -44,7 +44,7 @@
 %%
 
 BoundExpression : Definitions Expression { $$ = new($2, $1); }
-   | Expression { $$ = new($1, ImmutableList<Definition>.Empty); };
+   | Expression;
 
 Definitions :
    Definition { $$ = ImmutableList<Definition>.Empty.Add($1); }
@@ -57,70 +57,46 @@ Definition :
 
 Expression :
    Expression OR Expression {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => left.Evaluate(env) || right.Evaluate(env));
+      $$ = new(env => $1.Evaluate(env) || $3.Evaluate(env));
    }
    | Expression AND Expression {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => left.Evaluate(env) && right.Evaluate(env));
+      $$ = new(env => $1.Evaluate(env) && $3.Evaluate(env));
    }
    | Expression XOR Expression {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => left.Evaluate(env) != right.Evaluate(env));
+      $$ = new(env => $1.Evaluate(env) != $3.Evaluate(env));
    }
    | NOT Expression {
-      var exp = $2;
-      $$ = new(env => !exp.Evaluate(env));
+      $$ = new(env => !$2.Evaluate(env));
    }
    | PARENTHESIS_OPEN Expression PARENTHESIS_CLOSE {
       $$ = $2; 
    }
    | Argument GREATER Argument {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) > 0);
+      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) > 0);
    }
    | Argument GREATER_EQUAL Argument {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) >= 0);
+      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) >= 0);
    }
    | Argument EQUAL Argument {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) == 0);
+      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) == 0);
    }
    | Argument LESS_EQUAL Argument {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) <= 0);
+      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) <= 0);
    }
    | Argument LESS Argument {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) < 0);
+      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) < 0);
    }
    | Argument DIFFERENT Argument {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) != 0);
+      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) != 0);
    }
    | Argument DIFFERENT2 Argument {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) != 0);
+      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) != 0);
    }
    | Argument MATCHES Argument {
-      var left = $1;
-      var right = $3;
-      $$ = new(env => Arguments.ArgumentMatcher.Match(left, right, env));
+      $$ = new(env => Arguments.ArgumentMatcher.Match($1, $3, env));
    }
    | Argument {
-      var arg = $1;
-      $$ = new(env => (bool)arg.GetValue(env));
+      $$ = new(env => (bool)$1.GetValue(env));
    }
    ;
 

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnExpression">
-    <Scanner Include="ColumnExpressionTokens.lex" />
-    <Parser Include="ColumnExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -1,62 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-public class ColumnExpression {
-   public ColumnExpression(string name, Type? type = null, object? value = null) {
-      Name = name;
-      Type = type
-         ?? value?.GetType()
-         ?? throw new ArgumentException(
-            "Type must be specified explicitly when value is null", nameof(type));
-      Value = value
-         ?? DBNull.Value;
-   }
-
-   public string Name { get; }
-
-   public Type Type { get; }
-
-   public object Value { get; }
-
-   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
-      // Try to convert value to 'type' to verify they are compatible
-      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
-      // account for implicit conversions (e.g., int -> long), just inheritance.
-      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
-         throw new ArgumentException(
-            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
-      }
-      return new(name, type ?? typeof(T), value);
-   }
-
-   internal static Type ParseType(string typeId) {
-      return Candidates().ExceptNull().FirstOrDefault() ??
-         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
-
-      IEnumerable<Type?> Candidates() {
-         if (typeId.Length == 0) {
-            yield break;
-         }
-         yield return Type.GetType(typeId);
-         yield return Type.GetType($"System.{typeId}");
-         if (typeId.Length > 1) {
-            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
-         }
-         yield return typeId.ToLower() switch {
-            "float64" or "double" or "real64" or "number" => typeof(double),
-            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
-            "int16" or "short" or "integer16" or "signed16" => typeof(short),
-            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
-            "int64" or "long" or "integer64" or "signed64" => typeof(long),
-            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
-            "uint32" or "uint" or "unsigned" => typeof(uint),
-            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
-            "char" => typeof(char),
-            "byte" or "uint8" => typeof(byte),
-            "bool" or "boolean" => typeof(bool),
-            "string" or "text" => typeof(string),
-            "date" or "datetime" => typeof(DateTime),
-            _ => null
-         };
-      }
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -1,49 +0,0 @@
-%namespace Bts.ColumnExpressions
-%partial
-%visibility internal
-%tokentype ColumnExpressionToken
-%parsertype ColumnExpressionParser
-
-%union {
-   public int Integer;
-   public double Real;
-   public bool Boolean;
-   public string String;
-   public string Identifier;
-   public string TypeId;
-   public DateTime DateTime;
-   public Tuple<string, Type> ColumnDefinition;
-   public ColumnExpression ColumnExpression;
-   public ImmutableList<ColumnExpression> ColumnExpressions;
-}
-
-%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
-%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
-
-%nonassoc EQUALS
-
-%type <ColumnDefinition> ColumnDefinition
-%type <ColumnExpression> ColumnExpression
-%type <ColumnExpressions> ColumnExpressions
-
-%%
-
-ColumnExpressions :
-   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
-   ;
-
-ColumnExpression :
-   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
-   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
-   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
-   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
-   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
-   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
-   ;
-
-ColumnDefinition :
-   IDENTIFIER { $$ = new($1.Identifier, null); }
-   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
-   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
-   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnExpressionInterpreter {
-   public static IList<ColumnExpression> Evaluate(Stream stream) {
-      var parser = new ColumnExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
-         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.Text;
-
-partial class ColumnExpressionParser {
-   public ColumnExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-partial class ColumnExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -1,85 +0,0 @@
-%namespace Bts.ColumnExpressions
-%scannertype ColumnExpressionScanner
-%visibility internal
-%tokentype ColumnExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-BooleanConstant          true|false
-StringConstant           '([^\'\\]*|\\.)*'
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
-IntegerConstant          -?[0-9]+
-NullConstant             'null'
-RealConstant             -?[0-9]+\.[0-9]+
-
-/* Relational Operators */
-Equals                   =
-OfType                   /
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-NamespacePrefix          {Identifier}\.
-TypeId                   {NamespacePrefix}*{Identifier}
-
-/* Utility Tokens */
-OpenParenthesis          \(
-CloseParenthesis         \)
-Comma                    ,
-QuoteOpen                \[
-QuoteClose               \]
-
-/* Comments */
-LineComment \/\/[^\n\r]*(\n|\r|\r\n)
-
-%%
-
-{Equals} {
-   return (int)ColumnExpressionToken.EQUALS;
-}
-
-{OfType} {
-   return (int)ColumnExpressionToken.OF_TYPE;
-}
-
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
-   return (int)ColumnExpressionToken.STRING_CONSTANT;
-}
-
-{BooleanConstant} {
-   yylval.Boolean = yytext.ToLower() switch {
-      "true" => true,
-      "false" => false,
-      _ => throw new Exception(
-         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
-   };
-   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
-}
-
-{DateTimeConstant} {
-   yylval.DateTime = DateTime.Parse(yytext);
-   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
-}
-
-{IntegerConstant} {
-   yylval.Integer = int.Parse(yytext);
-   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
-}
-
-{RealConstant} {
-   yylval.Real = double.Parse(yytext);
-   return (int)ColumnExpressionToken.REAL_CONSTANT;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnExpressionToken.IDENTIFIER;
-}
-
-{TypeId} {
-   yylval.TypeId = yytext;   
-   return (int)ColumnExpressionToken.TYPE_ID;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnSortExpression">
-    <Scanner Include="ColumnSortExpressionTokens.lex" />
-    <Parser Include="ColumnSortExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -1,11 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-public class ColumnSortExpression {
-   public ColumnSortExpression(string name, bool isAscending = true) {
-      Name = name;
-      IsAscending = isAscending;
-   }
-
-   public string Name { get; }
-   public bool IsAscending { get; }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -1,29 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%partial
-%visibility internal
-%tokentype ColumnSortExpressionToken
-%parsertype ColumnSortExpressionParser
-
-%union {
-   public string Identifier;
-   public ColumnSortExpression Column;
-   public ImmutableList<ColumnSortExpression> Columns;
-}
-
-%token ASCENDING, DESCENDING, IDENTIFIER
-
-%type <Column> Column
-%type <Columns> Columns
-
-%%
-
-Columns :
-   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | Columns Column { $$ = $1.Add($2); }
-   ;
-
-Column :
-   IDENTIFIER { $$ = new($1.Identifier, true); }
-   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
-   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
-   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnSortExpressionInterpreter {
-   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
-      var parser = new ColumnSortExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
-         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnSortExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.Text;
-
-partial class ColumnSortExpressionParser {
-   public ColumnSortExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnSortExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-partial class ColumnSortExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -1,30 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%scannertype ColumnSortExpressionScanner
-%visibility internal
-%tokentype ColumnSortExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-Ascending                +
-Descending               -
-
-%%
-
-{Ascending} {
-   return (int)ColumnSortExpressionToken.ASCENDING;
-}
-
-{Descending} {
-   return (int)ColumnSortExpressionToken.DESCENDING;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnSortExpressionToken.IDENTIFIER;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -7,19 +7,11 @@
 
   <ItemGroup>
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
+    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
-    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-  </ItemGroup>
-
 </Project>

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,62 +0,0 @@
-// using System;
-// using System.Diagnostics;
-
-// using BCG;
-// using BCG.Utilities.General;
-
-
-// namespace bts {
-
-//     public class PandasStream : ViewStream, IWorker {
-//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
-
-//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
-
-//         }
-
-//         public PandasStream(ViewStream vs, string pandas_code) :
-//         this(vs, ViewStreamDatabase.instance, pandas_code) {
-//         }
-
-
-//         protected override void myOnFirstSubscriber() {
-
-//             // string python_code = @"
-//             //    import winsound
-//             //    frequency = 2500  # Set frequency (in Hz)
-//             //    duration = 1000   # Set duration (in milliseconds)
-//             //    winsound.Beep(frequency, duration)
-//             // ";
-//             string python_code = @"
-//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
-//             with open(file_path, \"w\") as f:
-//                 f.write(\"Hello, world!\n\")
-//             ";
-
-//             execute_python_code(python_code);
-//         }
-
-//         static void execute_python_code(string python_code) {
-//             ProcessStartInfo start = new ProcessStartInfo();
-//             start.FileName = "python"; // Python executable
-//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
-//             start.UseShellExecute = false;
-//             start.RedirectStandardOutput = true;
-
-//             using (Process process = Process.Start(start)) {
-//                 using (System.IO.StreamReader reader = process.StandardOutput) {
-//                     string result = reader.ReadToEnd();
-//                     Console.Write(result);
-//                 }
-//             }
-//         }
-
-//         protected override void myOnLastSubscriber() {
-//         }
-
-//         public void doWork() {
-
-//         }
-
-//     }
-// }

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -1,79 +0,0 @@
-using BCG.Utilities.General;
-
-using System;
-
-namespace BCG {
-    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
-        class State {
-            public int n_sends = 0;
-            public int n_recvs = 0;
-        }
-        private readonly string _ip;
-        private readonly int _port;
-        private readonly ViewStreamDatabase _vsd;
-        State _state = new State();
-        private readonly int _msg_size;
-
-        private IBlueTableSchema _schema;
-        private TimeSpan _send_period;
-        private ITCPConnection _connection = null;
-        private byte[] _buff;
-
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
-        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
-            _ip = ip;
-            _port = port;
-            _msg_size = msg_size;
-            _vsd = vsd;
-            _schema = BlueDataUtility.form_schema("TCPClient",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("n_sends", TypeCode.UInt32),
-            Tuple.Create("n_recvs", TypeCode.UInt32));
-            _send_period = TimeSpan.FromSeconds(send_period_seconds);
-            _buff = new byte[msg_size];
-            for (int i = 0; i < _buff.Length; i++) {
-                _buff[i] = (byte)i;
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_address = new BlutoSubAddress(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            if (_connection != null) {
-                _connection.send(_buff);
-                _state.n_sends++;
-            }
-            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
-            BlueTable rows = new BlueTable(r.schema);
-            rows.addRow(r);
-            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        public void onConnection(ITCPConnection connection) {
-            _connection = connection;
-            _connection.register(this);
-        }
-        public bool isDisposed { get; }
-
-        public void onData(RemoteMessage msg) {
-            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
-            _state.n_recvs++;
-        }
-
-        public void Dispose() {
-
-        }
-
-
-    }
-}

src/DataStreamSuite/DataStream/Misc/test.py

@@ -1,3 +0,0 @@
-file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
-with open(file_path, "w") as f:
-    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -1,32 +0,0 @@
-using BCG;
-
-namespace BCG
-{
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("add_to_cloud")]
-   public class AddToCloudStream : ViewStream
-   {
-      private readonly string _name, _stream_exp;
-      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
-      public AddToCloudStream(string name, params string[] lines)
-      : this(name, 0, lines)
-      {
-      }
-
-      public AddToCloudStream(string name, int persist, params string[] lines)
-      {
-         _name = name;
-         _stream_exp = getExpression(lines);
-         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
-         mySetRealSource(new TableStream("cloud_streams"));
-      }
-
-      static string getExpression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,100 +0,0 @@
-using BCG;
-
-using System;
-using System.Collections.Generic;
-
-using mbts.config;
-using bts.utils;
-using System.Threading.Tasks;
-using Bts.DataStreamExpressions;
-using System.Collections.Generic;
-using BCG.Utilities.General;
-
-namespace BCG
-{
-   // listens to the requests posted on "cloud_streams" request table, and 
-   // if unfulfilled starts a ViewPubsliher
-   public class CloudStreamManager : IWorker
-   {
-      public class StreamPubUnit
-      {
-         public string name;
-         public string stream_expression;
-         public ViewStream stream;
-         public bool persist;
-         public ViewPublisher view_publisher;
-
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
-         {
-            name = name_;
-            stream_expression = stream_expression_;
-            stream = stream_;
-            persist = persist_;
-            view_publisher = view_publisher_;
-         }
-      }
-      static ViewStream build_stream(string stream_expression)
-      {
-         try
-         {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-            return rv;
-
-         }
-         catch (Exception ex)
-         {
-            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-      private IDictionary<string, StreamPubUnit>
-         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
-
-      public CloudStreamManager()
-      {
-         _stream.onChange += _on_str;
-
-      }
-      private void _on_str(object obj)
-      {
-         DataPacket dp = obj as DataPacket;
-         foreach (IBlueRow r in dp.newRows)
-         {
-            string name = r.get<string>("name");
-            bool persist = r.get<bool>("persist");
-            string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name))
-            {
-               _assign(name, stream_exp, persist);
-            }
-         }
-      }
-
-      private void _assign(string name, string stream_expression, bool persist)
-      {
-         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         ViewStream stream = build_stream(stream_expression);
-         _expose_to_ws(name, stream);
-         if (persist)
-         {
-            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
-         }
-         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
-         StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
-         _stream_pub_map.Add(name, u);
-      }
-
-      private void _expose_to_ws(string name, ViewStream stream)
-      {
-         // TODO: john
-      }
-
-      public void doWork()
-      {
-
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -1,59 +0,0 @@
-using BCG;
-
-using System.Linq;
-
-namespace BCG
-{
-   public class StreamOffloadRequestor
-   {
-      public class OffloadOp
-      {
-         string name;
-         string stream_exp;
-      }
-      private IBlueTableSchema _schema;
-      private IBlueRowCollection _rows;
-      private PubTableBroker _broker = new PubTableBroker();
-      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-      public StreamOffloadRequestor()
-      {
-         _schema = new BlueTableSchema("cloud_streams",
-                                   new ColumnInfo("id", 0, typeof(int)),
-                                   new ColumnInfo("name", 1, typeof(string)),
-                                   new ColumnInfo("persist", 2, typeof(bool)),
-                                   new ColumnInfo("stream_exp", 3, typeof(string)));
-         _existing_offloads_vs.onChange += _on_str;
-      }
-      private void _on_str(object obj)
-      {
-         DataPacket dp = obj as DataPacket;
-         _rows = dp.newRows;
-         // this is needed so the broker knows the state of the table
-         _broker.createTable(_rows);
-      }
-      public void add_to_cloud(string name, string stream_exp, bool persist)
-      {
-         // check if it exists
-         if (_rows == null)
-         {
-            BlueTable table = new BlueTable(_schema);
-            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-            table.addRow(nr);
-            _broker.createTable(table);
-            // create table  
-         }
-         else
-         {
-            if (exists(name))
-               return;
-            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-            _broker.addRow(nr);
-         }
-      }
-
-      bool exists(string name)
-      {
-         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,6 +1,13 @@
-namespace BCG;
-
-public class AddColumnsStream : MapStream {
-   public AddColumnsStream(ViewStream vs, string columnDef)
-      : base(vs, new ColumnAdder(columnDef)) { }
-}
+// using System;
+// using System.Data;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Utilities.General;
+// namespace BCG
+// {
+
+//    public class AddColumnsStream : MapStream
+//    {
+//       public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef))  {}    
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -1,12 +0,0 @@
-namespace bts {
-    using BCG;
-
-    using System;
-    using System.Collections;
-
-    public class AddConstantColumnRow {
-        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
-
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,52 +1,73 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-
-using Bts.ColumnExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Provides methods to add columns to the schema
-/// </summary>
-/// <param name="columnsExpression">
-/// String represting the columns to add to the output table
-/// </param>
-public class ColumnAdder : IStreamMapper {
-   // Store tuples of (column, index) to preserve order
-   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
-   protected IBlueTableSchema? mySchema;
-
-   public ColumnAdder(string columnsExpression) =>
-      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
-         throw new Exception("Invalid columnn expression: " + columnsExpression))
-         .Select((c, i) => (c, i))
-         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
-
-   [MemberNotNull(nameof(mySchema))]
-   public virtual IBlueRow map(IBlueRow row) {
-      // Only rebuild schema if necessary (i.e., if it's null)
-      mySchema ??= buildSchema(row.schema, myNewColumns);
-      return new AddColumnsRow(mySchema, row);
-   }
-
-   [MemberNotNull(nameof(mySchema))]
-   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
-      // Always rebuild schema (even if it's already non-null)
-      mySchema = buildSchema(inputSchema, myNewColumns);
-
-   [MemberNotNull(nameof(mySchema))]
-   static BlueTableSchema buildSchema(
-      IBlueTableSchema schema,
-      IDictionary<string, (ColumnExpression column, int index)> newColumns
-   ) {
-      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
-      schema.columnInfo.CopyTo(cis, 0);
-      foreach (var (col, index) in newColumns.Values) {
-         var schemaIdx = schema.columnInfo.Length + index;
-         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
-      }
-      return new BlueTableSchema(cis, schema.tableName);
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.AddColumnsExpression;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Provides methods to add columns to the schema
+//    /// </summary>
+//    public class ColumnAdder : IStreamMapper
+//    {
+//       protected IDictionary myNewColumns;
+//       protected IBlueTableSchema mySchema;
+
+//       static syntax myParser =
+//          new syntax();
+
+//       /// <summary>
+//       /// string represting the columns to be added to the output table. 
+//       /// </summary>
+//       /// <param name="colAddExpression"></param>
+//       public ColumnAdder (string columnDef)
+//       {
+//          AddColumnsExpression ace;
+
+//          if (DataStreamLoadFileConfig.instance.useStaticParser)
+//          {
+//             lock (myParser)
+//             {
+//                ace =
+//                (AddColumnsExpression)myParser.Parse(columnDef);
+//             }
+//          }
+//          else
+//          {
+//             syntax parser = new syntax();
+//             ace =
+//             (AddColumnsExpression)parser.Parse(columnDef);
+//          }
+
+// 	 if (ace == null)
+// 	   throw new Exception ("Invalid columnn expression: " + columnDef);
+//          myNewColumns = ace.columns;
+//       }
+         
+//       public virtual IBlueRow map (IBlueRow row)
+//       {
+//          if (mySchema == null)
+//             myConstructSchema (row.schema);
+//          return new AddColumnsRow (mySchema, row);
+//       }
+
+//       public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
+//       {
+//          myConstructSchema (inputSchema);
+//          return mySchema;
+//       }
+
+//       private void myConstructSchema (IBlueTableSchema schema)
+//       {
+//          ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
+//          int ciIndex = 0;
+//          foreach (ColumnInfo ci in schema.columnInfo)
+//             cis[ciIndex++] = ci;
+//          foreach (DictionaryEntry de in myNewColumns)
+//             cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++, 
+//                ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
+//          mySchema = new BlueTableSchema (cis, schema.tableName);
+//       }
+
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/CaptureSnapshotStream.cs

@@ -0,0 +1,40 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+
+// namespace BCG
+// {
+//    public class CaptureSnapshotStream : ViewStream
+//    {
+//       public CaptureSnapshotStream(string basePath,ViewStream sourceStream)
+//       {
+//          myBasePath = basePath;
+//          mySource = sourceStream;
+//          Directory.CreateDirectory(myBasePath);
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//       }
+
+//       void mySource_onChange(object newValue)
+//       {
+//          DateTime now = DateTime.Now;
+//          new ViewStreamWaiter(new SaverStream(new BlueRowCollectionStaticStream(((DataPacket)newValue).newRows),
+//              Path.Combine(myBasePath, String.Format("{0}_{1}.ds", now.ToString("yyyyMMdd_HHmmss"),
+//              now.Millisecond)))).waitUntilArrived();
+//          myFireEvent(newValue);
+//       }
+
+//       private string myBasePath;
+//       private ViewStream mySource;
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -1,135 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using Google.Protobuf;
-using System.Globalization;
-
-namespace BCG {
-   // Writes all the udpates to a stream in a file.
-   // File naming:
-   // token_epoch_yyyyMMdd_HHmmss
-   // File location: directory(passed)\token\
-   // directory passed on, it creates a folder with the token passed
-   // data:
-   // epochtime, size, serialized data
-   [DataStreamUserRespresentation("pers")]
-   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
-   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
-   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
-   // file is created. The file names are token_epoch_datetime
-   // the first entry in a file is the complete state
-   public class PersistorStream : ViewStream {
-      private ViewStream _source_stream;
-      private string _token;
-      private FileStream _out_fs;
-      private IBlueDataSerializer _serializer;
-      private string _file_name = String.Empty;
-      private long _rollover_size;
-      private readonly string _directory_path;
-      private int _n_updates_in_current_file = 0;
-
-      public PersistorStream(ViewStream src, string token, string directory, int rollover_size) {
-         _source_stream = src;
-         _token = token;
-         _directory_path = directory + $@"\{token}\";
-         _open_new_file();
-         _rollover_size = rollover_size;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _source_stream.onChange += _on_change;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _source_stream.onChange -= _on_change;
-         _out_fs.Close();
-      }
-
-
-      public static DateTime parse_datetime_from_filename(string s) {
-         // allbooks_1697054438_2023-10-11_16-00-38
-         string[] toks = s.Split("_");
-         bts.Log.assert(toks.Length == 4);
-         string dt = $"{toks[2]}_{toks[3]}";
-         return parse_datetime(dt);
-      }
-
-      // this is used to parse the string passed the datetime texts passed to cons of the stream 
-      public static DateTime parse_datetime(string s) {
-         // for whatever reason "yyyy-MM-dd_HH-mm-ss" is not parsable using DateTime.ParseExact (it only gets
-         // the Date not the time), "yyyy-MM-dd_HH:mm:ss" is parsable
-         // however a file name can not have : in its name, therefore we need to parse "yyyy-MM-dd_HH-mm-ss"
-         DateTime rv = DateTime.ParseExact(s, PersistorStream.DateTime_format, CultureInfo.InvariantCulture);
-         string[] toks = s.Split("_");
-         string time_str = toks[1];
-         string[] time_toks = time_str.Split("-");
-         bts.Log.assert(time_toks.Length == 3);
-         int hh = Int32.Parse(time_toks[0]);
-         int mm = Int32.Parse(time_toks[1]);
-         int ss = Int32.Parse(time_toks[2]);
-         rv = rv.Date;
-         rv = rv.AddHours(hh);
-         rv = rv.AddMinutes(mm);
-         rv = rv.AddSeconds(ss);
-         return rv;
-      }
-
-      public static string DateTime_format {
-         get {
-            return "yyyy-MM-dd_HH-mm-ss";
-         }
-      }
-
-      public static long dt_to_epoch(DateTime dt) {
-         DateTimeOffset dtos = new DateTimeOffset(dt, TimeSpan.Zero);
-         return dtos.ToUnixTimeSeconds();
-      }
-      public static long epoch_now() {
-         return DateTimeOffset.UtcNow.ToUnixTimeSeconds();
-      }
-
-      void _on_change(object new_value) {
-         DataPacket dp = new_value as DataPacket;
-         byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
-         if (_write_create) {
-            // new files always starts with create
-            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
-         } else
-            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
-
-         BinaryWriter bw = new BinaryWriter(_out_fs);
-         long utc_epoch = epoch_now();
-
-         //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
-         bw.Write(serialized_update);
-         _out_fs.Flush();
-         myFireEvent(dp);
-         _n_updates_in_current_file++;
-         if (_out_fs.Length > _rollover_size) {
-            _close_current_and_open_new();
-         }
-      }
-
-      void _close_current_and_open_new() {
-         _out_fs.Close();
-         _open_new_file();
-      }
-
-      void _open_new_file() {
-         var epoch = epoch_now();
-         _n_updates_in_current_file = 0;
-         // Check if the directory was successfully created
-         if (!Directory.Exists(_directory_path)) {
-            Directory.CreateDirectory(_directory_path);
-         }
-         _file_name = $"{_directory_path}{_token}_{epoch}_{DateTime.Now.ToString(DateTime_format)}";
-         _out_fs = new FileStream(_file_name, FileMode.OpenOrCreate);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -1,190 +1,117 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Runtime.Serialization.Formatters.Binary;
+using BCG.Utilities.General;
+using BCG.Utilities;
+
+namespace BCG
+{
+   public delegate void PlaybackStreamProgress(long readFar, long totalBytesRead);
+   public class PlaybackStream : ViewStream, IWorker
+   {
+
+      public PlaybackStream(string fileName) : this(fileName, 1.0) { }
+
+      private string myFileName;
+      private FileStream myFileReader;
+      private double myPlaybackSpeed;
+      private DateTime myPlaybackStart;
+      private DateTime myNextPacketTime;
+      private IClientDeserializer myDeserializer;
+      private ITableAccumulator myAccumulator;
+
+      private BlueTable myLastTableVersion;
+
+      public PlaybackStream(string fileName, double playbackSpeed)
+      {
+         myFileName = fileName;
+         myPlaybackSpeed = playbackSpeed;
+      }
+
+      protected override void myOnFirstSubscriber()
+      {         myPlaybackStart = DateTime.Now;
+         myNextPacketTime = DateTime.MinValue;
+         myFileReader = new FileStream(myFileName, FileMode.Open, FileAccess.Read, FileShare.Read);
+         myReadNextPacketTime();
+         ViewStreamDatabase.instance.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         if (myFileReader != null)
+            myFileReader.Close();
+         ViewStreamDatabase.instance.removeWorker(this);
+      }
+
+      protected void myReadNextPacketTime()
+      {
+         if (myFileReader.Position == myFileReader.Length)
+         {
+            myNextPacketTime = DateTime.MaxValue;
+            myFileReader.Dispose();
+            if (onFileExhausted != null)
+               onFileExhausted();
+         }
+         else
+         {
+            double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
+            myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
+            if (onProgress != null)
+               onProgress(myFileReader.Position, myFileReader.Length);
+         }
+      }
+
+      public event PlaybackStreamProgress onProgress;
+      public event Action onFileExhausted;
 
 
-// namespace BCG {
+      #region IWorker Members
 
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
+      public void doWork()
+      {
+         if (myNextPacketTime <= DateTime.Now)
+         {
+            if (myDeserializer == null)
+            {
+               myDeserializer = new ClientUniversalDeserializer();
+               myAccumulator = new TableAccumulator(myFileName, null);
+            }
 
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
+            BinaryReader reader = new BinaryReader(myFileReader);
+            int numBytes = reader.ReadInt32();
+            byte[] thisData = reader.ReadBytes(numBytes);
+            byte[] ops;
+            Array resultRowCollections;
+            myDeserializer.deserialize(new Stream[] { new MemoryStream(thisData) }, out ops, out resultRowCollections);
 
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
+            //there should only be one op in there becuae we just read a single update.
+            // BlueEventLog.logAssert(ops.Length == 1,"Playback stream got " +ops.Length.ToString() + " op codes in one update, from stream file "+myFileName);
+            //there shoudl only be one set of collections becaue there was just a single op code 
+            //(those two arrays are parallel)
+            // BlueEventLog.logAssert(resultRowCollections.Length == 1, "StreamOpCode was create, serializer should return exactly 1 set of rows");
 
+            if (ops[0] == (byte)StreamOpCode.create)
+            {
+               myLastTableVersion = new BlueTable((resultRowCollections.GetValue(0) as IBlueRowCollection[])[0]);
+               myFireEvent(new DataPacket(myLastTableVersion, new BlueTable(myLastTableVersion.schema), new BlueTable(myLastTableVersion.schema)));
+            }
+            else
+            {
+               // BlueEventLog.logAssert(ops[0] == (byte)StreamOpCode.multipleRemoveAdd, "Playback stream got unexpected open code " + ops[0].ToString() + " from file " + myFileName);
+               // BlueEventLog.logAssert(myLastTableVersion != null, "Playback stream for file "+myFileName + "got a non-create before a create");
+               IBlueRowCollection[] deletedAndAdded = resultRowCollections.GetValue(0) as IBlueRowCollection[];
+               BlueDataUtility.absorbChanges(myLastTableVersion, deletedAndAdded[0], deletedAndAdded[1]);
+               myFireEvent(new DataPacket(myLastTableVersion, deletedAndAdded[1], deletedAndAdded[0]));
+            }
+            
+            myReadNextPacketTime();
+         }
 
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
+      }
 
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -1,129 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using System.Runtime.Serialization.Formatters.Binary;
-using BCG.Utilities.General;
-using bts.collections;
-using System.Reflection.Metadata;
-
-
-namespace BCG {
-   public class SnapshotStream : ViewStream {
-      public enum SpecificSnapshot {
-         FirstOfDay,
-         LastOfDay
-      };
-      private BlueTable _current_table;
-      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-      public DateTime _snapshot_time;
-      public string _snapshot_file;
-      public IEnumerable<PlaybackPacket> _packets;
-      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-         DateTime dt;
-         var parsed = DateTime.TryParse(snapshot_date, out dt);
-         dt = dt.Date;
-         bts.Log.assert(parsed);
-         SpecificSnapshot specific_snapshot_enum;
-         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-         bts.Log.assert(parsed);
-         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-         mySetRealSource(rs);
-      }
-
-      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-         string token_directory_name = $"{directory_name}\\{token}";
-         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-         _packets = StreamPersistorUtils.load_packets_from_file(fs);
-         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-         if (snapshot_packet != null) {
-            foreach (PlaybackPacket p in _packets) {
-               byte[] ops;
-               Array row_array;
-               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-               for (int i = 0; i < ops.Length; i++) {
-                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-                  byte op = ops[i];
-                  if (op == StreamOpCode.create) {
-                     IBlueRowCollection full_table = rows_args[0];
-                     _current_table = new BlueTable(full_table);
-                  } else if (op == StreamOpCode.multipleRemoveAdd) {
-                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
-                  } else {
-                     bts.Log.assert(false, $"Oprator: {op} is not expected");
-                  }
-                  if (p.idx == snapshot_packet.idx) {
-                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-                     break;
-                  }
-               }
-            }
-         }
-      }
-
-      private void _absorb_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-            } else {
-               bts.Log.assert(false, $"Operator: {op} is not expected");
-            }
-         }
-      }
-
-      protected override void myOnLastSubscriber() {
-      }
-
-      // public event SnapshotStream.Progress onProgress;
-      // public event Action onFileExhausted;
-      private PlaybackPacket _locate_packet_to_replay() {
-         foreach (var p in _packets) {
-            // returns the first non replayed packet
-            if (!p.replayed)
-               return p;
-         }
-         return null;
-      }
-
-      private void _fire_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-               //_current_table.addRows(full_table);
-               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-                                          (myLastValue as DataPacket).newRows));
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-               myFireEvent(new DataPacket(_current_table, adds, dels));
-            } else {
-               bts.Log.assert(false, $"Oprator: {op} is not expected");
-            }
-
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -1,159 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Globalization;
-using System.IO;
-
-
-namespace BCG {
-   // each stream op in the replay file is reprsented by a packet here
-   public class PlaybackPacket {
-      public readonly int idx;
-      public readonly long timestamp;
-      public readonly byte[] data;
-      public long replay_timestamp = 0; // non zero it has been replayed
-      public long offset_from_previous = 0;
-      public PlaybackPacket(int the_idx, long the_timestamp, byte[] the_data) {
-         this.idx = the_idx;
-         this.timestamp = the_timestamp;
-         this.data = the_data;
-      }
-      public bool replayed {
-         get { return replay_timestamp != 0; }
-      }
-   }
-
-   public enum PlaybackType {
-      Streaming = 0,
-      Snapshot
-   }
-   public class SnapshotData {
-      public DateTime snapshot_time;
-      public string snapshot_file;
-      public SnapshotData(DateTime snapshot_time_arg, string snapshot_file_arg) {
-         snapshot_time = snapshot_time_arg;
-         snapshot_file = snapshot_file_arg;
-      }
-      public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
-   }
-   public static class StreamPersistorUtils {
-      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
-         DateTime max_dt = DateTime.MinValue;
-         string rv = String.Empty;
-         string[] file_names = Directory.GetFiles(directory_name);
-         for (int i = 0; i < file_names.Count() - 1; i++) {
-            string fn = file_names[i];
-            var dt = parse_datetime_from_filename(fn);
-            if (dt > max_dt) {
-               max_dt = dt;
-               rv = fn;
-            }
-         }
-         return rv;
-      }
-      public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
-         string[] file_names = Directory.GetFiles(directory_name);
-         for (int i = 0; i < file_names.Count() - 1; i++) {
-            string fn = file_names[i];
-            string next_fn = file_names[i + 1];
-            DateTime f_dt = PersistorStream.parse_datetime_from_filename(fn);
-            DateTime next_f_dt = PersistorStream.parse_datetime_from_filename(next_fn);
-            if (snapshot_time >= f_dt && snapshot_time <= next_f_dt) {
-               return fn;
-            }
-         }
-         return String.Empty;
-      }
-      public static DateTime parse_datetime_from_filename(string s) {
-         // allbooks_1697054438_2023-10-11_16-00-38
-         string[] toks = s.Split("_");
-         bts.Log.assert(toks.Length == 4);
-         string dt = $"{toks[2]}_{toks[3]}";
-         return parse_datetime(dt);
-      }
-
-      // this is used to parse the string passed the datetime texts passed to cons of the stream 
-      public static DateTime parse_datetime(string s) {
-         // for whatever reason "yyyy-MM-dd_HH-mm-ss" is not parsable using DateTime.ParseExact (it only gets
-         // the Date not the time), "yyyy-MM-dd_HH:mm:ss" is parsable
-         // however a file name can not have : in its name, therefore we need to parse "yyyy-MM-dd_HH-mm-ss"
-         DateTime rv = DateTime.ParseExact(s, PersistorStream.DateTime_format, CultureInfo.InvariantCulture);
-         string[] toks = s.Split("_");
-         string time_str = toks[1];
-         string[] time_toks = time_str.Split("-");
-         bts.Log.assert(time_toks.Length == 3);
-         int hh = Int32.Parse(time_toks[0]);
-         int mm = Int32.Parse(time_toks[1]);
-         int ss = Int32.Parse(time_toks[2]);
-         rv = rv.Date;
-         rv = rv.AddHours(hh);
-         rv = rv.AddMinutes(mm);
-         rv = rv.AddSeconds(ss);
-         return rv;
-      }
-
-      public static string DateTime_format {
-         get {
-            return "yyyy-MM-dd_HH-mm-ss";
-         }
-      }
-
-      public static long dt_to_epoch(DateTime dt) {
-         DateTimeOffset dtos = new DateTimeOffset(dt, TimeSpan.Zero);
-         return dtos.ToUnixTimeSeconds();
-      }
-      public static long epoch_now() {
-         return DateTimeOffset.UtcNow.ToUnixTimeSeconds();
-      }
-      public static DateTime epoch_to_dt(long epoch_seconds) {
-         return DateTimeOffset.FromUnixTimeSeconds(epoch_seconds).DateTime;
-      }
-
-      public static PlaybackPacket find_packet_at_time(IEnumerable<PlaybackPacket> packets, DateTime snapshot_time) {
-         log_file_ts(packets, snapshot_time);
-         long snapshot_time_epoch = PersistorStream.dt_to_epoch(snapshot_time);
-         for (int i = 0; i < packets.Count() - 1; i++) {
-            PlaybackPacket curr_p = packets.ElementAt(i);
-            PlaybackPacket next_p = packets.ElementAt(i + 1);
-            if (curr_p.timestamp <= snapshot_time_epoch && next_p.timestamp >= snapshot_time_epoch) {
-               return curr_p;
-            }
-         }
-         // if for some reason it cant find it, send the last packet
-         return packets.Last();
-      }
-
-      private static void log_file_ts(IEnumerable<PlaybackPacket> packets, DateTime snapshot_time) {
-         string big_log_msg = String.Empty;
-         foreach (var p in packets) {
-            big_log_msg += $"packet idx: {p.idx} epoch_ts: {p.timestamp} ts: {epoch_to_dt(p.timestamp)} snapshot_time: {snapshot_time}";
-            big_log_msg += "\n";
-         }
-         bts.Log.Gen.info($"Packets: {big_log_msg}");
-      }
-
-      public static IEnumerable<PlaybackPacket> load_packets_from_file(FileStream fs) {
-         BinaryReader br = new BinaryReader(fs);
-         List<PlaybackPacket> rv = new List<PlaybackPacket>();
-         while (fs.Position < fs.Length) {
-            try {
-               long ts = br.ReadInt64();
-               int sz = br.ReadInt32();
-               byte[] data = br.ReadBytes(sz);
-               var pb = new PlaybackPacket(rv.Count, ts, data);
-               rv.Add(pb);
-            } catch (Exception ex) {
-               bts.Log.Gen.error($"ex: {ex}");
-               break;
-            }
-         }
-         // setting offset field
-         for (int i = 0; i < rv.Count; i++) {
-            if (i == 0)
-               continue;
-            rv[i].offset_from_previous = rv[i].timestamp - rv[i - 1].timestamp;
-         }
-         return rv;
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,42 +1,47 @@
-using System;
-using System.Collections;
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
 
-using BCG.Utilities;
+// namespace BCG
+// {
+// 	/// <summary>
+// 	/// Computes the max for a given column
+// 	/// </summary>
+// 	public class MaxGroupByEvaluator : GroupByEvaluator
+// 	{
 
-namespace BCG {
-    /// <summary>
-    /// Computes the max for a given column
-    /// </summary>
-    public class MaxGroupByEvaluator : GroupByEvaluator {
+//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
+//       {}
 
-        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
+//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
+//          : base (colMapEpression, outputCol, groupByCols)   {}
 
-        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
-
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
-            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
-            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
-            if (maxRow != null) {
-                bts.Log.assert(currMaxRow != null);
-                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
-                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
-                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
-                    if (maxRow == null)
-                        myOutputTable.removeRow(hashString);
-                    else
-                        myOutputTable.addRow(maxRow, hashString);
-                }
-            }
-            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
-            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-            // otherwise if maxRow > currMaxRow then its replaced
-            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
-                myOutputTable.addRow(maxRow, hashString);
-            return myProject(myOutputTable.getRow(hashString));
-        }
-    }
-}
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
+//             myOutputTable;
+//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
+//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
+//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
+//          if (maxRow != null)
+//          {
+//             BlueEventLog.logAssert (currMaxRow != null);
+//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
+//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
+//             {
+//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
+//                if (maxRow == null)
+//                   myOutputTable.removeRow (hashString);
+//                else
+//                   myOutputTable.addRow (maxRow, hashString);
+//             }
+//          }
+//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
+//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+//          // otherwise if maxRow > currMaxRow then its replaced
+//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
+//             myOutputTable.addRow (maxRow, hashString);
+//          return myProject (myOutputTable.getRow (hashString));
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,15 +1,20 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class MaxStream : ViewStream {
-        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
-        }
-        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
-            mySetRealSource(new GroupByStream(stream,
-                                                new MaxGroupByEvaluator(outputCol, inputCols)));
-        }
+// namespace BCG
+// {
+// 	public class MaxStream : ViewStream
+// 	{
+//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
+//       {
+//       }
+// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
+// 		{
+//          mySetRealSource (new GroupByStream (stream, 
+//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
+// 		}
 
-        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
-    }
-}
+//       public MaxStream (ViewStream stream, string outputCol)
+//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
+//       {}
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,36 +1,40 @@
-using System;
+// using System;
+// using BCG.Utilities;
 
-using BCG.Utilities;
+// namespace BCG
+// {
+//    public class MinGroupByEvaluator : MaxGroupByEvaluator
+//    {
+//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
+//          : base (outputCol, groupByCols)  {}
 
-namespace BCG {
-    public class MinGroupByEvaluator : MaxGroupByEvaluator {
-        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
-           : base(outputCol, groupByCols) { }
+//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
+//          base (colMapEpression, outputCol, groupByCols)  {}
 
-        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
-           base(colMapEpression, outputCol, groupByCols) { }
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
+//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
+//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
+//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
+//          if (minRow != null)
+//          {
+//             // BlueEventLog.logAssert (currMinRow != null);
+//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
+//             {
+//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
+//                if (minRow == null)
+//                   myOutputTable.removeRow (stringRep);
+//                else
+//                   myOutputTable.addRow (minRow, stringRep);
+//             }
+//          }
+//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
+//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
+//             myOutputTable.addRow (minRow, stringRep);
+//          return myOutputTable.getRow (stringRep);
+//       }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
-            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
-            if (minRow != null) {
-                // BlueEventLog.logAssert (currMinRow != null);
-                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
-                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
-                    if (minRow == null)
-                        myOutputTable.removeRow(stringRep);
-                    else
-                        myOutputTable.addRow(minRow, stringRep);
-                }
-            }
-            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
-            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
-                myOutputTable.addRow(minRow, stringRep);
-            return myOutputTable.getRow(stringRep);
-        }
-
-    }
-}
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,13 +1,17 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class MinStream : ViewStream {
-        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
-            mySetRealSource(new GroupByStream(stream,
-                              new MinGroupByEvaluator(outputCol, inputCol)));
-        }
+// namespace BCG
+// {
+//    public class MinStream : ViewStream
+//    {
+//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
+//       {
+//          mySetRealSource (new GroupByStream (stream, 
+//                            new MinGroupByEvaluator (outputCol, inputCol)));
+//       }
 
-        public MinStream(ViewStream stream, string outputCol)
-                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
-    }
-}
+//       public MinStream (ViewStream stream, string outputCol)
+//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
+//       {}
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,82 +1,94 @@
-using System;
+// using System;
 
-namespace BCG {
-    /// <summary>
-    /// Computes the sum for a given column
-    /// </summary>
-    public class SumGroupByEvaluator : GroupByEvaluator {
-        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Computes the sum for a given column
+//    /// </summary>
+//    public class SumGroupByEvaluator : GroupByEvaluator
+//    {
+//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
+//       {}
 
-        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
+//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
+//          : base (colMapEpression, outputCol, groupByCols)   {}
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-
-            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-            return sumRow;
-        }
-    }
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
+         
+//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+//          return sumRow;
+//       }
+//    }
 
 
-    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
-        private string[] myOutputCols, myGroupByCols;
-        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
-            myOutputCols = spaceSeperatedOutputCols.Split(' ');
-            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-        }
+//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
+//    {
+//       private string[] myOutputCols, myGroupByCols;
+//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
+//       {
+//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
+//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+//       }
 
-        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
-            myOutputCols = output_columns;
-            myGroupByCols = groupby_columns;
-        }
+//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
+//       {
+//          myOutputCols = output_columns;
+//          myGroupByCols = groupby_columns;
+//       }
 
-        #region IGroupByEvaluator Members
+//       #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
+//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
 
-        public string[] groupByColumns { get { return myGroupByCols; } }
+//       public string[] groupByColumns { get { return myGroupByCols; } }
 
-        public void clear() {/*stateless*/}
+//       public void clear()  {/*stateless*/}
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
 
-        #endregion
-    }
+//       #endregion
+//    }
 
-    public class SumGroupByEvaluator<T> : GroupByEvaluator {
-        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-           : base(outputCol, groupByCols) { }
+//    public class SumGroupByEvaluator<T> : GroupByEvaluator
+//    {
+//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+//          : base(outputCol, groupByCols)
+//       { }
 
-        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
+//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+//          : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+//       public override IBlueRow evaluate(DataPacket dp)
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-            return sumRow;
-        }
-    }
+//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+//          return sumRow;
+//       }
+//    }
 
-    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
-        private string[] myOutputCols, myGroupByCols;
-        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
-            myOutputCols = spaceSeperatedOutputCols.Split(' ');
-            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-        }
+//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
+//    {
+//       private string[] myOutputCols, myGroupByCols;
+//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
+//       {
+//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
+//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+//       }
 
-        #region IGroupByEvaluator Members
+//       #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
+//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
 
-        public string[] groupByColumns { get { return myGroupByCols; } }
+//       public string[] groupByColumns { get { return myGroupByCols; } }
 
-        public void clear() {/*stateless*/}
+//       public void clear()  {/*stateless*/}
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
 
-        #endregion
-    }
+//       #endregion
+//    }
 
-}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,89 +1,107 @@
-using System;
-using System.Collections.Generic;
+// using System;
+// using System.Collections.Generic;
 
-namespace BCG {
-    public class NullMapper : IStreamMapper {
-        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+// namespace BCG
+// {
+//    public class NullMapper : IStreamMapper
+//    {
+//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-        public NullMapper(string[] valid_cols) {
-            foreach (string c in valid_cols)
-                _valid_cols[c] = true;
-            _valid_cols["id"] = true;
-        }
+//       public NullMapper(string[] valid_cols)
+//       {
+//          foreach (string c in valid_cols)
+//             _valid_cols[c] = true;
+//          _valid_cols["id"] = true;
+//       }
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-        public IBlueRow map(IBlueRow row) {
-            MutableBlueRow out_row = new MutableBlueRow(row);
-            foreach (string c in row.schema.columnNames)
-                if (!_valid_cols.ContainsKey(c))
-                    out_row.setValue(c, DBNull.Value);
-            return out_row;
-        }
-    }
+//       public IBlueRow map(IBlueRow row)
+//       {
+//          MutableBlueRow out_row = new MutableBlueRow(row);
+//          foreach (string c in row.schema.columnNames)
+//             if (!_valid_cols.ContainsKey(c))
+//                out_row.setValue(c, DBNull.Value);
+//          return out_row;
+//       }
+//    }
 
-    public class SumStream : ViewStream {
-        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
-            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-        }
+//    public class SumStream : ViewStream
+//    {
+//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
+//       {
+//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+//          mySetRealSource(new GroupByStream(stream,
+//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+//       }
 
-        public SumStream(ViewStream stream, string[] output_columns) {
-            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-            rs = new ProjectStream(rs, "!fake_col");
-            mySetRealSource(rs);
-        }
+//       public SumStream(ViewStream stream, string[] output_columns)
+//       {
+//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+//          rs = new ProjectStream(rs, "!fake_col");
+//          mySetRealSource(rs);
+//       }
 
-        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
-            string[] output_columns = space_seperated_output_columns.Split(' ');
-            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
-            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
-        }
+//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
+//       {
+//          string[] output_columns = space_seperated_output_columns.Split(' ');
+//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
+//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
+//       }
 
-        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
-            ViewStream rs = null;
-            groupby_columns = groupby_columns ?? new string[] { };
-            if (groupby_columns.Length == 0) {
-                // rs = new AddColumnsStream(stream, "fake_col=0");
-                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
-                // rs = new ProjectStream(rs, "!fake_col");
-            } else {
-                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-            }
-            if (force_null_into_other_columns) {
-                List<string> valid_cols = new List<string>(output_columns);
-                valid_cols.AddRange(groupby_columns);
-                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-            }
-            mySetRealSource(rs);
-        }
+//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
+//       {
+//          ViewStream rs = null;
+//          groupby_columns = groupby_columns ?? new string[] {};
+//          if (groupby_columns.Length == 0)
+//          {
+//             rs = new AddColumnsStream(stream, "fake_col=0");
+//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
+//             rs = new ProjectStream(rs, "!fake_col");
+//          }
+//          else
+//          {
+//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+//          }
+//          if (force_null_into_other_columns)
+//          {
+//             List<string> valid_cols = new List<string>(output_columns);
+//             valid_cols.AddRange(groupby_columns);
+//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+//          }
+//          mySetRealSource(rs);
+//       }
 
-        public SumStream(ViewStream stream, string outputCol)
-        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
-        }
-    }
+//    public SumStream (ViewStream stream, string outputCol)
+//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
+//    {
+//    }
+// }
 
-    public class IntSumStream : ViewStream {
-        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
-            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-        }
+// public class IntSumStream : ViewStream
+// {
+//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
+//    {
+//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+//       mySetRealSource(new GroupByStream(stream,
+//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
+//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+//    }
 
-    }
+// }
 
 
-    public class SumStream<T> : ViewStream {
-        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
-            string[] inputCols = spaceSeparatedInputCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
-        }
-    }
-}
+//    public class SumStream<T> : ViewStream
+//    {
+//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
+//       {
+//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
+//          mySetRealSource(new GroupByStream(stream,
+//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -1,79 +0,0 @@
-using BCG;
-
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-using ZstdSharp.Unsafe;
-
-namespace bts {
-    public class RowDiffStream : ViewStream {
-        private ViewStream _vs;
-        private readonly string _order_col, _col_1, _col_2;
-        private readonly int _diff;
-        private IBlueTableSchema _out_schema;
-        private readonly string _diff_col_name;
-        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
-            _vs = vs;
-            _order_col = order_col;
-            _col_1 = col_1;
-            _col_2 = col_2;
-            _diff = diff;
-            _diff_col_name = diff_col_name;
-        }
-
-        private string _diff_col_name_perc {
-            get {
-                return $"{_diff_col_name}_percentage";
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _vs.onChange += _handle_vs;
-        }
-
-        protected override void myOnLastSubscriber() {
-            _vs.onChange -= _handle_vs;
-        }
-
-        private void _handle_vs(object dp_obj) {
-            var dp = dp_obj as DataPacket;
-            IBlueRowCollection nrows = dp.newRows;
-            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
-            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
-            List<IBlueRow> row_list = new List<IBlueRow>();
-            foreach (IBlueRow r in nrows)
-                row_list.Add(r);
-            //OrderBy sorts in ascedning order
-            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
-            var sorted_row_array = sorted_rows.ToArray();
-            // reverse it to make it descending    
-            sorted_rows.Reverse();
-            //sort rows so we can do the diff
-            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
-            int i = 0;
-            foreach (IBlueRow r in sorted_rows) {
-                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
-                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
-                // we assume x - y, and name variables like that
-                double x = r.get<double>(_col_1);
-                int y_row_idx = i + _diff;
-                double diff = System.Double.NaN;
-                double perc = System.Double.NaN;
-                if (y_row_idx < sorted_rows.Count()) {
-                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
-                    diff = x - y;
-                    perc = 100.0 * diff / y;
-                }
-                nr.setValue(_diff_col_name, diff);
-                nr.setValue(_diff_col_name_perc, perc);
-                rv.addRow(nr);
-                i++;
-            }
-
-            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -8,7 +8,6 @@ using mbts.spark;
 
 namespace BCG
 {
-   [DataStreamUserRespresentation("fastJoin")]
    public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
    {
       private readonly NonIncJoinSpec[] _join_specs;
@@ -24,7 +23,7 @@ namespace BCG
 
       public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
                               ViewStream right, string right_join_cols, string right_proj)
-         : this(new Triplet<ViewStream, string, string>[]
+         : this(new Triplet<ViewStream, string, string>[] 
                {
                   new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                   new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
@@ -32,7 +31,7 @@ namespace BCG
       { }
 
       public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
+         : this (NonIncJoinSpec.from_triplet (join_specs), null, 0)
       { }
 
       public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
@@ -103,8 +102,8 @@ namespace BCG
          {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
             foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-               d.constribute_cols_to_schema(cis);
-            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
+               d.constribute_cols_to_schema (cis);
+            _output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray());
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -1,31 +0,0 @@
-namespace BCG {
-    using System.Collections.Generic;
-
-    public class KeyValueToRelationalStream : ViewStream {
-        private readonly ViewStream _kv_str;
-        private readonly string _doc_id_col, _key_col, _val_col;
-        public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
-            _kv_str = kv_str;
-            _doc_id_col = doc_id_col;
-            _key_col = key_col;
-            _val_col = val_col;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _kv_str.onChange += _on_str_update;
-        }
-
-        protected override void myOnLastSubscriber() {
-            _kv_str.onChange -= _on_str_update;
-        }
-
-        private void _on_str_update(object dp_obj) {
-            DataPacket dp = dp_obj as DataPacket;
-            var rows = dp.newRows;
-            var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
-            var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-        }
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs

@@ -1,48 +0,0 @@
-
-
-using System;
-
-namespace BCG {
-   // Keeps adding changing rows to the stream, hence preserving the history
-   // and creating a time series, for example this can be used to make bbo updates
-   // into a time series of prices/qty 
-   public class DebugStream : ViewStream {
-
-      private ViewStream _stream;
-      private IBlueRowCollection _last_recv_rows = null;
-
-      public DebugStream(ViewStream stream) {
-         _stream = stream;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
-      }
-      protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
-      }
-
-      private void _onstream_change(object o) {
-         DataPacket dp = o as DataPacket;
-         BlueDataUtility.validate_table_schema(dp.newRows);
-         BlueDataUtility.validate_table_schema(dp.addedRows);
-         if (_last_recv_rows == null) {
-            _last_recv_rows = dp.newRows;
-         } else {
-            //schema 
-            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.newRows.schema));
-            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.addedRows.schema));
-            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.deletedRows.schema));
-            foreach (IBlueRow r in dp.deletedRows) {
-               bts.Log.assert(_last_recv_rows.containsID(r.get<int>("id")));
-            }
-            foreach (IBlueRow r in dp.addedRows) {
-               bts.Log.assert(dp.newRows.containsID(r.get<int>("id")));
-            }
-            _last_recv_rows = dp.newRows;
-         }
-         myFireEvent(o);
-
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -1,53 +0,0 @@
-using Google.Protobuf.WellKnownTypes;
-
-using System;
-using BCG.Utilities.General;
-
-namespace BCG {
-   // Keeps adding changing rows to the stream, hence preserving the history
-   // and creating a time series, for example this can be used to make bbo updates
-   // into a time series of prices/qty 
-   public class TimeSeriesStream : ViewStream {
-      private BlueTable _table;
-      private ViewStream _stream;
-      private int _id = 0;
-
-      public TimeSeriesStream(ViewStream stream) {
-         _stream = stream;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
-      }
-      protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
-      }
-
-
-      private void _onstream_change(object o) {
-         DataPacket dp = o as DataPacket;
-         DateTime now = DateTime.Now;
-         if (_table == null) {
-            IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
-            _table = new BlueTable(schema);
-         }
-         BlueTable added_rows = new BlueTable(_table.schema);
-         foreach (IBlueRow r in dp.addedRows) {
-            MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue("timestamp", now);
-            ar.setValue("id", _id++);
-            added_rows.addRow(ar);
-         }
-         _table.addRows(added_rows);
-         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
-
-      }
-   }
-
-
-
-}
-
-
-

src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs

@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Security.Cryptography.X509Certificates;
-
-using BCG;
-using BCG.Utilities.General;
-
-using HtmlAgilityPack;
-
-namespace BCG {
-    class Program {
-        static void Main() {
-            string url = "http://quotes.toscrape.com";
-            List<string> quotes = ScrapeQuotes(url);
-
-            Console.WriteLine("Quotes from http://quotes.toscrape.com:");
-            foreach (var quote in quotes) {
-                Console.WriteLine(quote);
-            }
-        }
-
-        static List<string> ScrapeQuotes(string url) {
-            List<string> quotes = new List<string>();
-
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-
-                // Extract quotes from the HTML document
-                var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
-                if (quoteNodes != null) {
-                    foreach (var quoteNode in quoteNodes) {
-                        string quote = quoteNode.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
-
-            return quotes;
-        }
-    }
-
-    public class WebScapperStream : ViewStream, IWorker {
-        private readonly string _url;
-        private readonly string _xpath;
-        private readonly ViewStreamDatabase _vsd;
-        private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
-        private readonly IBlueTableSchema _schema;
-
-        public WebScapperStream()
-        :
-        this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath)
-        : this(url, xpath, ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
-            _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("url", TypeCode.UInt32),
-                                                Tuple.Create("line", TypeCode.String));
-            _url = url;
-            _xpath = xpath;
-            _vsd = vsd;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            var lines = scrape_lines(_url, _xpath);
-            Func<string, int> get_id = (line) => {
-                int rv = 0;
-                if (!_line_id_map.TryGetValue(line, out rv))
-                    _line_id_map.Add(line, _line_id_map.Count);
-                return rv;
-            };
-            BlueTable nrows = new BlueTable(_schema);
-            foreach (var l in lines) {
-                bts.Log.Gen.info($"line: {l}");
-                var id = get_id(l);
-                BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
-                nrows.addRow(nr);
-            }
-            var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        static List<string> scrape_lines(string url, string xpath) {
-            List<string> quotes = new List<string>();
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-                // Extract quotes from the HTML document
-                var quote_nodes = document.DocumentNode.SelectNodes(xpath);
-                if (quote_nodes != null) {
-                    foreach (var qn in quote_nodes) {
-                        string quote = qn.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
-
-            return quotes;
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,212 +0,0 @@
-using System;
-using System.Collections.Generic;
-using MongoDB.Driver;
-using MongoDB.Bson;
-using BCG;
-using BCG.Utilities.General;
-using ZstdSharp.Unsafe;
-using System.Runtime.CompilerServices;
-using mbts.spark;
-
-
-namespace BCG {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
-
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
-
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
-
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            _read_mogodb();
-        }
-
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
-
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
-
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
-
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
-
-    [DataStreamUserRespresentation("mongodb")]
-    public class MongoDBStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
-
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
-
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
-
-        }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            _read_mogodb();
-        }
-
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
-
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_doc(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
-
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("doc_id", TypeCode.String),
-            Tuple.Create("key", TypeCode.String),
-            Tuple.Create("value", TypeCode.String));
-            _table = new BlueTable(schema);
-        }
-
-        private void _add_doc(BsonDocument doc) {
-            var doc_id_bson = doc["_id"];
-            var doc_id = doc_id_bson.ToString();
-            Func<string, string, int> get_id = (string doc_id, string key) => {
-                string hash_key = $"{doc_id}_{key}";
-                int id = 0;
-                if (!_id_map.TryGetValue(hash_key, out id)) {
-                    id = _id_map.Count;
-                    _id_map.Add(hash_key, id);
-                }
-                return id;
-            };
-            foreach (var n in doc.Names) {
-                string key = n;
-                var id = get_id(doc_id, key);
-                MutableBlueRow nr = new MutableBlueRow(_table.schema);
-                var val = doc[n].ToString();
-                nr.setValue("id", id);
-                nr.setValue("doc_id", doc_id);
-                nr.setValue("key", key);
-                nr.setValue("value", val);
-                _table.addRow(nr);
-            }
-        }
-    }
-
-}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -6,7 +6,7 @@ using BCG.Utilities;
 
 namespace BCG
 {
-   [DataStreamUserRespresentation("CSVSaver")]
+   [DataStreamUserRespresentation("csvSaver")]
    public class CSVSaverStream : ViewStream, ISideEffectStream
    {
       private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -1,34 +0,0 @@
-using BCG;
-
-namespace bts {
-    public class SchemaGuesserStream : ViewStream {
-
-        private IBlueRowCollection _rows = null;
-        private ViewStream _vs;
-        public SchemaGuesserStream(ViewStream vs) {
-            _vs = vs;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _vs.onChange += _on_data;
-        }
-        protected override void myOnLastSubscriber() {
-            _vs.onChange -= _on_data;
-        }
-
-        private void _on_data(object obj) {
-            DataPacket dp = obj as DataPacket;
-            if (_rows == null) {
-                _rows = BlueDataUtility.get_typed_table(dp.newRows);
-            }
-            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-
-
-        }
-
-
-
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -1,107 +1,115 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace BCG;
-
-public class SortRow : IComparable, IBlueRow {
-   readonly IBlueRow row;
-   readonly IEnumerable<ColumnSortExpression>? columns;
-   readonly IComparer? comparer;
-
-   public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) {
-      this.row = row;
-      this.columns = columns;
-   }
-
-   public SortRow(IBlueRow row, IComparer comparer) {
-      this.row = row;
-      this.comparer = comparer;
-   }
-
-   public IBlueRow internal_row { get { return row; } }
-
-   #region IComparable method
-   public int CompareTo(object? obj) {
-      if (obj is not IBlueRow otherRow) {
-         throw new ArgumentException(
-            $"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})",
-            nameof(obj));
-      }
-      if (comparer is not null) {
-         return comparer.Compare(this, otherRow);
-      }
-      return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0);
-      int Compare(ColumnSortExpression col) {
-         var otherVal = otherRow[col.Name];
-         int rv = myIsNull(this[col.Name]) ?
-            (myIsNull(otherVal) ? 0 : -1) :
-            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
-         rv *= col.IsAscending ? 1 : -1;
-         return rv;
-      }
-   }
-   #endregion
-
-   #region IBlueRow members
-   public IBlueTableSchema schema { get { return row.schema; } }
-   #endregion
-
-   #region ICollection members
-   public int Count { get { return row.Count; } }
-
-   public bool IsSynchronized { get { return false; } }
-
-   public object SyncRoot { get { return this; } }
-
-   public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
-   #endregion
-
-   #region IDictionary members
-   public bool IsFixedSize { get { return true; } }
-
-   public bool IsReadOnly { get { return true; } }
-
-   /// <summary>
-   /// The key can either be the string corrsponding to the column name or
-   /// the order of the column
-   /// </summary>
-   public object this[object key] {
-      get {
-         if (key is string)
-            return this[(string)key];
-         return this[(int)key];
-      }
-      set { throw new NotSupportedException("The IBlueRow is readonly"); }
-   }
-
-   public object this[int index] => row[index];
-
-   public object this[string colName] => row[colName];
-
-   public ICollection Keys => row.Keys;
-
-   public ICollection Values => row.Values;
-
-   public bool Contains(object key) => row.Contains(key);
-
-   void IDictionary.Add(object key, object? value) =>
-      throw new NotSupportedException("The IBlueRow is readonly");
-
-   void IDictionary.Clear() =>
-      throw new NotSupportedException("The IBlueRow is readonly");
-
-   IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); }
-
-   IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); }
-
-   public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-   #endregion
-
-   bool myIsNull(object? o) =>
-      o is null || o == null || o is DBNull;
-
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.SortExpression;
+
+
+// namespace BCG
+// {
+//    public class SortRow : IComparable, IBlueRow
+//    {
+//       private IBlueRow myRow;
+//       private IEnumerable myColumns;
+//       private IComparer myComparer;
+
+//       public SortRow (IBlueRow row, SortExpression sortExpression)
+//       {
+//          myRow = row;
+//          myColumns = sortExpression.columns;
+//       }
+
+//       public SortRow (IBlueRow row, IEnumerable columns)
+//       {
+//          myRow = row;
+//          myColumns = columns;
+//       }
+
+//       public SortRow (IBlueRow row, IComparer comparer)
+//       {
+//          myRow = row;
+//          myComparer = comparer;
+//       }
+
+//       public IBlueRow internal_row { get { return myRow; } }
+
+//       #region IComparable method
+//       public int CompareTo (object obj)
+//       {  
+//          IBlueRow otherRow = (IBlueRow)obj;
+//          if (myComparer == null)
+//          {
+//             foreach (SortExpression.ColumnCriteria cc in myColumns)
+//             {
+//                int retVal = myIsNull (this[cc.columnName]) ?
+//                   myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
+//                   myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
+//                retVal *= cc.ascending ? 1 : -1;
+//                if (retVal != 0)
+//                   return retVal;
+//             }
+//             return 0;
+//          }
+//          else
+//             return myComparer.Compare (this, otherRow);
+//       }
+//       #endregion
+
+//       #region IBlueRow members
+//       public IBlueTableSchema schema{get{return myRow.schema;}}
+//       #endregion
+
+//       #region ICollection members
+//       public int Count {get{return myRow.Count;}}
+
+//       public bool IsSynchronized {get{return false;}}
+
+//       public object SyncRoot {get{return this;}}
+
+//       public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
+//       #endregion
+
+//       #region IDictionary members
+//       public bool IsFixedSize {get{return true;}}
+
+//       public bool IsReadOnly {get{return true;}}
+
+//       /// <summary>
+//       /// The key can either be the string corrsponding to the column name or
+//       /// the order of the column
+//       /// </summary>
+//       public object this [object key]
+//       {
+//          get
+//          {
+//             if (key is string)
+//                return this[(string)key];
+//             return this[(int)key];
+//          }
+//          set {throw new NotSupportedException ("The IBlueRow is readonly");}
+//       }
+
+//       public object this [int index]   {get{return myRow[index];}}
+
+//       public object this [string colName] {get{return myRow[colName];}}
+
+//       public ICollection Keys{get{return myRow.Keys;}}
+
+//       public ICollection Values{get{return myRow.Values;}}
+
+//       public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+//       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+//       public bool Contains (object key){return myRow.Contains (key);}
+
+//       IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
+
+//       IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
+
+//       public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+//       #endregion
+
+//       private bool myIsNull (object o) {return o == null || o is DBNull;}
+
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,99 +1,129 @@
-using Bts.ColumnSortExpressions;
+// using System;
+// using System.Collections;
+// using BCG.Evaluator.SortExpression;
 
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 
-namespace BCG;
+// namespace BCG
+// {
+// 	/// <summary>
+// 	/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+// 	/// to merge the additions to the accumulates table
+// 	/// </summary>
+// 	public class SortStream : ViewStream
+// 	{
+//       private ViewStream myStream;
+//       private IEnumerable mySortCols;
+//       private IComparer myComparer;
+//       private DataPacket myLastFiredPacket;
+//       private DataPacket myLastRecievedPacket;
 
-/// <summary>
-/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-/// to merge the additions to the accumulates table
-/// </summary>
-public class SortStream : ViewStream {
-   ViewStream stream;
-   IList<ColumnSortExpression> sortCols;
-   IComparer? comparer;
-   DataPacket? lastFiredPacket;
-   DataPacket? lastReceivedPacket;
+//       public SortStream (ViewStream stream, string sortExpression)
+//       {
+//          myStream = stream;
+//          mySetSortExpression (sortExpression);
+//       }
 
-   public SortStream(ViewStream stream, string sortExpression) {
-      this.stream = stream;
-      SortExpression = sortExpression;
-   }
+//       public SortStream (ViewStream vs, string colName, bool isAsc)
+//       {
+//          myStream = vs;
+//          string sortExpression = isAsc ? "+" + colName : "-" + colName;
+//          mySetSortExpression (sortExpression);
+//       }
 
-   public SortStream(ViewStream vs, string colName, bool isAsc) {
-      stream = vs;
-      SortExpression = isAsc ? "+" + colName : "-" + colName;
-   }
+//       public SortStream (ViewStream vs, IComparer comparer)
+//       {
+//          myStream = vs;
+//          myComparer = comparer;
+//       }
 
-   public SortStream(ViewStream stream, IComparer comparer) {
-      this.stream = stream;
-      this.comparer = comparer;
-      sortCols = Array.Empty<ColumnSortExpression>();
-   }
+//       public string sortExpression
+//       {
+//          set
+//          {
+//             myComparer = null;
+//             myLastFiredPacket = null;
+//             mySetSortExpression (value);   
+//             myChangeHandler (myLastRecievedPacket);
+//          }
+//       }
 
-   [MemberNotNull(nameof(sortCols))]
-   public string SortExpression {
-      set {
-         comparer = null;
-         lastFiredPacket = null;
-         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
+//       public IComparer comparer
+//       {
+//          set
+//          {
+//             myComparer = value;
+//             myLastFiredPacket = null;
+//             myChangeHandler(myLastRecievedPacket);
+//          }
+//       }
 
-   public IComparer Comparer {
-      set {
-         comparer = value;
-         lastFiredPacket = null;
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
+//       private void mySetSortExpression (string sortExpression)
+//       {
+//          syntax parser = 
+//             new syntax();
+         
+//          mySortCols = ((SortExpression)
+//             parser.Parse (sortExpression)).columns;
+//       }
 
-   public void myChangeHandlerTest(object o) =>
-      myChangeHandler(o);
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); 
+//       }
 
-   [MemberNotNull(nameof(lastFiredPacket))]
-   internal void myChangeHandler(object o) {
-      lastReceivedPacket = (DataPacket)o;
-      // First callback, so we need to sort the whole table, this is the
-      // only time we will do that for all the other sorting operations would
-      // be sorting the increments and inserting them into the whole list
-      if (lastFiredPacket is null) {
-         lastFiredPacket = new() {
-            newRows = comparer is null ?
-               new SortedTable(lastReceivedPacket.newRows, sortCols) :
-               new SortedTable(lastReceivedPacket.newRows, comparer)
-         };
-         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-         myFireEvent(lastFiredPacket.Clone());
-         return;
-      }
-      // Deletes
-      lastFiredPacket.deletedRows =
-         comparer == null ?
-         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-         new SortedTable(lastReceivedPacket.deletedRows, comparer);
-      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-      // Adds
-      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-      myFireEvent(lastFiredPacket.Clone());
-   }
+//       protected override void myOnLastSubscriber()
+//       {
+//          myInit ();
+//          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); 
+//       }
 
-   protected override void myOnFirstSubscriber() =>
-      stream.onChange += myChangeHandler;
+//       internal void myChangeHandler (object o)
+//       {
+//          myLastRecievedPacket = (DataPacket)o;
+//          //first callback, so we need to sort the whole table, this is the
+//          //only time we will do that for all the other sorting operations would
+//          //be sorting the increments and inserting them into the whole list
+//          if (myLastFiredPacket == null)
+//          {
+//             myLastFiredPacket = new DataPacket ();
+//             myLastFiredPacket.newRows = 
+//                myComparer == null ?
+//                new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
+//                new SortedTable (myLastRecievedPacket.newRows, myComparer);
+//             myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+//             myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+//             myFireEvent (myLastFiredPacket.Clone());
+//             return;
+//          }
+//          myHandleDeletes (myLastRecievedPacket);
+//          myHandleAdds (myLastRecievedPacket);
+//          myFireEvent (myLastFiredPacket.Clone());
+//       }
 
-   protected override void myOnLastSubscriber() {
-      lastFiredPacket = null;
-      lastReceivedPacket = null;
-      stream.onChange -= myChangeHandler;
-   }
-}
+//       public void myChangeHandlerTest(object o)
+//       {
+//           myChangeHandler(o);
+//       }
+
+//       private void myHandleDeletes (DataPacket dp)
+//       {
+//          myLastFiredPacket.deletedRows = 
+//             myComparer == null ?
+//             new SortedTable (dp.deletedRows, mySortCols) :
+//             new SortedTable (dp.deletedRows, myComparer);
+//          ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
+//       }
+
+//       private void myHandleAdds (DataPacket dp)
+//       {
+//          myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
+//          ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
+//       }
+
+//       private void myInit ()
+//       {
+//          myLastFiredPacket = null;
+//          myLastRecievedPacket = null;
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs

@@ -1,108 +1,136 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-using Bts.ColumnSortExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Represents an IBlueRowCollection with the rows sorted according to the
-/// </summary>
-public class SortedTable : IBlueRowCollection {
-   readonly List<SortRow> rows = new(0);
-   readonly Dictionary<int, IBlueRow> rowIndex = new(0);
-   readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>();
-   readonly IComparer? comparer;
-
-   /// <summary>
-   /// Sorts the rows using sortExpression as the sorting criteria
-   /// </summary>
-   /// <param name="rows">Rows to be inserted into the table</param>
-   /// <param name="sortCols">Sorting criteria</param>
-   public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) {
-      this.sortCols = sortCols;
-      init(rows);
-   }
-
-   public SortedTable(IBlueRowCollection rows, IComparer comparer) {
-      this.comparer = comparer;
-      init(rows);
-   }
-
-   public int Count => rows.Count;
-   public bool IsSynchronized => false;
-   public IBlueTableSchema schema { get; private set; }
-   public object SyncRoot => this;
-   public IBlueRow this[int id] => rowIndex[id]
-      ?? throw new BlueDataException(
-         $"ID '{id}' does not exist in table '{schema?.tableName}'");
-
-   public void addRows(IBlueRowCollection rows) {
-      // Requires merger of the addedRows to the already existing ones
-      if (rows.Count == 0) {
-         // At times there are calls with 0 additions, in that case
-         // we of course need no processing
-         return;
-      }
-      SortedTable addedRows =
-         comparer == null ? new SortedTable(rows, sortCols) :
-            new SortedTable(rows, comparer);
-      int addedRowsIndex = 0;
-      SortRow newRow = addedRows.getRowByIndex(addedRowsIndex);
-      for (int i = 0; i < this.rows.Count; i++) {
-         if (newRow.CompareTo(this.rows[i]) <= 0) {
-            this.rows.Insert(i, newRow);
-            rowIndex[(int)newRow["id"]] = newRow;
-            if (++addedRowsIndex == addedRows.Count) {
-               return;
-            }
-            newRow = addedRows.getRowByIndex(addedRowsIndex);
-         }
-      }
-      if (addedRowsIndex < addedRows.Count) {
-         for (int i = addedRowsIndex; i < addedRows.Count; i++) {
-            newRow = addedRows.getRowByIndex(i);
-            this.rows.Add(newRow);
-            rowIndex[(int)newRow["id"]] = newRow;
-         }
-      }
-   }
-
-   public void deleteRows(IBlueRowCollection deletedRows) {
-      foreach (IBlueRow deletedRow in deletedRows) {
-         var row = rowIndex[(int)deletedRow["id"]!] as SortRow
-            ?? throw new InvalidOperationException("Index did not return IBlueRow");
-         rows.Remove(row);
-         rowIndex.Remove((int)row["id"]!);
-      }
-   }
-
-   public SortRow getRowByIndex(int index) =>
-      rows[index]!;
-
-   public bool containsID(int id) =>
-      rowIndex[id] is not null;
-
-   public IEnumerator GetEnumerator() =>
-      rows.GetEnumerator();
-
-   public void CopyTo(Array array, int index) {
-      for (int i = 0; i < Count; i++) {
-         array.SetValue(rows[i], index + i);
-      }
-   }
-
-   [MemberNotNull(nameof(schema))]
-   void init(IBlueRowCollection rows) {
-      schema = rows.schema;
-      foreach (IBlueRow row in rows) {
-         SortRow newRow = comparer is null ?
-            new(row, sortCols) : new(row, comparer);
-         this.rows.Add(newRow);
-         rowIndex[(int)row["id"]!] = newRow;
-      }
-      this.rows.Sort();
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Evaluator.SortExpression;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Represents an IBlueRowCollection with the rows sorted according to the
+//    /// 
+//    /// </summary>
+//    public class SortedTable : IBlueRowCollection 
+//    {
+//       private ArrayList myRows = new ArrayList ();
+//       private Hashtable myIndex = new Hashtable ();
+//       private IBlueTableSchema mySchema;
+//       private IEnumerable mySortCols;
+//       private IComparer myComparer;
+
+//       #region SortedTable method(s)
+//       public void addRows (IBlueRowCollection rows)
+//       {
+//          //requires merger of the addedRows to the already existing ones
+//          if (rows.Count == 0)
+//             //at times there are calls with 0 additions, in that case
+//             //we ofcourse need no processing
+//             return;
+//          SortedTable addedRows = 
+//             myComparer == null ? new SortedTable (rows, mySortCols) :
+//                new SortedTable (rows, myComparer);
+//          int addedRowsIndex = 0;
+//          SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
+//          for (int i = 0; i < myRows.Count; i ++)
+//             if (newRow.CompareTo(myRows[i]) <= 0)
+//             {
+//                myRows.Insert (i, newRow);
+//                myIndex[newRow["id"]] = newRow;
+//                if (++addedRowsIndex == addedRows.Count)
+//                   return;
+//                newRow = addedRows.getRowByIndex (addedRowsIndex);
+//             }
+//          if (addedRowsIndex < addedRows.Count)
+//             for (int i = addedRowsIndex; i < addedRows.Count; i++)
+//             {
+//                newRow = addedRows.getRowByIndex (i);
+//                myRows.Add (newRow);
+//                myIndex[newRow["id"]] = newRow;
+//             }
+//       }
+
+//       public void deleteRows (IBlueRowCollection deletedRows)
+//       {
+//          foreach (IBlueRow deletedRow in deletedRows)
+//          {
+//             IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
+//             myRows.Remove (row);
+//             myIndex.Remove (row["id"]);
+//          }
+//       }
+
+//       public SortRow getRowByIndex (int index)  {return (SortRow)myRows[index];}
+//       #endregion
+
+//       #region Constructor(s)
+//       /// <summary>
+//       /// Sorts the rows using sortExpression as the sorting criteria
+//       /// </summary>
+//       /// <param name="rows">
+//       /// Rows to be inserted into the table
+//       /// </param>
+//       /// <param name="sortExpression">
+//       /// Sorting criteria
+//       /// </param>
+//       public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
+//       {
+//          mySortCols = sortExpression.columns;
+//          myConstructorHelper (rows);
+//       }
+
+//       public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
+//       {
+//          mySortCols = sortCols;
+//          myConstructorHelper (rows);
+//       }
+
+//       public SortedTable (IBlueRowCollection rows, IComparer comparer)
+//       {
+//          myComparer = comparer;
+//          myConstructorHelper (rows);
+//       }
+
+//       private void myConstructorHelper (IBlueRowCollection rows)
+//       {
+//          mySchema = rows.schema;
+//          foreach (IBlueRow row in rows)
+//          {
+//             SortRow newRow = 
+//                myComparer == null ?
+//                new SortRow (row, mySortCols) : new SortRow (row, myComparer);
+//             myRows.Add (newRow);
+//             myIndex[row["id"]] = newRow;
+//          }
+//          myRows.Sort ();
+//       }
+
+//       #endregion
+
+//       #region IBlueRowCollections members
+//       public IBlueRow this[int id] 
+//       {
+//          get
+//          {
+//             if (myIndex[id] == null)
+//                throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
+//             return (IBlueRow)myIndex[id];
+//          }
+//       }
+//       public IBlueTableSchema schema {get{return mySchema;}}
+
+//       public bool containsID (int id)  {return myIndex[id] != null;}
+
+//       #endregion
+
+//       #region ICollection members
+//       public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
+
+//       public bool IsSynchronized {get{return false;}}
+
+//       public object SyncRoot {get{return this;}}
+
+//       public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
+
+//       public int Count{get{return myRows.Count;}}
+//          #endregion
+//    }
+
+// }

src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs

@@ -1,143 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-
-namespace BCG
-{
-   /// <summary>
-   /// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
-   /// </summary>
-   public enum DateConversion
-   {
-      ConvertDoubleToDateTime = 0,
-      ChopTime,
-      UTCToCurrent,
-      ConvertStringToDateTime,
-      ConvertDateTimeToMilliseconds,
-      SecondsOfDay
-   }
-
-   [DataStreamUserRespresentation("dateStandardize")]
-   public class DateStandardizationStream : ViewStream, IStreamMapper
-   {
-      private IBlueTableSchema mySchema;
-      private DateConversion myConversion;
-      private string myConversionColName;
-      private readonly Dictionary<DateConversion, Type> _date_conversion_to_type_map = new Dictionary<DateConversion, Type>()
-      {
-         {DateConversion.ConvertDoubleToDateTime      , typeof(DateTime)},
-         {DateConversion.ChopTime                     , typeof(DateTime)},
-         {DateConversion.UTCToCurrent                 , typeof(DateTime)},
-         {DateConversion.ConvertStringToDateTime      , typeof(DateTime)},
-         {DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
-         {DateConversion.SecondsOfDay                 , typeof(Double)}
-      };
-
-
-      private static DateConversion _parse(string s)
-      {
-         DateConversion rv = DateConversion.ChopTime;
-         Enum.TryParse<DateConversion>(s, false, out rv);
-         return rv;
-      }
-      /// <summary>
-      /// for DSQA
-      /// </summary>
-      public DateStandardizationStream(ViewStream vs, string colName, string conversion)
-         : this(vs, colName, _parse(conversion))
-      { }
-
-      public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion)
-      {
-         myConversionColName = colName;
-         myConversion = conversion;
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            foreach (ColumnInfo c in inputSchema.columnInfo)
-            {
-               if (c.colName == myConversionColName)
-               {
-                  Type convType;
-                  if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType))
-                  {
-                     //BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
-                     convType = typeof(DateTime);
-                  }
-                  cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
-               }
-               else
-                  cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
-            }
-            mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row)
-      {
-         return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
-      }
-
-      #endregion
-
-      public static object mapDateTime(object objIn, DateConversion conv)
-      {
-         if (objIn == DBNull.Value || objIn == null)
-         {
-            return DBNull.Value;
-         }
-         else
-         {
-            if (conv == DateConversion.ChopTime)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.Date;
-            }
-            else if (conv == DateConversion.ConvertDoubleToDateTime)
-            {
-               double dtIn = (double)objIn;
-               return DateTime.FromOADate(dtIn);
-            }
-            else if (conv == DateConversion.UTCToCurrent)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.ToLocalTime();
-            }
-            else if (conv == DateConversion.ConvertStringToDateTime)
-            {
-               return DateTime.Parse((string)objIn);
-            }
-            else if (conv == DateConversion.ConvertDateTimeToMilliseconds)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.TimeOfDay.TotalMilliseconds;
-            }
-            else if (conv == DateConversion.SecondsOfDay)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               DateTime start_of_day = dtIn.Date;
-               TimeSpan day_period = dtIn - start_of_day;
-               return day_period.TotalSeconds;
-            }
-            else
-            {
-               bts.Log.Gen.info("incorrect type");
-               return null;
-            }
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Time/TimeBucketStream

@@ -1,51 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using System.Collections;
-
-namespace BCG
-{
-   [DataStreamUserRepresentation("timeBucket")]
-   public class TimeBucketStream : ViewStream, IStreamMapper
-   {
-      private ViewStream _src;
-      private readonly string _time_stamp_col;
-      private IBlueTableSchema _schema;
-      private uint _bucket_size_seconds;
-
-      public TimeBucketStream(ViewStream input, string time_stamp_col, uint bucket_size_seconds)
-      {
-         _time_stamp_col = time_stamp_col;
-         _bucket_size_seconds = bucket_size_seconds;
-      }
-
-      #region IStreamMapper Members
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
-      {
-         IBlueTableSchema rv = input_schema;
-         //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
-         rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
-         return rv;
-
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row)
-      {
-         DateTime ts = row.get<DateTime>(_time_stamp_col);
-         TimeSpan time_of_day = (ts - ts.Date)
-         int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
-         int buckets = seconds / _bucket_size_seconds;
-         int bucket_number = Math.Floor(buckets);
-         MutableBlueRow rv = new MutableBlueRow(row, getOutputSchema(row.schema))
-         rv.setValue("bucket_number", bucket_number);
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs

@@ -1,9 +1,10 @@
 using System;
-
 using BCG.Utilities.General;
 
-namespace BCG {
-   public abstract class TimeUnitStream : ViewStream, IWorker {
+namespace BCG
+{
+	public abstract class TimeUnitStream : ViewStream, IWorker
+	{
       protected TimeSpan myOffset;
       protected DateTime myConstructionTime;
       protected int mySizeOfStream;
@@ -12,80 +13,87 @@ namespace BCG {
       protected ViewStreamDatabase myDb;
       protected int myId = 0;
 
-      public TimeUnitStream(DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) {
+      public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db)
+		{
          myConstructionTime = DateTime.Now;
          myZeroTime = zeroTime;
          myOffset = offSet;
          mySizeOfStream = sizeOfStream;
          myDb = db;
-         myCollection = new CircularRowCollection(
-            new BlueTableSchema("Time",
-            new ColumnInfo("id", 0, typeof(Int32)), new ColumnInfo("time", 1, typeof(DateTime))),
+         myCollection = new CircularRowCollection (
+            new BlueTableSchema ("Time",
+            new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))), 
             mySizeOfStream);
-      }
+		}
 
       #region ViewStream overriden methods
 
-      protected override void myOnFirstSubscriber() {
-         myDb.addWorker(this);
-         myInit();
+      protected override void myOnFirstSubscriber()
+      {
+         myDb.addWorker (this);
+         myInit ();
       }
 
-      protected override void myOnLastSubscriber() {
-         myDb.removeWorker(this);
+      protected override void myOnLastSubscriber()
+      {
+         myDb.removeWorker (this);
       }
 
       #endregion
 
-      protected virtual void myInit() {
+      protected virtual void myInit ()
+      {
          int periods = (int)
-            (DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
+            (DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds;
+            
 
-
-         DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
-            myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
+         DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) : 
+            myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods));
 
 
          if (myGetLatestRow() == null ||
-            !myGetLatestRow()["time"].Equals(dt)) {
-            IBlueRow ar = new BlueRow(myCollection.schema,
+            !myGetLatestRow()["time"].Equals(dt))
+         {
+            IBlueRow ar = new BlueRow (myCollection.schema,
                myId++, dt);
-            IBlueRow dr = myCollection.addRowWithReturn(ar);
-            if (n_updates <= 3000)
-               myFireEvent(new DataPacket(myCollection,
-                  myConvertToCollection(ar),
-                  myConvertToCollection(dr)));
+            IBlueRow dr = myCollection.addRowWithReturn (ar);
+            myFireEvent (new DataPacket (myCollection,
+               myConvertToCollection (ar),
+               myConvertToCollection (dr)));
          }
       }
 
       #region IWorker methods
 
-      public void doWork() {
-         myInit();
+      public void doWork ()
+      {
+         myInit ();
       }
 
       #endregion
 
-      protected abstract DateTime myRoundTime(DateTime dt);
-
+      protected abstract DateTime myRoundTime (DateTime dt);
+         
 
       #region Private Helper methods
 
-      private IBlueRow myGetLatestRow() {
+      private IBlueRow myGetLatestRow ()
+      {
          return myCollection.Count == 0 ?
             null :
-            BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
+            BlueDataUtility.extractRow (myCollection, myCollection.Count - 1);
       }
 
-      private IBlueRowCollection myConvertToCollection(IBlueRow r) {
-         BlueTable retVal = new BlueTable(myCollection.schema);
+      private IBlueRowCollection myConvertToCollection (IBlueRow r)
+      {
+         BlueTable retVal = new BlueTable (myCollection.schema);
          if (r != null)
-            retVal.addRow(r);
+            retVal.addRow (r);
          return retVal;
       }
 
       #endregion
 
-   }
+	}
 
 }

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,43 +1,49 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class TopStream : MethodDrivenStream {
-        private int myTopCount;
+// namespace BCG
+// {
+//    public class TopStream : MethodDrivenStream
+//    {
+//       private int myTopCount;
 
-        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
+//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
 
-        public TopStream(ViewStream stream,
-           int topCount,
-           string sortExpression)
-           :
-           this(sortExpression == null ?
-           stream :
-           new SortStream(stream, sortExpression), topCount) { }
+//       public TopStream (ViewStream stream, 
+//          int topCount, 
+//          string sortExpression) 
+//          :
+//          this (sortExpression == null? 
+//          stream :
+//          new SortStream (stream, sortExpression), topCount)
+//       {}
 
-        protected override void myOnTableClear() {
-            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-        }
+//       protected override void myOnTableClear () 
+//       {
+//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
+//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
+//       }
 
-        protected override void myHandleFirstCall() {
-            myLastFiredPacket = new DataPacket();
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
-        }
+//       protected override void myHandleFirstCall () 
+//       {
+//          myLastFiredPacket = new DataPacket ();
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
+//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
+//       }
 
-        protected override void myHandleIncrementalAdd() { }
+//       protected override void myHandleIncrementalAdd ()     {}
 
-        protected override void myHandleIncrementalDelete() {
-            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
-               oldRows,
-               myLastFiredPacket.schema.columnNames,
-               out myLastFiredPacket.deletedRows,
-               out myLastFiredPacket.addedRows);
-            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-        }
-    }
-}
+//       protected override void myHandleIncrementalDelete()   
+//       {
+//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
+//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
+//             oldRows,
+//             myLastFiredPacket.schema.columnNames,
+//             out myLastFiredPacket.deletedRows,
+//             out myLastFiredPacket.addedRows);
+//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs

@@ -1,8 +1,6 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Linq.Expressions;
-using System.Xml.Serialization;
 
 using BCG.Utilities;
 
@@ -55,12 +53,7 @@ namespace BCG {
       }
 
       public virtual void createTable(IBlueRowCollection newTable) {
-         string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
-         foreach (IBlueRow r in newTable) {
-            log_msg += $"id: {r["id"]}";
-         }
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(newTable.schema);
+         bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}");
          myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = newTable.schema.tableName.ToLower();
@@ -78,9 +71,6 @@ namespace BCG {
       /// </summary>
       /// <param name="r"></param>
       public virtual void addRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
          myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = r.schema.tableName.ToLower();
@@ -96,9 +86,6 @@ namespace BCG {
       /// </summary>
       /// <param name="r"></param>
       public virtual void deleteRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
          myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = r.schema.tableName.ToLower();
@@ -114,9 +101,6 @@ namespace BCG {
       /// </summary>
       /// <param name="r"></param>
       public virtual void changeRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
          myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = r.schema.tableName.ToLower();
@@ -134,16 +118,7 @@ namespace BCG {
       public virtual void makeBatchChanges(IBlueRowCollection newRows,
                                              IBlueRowCollection deletedRows,
                                              IBlueRowCollection addedRows) {
-         string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
-         foreach (IBlueRow r in deletedRows)
-            log_msg += $"id: {r["id"]} ";
-         log_msg += "addedRows: ";
-         foreach (IBlueRow r in addedRows)
-            log_msg += $"id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(newRows.schema);
-         _check_schema_consistency(deletedRows.schema);
-         _check_schema_consistency(addedRows.schema);
+         myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = newRows.schema.tableName.ToLower();
          if (!_publisher_map.TryGetValue(name_key, out table_publisher))
@@ -169,16 +144,8 @@ namespace BCG {
       }
 
       protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
-         bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
          return new PubTablePublisher(tag, _publisher_core);
       }
 
-      private void _check_schema_consistency(IBlueTableSchema schema_arg) {
-         IBlueTableSchema schema;
-         if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
-            bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
-         }
-      }
-
    }
 }

src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs

@@ -14,7 +14,7 @@ namespace BCG {
       /// is not uniquely chosen)
       public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
          myName = name;
-         myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
+         myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
          myBroker = br;
          myFirstTime = true;
          myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);

src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs

@@ -5,12 +5,14 @@ using System.Data;
 //using BCG.Utilities;
 using System.Linq;
 
-namespace BCG {
+namespace BCG
+{
    /// <summary>
    /// Anologue of the dotNet DataTable.
    /// </summary>
    [Serializable]
-   public class BlueTable : IBlueRowCollection {
+   public class BlueTable : IBlueRowCollection
+   {
       /// <summary>
       /// Hashtable indexed on "id" to speed up removes
       /// </summary>
@@ -27,19 +29,22 @@ namespace BCG {
       /// all the data over
       /// </summary>
       /// <param name="table"></param>
-      public BlueTable(DataTable table) {
+      public BlueTable(DataTable table)
+      {
          ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
          int i = 0;
          foreach (DataColumn col in table.Columns)
             cis[i] = new ColumnInfo(col.ColumnName, i++, col.DataType);
          mySchema = new BlueTableSchema(cis, table.TableName);
-         foreach (DataRow row in table.Rows) {
+         foreach (DataRow row in table.Rows)
+         {
             IBlueRow newRow = new BlueRow(mySchema, row.ItemArray);
             myIdIndex[newRow["id"]] = newRow;
          }
       }
 
-      public BlueTable(IBlueRowCollection table) {
+      public BlueTable(IBlueRowCollection table)
+      {
          mySchema = table.schema;
          foreach (IBlueRow row in table)
             addRow(row);
@@ -47,10 +52,12 @@ namespace BCG {
 
       public BlueTable(string tableName,
           IEnumerable objects,
-          params CMap[] columnMappers) {
+          params CMap[] columnMappers)
+      {
          // extract schema from mappers
          ColumnInfo[] cols = new ColumnInfo[columnMappers.Length];
-         for (int i = 0; i < columnMappers.Length; i++) {
+         for (int i = 0; i < columnMappers.Length; i++)
+         {
             cols[i] = new ColumnInfo(
                 columnMappers[i].getColumnName(), i,
                 columnMappers[i].getColumnType());
@@ -59,7 +66,8 @@ namespace BCG {
 
          // load rows
          int rowNumber = 0;
-         foreach (object rowObject in objects) {
+         foreach (object rowObject in objects)
+         {
             object[] values = new object[columnMappers.Length];
             for (int cn = 0; cn < columnMappers.Length; cn++)
                values[cn] = columnMappers[cn].getColumnValue(rowNumber, rowObject);
@@ -79,8 +87,10 @@ namespace BCG {
       /// <summary>
       /// Returns the row by the index, interface property
       /// </summary>
-      public virtual IBlueRow this[int id] {
-         get {
+      public virtual IBlueRow this[int id]
+      {
+         get
+         {
             object row = myIdIndex[id];
             if (row == null)
                throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
@@ -95,7 +105,8 @@ namespace BCG {
 
       public virtual bool containsID(int id) { return myIdIndex[id] != null; }
 
-      public virtual bool tryGetValue(int id, out IBlueRow row) {
+      public virtual bool tryGetValue(int id, out IBlueRow row)
+      {
          row = (IBlueRow)myIdIndex[id];
          return (row != null);
       }
@@ -103,7 +114,8 @@ namespace BCG {
       #endregion
 
       #region BlueTable members
-      public virtual void clear() {
+      public virtual void clear()
+      {
          myIdIndex.Clear();
       }
 
@@ -111,7 +123,8 @@ namespace BCG {
       /// If a row with the same id exists, it replaces it
       /// </summary>
       /// <param name="r"></param>
-      public virtual void forceAddRow(IBlueRow r) {
+      public virtual void forceAddRow(IBlueRow r)
+      {
          myIdIndex[r["id"]] = r;
       }
 
@@ -119,41 +132,43 @@ namespace BCG {
       /// Adds a new row to the table. 
       /// </summary>
       /// <param name="row"></param>
-      public virtual void addRow(IBlueRow row) {
+      public virtual void addRow(IBlueRow row)
+      {
          object id = row["id"];
 
          if (id == null || id == DBNull.Value)
             throw new BlueDataException("The ID of the row being added is null/DNBull");
-         try {
+         try
+         {
             myIdIndex.Add(id, row);
-         } catch (ArgumentException) {
-            throw new BlueDataException($"ID: {id} already exists in the table: {mySchema.tableName}");
+         }
+         catch (ArgumentException)
+         {
+            throw new BlueDataException($"ID {id} already exists in the table: {mySchema.tableName} , duplication of field ID not allowed");
          }
 
       }
 
-      public virtual void addRow(DataRow row) {
+      public virtual void addRow(DataRow row)
+      {
          object val = row["id"];
 
          if (val == null || val == DBNull.Value)
             throw new BlueDataException("The ID of the row being added is null/DNBull");
-         if (myIdIndex[val] != null) {
-            string err_msg = $"ID: {val} already exists in the table: {mySchema.tableName} duplication of field ID not allowed";
-            // temp TODOds remove it
-            bts.Log.assert(false, err_msg);
-            throw new BlueDataException(err_msg);
-
-         }
+         if (myIdIndex[val] != null)
+            throw new BlueDataException("ID already exists in the table, duplication of field ID not allowed");
          IBlueRow newRow = new BlueRow(mySchema, row);
          myIdIndex[val] = newRow;
       }
 
-      public virtual void addRows(IBlueRowCollection rows) {
+      public virtual void addRows(IBlueRowCollection rows)
+      {
          foreach (IBlueRow nr in rows)
             addRow(nr);
       }
 
-      public virtual void addRows(IEnumerable rows) {
+      public virtual void addRows(IEnumerable rows)
+      {
          foreach (IBlueRow nr in rows)
             addRow(nr);
       }
@@ -170,32 +185,37 @@ namespace BCG {
       /// Removes a row specified by the id. The index helps in making this operation O(1)
       /// </summary>
       /// <param name="id"></param>
-      public virtual bool deleteRow(int id) {
+      public virtual bool deleteRow(int id)
+      {
          if (myIdIndex[id] == null)
             return false;
          myIdIndex.Remove(id);
          return true;
       }
 
-      public virtual bool deleteRows(IBlueRowCollection rows) {
+      public virtual bool deleteRows(IBlueRowCollection rows)
+      {
          bool retVal = true;
          foreach (IBlueRow r in rows)
             retVal &= deleteRow(r);
          return retVal;
       }
 
-      public virtual bool deleteRows(IEnumerable rows) {
+      public virtual bool deleteRows(IEnumerable rows)
+      {
          bool retVal = true;
          foreach (IBlueRow r in rows)
             retVal &= deleteRow(r);
          return retVal;
       }
 
-      public virtual void changeRow(IBlueRow r) {
+      public virtual void changeRow(IBlueRow r)
+      {
          myIdIndex[r["id"]] = r;
       }
 
-      public virtual void changeRows(IBlueRowCollection rows) {
+      public virtual void changeRows(IBlueRowCollection rows)
+      {
          foreach (IBlueRow r in rows)
             changeRow(r);
       }

src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs

@@ -11,17 +11,17 @@ namespace BCG
    public class CircularRowCollection : OrderedBlueTable
    {
       private int myBufferSize;
-      public CircularRowCollection(IBlueTableSchema schema, int bufferSize)
-         : base(schema)
-      {
-         myBufferSize = bufferSize;
+      public CircularRowCollection (IBlueTableSchema schema, int bufferSize)
+         : base (schema)
+      {		
+         myBufferSize = bufferSize ;
       }
 
-      public CircularRowCollection(IBlueTableSchema schema, IEnumerable<IBlueRow> rows, int bufferSize)
-         : this(schema, bufferSize)
+      public CircularRowCollection (IBlueRowCollection rows, int bufferSize)
+         : this (rows.schema, bufferSize)
       {
          foreach (IBlueRow r in rows)
-            addRow(r);
+            addRow (r);
       }
 
       public int bufferSize
@@ -39,30 +39,30 @@ namespace BCG
       /// </summary>
       /// <param name="r"></param>
       /// <returns></returns>
-      public IBlueRow addRowWithReturn(IBlueRow r)
+      public IBlueRow addRowWithReturn (IBlueRow r)
       {
-         base.addRow(r);
+         base.addRow (r);
          if (myOrderedRows.Count > myBufferSize)
          {
-            IBlueRow retVal = myOrderedRows[0] as IBlueRow;
-            deleteRow(retVal);
+            IBlueRow retVal = myOrderedRows [0] as IBlueRow;
+            deleteRow (retVal);
             return retVal;
          }
          return null;
       }
 
-      public override void addRow(IBlueRow row)
+      public override void addRow (IBlueRow row)
       {
-         base.addRow(row);
+         base.addRow (row);
          if (myOrderedRows.Count > myBufferSize)
-            deleteRow(myOrderedRows[0] as IBlueRow);
+            deleteRow (myOrderedRows [0] as IBlueRow);
       }
 
-      public override void addRow(DataRow row)
+      public override void addRow (DataRow row)
       {
-         base.addRow(row);
+         base.addRow (row);
          if (myOrderedRows.Count > myBufferSize)
-            deleteRow(myOrderedRows[0] as IBlueRow);
+            deleteRow (myOrderedRows [0] as IBlueRow);
       }
 
       public void add_rows(IEnumerable<IBlueRow> rows_to_add, out IBlueRowCollection added_rows, out IBlueRowCollection deleted_rows)

src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs

@@ -3,7 +3,8 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BCG {
+namespace BCG
+{
    /*
       These fields are first going to be used by  BlueRay to help it determine what differnet tags mean, but eventually
     * the goal is that they will be used for all services in this system. 
@@ -11,12 +12,11 @@ namespace BCG {
     * not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service
     * probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated
     */
-   public enum MetaDataField {
-      Protocol,       // is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
-      Type,           // if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
-      Symbol,
-      Exchange,
-      MarketDataType, // BAM (bid, ask, mid), last, etc
-      RealName
-   }; //does this thing have a better name we could call it
+   public enum MetaDataField {   Protocol,  //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
+                                 Type,   //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
+                                 Symbol, 
+                                 Exchange,
+                                 Platform,
+                                 MarketDataType, // BAM (bid, ask, mid), last, etc
+                                 RealName }; //does this thing have a better name we could call it
 }

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -1,316 +1,316 @@
-using System;
-using System.Collections;
-using System.Data;
-using BCG.Utilities.General;
+// using System;
+// using System.Collections;
+// using System.Data;
+// using BCG.Utilities.General;
 
-namespace BCG
-{
-	[Serializable]
-   public class DataPacket : ICloneable, IDataPacket
-   {      
-      public class BlueRowIdComparator
-      {
-         private string[] myIdColumns = new string[]{"id"};
+// namespace BCG
+// {
+// 	[Serializable]
+//    public class DataPacket : ICloneable, IDataPacket
+//    {      
+//       public class BlueRowIdComparator
+//       {
+//          private string[] myIdColumns = new string[]{"id"};
 
-         public BlueRowIdComparator(string[] idCols)
-         {
-            if ( idCols.Length != 0 ) 
-            {
-               myIdColumns = idCols;
-            }
-         }
-                    
-         public  bool blueRowIdEqualEvaluator(object a, object b)
-         {
-            if ( a == null || b == null )
-            {
-               return false;
-            }
+//          public BlueRowIdComparator(string[] idCols)
+//          {
+//             if ( idCols.Length != 0 ) 
+//             {
+//                myIdColumns = idCols;
+//             }
+//          }
 
-            if ( !(a is IBlueRow) || !(b is IBlueRow) )
-            {
-               return false;
-            }
+//          public  bool blueRowIdEqualEvaluator(object a, object b)
+//          {
+//             if ( a == null || b == null )
+//             {
+//                return false;
+//             }
 
-            IBlueRow lhs = (IBlueRow)a;
-            IBlueRow rhs = (IBlueRow)b;
-                    
-            bool result = true;
-            foreach(string idColumn in myIdColumns)
-            {
-               result = result && lhs[idColumn].Equals(rhs[idColumn]);
-               if ( !result )
-                  break;
-            }
+//             if ( !(a is IBlueRow) || !(b is IBlueRow) )
+//             {
+//                return false;
+//             }
 
-            return result;            
-         }
-      }
+//             IBlueRow lhs = (IBlueRow)a;
+//             IBlueRow rhs = (IBlueRow)b;
 
-      public IBlueRowCollection newRows, addedRows, deletedRows;
-      public DataPacket () {}
-      public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
-      {
-         this.newRows = newRows;
-         this.addedRows = addedRows;
-         this.deletedRows = deletedRows;
-      }
+//             bool result = true;
+//             foreach(string idColumn in myIdColumns)
+//             {
+//                result = result && lhs[idColumn].Equals(rhs[idColumn]);
+//                if ( !result )
+//                   break;
+//             }
 
-      public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
-      {
-         new_rows = newRows;
-         added_rows = addedRows;
-         deleted_rows = deletedRows;
-      }
+//             return result;            
+//          }
+//       }
 
-      public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
-      {                 
-         BlueTable truly = new BlueTable(addedRows.schema);
-         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+//       public IBlueRowCollection newRows, addedRows, deletedRows;
+//       public DataPacket () {}
+//       public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
+//       {
+//          this.newRows = newRows;
+//          this.addedRows = addedRows;
+//          this.deletedRows = deletedRows;
+//       }
 
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
+//       {
+//          new_rows = newRows;
+//          added_rows = addedRows;
+//          deleted_rows = deletedRows;
+//       }
 
-         ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
-           
-         foreach(IBlueRow r in ta)
-         {
-            truly.addRow(r);
-         }
-           
-         return truly;
-                
-      }
+//       public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
+//       {                 
+//          BlueTable truly = new BlueTable(addedRows.schema);
+//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-      public virtual IBlueRowCollection getTrulyAddedRows ()
-      {
-         BlueTable retVal = new BlueTable (schema);
-         foreach (IBlueRow r in addedRows)
-            if (!deletedRows.containsID ((int)r["id"]))
-               retVal.addRow (r);
-         return retVal;
-      }
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-      public virtual IBlueRowCollection getTrulyDeletedRows ()
-      {
-         BlueTable retVal = new BlueTable (schema);
-         foreach (IBlueRow r in deletedRows)
-            if (!addedRows.containsID ((int)r["id"]))
-               retVal.addRow (r);
-         return retVal;
-      }
+//          ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
 
-      public virtual IBlueRowCollection getTrulyChangedRows ()
-      {
-         BlueTable retVal = new BlueTable (schema);
-         foreach (IBlueRow r in addedRows)
-            if (deletedRows.containsID ((int)r["id"]))
-               retVal.addRow (r);
-         return retVal;
-      }
+//          foreach(IBlueRow r in ta)
+//          {
+//             truly.addRow(r);
+//          }
 
-      
-      public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
-      {        
-         BlueTable truly = new BlueTable(deletedRows.schema);
-         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+//          return truly;
 
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       }
 
-         ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
-                      
-         foreach(IBlueRow r in td)
-         {
-            truly.addRow(r);
-         }
+//       public virtual IBlueRowCollection getTrulyAddedRows ()
+//       {
+//          BlueTable retVal = new BlueTable (schema);
+//          foreach (IBlueRow r in addedRows)
+//             if (!deletedRows.containsID ((int)r["id"]))
+//                retVal.addRow (r);
+//          return retVal;
+//       }
 
-         return truly;      
-      }
-      
-      public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
-      {
-       
-         BlueTable truly = new BlueTable(addedRows.schema);
-         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+//       public virtual IBlueRowCollection getTrulyDeletedRows ()
+//       {
+//          BlueTable retVal = new BlueTable (schema);
+//          foreach (IBlueRow r in deletedRows)
+//             if (!addedRows.containsID ((int)r["id"]))
+//                retVal.addRow (r);
+//          return retVal;
+//       }
 
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       public virtual IBlueRowCollection getTrulyChangedRows ()
+//       {
+//          BlueTable retVal = new BlueTable (schema);
+//          foreach (IBlueRow r in addedRows)
+//             if (deletedRows.containsID ((int)r["id"]))
+//                retVal.addRow (r);
+//          return retVal;
+//       }
 
-         ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
-         ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
-         
-         foreach(IBlueRow r in tc)
-         {
-            truly.addRow(r);
-         }   
-        
-         return truly;      
-      }
-      
-      public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
-      {        
-         BlueTable truly = new BlueTable(deletedRows.schema);
-         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
+//       {        
+//          BlueTable truly = new BlueTable(deletedRows.schema);
+//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-         ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
-         ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
-         
-         foreach(IBlueRow r in tco)
-         {
-            truly.addRow(r);
-         }         
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-         return truly;         
-      }
-      
-      public TablePacket toTablePacket ()
-      {
-         return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
-                                 BlueDataUtility.toDataTable (deletedRows));
-                                 
-      }
+//          ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
 
-      public IBlueTableSchema schema 
-      {
-         get
-         {
-            IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
-               addedRows.schema != null ? addedRows.schema : deletedRows.schema;
-            return retVal;
-         }
-      }
+//          foreach(IBlueRow r in td)
+//          {
+//             truly.addRow(r);
+//          }
 
-      /*          
-       status  D total  A total  D this   A This   D' total A' Total 
-         x  0  0  0  0  0  0  
-            0  0  0  1     1 this+  
-            0  0  1  0  1this    
-            0  0  1  1  1 this   1 this+  
-         x  0  1  0  0        
-         error 0  1  0  1        
-            0  1  1  0     1 total- 
-            0  1  1  1     1 total-, 1 this +   
-         x  1  0  0  0        
-            1  0  0  1     1 this+  
-         error 1  0  1  0        
-         error 1  0  1  1        
-         x  1  1  0  0        
-         error 1  1  0  1        
-            1  1  1  0     1 total -1  keep the old delete and remove the old add
-            1  1  1  1     1 total-, 1 this +   
-      
-         d' = dthis - (dthis intersect a) + d // keep the old delete
-         a' = a - (dthis intersect a) + athis // keep the new add       
-       * */
-      public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
-      {       
-         if ( packets.Count == 0 )
-         {
-            return new DataPacket(
-               new BlueTable(new DataTable()),
-               new BlueTable(new DataTable()),
-               new BlueTable(new DataTable()));                    
-         }
+//          return truly;      
+//       }
 
-         DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
-         
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
+//       {
 
-         ArrayList added = new ArrayList();
-         ArrayList deleted = new ArrayList();
-         BlueTable dt = null;
-         BlueTable at = null;
-         IBlueRowCollection nt = null;
+//          BlueTable truly = new BlueTable(addedRows.schema);
+//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-         foreach(DataPacket dp in packets)
-         {               
-            ArrayList oldDeleted = deleted;
-            ArrayList oldAdded = added;
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-            added = new ArrayList();
-            deleted = new ArrayList();
-            if ( dt == null )
-            {
-               dt = new BlueTable(dp.schema);
-               at = new BlueTable(dp.schema);
-            }
-            
-            deleted.AddRange(oldDeleted);
-            deleted.AddRange( CollectionHelpers.difference(
-               CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
-               oldDeleted, ed) );
+//          ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
+//          ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
 
-            added.AddRange(dp.addedRows);
-            added.AddRange( CollectionHelpers.difference(
-               CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
-               added, ed) );
-            nt = dp.newRows;
-         }        
-         
-         foreach ( IBlueRow dr in added )
-         {
-            at.addRow(dr);
-         }
+//          foreach(IBlueRow r in tc)
+//          {
+//             truly.addRow(r);
+//          }   
 
-         foreach ( IBlueRow dr in deleted )
-         {
-            dt.addRow(dr);
-         }
+//          return truly;      
+//       }
 
-         return new DataPacket(nt,at,dt);            
-      }
+//       public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
+//       {        
+//          BlueTable truly = new BlueTable(deletedRows.schema);
+//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-      public DataPacket deepCopy ()
-      {
-         return new DataPacket (new OrderedBlueTable (newRows),
-            new OrderedBlueTable (newRows),
-            new BlueTable (deletedRows));
-      }
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-      public virtual DataPacket createImmutable ()
-      {
-         return 
-         new DataPacket(
-               newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
-               addedRows, 
-               deletedRows);
-      }
-      #region ICloneable Members
+//          ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
+//          ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
 
-      public object Clone()
-      {
-         return MemberwiseClone();
-      }
+//          foreach(IBlueRow r in tco)
+//          {
+//             truly.addRow(r);
+//          }         
 
-      #endregion
+//          return truly;         
+//       }
 
-      public string toIncrementalIDString()
-      {
-         System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
-         sb.Append ("\ndeleted : ");
-         foreach (IBlueRow r in deletedRows)
-         {
-            sb.Append(r["id"]);
-            sb.Append(',');
-         }
-         sb.Append("\nadded : ");
-         foreach (IBlueRow r in addedRows)
-         {
-            sb.Append(r["id"]);
-            sb.Append(',');
-         }
-         sb.Append("\n");
-         return sb.ToString();
-      }
+//       public TablePacket toTablePacket ()
+//       {
+//          return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
+//                                  BlueDataUtility.toDataTable (deletedRows));
 
-      public override string ToString()
-      {
-         return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
-      }
-   }
-}
+//       }
+
+//       public IBlueTableSchema schema 
+//       {
+//          get
+//          {
+//             IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
+//                addedRows.schema != null ? addedRows.schema : deletedRows.schema;
+//             return retVal;
+//          }
+//       }
+
+//       /*          
+//        status  D total  A total  D this   A This   D' total A' Total 
+//          x  0  0  0  0  0  0  
+//             0  0  0  1     1 this+  
+//             0  0  1  0  1this    
+//             0  0  1  1  1 this   1 this+  
+//          x  0  1  0  0        
+//          error 0  1  0  1        
+//             0  1  1  0     1 total- 
+//             0  1  1  1     1 total-, 1 this +   
+//          x  1  0  0  0        
+//             1  0  0  1     1 this+  
+//          error 1  0  1  0        
+//          error 1  0  1  1        
+//          x  1  1  0  0        
+//          error 1  1  0  1        
+//             1  1  1  0     1 total -1  keep the old delete and remove the old add
+//             1  1  1  1     1 total-, 1 this +   
+
+//          d' = dthis - (dthis intersect a) + d // keep the old delete
+//          a' = a - (dthis intersect a) + athis // keep the new add       
+//        * */
+//       public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
+//       {       
+//          if ( packets.Count == 0 )
+//          {
+//             return new DataPacket(
+//                new BlueTable(new DataTable()),
+//                new BlueTable(new DataTable()),
+//                new BlueTable(new DataTable()));                    
+//          }
+
+//          DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
+
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+
+//          ArrayList added = new ArrayList();
+//          ArrayList deleted = new ArrayList();
+//          BlueTable dt = null;
+//          BlueTable at = null;
+//          IBlueRowCollection nt = null;
+
+//          foreach(DataPacket dp in packets)
+//          {               
+//             ArrayList oldDeleted = deleted;
+//             ArrayList oldAdded = added;
+
+//             added = new ArrayList();
+//             deleted = new ArrayList();
+//             if ( dt == null )
+//             {
+//                dt = new BlueTable(dp.schema);
+//                at = new BlueTable(dp.schema);
+//             }
+
+//             deleted.AddRange(oldDeleted);
+//             deleted.AddRange( CollectionHelpers.difference(
+//                CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
+//                oldDeleted, ed) );
+
+//             added.AddRange(dp.addedRows);
+//             added.AddRange( CollectionHelpers.difference(
+//                CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
+//                added, ed) );
+//             nt = dp.newRows;
+//          }        
+
+//          foreach ( IBlueRow dr in added )
+//          {
+//             at.addRow(dr);
+//          }
+
+//          foreach ( IBlueRow dr in deleted )
+//          {
+//             dt.addRow(dr);
+//          }
+
+//          return new DataPacket(nt,at,dt);            
+//       }
+
+//       public DataPacket deepCopy ()
+//       {
+//          return new DataPacket (new OrderedBlueTable (newRows),
+//             new OrderedBlueTable (newRows),
+//             new BlueTable (deletedRows));
+//       }
+
+//       public virtual DataPacket createImmutable ()
+//       {
+//          return 
+//          new DataPacket(
+//                newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
+//                addedRows, 
+//                deletedRows);
+//       }
+//       #region ICloneable Members
+
+//       public object Clone()
+//       {
+//          return MemberwiseClone();
+//       }
+
+//       #endregion
+
+//       public string toIncrementalIDString()
+//       {
+//          System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
+//          sb.Append ("\ndeleted : ");
+//          foreach (IBlueRow r in deletedRows)
+//          {
+//             sb.Append(r["id"]);
+//             sb.Append(',');
+//          }
+//          sb.Append("\nadded : ");
+//          foreach (IBlueRow r in addedRows)
+//          {
+//             sb.Append(r["id"]);
+//             sb.Append(',');
+//          }
+//          sb.Append("\n");
+//          return sb.ToString();
+//       }
+
+//       public override string ToString()
+//       {
+//          return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -1,4 +1,5 @@
-namespace BCG {
+namespace BCG
+{
    using Bts.DataStream;
    using bts;
 
@@ -9,26 +10,28 @@ namespace BCG {
    using System.Reflection;
    using System.IO;
 
-   public class PrimitiveManager {
+   public class PrimitiveManager
+   {
       private IDictionary myTypeLookupTable = new Hashtable();
       private static PrimitiveManager mySelf;
       private ArrayList myAssemblyNames = new ArrayList();
 
-      private PrimitiveManager() {
+      private PrimitiveManager()
+      {
          myLoadAllTypes();
       }
 
-      private void myLoadAllTypes() {
+      private void myLoadAllTypes()
+      {
          AppDomain currentDomain = AppDomain.CurrentDomain;
          currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
          loadAssembly(this.GetType().Assembly.FullName);
-         var assemblies = new string[] { "datastream.dll", "sharedstreams.dll" };
-         var config_assemblies = Configuration.get<DataStreamConfig>().Assemblies;
-         bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}");
-         foreach (string assemblyName in assemblies/*config_assemblies*/) {
-            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
+
+         foreach (string assemblyName in Configuration.get<DataStreamConfig>().Assemblies)
+         {
             string resolvedAssemblyPath = assemblyName;
-            if (!System.IO.Path.IsPathRooted(assemblyName)) {
+            if (!System.IO.Path.IsPathRooted(assemblyName))
+            {
                string appPath = Path.GetDirectoryName(
                               Assembly.GetAssembly(typeof(PrimitiveManager)).CodeBase.Substring("file:///".Length));
                resolvedAssemblyPath = System.IO.Path.GetFullPath(System.IO.Path.Combine(appPath, assemblyName));
@@ -37,7 +40,8 @@ namespace BCG {
          }
       }
 
-      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
+      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args)
+      {
          foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
             if (args.Name.Equals(a.FullName))
                return a;
@@ -48,8 +52,8 @@ namespace BCG {
       /// Loads all the Stream types associated with the assembly
       /// </summary>
       /// <param name="assemblyName"></param>
-      public void loadAssembly(string assemblyName) {
-         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
+      public void loadAssembly(string assemblyName)
+      {
          if (myAssemblyNames.Contains(assemblyName))
             return;
          myAssemblyNames.Add(assemblyName);
@@ -61,11 +65,13 @@ namespace BCG {
             if (!type.IsInterface
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
-               && type.IsPublic) {
+               && type.IsPublic)
+            {
                bool isVisible = true;
                string userRep = createDefaultName(type.Name);
                foreach (Attribute att in type.GetCustomAttributes(false))
-                  if (att is DataStreamUserRespresentation) {
+                  if (att is DataStreamUserRespresentation)
+                  {
                      DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
                      isVisible = dsatt.isVisible;
                      userRep = isVisible ?
@@ -73,14 +79,16 @@ namespace BCG {
                         : null;
                      break;
                   }
-               if (isVisible) {
-                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+               if (isVisible)
+               {
+                  Log.assert(myTypeLookupTable[userRep] == null);
                   myTypeLookupTable[userRep] = type;
                }
             }
       }
 
-      public static string createDefaultName(string typeName) {
+      public static string createDefaultName(string typeName)
+      {
          if (!typeName.EndsWith("Stream"))
             return typeName;
          string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
@@ -92,7 +100,8 @@ namespace BCG {
 
 
 
-      public static PrimitiveManager instance {
+      public static PrimitiveManager instance
+      {
          get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
       }
 
@@ -101,7 +110,8 @@ namespace BCG {
       /// </summary>
       /// <param name="friendlyName"></param>
       /// <returns></returns>
-      public string find(string friendlyName) {
+      public string find(string friendlyName)
+      {
          if (myTypeLookupTable[friendlyName] == null)
             throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
          return (myTypeLookupTable[friendlyName] as Type).Name;
@@ -114,9 +124,11 @@ namespace BCG {
       /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
       /// is the desired one</param>
       /// <returns></returns>
-      public ViewStream build(string friendlyName, ICollection args) {
+      public ViewStream build(string friendlyName, ICollection args)
+      {
          object[] constructorArgs = null;
-         if (args != null && args.Count > 0) {
+         if (args != null && args.Count > 0)
+         {
             constructorArgs = new object[args.Count];
             args.CopyTo(constructorArgs, 0);
          }
@@ -132,7 +144,8 @@ namespace BCG {
       /// Returns the name of all the primitives 
       /// </summary>
       /// <returns></returns>
-      public string[] getPrimitiveNames() {
+      public string[] getPrimitiveNames()
+      {
          string[] retVal = new string[myTypeLookupTable.Count];
          myTypeLookupTable.Keys.CopyTo(retVal, 0);
          return retVal;
@@ -142,17 +155,20 @@ namespace BCG {
       /// Return the types of all the streams
       /// </summary>
       /// <returns></returns>
-      public Type[] getStreamTypes() {
+      public Type[] getStreamTypes()
+      {
          Type[] retVal = new Type[myTypeLookupTable.Count];
          myTypeLookupTable.Values.CopyTo(retVal, 0);
          return retVal;
       }
 
-      public ICollection getConstructorSigns(string primitiveName) {
+      public ICollection getConstructorSigns(string primitiveName)
+      {
          ArrayList retVal = new ArrayList();
          Log.assert(myTypeLookupTable[primitiveName] != null);
          Type t = myTypeLookupTable[primitiveName] as Type;
-         foreach (ConstructorInfo ci in t.GetConstructors()) {
+         foreach (ConstructorInfo ci in t.GetConstructors())
+         {
             string sign = "";
             sign = t.Name + " (";
             foreach (ParameterInfo pi in ci.GetParameters())
@@ -165,7 +181,8 @@ namespace BCG {
          return retVal;
       }
 
-      public string getAssemblyName(string primitiveName) {
+      public string getAssemblyName(string primitiveName)
+      {
          Log.assert(myTypeLookupTable[primitiveName] != null);
          return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
       }

src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs

@@ -42,7 +42,7 @@ namespace BCG
                Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate;
             if (myStateTable != null)
                value (myStateTable, myStateTable, new BlueTable (myStateTable.schema));
-      }
+         }
          remove
          {
             // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
@@ -64,7 +64,6 @@ namespace BCG
 
       public void update(Stream[] streams)
       {
-         bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
          //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
          //data element for that tag in the ViewStreamDatabase
          if (myOnUpdateDelegate != null)

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs

@@ -3,15 +3,16 @@ using System.IO;
 using System.Collections;
 using BCG.Utilities;
 
-namespace BCG {
+namespace BCG
+{
    /// <summary>
    /// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
    /// </summary>
-   public class ClientDeserializationController : IClientDeserializer {
+   public class ClientDeserializationController : IClientDeserializer
+   {
       private IClientDeserializer myDeserializer;
-      private int _n_desers = 0;
-
-      public ClientDeserializationController() { }
+		
+      public ClientDeserializationController () {}
 
       #region IClientDeserializer Members
 
@@ -22,41 +23,42 @@ namespace BCG {
       /// <param name="streams"></param>
       /// <param name="theOps"></param>
       /// <param name="theRows"></param>
-      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
-         bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}");
-         if (myDeserializer != null) {
-            myDeserializer.deserialize(streams, out theOps, out theRows);
-         } else {
-            Stream totalStream = UtilityMethods.getAggregatedStream(streams);
+      public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
+      {
+         if (myDeserializer != null)
+         {
+            myDeserializer.deserialize (streams, out theOps, out theRows);
+         }
+         else 
+         {
+            Stream totalStream = UtilityMethods.getAggregatedStream (streams);
             long initialPos = totalStream.Position;
-            BinaryReader br = new BinaryReader(totalStream);
+            BinaryReader br = new BinaryReader (totalStream);
             byte op;
-            try {
-               op = br.ReadByte();
-            } catch (EndOfStreamException) {
+            try 
+            {
+               op = br.ReadByte ();
+            }
+            catch (EndOfStreamException)
+            {
                op = Byte.MaxValue;
             }
-            if (op == StreamOpCode.create) {
-               br.ReadString(); //pubsubclientdeserializer
-               string deserName = br.ReadString();
-               myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
+            if (op == StreamOpCode.create)
+            {
+               br.ReadString (); //pubsubclientdeserializer
+               string deserName = br.ReadString ();   
+               myDeserializer = Activator.CreateInstance (Type.GetType (deserName))
                   as IClientDeserializer;
                totalStream.Position = initialPos;
-               myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
-            } else {
+               myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows);
+            }
+            else
+            {
                // BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
-               theOps = new byte[] { };
-               theRows = new object[] { };
+               theOps = new byte[]{};
+               theRows = new object[]{};
             }
          }
-         var msgs = StreamIOMessage.assemble_msgs(theOps, theRows);
-         int i = 0;
-         foreach (var m in msgs) {
-            bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}");
-         }
-         _n_desers++;
-         // log it here
-
       }
 
 

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs

@@ -5,61 +5,61 @@ using System.IO;
 using BCG.Utilities;
 //using BCG.Utilities.BlueConfiguration;
 
-namespace BCG {
+namespace BCG
+{
    /// <summary>
    /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
    /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) 
    /// </summary>
-   internal class ClientUniversalRowDeserializer : IClientRowDeserializer {
+   internal class ClientUniversalRowDeserializer : IClientRowDeserializer
+   {
       private IBlueTableSchema mySchema;
       private int myNullMapLength = 0;
       private PrimitiveDeserializationHelper myDeserializationHelper;
       private BitArray myNullMap;
       private int[] mySerIndexToSchemaIndexMap;
+		
+      public ClientUniversalRowDeserializer (IBlueTableSchema schema)
+      {
+         myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema);
+         myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
+         myInitReorderSchema ();
 
-      public ClientUniversalRowDeserializer(IBlueTableSchema schema) {
-         myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema);
-         myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8)));
-         myInitReorderSchema();
-
-
+         
       }
 
       #region Schema Reordering initialization
-      private void myInitReorderSchema() {
-         try {
-            int idColIndex = mySchema["id"].colNo;
-            int[] temp = new int[mySchema.columnInfo.Length];
-            foreach (ColumnInfo ci in mySchema.columnInfo) {
-               temp[ci.colNo] =
-                  ci.colNo == idColIndex ? 0 :
-                  ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
-            }
-            mySerIndexToSchemaIndexMap = new int[temp.Length];
-            for (int i = 0; i < temp.Length; i++)
-               mySerIndexToSchemaIndexMap[temp[i]] = i;
-         } catch (Exception ex) {
-            string schema_name = mySchema == null ? "NULL" : mySchema.tableName;
-            string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}";
-            bts.Log.Gen.error(err_msg);
-            throw new ApplicationException(err_msg);
+      private void myInitReorderSchema ()
+      {
+         int idColIndex = mySchema["id"].colNo;
+         int[] temp = new int [mySchema.columnInfo.Length];
+         foreach (ColumnInfo ci in mySchema.columnInfo)
+         {
+            temp[ci.colNo] =
+               ci.colNo == idColIndex ? 0 :
+               ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
          }
+         mySerIndexToSchemaIndexMap = new int[temp.Length];
+         for (int i = 0; i < temp.Length; i ++)
+            mySerIndexToSchemaIndexMap [temp[i]] = i;
       }
       #endregion
 
       #region IClientRowDeserializer Members
 
-      public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) {
-         br.ReadInt32();  //length information for the row not needed
-         myNullMap = new BitArray(br.ReadBytes(myNullMapLength));
-         object[] values = new object[mySchema.columnNames.Length];
-         for (int i = 0; i < values.Length; i++) {
+      public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
+      {
+         br.ReadInt32 ();  //length information for the row not needed
+         myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
+         object[] values = new object [mySchema.columnNames.Length];
+         for (int i = 0; i < values.Length; i++)
+         {
             int colNo = mySerIndexToSchemaIndexMap[i];
-            values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br);
+            values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br);
          }
 
 
-         r = new BlueRow(mySchema, values);
+             r = new BlueRow(mySchema, values);
 
       }
 

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs

@@ -1,149 +1,22 @@
 using System;
-using System.Collections.Generic;
+using System.Collections;
 using System.IO;
-using System.Linq;
-
-namespace BCG {
-   public class StreamIOMessage {
-      public enum StreamOp {
-         Create = 0,
-         Add = 1,             //adds a single row
-         Remove = 2,          //removes a single row
-         Change = 3,          //changes a single row
-         MultipleRemoveAdd = 4,//removes and adds multiple rows
-         MultipleChange = 5 //changes multiples rows          
-      }
-      public class CreateMessage {
-         public IBlueRowCollection rows;
-         public override string ToString() {
-            string rv = "Create message ";
-            if (rows == null) {
-               rv += " rows: null";
-               return rv;
-            }
-            foreach (IBlueRow r in rows) {
-               rv += $"id: {r["id"]} ";
-            }
-            return rv;
-         }
-      }
-      public class AddMessage {
-         public IBlueRow added_row;
-      }
-      public class RemoveMessage {
-         public IBlueRow removed_row;
-      }
-      public class ChangeMessage {
-         public IBlueRow changed_row;
-      }
-      public class MultipleRemoveAddMessage {
-         public IBlueRowCollection removed;
-         public IBlueRowCollection added;
-         public override string ToString() {
-            string rv = "MultipleRemoveAdd removed: ";
-            foreach (IBlueRow r in removed) {
-               rv += $"id: {r["id"]} ";
-            }
-            rv += "added: ";
-            foreach (IBlueRow r in added) {
-               rv += $"id: {r["id"]} ";
-            }
-            return rv;
-         }
-      }
-      public class MultipleChangeMessage {
-         public IBlueRowCollection changed;
-      }
-
-      public StreamOp op;
-      public CreateMessage create_message = null;
-      public AddMessage add_message = null;
-      public RemoveMessage remove_message = null;
-      public ChangeMessage change_message = null;
-      public MultipleRemoveAddMessage multiple_remove_add_message = null;
-      public MultipleChangeMessage multiple_change_message = null;
-      public StreamIOMessage(byte op_, object data) {
-         op = (StreamIOMessage.StreamOp)(op_);
-         switch (op) {
-            case StreamIOMessage.StreamOp.Create:
-               IBlueRowCollection state = null;
-               // TODO, not sure why it can be an array and by itself
-               if (data is IBlueRowCollection) state = data as IBlueRowCollection;
-               else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0);
-               create_message = new CreateMessage() { rows = state };
-               break;
-            case StreamIOMessage.StreamOp.Add:
-               bts.Log.assert(false);
-               break;
-            case StreamIOMessage.StreamOp.Remove:
-               bts.Log.assert(false);
-               break;
-            case StreamIOMessage.StreamOp.Change:
-               bts.Log.assert(false);
-               break;
-            case StreamIOMessage.StreamOp.MultipleRemoveAdd:
-               IBlueRowCollection[] row_args = data as IBlueRowCollection[];
-               multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] };
-               break;
-            case StreamIOMessage.StreamOp.MultipleChange:
-               //IBlueRowCollection[] row_args = data as IBlueRowCollection[];
-               multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] };
-               break;
-         }
-      }
-
-
-      public override string ToString() {
-         switch (op) {
-            case StreamIOMessage.StreamOp.Create:
-               return create_message.ToString();
-               break;
-            case StreamIOMessage.StreamOp.Add:
-               bts.Log.assert(false);
-               return String.Empty;
-               break;
-            case StreamIOMessage.StreamOp.Remove:
-               bts.Log.assert(false);
-               return String.Empty;
-               break;
-            case StreamIOMessage.StreamOp.Change:
-               bts.Log.assert(false);
-               return String.Empty;
-               break;
-            case StreamIOMessage.StreamOp.MultipleRemoveAdd:
-               return multiple_remove_add_message.ToString();
-               break;
-            case StreamIOMessage.StreamOp.MultipleChange:
-               return multiple_change_message.ToString();
-               break;
-         }
-         bts.Log.assert(false);
-         return String.Empty;
-      }
-      public static IEnumerable<StreamIOMessage> assemble_msgs(byte[] ops, Array op_data) {
-         int i = 0;
-         List<StreamIOMessage> rv = new List<StreamIOMessage>();
-         foreach (byte op in ops) {
-            object data = op_data.GetValue(i++);
-            rv.Add(new StreamIOMessage(op, data));
-         }
-         return rv;
-      }
-   }
 
+namespace BCG
+{
    /// <summary>
    /// Performs deserialization inside the client app. The client gets either the complete data set or the
    /// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
    /// </summary>
-   public interface IClientDeserializer {
+   public interface IClientDeserializer 
+   {
       /// <summary>
       /// deserializes the data received by the client from the pubsub
       /// </summary>
       /// <param name="serializedData">the data in byte format received from pubsub</param>
       /// <param name="theOps">the operations that the data maps onto</param>
       /// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
-      void deserialize(Stream[] streams, out byte[] theOps, out Array theRows);
+      void deserialize (Stream[] streams, out byte[] theOps, out Array theRows);
    }
+
 }
-
-

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs

@@ -2,11 +2,13 @@ using System;
 using System.IO;
 using System.Collections;
 
-namespace BCG {
+namespace BCG
+{
    /// <summary>
    /// Deserializes one row. the current state is also provided as some deserialization schemes might use it. 
    /// </summary>
-   public interface IClientRowDeserializer {
-      void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
+   public interface IClientRowDeserializer
+   {
+      void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs

@@ -16,7 +16,7 @@ namespace BCG {
 
       public PubTablePublisher(string tag, PublisherCore publisherCore) {
          _tag = tag;
-         bts.Log.Gen.info($"PubTablePublisher.cons created for tag: {tag}");
+         bts.Log.Gen.info($"PubTablePublisher created for tag: {tag}");
          myPubCore = publisherCore;
          mySerializer = new BlueDataSerializationController();
       }
@@ -54,14 +54,10 @@ namespace BCG {
                                  out data);
          bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}");
          myPublish(data, rows.schema.tableName);
-         // byte[] ops;
-         // Array rows_ds;
-         // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
       }
 
-      //private ClientDeserializationController _test_deserializer = new ClientDeserializationController();
       public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
-         byte[] data = null;
+         byte[] data;
          if (myIsRealChange(delRows, addRows))
             batchChange(addRows);
          else {
@@ -70,9 +66,6 @@ namespace BCG {
                                     out data);
             myPublish(data, delRows.schema.tableName);
          }
-         // byte[] ops;
-         // Array rows_ds;
-         // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
       }
 
       public virtual void batchChange(IBlueRowCollection rows) {

src/DataStreamSuite/DataStreamCore/Misc/IPubSubRouter.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+   public delegate void PubSubTagURLChangeDelegate(List<string> tagsMoving, string newURL);
+   
+   public interface IPubSubRouter
+   {
+
+      event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
+      event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
+      List<string> getAllPubSubSubURLs();
+      List<string> getAllPubSubPubURLs();
+      string getSubURLForTag(string tag);
+      string getPubURLForTag(string tag);
+   }
+}

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionGrammar.y

@@ -23,7 +23,6 @@
 
 %token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, CS_STRING_CONSTANT, NULL_CONSTANT
 %token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, IDENTIFIER, EQUALS, QUOTE_OPEN, QUOTE_CLOSE
-%token LINE_COMMENT
 
 %nonassoc EQUALS
 
@@ -50,12 +49,6 @@ Statement :
    | IDENTIFIER EQUALS Statement {
       $$ = new Assignment($1.Identifier, $3);
    }
-   | LINE_COMMENT Statement {
-      $$ = $2;
-   }
-   | Statement LINE_COMMENT {
-      $$ = $1;
-   }
    ;
 
 Arguments :

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex

@@ -26,9 +26,6 @@ Comma                    ,
 QuoteOpen                \[
 QuoteClose               \]
 
-/* Comments */
-LineComment \/\/[^\n\r]*(\n|\r|\r\n)
-
 %%
 
 {Equals} {
@@ -95,8 +92,4 @@ LineComment \/\/[^\n\r]*(\n|\r|\r\n)
    return (int)DataStreamExpressionToken.IDENTIFIER;
 }
 
-{LineComment} {
-   return (int)DataStreamExpressionToken.LINE_COMMENT;
-}
-
 %%

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs

@@ -48,26 +48,14 @@ public class PrimitiveExpression : StatementExpression {
       return PrimitiveManager.instance.build(myName, resolvedArgs);
    }
 
-   string myPrintSingleArg(int atColumn, object? o, IList<string> activeBindings) {
-      /// Unwrap <see cref="Argument"/> to get its underlying value
-      while (o is Argument arg) {
-         o = arg.Obj;
-         // Safeguard against infinite loop
-         if (ReferenceEquals(o, arg)) {
-            throw new ArgumentException(
-               $"Detected circular reference in argument " +
-               $"('{nameof(arg)}.{nameof(arg.Obj)}' is '{nameof(arg)}' itself)");
-         }
-      }
 
-      string result = new string(' ', atColumn);
-      if (o is null) {
-         return result;
-      }
+   string myPrintSingleArg(int atColumn, object o, IList<string> activeBindings) {
       if (o is StatementExpression)
          return new string(' ', atColumn) +
             ((o as StatementExpression)?.print(atColumn, activeBindings) ?? string.Empty);
       else {
+         string result = new string(' ', atColumn);
+
          if (o is string)
             result += makeCSString((string)o);
          else if (o is DateTime)