fix build: Add a few includes, suppress some werrors

Not sure how this slipped through the cracks, but I'm guessing the build
broke when we upgraded the cpp compilers (to clang 17 and gcc 13).

* Include <cstdint> in a few places
* Suppress -Wstringop-overflow in sparkd-gcc-release build
* Change const char * param to explicit std::string for proper type
  deduction in call to 'std::make_tuple'. For some reason, the gcc build
  was failing with an array-bounds werror (changing the arg fixed it).

.editorconfig

@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles
 
-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case
 
-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion
 
-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion
 
-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion

.gitignore

@@ -47,8 +47,8 @@ trace_data/
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
 /src/server/sparkd/cmds/spartan/analysis/data
-*.tmp/*
-/.venv*/*
+.tmp/
+.venv*/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/

CMakePresets.json

@@ -29,8 +29,8 @@
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
       }
     },
 
@@ -143,7 +143,7 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
       }
     },
@@ -197,7 +197,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0"
       }
     },
 

docker/sparkd/dev/Dockerfile

@@ -1,65 +1,74 @@
-FROM ubuntu:latest
+FROM ubuntu:22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
-ENV TZ=America/New_York
 
-# light deps (should be quick to download/install)
-RUN apt-get update && apt-get install -y \
-    ca-certificates \
-    curl \
-    gnupg \
-    liblttng-ust-dev \
-    libpcap0.8-dev \
-    locales \
-    ninja-build \
-    software-properties-common \
-    sqlite3 \
-    tzdata
+# Note installations that take a while (more than 10s)
 
-# clang
-RUN apt-get update && apt-get install -y \
-    clang-13 \
-    clang-format-13 \
-    clangd-13 \
-    clang-15 \
-    clang-format-15 \
-    clangd-15
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
 
-# g++
-RUN apt-get update && apt-get install -y \
-    g++-11 \
-    g++-12
-
-# python
-RUN apt-get update && apt-get install -y \
+# Python: ~30s
+RUN apt-get install -y \
     python3-pip \
     python3-venv \
     && python3 -m pip install --upgrade pip
 
-# dotnet
-RUN apt-get update && apt-get install -y \
-    dotnet7
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-get update \
+    && apt-get install -y g++-$gcc_major
 
-# ssh
-RUN apt-get update && apt-get install -y \
-    libssh2-1-dev \
-    openssh-server
+# Clang
+ARG clang_major=17
+RUN curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+    | tee -a "/etc/apt/sources.list.d/llvm.list" \
+    && apt-get update \
+    && apt-get install -y \
+    clang-$clang_major \
+    clang-format-$clang_major \
+    clangd-$clang_major
 
-# cmake
-RUN \
-    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+# CMake
+RUN curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
     | gpg --dearmor - \
     | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
     && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
     | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
     && apt-get update \
-    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
-    && apt-get install -y kitware-archive-keyring \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
-    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
-# node
+# Dotnet
+RUN curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" \
+    && dpkg -i packages-microsoft-prod.deb \
+    && rm packages-microsoft-prod.deb \
+    && apt-get update \
+    && apt-get install -y dotnet-sdk-8.0
+
+# Node
 ARG node_major=18
 RUN mkdir -p /etc/apt/keyrings \
     && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
@@ -69,60 +78,35 @@ RUN mkdir -p /etc/apt/keyrings \
     && apt-get update \
     && apt-get install -y nodejs
 
-# ccache is done here so that ccache sees clang++ installed, not clang-10
-RUN apt-get update && apt-get install -y \
-    ccache 
-
-# emumba (light deps)
+# Emumba deps
 RUN apt-get update && apt-get install -y \
     build-essential \
     doxygen \
-    python3-sphinx \
     graphviz \
     libgtest-dev \
     libssl-dev \
+    pkg-config \
+    python3-sphinx \
     rapidjson-dev \
-    zlib1g-dev \
-    pkg-config
+    zlib1g-dev
 
-# emumba (boost)
-RUN apt-get update && apt-get install -y \
-    libboost-filesystem-dev \
-    libboost-program-options-dev \
-    libboost-test-dev \
-    libboost-date-time-dev
-
-# dev utils
-RUN apt-get update && apt-get install -y \ 
-    fish \
-    git-all \
-    sudo \
-    vim \
-    wget
-
-# set up default executables (update-alternatives)
-RUN \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
-    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+# Set up default executables (update-alternatives)
+RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
 
 # Add compiler symlinks for ccache 
-RUN \
-    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang; \
     ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc; \
     ln -s /usr/bin/ccache /usr/local/bin/g++; \
-    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
-    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
-    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
-    ln -s /usr/bin/ccache /usr/local/bin/g++-12
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-$gcc_major; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-$gcc_major
 
 RUN locale-gen en_US.UTF-8
 
@@ -130,8 +114,7 @@ ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
 
-RUN \
-    groupadd -g "$docker_uid" "$docker_user" \
+RUN groupadd -g "$docker_uid" "$docker_user" \
     && useradd \
     -m \
     -u "$docker_uid" \

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -42,7 +42,6 @@ OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
 Negate                   '!'
-MapsTo                   ':'
 IsOfType                 '/'
 
 %%
@@ -98,10 +97,6 @@ IsOfType                 '/'
    return (int)Token.NEGATE;
 }
 
-{MapsTo} {
-   return (int)Token.MAPS_TO;
-}
-
 {Equal} {
    return (int)Token.EQUAL;
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnExpression">
+    <Scanner Include="ColumnExpressionTokens.lex" />
+    <Parser Include="ColumnExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -0,0 +1,62 @@
+namespace Bts.ColumnExpressions;
+
+public class ColumnExpression {
+   public ColumnExpression(string name, Type? type = null, object? value = null) {
+      Name = name;
+      Type = type
+         ?? value?.GetType()
+         ?? throw new ArgumentException(
+            "Type must be specified explicitly when value is null", nameof(type));
+      Value = value
+         ?? DBNull.Value;
+   }
+
+   public string Name { get; }
+
+   public Type Type { get; }
+
+   public object Value { get; }
+
+   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
+      // Try to convert value to 'type' to verify they are compatible
+      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
+      // account for implicit conversions (e.g., int -> long), just inheritance.
+      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
+         throw new ArgumentException(
+            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
+      }
+      return new(name, type ?? typeof(T), value);
+   }
+
+   internal static Type ParseType(string typeId) {
+      return Candidates().ExceptNull().FirstOrDefault() ??
+         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
+
+      IEnumerable<Type?> Candidates() {
+         if (typeId.Length == 0) {
+            yield break;
+         }
+         yield return Type.GetType(typeId);
+         yield return Type.GetType($"System.{typeId}");
+         if (typeId.Length > 1) {
+            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
+         }
+         yield return typeId.ToLower() switch {
+            "float64" or "double" or "real64" or "number" => typeof(double),
+            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
+            "int16" or "short" or "integer16" or "signed16" => typeof(short),
+            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
+            "int64" or "long" or "integer64" or "signed64" => typeof(long),
+            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
+            "uint32" or "uint" or "unsigned" => typeof(uint),
+            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
+            "char" => typeof(char),
+            "byte" or "uint8" => typeof(byte),
+            "bool" or "boolean" => typeof(bool),
+            "string" or "text" => typeof(string),
+            "date" or "datetime" => typeof(DateTime),
+            _ => null
+         };
+      }
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -0,0 +1,49 @@
+%namespace Bts.ColumnExpressions
+%partial
+%visibility internal
+%tokentype ColumnExpressionToken
+%parsertype ColumnExpressionParser
+
+%union {
+   public int Integer;
+   public double Real;
+   public bool Boolean;
+   public string String;
+   public string Identifier;
+   public string TypeId;
+   public DateTime DateTime;
+   public Tuple<string, Type> ColumnDefinition;
+   public ColumnExpression ColumnExpression;
+   public ImmutableList<ColumnExpression> ColumnExpressions;
+}
+
+%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
+%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
+
+%nonassoc EQUALS
+
+%type <ColumnDefinition> ColumnDefinition
+%type <ColumnExpression> ColumnExpression
+%type <ColumnExpressions> ColumnExpressions
+
+%%
+
+ColumnExpressions :
+   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
+   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
+   ;
+
+ColumnExpression :
+   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
+   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
+   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
+   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
+   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
+   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
+   ;
+
+ColumnDefinition :
+   IDENTIFIER { $$ = new($1.Identifier, null); }
+   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
+   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
+   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnExpressions;
+
+using System.IO;
+using System.Text;
+
+public class ColumnExpressionInterpreter {
+   public static IList<ColumnExpression> Evaluate(Stream stream) {
+      var parser = new ColumnExpressionParser();
+      var res = parser.Parse(stream);
+      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
+         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
+   }
+
+   public static IList<ColumnExpression> Evaluate(string str) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
+      return Evaluate(ms);
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnExpressions;
+
+using System.Text;
+
+partial class ColumnExpressionParser {
+   public ColumnExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -0,0 +1,9 @@
+namespace Bts.ColumnExpressions;
+
+partial class ColumnExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -0,0 +1,85 @@
+%namespace Bts.ColumnExpressions
+%scannertype ColumnExpressionScanner
+%visibility internal
+%tokentype ColumnExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+BooleanConstant          true|false
+StringConstant           '([^\'\\]*|\\.)*'
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+IntegerConstant          -?[0-9]+
+NullConstant             'null'
+RealConstant             -?[0-9]+\.[0-9]+
+
+/* Relational Operators */
+Equals                   =
+OfType                   /
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+NamespacePrefix          {Identifier}\.
+TypeId                   {NamespacePrefix}*{Identifier}
+
+/* Utility Tokens */
+OpenParenthesis          \(
+CloseParenthesis         \)
+Comma                    ,
+QuoteOpen                \[
+QuoteClose               \]
+
+/* Comments */
+LineComment \/\/[^\n\r]*(\n|\r|\r\n)
+
+%%
+
+{Equals} {
+   return (int)ColumnExpressionToken.EQUALS;
+}
+
+{OfType} {
+   return (int)ColumnExpressionToken.OF_TYPE;
+}
+
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
+   return (int)ColumnExpressionToken.STRING_CONSTANT;
+}
+
+{BooleanConstant} {
+   yylval.Boolean = yytext.ToLower() switch {
+      "true" => true,
+      "false" => false,
+      _ => throw new Exception(
+         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
+   };
+   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
+}
+
+{DateTimeConstant} {
+   yylval.DateTime = DateTime.Parse(yytext);
+   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
+}
+
+{IntegerConstant} {
+   yylval.Integer = int.Parse(yytext);
+   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
+}
+
+{RealConstant} {
+   yylval.Real = double.Parse(yytext);
+   return (int)ColumnExpressionToken.REAL_CONSTANT;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnExpressionToken.IDENTIFIER;
+}
+
+{TypeId} {
+   yylval.TypeId = yytext;   
+   return (int)ColumnExpressionToken.TYPE_ID;
+}
+
+%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnSortExpression">
+    <Scanner Include="ColumnSortExpressionTokens.lex" />
+    <Parser Include="ColumnSortExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -0,0 +1,11 @@
+namespace Bts.ColumnSortExpressions;
+
+public class ColumnSortExpression {
+   public ColumnSortExpression(string name, bool isAscending = true) {
+      Name = name;
+      IsAscending = isAscending;
+   }
+
+   public string Name { get; }
+   public bool IsAscending { get; }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -0,0 +1,29 @@
+%namespace Bts.ColumnSortExpressions
+%partial
+%visibility internal
+%tokentype ColumnSortExpressionToken
+%parsertype ColumnSortExpressionParser
+
+%union {
+   public string Identifier;
+   public ColumnSortExpression Column;
+   public ImmutableList<ColumnSortExpression> Columns;
+}
+
+%token ASCENDING, DESCENDING, IDENTIFIER
+
+%type <Column> Column
+%type <Columns> Columns
+
+%%
+
+Columns :
+   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
+   | Columns Column { $$ = $1.Add($2); }
+   ;
+
+Column :
+   IDENTIFIER { $$ = new($1.Identifier, true); }
+   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
+   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
+   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnSortExpressions;
+
+using System.IO;
+using System.Text;
+
+public class ColumnSortExpressionInterpreter {
+   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
+      var parser = new ColumnSortExpressionParser();
+      var res = parser.Parse(stream);
+      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
+         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
+   }
+
+   public static IList<ColumnSortExpression> Evaluate(string str) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
+      return Evaluate(ms);
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnSortExpressions;
+
+using System.Text;
+
+partial class ColumnSortExpressionParser {
+   public ColumnSortExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnSortExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -0,0 +1,9 @@
+namespace Bts.ColumnSortExpressions;
+
+partial class ColumnSortExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -0,0 +1,30 @@
+%namespace Bts.ColumnSortExpressions
+%scannertype ColumnSortExpressionScanner
+%visibility internal
+%tokentype ColumnSortExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+Ascending                +
+Descending               -
+
+%%
+
+{Ascending} {
+   return (int)ColumnSortExpressionToken.ASCENDING;
+}
+
+{Descending} {
+   return (int)ColumnSortExpressionToken.DESCENDING;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnSortExpressionToken.IDENTIFIER;
+}
+
+%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.DataStream.Websocket/ClientSubscription.cs

@@ -0,0 +1,9 @@
+namespace Bts.DataStream.Websocket;
+
+class ClientSubscription(WebsocketClientID id, SubscriptionParams @params, IDisposable dispose) : IDisposable {
+   public SubscriptionParams Params => @params;
+   public WebsocketClientID Id => id;
+
+   public void Dispose() =>
+      dispose.Dispose();
+}

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -0,0 +1,301 @@
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.collections;
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+using Spark.Core.Json;
+
+using System.Buffers;
+using System.Reactive.Subjects;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+/// <summary>
+/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
+/// requests (from clients) as input, and emits data/notifications (to clients) as output.
+/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
+/// </summary>
+public class DataSync(JsonSerializerOptions? serializerOpts = null) :
+   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
+   protected readonly object lockObj = new();
+   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+
+   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
+
+   public void OnCompleted() =>
+      Log.Api.info("DataSync incoming request stream has completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"DataSync error: {error}");
+
+   public void OnNext(WebsocketConnectionUpdate value) {
+      switch (value.Type) {
+         case WebsocketConnectionUpdateType.IncomingMessage:
+            onIncomingMessage(value.IncomingMessage);
+            break;
+         case WebsocketConnectionUpdateType.Connect:
+            onConnect(value.Connect);
+            break;
+         case WebsocketConnectionUpdateType.Disconnect:
+            onDisconnect(value.Disconnect);
+            break;
+         default:
+            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
+            break;
+      }
+   }
+
+   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
+      lock (lockObj) {
+         requestSend += observer.OnNext;
+         return Disposable.Create(Dispose);
+      }
+      void Dispose() {
+         lock (lockObj) {
+            requestSend -= observer.OnNext;
+         }
+      }
+   }
+
+   static JsonRpcId? extractRpcId(string messageBody) {
+      try {
+         return JsonNode.Parse(messageBody)
+            ?.AsObject()
+            ?.GetValueOrDefault("id")
+            ?.Deserialize<JsonRpcId>();
+      } catch {
+         return null;
+      }
+   }
+
+   static string getRpcNotification(DataPacket dp) {
+      using (var buffer = new MemoryStream())
+      using (var writer = new Utf8JsonWriter(buffer))
+      using (writer.WriteObject()) {
+         writer.WriteString("method", "subscription");
+         using (writer.WriteObject("params")) {
+            writer.Write("tableName", dp.schema.tableName);
+            writer.Write("columns", dp.schema.columnInfo);
+            var cellJsonFactories = dp.schema.columnInfo.Select(getCellJsonFactory).ToList();
+            using (writer.WriteArray("added")) {
+               foreach (IBlueRow row in (dp.addedRows?.Cast<IBlueRow>()).empty_if_null()) {
+                  using (writer.WriteObject()) {
+                     for (int i = 0; i < cellJsonFactories.Count; i++) {
+                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
+                     }
+                  }
+               }
+            }
+            using (writer.WriteArray("deleted")) {
+               foreach (IBlueRow row in (dp.deletedRows?.Cast<IBlueRow>()).empty_if_null()) {
+                  using (writer.WriteObject()) {
+                     for (int i = 0; i < cellJsonFactories.Count; i++) {
+                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
+                     }
+                  }
+               }
+            }
+         }
+         writer.WriteString("jsonrpc", "2.0");
+         writer.Flush();
+         return Encoding.UTF8.GetString(buffer.ToArray());
+      }
+   }
+
+   static Func<IBlueRow, JsonNode?> getCellJsonFactory(ColumnInfo colInfo) {
+      var t = colInfo.colType;
+      if (t == typeof(int)) {
+         return NullCheckWrapper(v => (int)v);
+      }
+      if (t == typeof(uint)) {
+         return NullCheckWrapper(v => (uint)v);
+      }
+      if (t == typeof(long)) {
+         return NullCheckWrapper(v => (long)v);
+      }
+      if (t == typeof(ulong)) {
+         return NullCheckWrapper(v => (ulong)v);
+      }
+      if (t == typeof(short)) {
+         return NullCheckWrapper(v => (short)v);
+      }
+      if (t == typeof(ushort)) {
+         return NullCheckWrapper(v => (ushort)v);
+      }
+      if (t == typeof(double)) {
+         return NullCheckWrapper(v => (double)v);
+      }
+      if (t == typeof(float)) {
+         return NullCheckWrapper(v => (float)v);
+      }
+      if (t == typeof(bool)) {
+         return NullCheckWrapper(v => (bool)v);
+      }
+      return NullCheckWrapper(v => (string)v);
+
+      Func<IBlueRow, JsonNode?> NullCheckWrapper(Func<object, JsonNode> toJson) =>
+         r => {
+            var obj = r[colInfo.colNo];
+            if (obj is null) {
+               return null;
+            }
+            return toJson(obj);
+         };
+   }
+
+   void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, []);
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   void onIncomingMessage(WebsocketIncomingMessage message) {
+      var id = message.sender_client_id;
+      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
+         onBadRequest(message, err);
+         return;
+      }
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "publish":
+            Log.Api.error("DataStreamWebsocket api 'publish' not implemented.");
+            // TODO
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
+      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
+      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
+      lock (lockObj) {
+         if (requestSend is null) {
+            Log.Api.warn(
+               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
+            return false;
+         }
+         requestSend(messages);
+         return true;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += OnChange;
+      }
+
+      void OnChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var notif = getRpcNotification(dp);
+         send([new(id, notif)]);
+      }
+
+      void Dispose() =>
+         stream.onChange -= OnChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
+      var rpcId = extractRpcId(req.message_body);
+      var logMsg = $"Recieved invalid request:\n{req.message_body}";
+      if (errorMsg is not null) {
+         logMsg += $"\n\n**Exception**\n{errorMsg}";
+      }
+      Log.Api.error(logMsg);
+      if (rpcId is null) {
+         return;
+      }
+      // Send error back to client
+      var rpcErr = JsonRpcError.Create(
+         (long)JsonRpcErrorCode.InvalidRequest,
+         $"Received invalid request: {errorMsg ?? "unknown error"}",
+         req.message_body);
+      sendError(req.sender_client_id, rpcId.Value, rpcErr);
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/JObjectExtensions.cs

@@ -0,0 +1,34 @@
+namespace Bts.DataStream.Websocket;
+
+using Newtonsoft.Json.Linq;
+
+using System.Diagnostics.CodeAnalysis;
+
+public static class JObjectExtensions {
+   public static bool TryParse<T>(
+      this JObject filter, string key, [MaybeNull] out T obj, [NotNullWhen(false)] out string? error) {
+      obj = default;
+      var value = filter.GetValueOrDefault(key)?.ToString();
+      if (value == null) {
+         error = $"{key} not present";
+         return true;
+      }
+      return WebsocketJsonParsers.TryParse(value, key, out obj, out error);
+   }
+
+   public static bool TryParse<T>(
+     this JObject filter, string key, out T result, [NotNullWhen(false)] out string? error, T defaultValue) {
+      var value = filter.GetValueOrDefault(key)?.ToString();
+      if (value == null) {
+         result = defaultValue;
+         error = $"{key} not present";
+         return true;
+      }
+      if (WebsocketJsonParsers.TryParse<T>(value, key, out var maybeRes, out error)) {
+         result = maybeRes ?? defaultValue;
+         return true;
+      }
+      result = defaultValue;
+      return false;
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs

@@ -0,0 +1,48 @@
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class MultilineStringJsonConverter : JsonConverter<string> {
+   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (typeToConvert != typeof(string)) {
+         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+      }
+      return reader.TokenType switch {
+         JsonTokenType.String => StringImpl(ref reader),
+         JsonTokenType.StartArray => ArrayImpl(ref reader),
+         _ => throw new ArgumentException(
+            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+      };
+
+      static string? StringImpl(ref Utf8JsonReader reader) =>
+         new(reader.GetString()!);
+
+      static string? ArrayImpl(ref Utf8JsonReader reader) {
+         var lines = new List<string>(2);
+         while (reader.Read()) {
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  lines.Add(reader.GetString()!);
+                  break;
+               case JsonTokenType.EndArray:
+                  return string.Join("\n", lines);
+            }
+         }
+         throw new JsonException(
+            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
+      var lines = value.Split("\n");
+      if (lines.Length < 2) {
+         writer.WriteRawValue(value);
+         return;
+      }
+      writer.WriteRawValue($"[{", ".join(lines)}]");
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/RequestType.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStream.Websocket;
+
+public enum RequestType {
+   Unspecified,
+   Subscribe,
+   Unsubscribe,
+   UnsubscribeAll,
+}

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -0,0 +1,25 @@
+namespace Bts.DataStream.Websocket;
+
+using bts;
+
+using Google.Protobuf;
+
+using System.Text.Json;
+
+public static class Session {
+   static int sessionIdCounter;
+
+   public static async Task Start(
+      WebsocketServerConfig config, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      var sessionId = Interlocked.Increment(ref sessionIdCounter);
+      var gateway = new WebsocketClient(config);
+      var sync = new DataSync(opts);
+      using (gateway.Subscribe(sync))
+      using (sync.Subscribe(gateway)) {
+         Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
+         Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
+         await gateway.ConnectAsync();
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -0,0 +1,112 @@
+namespace Bts.DataStream.Websocket;
+
+using bts.utils;
+
+using Spark.Core;
+using Spark.Core.Json;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionParams> {
+   static class PropNames {
+      public const string QUERY = "query";
+   }
+
+   // Serializer supports plain string, or an array of lines for convenience
+   [MaybeNull]
+   [JsonPropertyName(PropNames.QUERY)]
+   [JsonConverter(typeof(MultilineStringJsonConverter))]
+   public string? Query { get; set; } = query;
+
+   public bool Equals(SubscriptionParams? other) =>
+      other is not null &&
+      // Not the most elegant equals impl, but it'll work
+      Query.safe_equals(other.Query, (a, b) => a?.ToString() == b?.ToString());
+
+   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
+      public override bool CanConvert(Type typeToConvert) =>
+         typeToConvert == typeof(SubscriptionParams);
+
+      public override SubscriptionParams Read(
+         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+         if (typeToConvert != typeof(SubscriptionParams)) {
+            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+         }
+         return read(ref reader);
+      }
+
+      static SubscriptionParams read(ref Utf8JsonReader reader) {
+         return reader.TokenType switch {
+            JsonTokenType.String => StringImpl(ref reader),
+            JsonTokenType.StartArray => ArrayImpl(ref reader),
+            JsonTokenType.StartObject => ObjectImpl(ref reader),
+            _ => throw new ArgumentException(
+               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+         };
+
+         static SubscriptionParams StringImpl(ref Utf8JsonReader reader) =>
+            new(reader.GetString()!);
+
+         static SubscriptionParams ArrayImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+
+         static SubscriptionParams ObjectImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.PropertyName:
+                     var propName = reader.GetString()!;
+                     if (propName == PropNames.QUERY) {
+                        switch (reader.TokenType) {
+                           case JsonTokenType.StartArray:
+                              return ArrayImpl(ref reader);
+                           case JsonTokenType.String:
+                              return StringImpl(ref reader);
+                        }
+                     }
+                     break;
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+      }
+
+      public override void Write(
+         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
+         var q = value.Query;
+         var lines = q?.Split("\n");
+         if (lines is null || lines.Length < 2) {
+            writer.WriteString(PropNames.QUERY, value.Query);
+            return;
+         }
+         writer.WriteArray(PropNames.QUERY, lines);
+      }
+   }
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as SubscriptionParams);
+
+   public override int GetHashCode() =>
+      Query?.Length ?? typeof(SubscriptionParams).GetHashCode();
+}

src/DataStreamSuite/Bts.DataStream.Websocket/Usings.cs

@@ -0,0 +1,5 @@
+global using bts.websocket_core;
+
+global using System;
+
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs

@@ -0,0 +1,41 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class ViewStreamUpdate {
+   [JsonPropertyName("tableName")]
+   public string? TableName { get; set; }
+
+   [JsonPropertyName("columns")]
+   public List<ColumnInfo>? Columns { get; set; }
+
+   [JsonPropertyName("addedRows")]
+   public List<Dictionary<string, object?>>? AddedRows { get; set; }
+
+   [JsonPropertyName("deletedRows")]
+   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
+
+   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
+      var schema = new BlueTableSchema([.. Columns!], TableName);
+      var addedRowsTable = new BlueTable(schema);
+      var deletedRowsTable = new BlueTable(schema);
+      if (applyChanges) {
+         state.deleteRows(deletedRowsTable);
+         state.addRows(addedRowsTable);
+      }
+      return new(state, addedRowsTable, deletedRowsTable);
+   }
+
+   public static string ToJson() {
+      return "";
+   }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStream.Websocket\Bts.DataStream.Websocket.csproj" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServer.conf

@@ -0,0 +1,7 @@
+{
+  "host": "localhost",
+  "port": 8080,
+  "scheme": "ws",
+  "default_end_point": "/",
+  "keep_alive_interval_secs": 1
+}

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs

@@ -0,0 +1,76 @@
+using bts.api.core;
+
+using Bts.DataStream.Websocket;
+
+using Google.Protobuf;
+
+using System.CommandLine.Parsing;
+using System.Text.Json;
+
+static WebsocketServerConfig readConfigFile(string path) {
+   using var fs = File.OpenRead(path);
+   using var sr = new StreamReader(fs);
+   return JsonParser.Default.Parse<WebsocketServerConfig>(sr);
+}
+
+static WebsocketServerConfig? readConfigArg(ArgumentResult result) {
+   var path = result.Tokens.Select(t => t.Value).FirstOrDefault() ?? string.Empty;
+   return readConfigFile(path);
+}
+
+static WebsocketServerConfig getDefaultConfig() {
+   const string defaultCfgPath = "DataStreamWebsocketServer.conf";
+   if (File.Exists(defaultCfgPath)) {
+      return readConfigFile(defaultCfgPath);
+   }
+   return new() {
+      Scheme = "http",
+      Host = "localhost",
+      Port = 8080,
+      DefaultEndPoint = "/",
+      KeepAliveIntervalSecs = 1
+   };
+}
+   
+static async Task run(
+   WebsocketServerConfig? wssConfig, string user, string password, string? sparkdConfigSource) {
+   var opts = new JsonSerializerOptions(JsonSerializerDefaults.Web);
+   wssConfig ??= getDefaultConfig();
+   Console.WriteLine("Starting DataStreamWebsocketServer...");
+   var serviceCtx = ServiceContext.create(sparkdConfigSource);
+   await serviceCtx.login(user, password);
+   await Session.Start(wssConfig, opts);
+   // if (sparkdConfigSource is not null && sparkdConfigSource.Length > 9999) {
+   // } else {
+   //    await ChatGptSession.Start();
+   // }
+}
+
+var userOpt = new Option<string>("--user", () => "bts");
+var passOpt = new Option<string>("--password", () => string.Empty);
+var sparkdConfigSourceOpt = new Option<string>(
+   "--sparkd", description: "The sparkd config server url");
+// var bling = new ParseArgument<WebsocketServerConfig?>(parseWsConfig);
+
+// WebsocketServerConfig? parseWsConfig(ArgumentResult result) {
+//    Token t;
+//    TokenType tt;
+//    Symbol s;
+//    throw new NotImplementedException();
+// }
+
+var configOpt = new Option<WebsocketServerConfig?>(
+   "--config", readConfigArg, description: "WebsocketServerConfig file path");
+configOpt.SetDefaultValue(new());
+var rootCommand = new RootCommand("Starts a DataStream Websocket Server");
+rootCommand.AddOption(configOpt);
+rootCommand.AddOption(userOpt);
+rootCommand.AddOption(passOpt);
+rootCommand.SetHandler(run, configOpt, userOpt, passOpt, sparkdConfigSourceOpt);
+
+try {
+   return await rootCommand.InvokeAsync(args);
+} catch (Exception ex) {
+   Console.WriteLine($"Error: {ex}");
+}
+return 1;

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs

@@ -0,0 +1,6 @@
+global using bts.websocket_core;
+global using bts.websocket_core.json_rpc;
+
+global using System.CommandLine;
+
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.2' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>false</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\dist</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-use-websocket": "^4.5.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.2.43",
+    "@types/react-dom": "^18.2.17",
+    "@vitejs/plugin-react": "^4.2.1",
+    "eslint": "^8.55.0",
+    "eslint-plugin-react": "^7.33.2",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.5",
+    "vite": "^5.0.8"
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -0,0 +1,44 @@
+#root {
+    max-width: 1280px;
+    margin: 0 auto;
+    padding: 2rem;
+    text-align: center;
+    --cell-background-color: #293136;
+    --cell-color: #cbe3ef;
+}
+
+tr:nth-child(even) {
+    background: #F2F2F2;
+}
+
+tr:nth-child(odd) {
+    background: #FFF;
+}
+
+th, td {
+    padding-left: 1rem;
+    padding-right: 1rem;
+    background-color: var(--cell-background-color);
+    color: var(--cell-color);
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -0,0 +1,88 @@
+import { Component } from "react";
+import "./App.css";
+
+export class Home extends Component {
+  static displayName = Home.name;
+  static sparkdHost = "localhost";
+
+  render() {
+    let socket = null;
+    const wsUrl = `ws://localhost:5268/ws`;
+
+    function connectToSparkd() {
+      if (socket == null) {
+        socket = new WebSocket(wsUrl);
+      }
+      if (socket && socket.readyState === WebSocket.OPEN) {
+        return;
+      }
+
+      const output = document.getElementById("query-output");
+      socket.onopen = (event) => {
+        console.log(`Connected to ${wsUrl}`);
+      };
+
+      socket.onmessage = (event) => {
+        console.log(`Received message: ${event.data}`);
+        output.innerHTML = event.data;
+      };
+
+      socket.onerror = (error) => {
+        console.log(`Error on websocket ${wsUrl}: ${error}`);
+      };
+
+      socket.onclose = (event) => {
+        if (event.wasClean) {
+          const msg = `Connection to ${wsUrl} closed cleanly, code=${event.code}, reason=${event.reason}`;
+          console.log(msg);
+        } else {
+          const msg = `Connection to ${wsUrl} abruptly closed`;
+          console.log(msg);
+        }
+      };
+    }
+
+    function executeQuery() {
+      if (socket == null || !socket) {
+        connectToSparkd();
+      }
+      if (socket == null || !socket) {
+        console.log("Failed to connect to websocket server");
+        return;
+      }
+      if (socket.readyState === WebSocket.OPEN) {
+        const queryInput = document.getElementById("query-input");
+        const rpcReq = {
+          jsonrpc: "2.0",
+          method: "subscribe",
+          params: {
+            query: queryInput.value,
+          },
+          id: 1,
+        };
+        socket.send(JSON.stringify(rpcReq));
+      }
+    }
+
+    return (
+      <div>
+        <h1>Data Stream Query Analyzer</h1>
+        <p>Run queries against the BTS Cloud</p>
+        <div>
+          <textarea id="query-input" placeholder="Enter Query"></textarea>
+          <button id="query-button" onClick={executeQuery}>
+            Execute
+          </button>
+        </div>
+        <table id="query-output"></table>
+        <p id="footer">
+          The <code>ClientApp</code> subdirectory is a standard React
+          application based on the <code>create-react-app</code> template. If
+          you open a command prompt in that directory, you can run{" "}
+          <code>npm</code> commands such as <code>npm test</code> or{" "}
+          <code>npm install</code>.
+        </p>
+      </div>
+    );
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -0,0 +1,68 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -0,0 +1,10 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import {Home} from './App.jsx'
+import './index.css'
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <Home />
+  </React.StrictMode>,
+)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -0,0 +1,60 @@
+import { fileURLToPath, URL } from 'node:url';
+
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+
+const baseFolder =
+    process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+        ? `${process.env.APPDATA}/ASP.NET/https`
+        : `${process.env.HOME}/.aspnet/https`;
+
+const certificateArg = process.argv.map(arg => arg.match(/--name=(?<value>.+)/i)).filter(Boolean)[0];
+const certificateName = certificateArg ? certificateArg.groups.value : "bts.datastreamwebapp.client";
+
+if (!certificateName) {
+    console.error('Invalid certificate name. Run this script in the context of an npm/yarn script or pass --name=<<app>> explicitly.')
+    process.exit(-1);
+}
+
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+    if (0 !== child_process.spawnSync('dotnet', [
+        'dev-certs',
+        'https',
+        '--export-path',
+        certFilePath,
+        '--format',
+        'Pem',
+        '--no-password',
+    ], { stdio: 'inherit', }).status) {
+        throw new Error("Could not create certificate.");
+    }
+}
+
+// https://vitejs.dev/config/
+export default defineConfig({
+    plugins: [plugin()],
+    resolve: {
+        alias: {
+            '@': fileURLToPath(new URL('./src', import.meta.url))
+        }
+    },
+    server: {
+        proxy: {
+            '^/weatherforecast': {
+                target: 'https://localhost:7128/',
+                secure: false
+            }
+        },
+        port: 5173,
+        https: {
+            key: fs.readFileSync(keyFilePath),
+            cert: fs.readFileSync(certFilePath),
+        }
+    }
+})

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy">
+      <Version>8.*-*</Version>
+    </PackageReference>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.http

@@ -0,0 +1,6 @@
+@Bts.DataStreamWebApp.Server_HostAddress = http://localhost:5268
+
+GET {{Bts.DataStreamWebApp.Server_HostAddress}}/weatherforecast/
+Accept: application/json
+
+###

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -0,0 +1,107 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.api.core;
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using Microsoft.AspNetCore.Mvc;
+
+using Spark.Core.Disposables;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
+   readonly ILogger<DataStreamSubscriptionController> logger = logger;
+   IDisposable? streamSubscription;
+
+   [Route("/ws")]
+   public async Task Get() {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      Console.WriteLine("Establishing websocket connection...");
+      using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+      Console.WriteLine($"Websocket connection established.");
+      await runWebSocketServer(null, "bts", string.Empty, webSocket);
+   }
+
+   public static async Task Echo(WebSocket webSocket) {
+      var buffer = new byte[1024 * 4];
+      var receiveResult = await webSocket.ReceiveAsync(
+          new ArraySegment<byte>(buffer), CancellationToken.None);
+
+      while (!receiveResult.CloseStatus.HasValue) {
+         await webSocket.SendAsync(
+             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+
+         receiveResult = await webSocket.ReceiveAsync(
+             new ArraySegment<byte>(buffer), CancellationToken.None);
+      }
+
+      await webSocket.CloseAsync(
+          receiveResult.CloseStatus.Value,
+          receiveResult.CloseStatusDescription,
+          CancellationToken.None);
+   }
+
+   async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
+      var context = ServiceContext.create(sparkd);
+      await context.login(user, password);
+      var buffer = new byte[1024 * 4];
+      var receiveResult = await webSocket.ReceiveAsync(
+          new ArraySegment<byte>(buffer), CancellationToken.None);
+
+      try {
+         while (!receiveResult.CloseStatus.HasValue) {
+            receiveResult = await webSocket.ReceiveAsync(
+                new ArraySegment<byte>(buffer), CancellationToken.None);
+            var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
+            var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
+            var query = rpcReq?.Params?.Query ?? throw new InvalidOperationException(
+               "Failed to parse JsonRpcRequest.Params");
+            var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+            if (exp is null) {
+               Console.WriteLine($"Invalid query expression: {query}.");
+               continue;
+            }
+            var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
+            Disposable.dispose(ref streamSubscription);
+            vs.onChange += OnStreamChanged;
+            streamSubscription = Disposable.Create(() => vs.onChange -= OnStreamChanged);
+         }
+      } catch (Exception ex) {
+         Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
+         return 1;
+      }
+
+      return 0;
+
+      async void OnStreamChanged(object newValue) {
+         if (newValue is not DataPacket dp) {
+            Console.WriteLine(
+               $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
+            return;
+         }
+         Console.WriteLine("Received stream update.");
+         var added_rows_str = dp.addedRows.get_printed_rows(val => val.Length);
+         var msg = $"Received new rows:\n{added_rows_str}";
+         var encoded = Encoding.UTF8.GetBytes(msg);
+         var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
+         await webSocket.SendAsync(
+             payload,
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/WeatherForecastController.cs

@@ -0,0 +1,22 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+using Microsoft.AspNetCore.Mvc;
+
+[ApiController]
+[Route("[controller]")]
+public class WeatherForecastController(ILogger<WeatherForecastController> logger) : ControllerBase {
+   static readonly string[] Summaries = [
+      "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
+   ];
+
+   readonly ILogger<WeatherForecastController> _logger = logger;
+
+   [HttpGet(Name = "GetWeatherForecast")]
+   public IEnumerable<WeatherForecast> Get() {
+      return Enumerable.Range(1, 5).Select(index => new WeatherForecast {
+         Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)),
+         TemperatureC = Random.Shared.Next(-20, 55),
+         Summary = Summaries[Random.Shared.Next(Summaries.Length)]
+      })
+      .ToArray();
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using System.Text.Json.Serialization;
+
+public class DataStreamSubscription {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamUpdate.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using BCG;
+
+class DataStreamUpdate {
+   public List<IBlueRow> addedRows { get; set; } = new(0);
+   public List<IBlueRow> deletedRows { get; set; } = new(0);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerOptions.cs

@@ -0,0 +1,14 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using CommandLine;
+
+class DataStreamWebAppServerOptions {
+   [Option("sparkd")]
+   public string? sparkd { get; set; }
+
+   [Option("user", Default = "bts")]
+   public string? user { get; set; }
+
+   [Option("password", Default = "")]
+   public string? password { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebsocketHandler.cs

@@ -0,0 +1,58 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Net.WebSockets;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Bts.DataStreamWebApp.Server {
+   // public class DataStreamWebSocketHandler {
+   //    readonly WebSocketConnectionManager _connectionManager = new();
+   //    readonly ILogger<DataStreamWebSocketHandler> _logger;
+
+   //    public DataStreamWebSocketHandler(
+   //       WebSocketConnectionManager connectionManager,
+   //       ILogger<DataStreamWebSocketHandler> logger
+   //    ) {
+   //       _connectionManager = connectionManager;
+   //       _logger = logger;
+   //    }
+
+   //    public async Task HandleWebSocket(HttpContext context, WebSocket webSocket) {
+   //       var socketId = _connectionManager.AddSocket(webSocket);
+
+   //       _logger.LogInformation($"WebSocket connection established with ID {socketId}");
+
+   //       while (webSocket.State == WebSocketState.Open) {
+   //          var message = await ReceiveMessageAsync(webSocket);
+   //          if (message != null) {
+   //             _logger.LogInformation($"Received message from ID {socketId}: {message}");
+   //             await BroadcastMessageAsync(message);
+   //          }
+   //       }
+
+   //       _connectionManager.RemoveSocket(socketId);
+   //       _logger.LogInformation($"WebSocket connection closed with ID {socketId}");
+   //    }
+
+   //    async Task<string?> ReceiveMessageAsync(WebSocket webSocket) {
+   //       var buffer = new byte[1024];
+   //       var result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+
+   //       if (result.CloseStatus.HasValue) {
+   //          return null;
+   //       }
+
+   //       return System.Text.Encoding.UTF8.GetString(buffer, 0, result.Count);
+   //    }
+
+   //    async Task BroadcastMessageAsync(string message) {
+   //       foreach (var socket in _connectionManager.GetAllSockets()) {
+   //          if (socket.Value.State == WebSocketState.Open) {
+   //             await socket.Value.SendAsync(System.Text.Encoding.UTF8.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
+   //          }
+   //       }
+   //    }
+   // }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -0,0 +1,159 @@
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.utils;
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+using Bts.DataStreamWebApp.Server;
+using Bts.DataStreamWebApp.Server.Controllers;
+
+using CommandLine;
+
+using mbts.config;
+
+using System.Net.WebSockets;
+using System.Reactive.Disposables;
+using System.Reactive.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+static string fmtReq(HttpRequest req) =>
+   $"{req.Protocol}://{req.Host}/{req.Path}?{req.QueryString}";
+
+static string fmtDataPacket(DataPacket dp) {
+   var sb = new StringBuilder();
+   var rows = dp.addedRows.Cast<IBlueRow>().ToList();
+   var colWidths = dp.schema.columnInfo.Select(c => rows.Max(r => r[c.colNo]?.ToString()?.Length ?? 0)).ToArray();
+   foreach (IBlueRow r in dp.addedRows) {
+      for (int i = 0; i < colWidths.Length; i++) {
+         var str = r[i]?.ToString() ?? string.Empty;
+         if (i > 0) {
+            sb.Append(" | ");
+         }
+         sb.Append(str);
+         var width = colWidths[i];
+         if (str.Length < width) {
+            sb.Append(new string(' ', width - str.Length));
+         }
+      }
+      sb.AppendLine();
+   }
+   return sb.ToString();
+}
+
+var parser = new Parser(ps => {
+   ps.CaseInsensitiveEnumValues = true;
+   ps.HelpWriter = Console.Out;
+});
+
+await parser.ParseArguments<DataStreamWebAppServerOptions>(args).
+   MapResult(run, errors => Task.FromResult(1));
+
+async Task<int> run(DataStreamWebAppServerOptions opts) {
+   BTSNativeMethods.provide_command_line(args, args.Length);
+   Configuration.initialize();
+   var context = ServiceContext.create(opts.sparkd);
+   await context.login(opts.user!, opts.password!);
+   var vsWorkLoop = Observable.Interval(TimeSpan.FromMilliseconds(200)).Subscribe(_ => {
+      ViewStreamDatabase.instance.doWork();
+      MasterWorker.getInstance(false).doWork();
+   });
+   var builder = WebApplication.CreateBuilder(args);
+   builder.Services.AddDirectoryBrowser();
+   var app = builder.Build();
+   var wsOpts = new WebSocketOptions {
+      KeepAliveInterval = TimeSpan.FromMinutes(2)
+   };
+   app.UseWebSockets(wsOpts);
+   app.UseDefaultFiles();
+   app.UseStaticFiles();
+   app.MapFallbackToFile("/index.html");
+
+   app.Use(async (context, next) => {
+      var reqDesc = fmtReq(context.Request);
+      Console.WriteLine($"Webserver received: {reqDesc}");
+      if (context.Request.Path == "/ws") {
+         if (context.WebSockets.IsWebSocketRequest) {
+            Console.WriteLine("Accepting websocket connection...");
+            using var webSocket = await context.WebSockets.AcceptWebSocketAsync();
+            Console.WriteLine("Websocket connection accepted.");
+            await runWebSocketServer(
+               opts.sparkd, opts.user!, opts.password!, webSocket);
+         } else {
+            context.Response.StatusCode = StatusCodes.Status400BadRequest;
+         }
+      } else {
+         Console.WriteLine($"Webserver ignoring req: {reqDesc}");
+         await next(context);
+      }
+   });
+
+   try {
+      await app.RunAsync();
+   } catch (Exception ex) {
+      Console.WriteLine($"Exception on DataStreamWebsocketServer thread: {ex}");
+      return 1;
+   }
+   Spark.Core.Disposables.Disposable.dispose(ref vsWorkLoop);
+   return 0;
+}
+
+async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
+   var buffer = new byte[1024 * 4];
+   var receiveResult = await webSocket.ReceiveAsync(
+       new ArraySegment<byte>(buffer), CancellationToken.None);
+
+   try {
+      while (!receiveResult.CloseStatus.HasValue) {
+         Console.WriteLine("Awaiting websocket message from client");
+         receiveResult = await webSocket.ReceiveAsync(
+             new ArraySegment<byte>(buffer), CancellationToken.None);
+         var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
+         var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
+         var queryArgs = rpcReq!.Params!;
+         // var queryArgs = new DataStreamSubscription { Query = msg };
+         var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
+         if (exp is null) {
+            Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
+            continue;
+         }
+         var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
+         Console.WriteLine("Subscribing to stream...");
+         vs.onChange += onStreamChanged;
+
+         await webSocket.SendAsync(
+             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+      }
+   } catch (Exception ex) {
+      Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
+      return 1;
+   }
+
+   return 0;
+
+   async void onStreamChanged(object newValue) {
+      if (newValue is not DataPacket dp) {
+         Console.WriteLine(
+            $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
+         return;
+      }
+      Console.WriteLine("Received stream update.");
+      var sb = new StringBuilder();
+      var updateDesc = fmtDataPacket(dp);
+      var msg = $"Received new rows:\n{updateDesc}";
+      // var payload = new ReadOnlyMemory<byte>([.. msg]);
+      var encoded = Encoding.UTF8.GetBytes(msg);
+      var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
+      await webSocket.SendAsync(
+          payload,
+          receiveResult.MessageType,
+          receiveResult.EndOfMessage,
+          CancellationToken.None);
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -0,0 +1,48 @@
+{
+  "profiles": {
+    "default": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5400"
+    },
+    "http": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "http://localhost:5268"
+    },
+    "https": {
+      "commandName": "Project",
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "https://localhost:7128;http://localhost:5268"
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      }
+    }
+  },
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:2083",
+      "sslPort": 44383
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WeatherForecast.cs

@@ -0,0 +1,12 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class WeatherForecast
+{
+   public DateOnly Date { get; set; }
+
+   public int TemperatureC { get; set; }
+
+   public int TemperatureF => 32 + (int)(TemperatureC / 0.5556);
+
+   public string? Summary { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WebsocketConnectionManager.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Concurrent;
+using System.Net.WebSockets;
+
+namespace Bts.DataStreamWebApp.Server {
+   // public class WebSocketConnectionManager {
+   //    private readonly ConcurrentDictionary<Guid, WebSocket> _sockets = new ConcurrentDictionary<Guid, WebSocket>();
+
+   //    public Guid AddSocket(WebSocket socket) {
+   //       var socketId = Guid.NewGuid();
+   //       _sockets.TryAdd(socketId, socket);
+   //       return socketId;
+   //    }
+
+   //    public WebSocket? GetSocket(Guid socketId) {
+   //       _sockets.TryGetValue(socketId, out var socket);
+   //       return socket;
+   //    }
+
+   //    public ConcurrentDictionary<Guid, WebSocket>.ValueCollection GetAllSockets() {
+   //       return _sockets.Values;
+   //    }
+
+   //    public Guid? GetSocketId(WebSocket socket) {
+   //       foreach (var (key, value) in _sockets) {
+   //          if (value == socket) {
+   //             return key;
+   //          }
+   //       }
+   //       return null;
+   //    }
+
+   //    public void RemoveSocket(Guid socketId) {
+   //       _sockets.TryRemove(socketId, out _);
+   //    }
+   // }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
@@ -9,12 +9,15 @@
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
   </ItemGroup>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -0,0 +1,26 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Reactive.Linq;
+
+public static class DataStreamExtensions {
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
+      source as ViewStream ?? new ObservableViewStream(source);
+
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
+         source.onChange += OnChange;
+         return Disposable.Create(() => source.onChange -= OnChange);
+
+         void OnChange(object? newValue) =>
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
+      }
+   }
+}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -0,0 +1,62 @@
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+//     public class PandasStream : ViewStream, IWorker {
+//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+//         }
+
+//         public PandasStream(ViewStream vs, string pandas_code) :
+//         this(vs, ViewStreamDatabase.instance, pandas_code) {
+//         }
+
+
+//         protected override void myOnFirstSubscriber() {
+
+//             // string python_code = @"
+//             //    import winsound
+//             //    frequency = 2500  # Set frequency (in Hz)
+//             //    duration = 1000   # Set duration (in milliseconds)
+//             //    winsound.Beep(frequency, duration)
+//             // ";
+//             string python_code = @"
+//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
+//             with open(file_path, \"w\") as f:
+//                 f.write(\"Hello, world!\n\")
+//             ";
+
+//             execute_python_code(python_code);
+//         }
+
+//         static void execute_python_code(string python_code) {
+//             ProcessStartInfo start = new ProcessStartInfo();
+//             start.FileName = "python"; // Python executable
+//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+//             start.UseShellExecute = false;
+//             start.RedirectStandardOutput = true;
+
+//             using (Process process = Process.Start(start)) {
+//                 using (System.IO.StreamReader reader = process.StandardOutput) {
+//                     string result = reader.ReadToEnd();
+//                     Console.Write(result);
+//                 }
+//             }
+//         }
+
+//         protected override void myOnLastSubscriber() {
+//         }
+
+//         public void doWork() {
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -0,0 +1,79 @@
+using BCG.Utilities.General;
+
+using System;
+
+namespace BCG {
+    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
+        class State {
+            public int n_sends = 0;
+            public int n_recvs = 0;
+        }
+        private readonly string _ip;
+        private readonly int _port;
+        private readonly ViewStreamDatabase _vsd;
+        State _state = new State();
+        private readonly int _msg_size;
+
+        private IBlueTableSchema _schema;
+        private TimeSpan _send_period;
+        private ITCPConnection _connection = null;
+        private byte[] _buff;
+
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
+        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
+            _ip = ip;
+            _port = port;
+            _msg_size = msg_size;
+            _vsd = vsd;
+            _schema = BlueDataUtility.form_schema("TCPClient",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("n_sends", TypeCode.UInt32),
+            Tuple.Create("n_recvs", TypeCode.UInt32));
+            _send_period = TimeSpan.FromSeconds(send_period_seconds);
+            _buff = new byte[msg_size];
+            for (int i = 0; i < _buff.Length; i++) {
+                _buff[i] = (byte)i;
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, _send_period);
+            var server_address = new BlutoSubAddress(_ip, _port);
+            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            if (_connection != null) {
+                _connection.send(_buff);
+                _state.n_sends++;
+            }
+            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
+            BlueTable rows = new BlueTable(r.schema);
+            rows.addRow(r);
+            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        public void onConnection(ITCPConnection connection) {
+            _connection = connection;
+            _connection.register(this);
+        }
+        public bool isDisposed { get; }
+
+        public void onData(RemoteMessage msg) {
+            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
+            _state.n_recvs++;
+        }
+
+        public void Dispose() {
+
+        }
+
+
+    }
+}

src/DataStreamSuite/DataStream/Misc/test.py

@@ -0,0 +1,3 @@
+file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
+with open(file_path, "w") as f:
+    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -0,0 +1,34 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
+   IDisposable? subscription;
+
+   public ObservableViewStream(IObservable<DataPacket> source) =>
+      this.source = source;
+
+   public void Dispose() {
+      subscription?.Dispose();
+      subscription = null;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(subscription);
+      subscription = source.Subscribe(myFireEvent);
+   }
+
+   protected override void myOnLastSubscriber() {
+      Log.notnull(subscription);
+      subscription.Dispose();
+      subscription = null;
+   }
+
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
+      source.Subscribe(observer);
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,11 +1,6 @@
-// using System;
-// using System.Data;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Utilities.General;
-// namespace BCG {
-
-//     public class AddColumnsStream : MapStream {
-//         public AddColumnsStream(ViewStream vs, string columnDef) : base(vs, new ColumnAdder(columnDef)) { }
-//     }
-// }
+namespace BCG;
+
+public class AddColumnsStream : MapStream {
+   public AddColumnsStream(ViewStream vs, string columnDef)
+      : base(vs, new ColumnAdder(columnDef)) { }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -0,0 +1,12 @@
+namespace bts {
+    using BCG;
+
+    using System;
+    using System.Collections;
+
+    public class AddConstantColumnRow {
+        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,63 +1,52 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Evaluator.AddColumnsExpression;
-
-// namespace BCG {
-//     /// <summary>
-//     /// Provides methods to add columns to the schema
-//     /// </summary>
-//     public class ColumnAdder : IStreamMapper {
-//         protected IDictionary myNewColumns;
-//         protected IBlueTableSchema mySchema;
-
-//         static syntax myParser =
-//            new syntax();
-
-//         /// <summary>
-//         /// string represting the columns to be added to the output table. 
-//         /// </summary>
-//         /// <param name="colAddExpression"></param>
-//         public ColumnAdder(string columnDef) {
-//             AddColumnsExpression ace;
-
-//             if (DataStreamLoadFileConfig.instance.useStaticParser) {
-//                 lock (myParser) {
-//                     ace =
-//                     (AddColumnsExpression)myParser.Parse(columnDef);
-//                 }
-//             } else {
-//                 syntax parser = new syntax();
-//                 ace =
-//                 (AddColumnsExpression)parser.Parse(columnDef);
-//             }
-
-//             if (ace == null)
-//                 throw new Exception("Invalid columnn expression: " + columnDef);
-//             myNewColumns = ace.columns;
-//         }
-
-//         public virtual IBlueRow map(IBlueRow row) {
-//             if (mySchema == null)
-//                 myConstructSchema(row.schema);
-//             return new AddColumnsRow(mySchema, row);
-//         }
-
-//         public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-//             myConstructSchema(inputSchema);
-//             return mySchema;
-//         }
-
-//         private void myConstructSchema(IBlueTableSchema schema) {
-//             ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
-//             int ciIndex = 0;
-//             foreach (ColumnInfo ci in schema.columnInfo)
-//                 cis[ciIndex++] = ci;
-//             foreach (DictionaryEntry de in myNewColumns)
-//                 cis[ciIndex] = new ColumnInfo((string)de.Key, ciIndex++,
-//                    ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
-//             mySchema = new BlueTableSchema(cis, schema.tableName);
-//         }
-
-//     }
-// }
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+using Bts.ColumnExpressions;
+
+namespace BCG;
+
+/// <summary>
+/// Provides methods to add columns to the schema
+/// </summary>
+/// <param name="columnsExpression">
+/// String represting the columns to add to the output table
+/// </param>
+public class ColumnAdder : IStreamMapper {
+   // Store tuples of (column, index) to preserve order
+   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
+   protected IBlueTableSchema? mySchema;
+
+   public ColumnAdder(string columnsExpression) =>
+      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
+         throw new Exception("Invalid columnn expression: " + columnsExpression))
+         .Select((c, i) => (c, i))
+         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
+
+   [MemberNotNull(nameof(mySchema))]
+   public virtual IBlueRow map(IBlueRow row) {
+      // Only rebuild schema if necessary (i.e., if it's null)
+      mySchema ??= buildSchema(row.schema, myNewColumns);
+      return new AddColumnsRow(mySchema, row);
+   }
+
+   [MemberNotNull(nameof(mySchema))]
+   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
+      // Always rebuild schema (even if it's already non-null)
+      mySchema = buildSchema(inputSchema, myNewColumns);
+
+   [MemberNotNull(nameof(mySchema))]
+   static BlueTableSchema buildSchema(
+      IBlueTableSchema schema,
+      IDictionary<string, (ColumnExpression column, int index)> newColumns
+   ) {
+      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
+      schema.columnInfo.CopyTo(cis, 0);
+      foreach (var (col, index) in newColumns.Values) {
+         var schemaIdx = schema.columnInfo.Length + index;
+         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
+      }
+      return new BlueTableSchema(cis, schema.tableName);
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -30,8 +30,8 @@ namespace BCG
            {
               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
               {
-                  BinaryFormatter bf = new BinaryFormatter();
 #pragma warning disable SYSLIB0011
+                  BinaryFormatter bf = new BinaryFormatter();
                   BlueTable bt = (BlueTable)bf.Deserialize(fs);
 #pragma warning restore SYSLIB0011
                   mySetRealSource(new BlueRowCollectionStaticStream(bt));

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -15,6 +15,11 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
+   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
+   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
+   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
+   // file is created. The file names are token_epoch_datetime
+   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -11,13 +11,27 @@ using System.Reflection.Metadata;
 
 namespace BCG {
    public class SnapshotStream : ViewStream {
-
+      public enum SpecificSnapshot {
+         FirstOfDay,
+         LastOfDay
+      };
       private BlueTable _current_table;
       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
       public DateTime _snapshot_time;
       public string _snapshot_file;
       public IEnumerable<PlaybackPacket> _packets;
-
+      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+         DateTime dt;
+         var parsed = DateTime.TryParse(snapshot_date, out dt);
+         dt = dt.Date;
+         bts.Log.assert(parsed);
+         SpecificSnapshot specific_snapshot_enum;
+         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+         bts.Log.assert(parsed);
+         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+         mySetRealSource(rs);
+      }
 
       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -37,6 +37,20 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
+         DateTime max_dt = DateTime.MinValue;
+         string rv = String.Empty;
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            var dt = parse_datetime_from_filename(fn);
+            if (dt > max_dt) {
+               max_dt = dt;
+               rv = fn;
+            }
+         }
+         return rv;
+      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,47 +1,42 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
+using System;
+using System.Collections;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Computes the max for a given column
-// 	/// </summary>
-// 	public class MaxGroupByEvaluator : GroupByEvaluator
-// 	{
+using BCG.Utilities;
 
-//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the max for a given column
+    /// </summary>
+    public class MaxGroupByEvaluator : GroupByEvaluator {
 
-//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
-//             myOutputTable;
-//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
-//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
-//          if (maxRow != null)
-//          {
-//             BlueEventLog.logAssert (currMaxRow != null);
-//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
-//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
-//             {
-//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
-//                if (maxRow == null)
-//                   myOutputTable.removeRow (hashString);
-//                else
-//                   myOutputTable.addRow (maxRow, hashString);
-//             }
-//          }
-//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
-//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-//          // otherwise if maxRow > currMaxRow then its replaced
-//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
-//             myOutputTable.addRow (maxRow, hashString);
-//          return myProject (myOutputTable.getRow (hashString));
-//       }
-// 	}
-// }
+        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
+
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
+            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
+            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
+            if (maxRow != null) {
+                bts.Log.assert(currMaxRow != null);
+                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
+                    if (maxRow == null)
+                        myOutputTable.removeRow(hashString);
+                    else
+                        myOutputTable.addRow(maxRow, hashString);
+                }
+            }
+            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
+            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+            // otherwise if maxRow > currMaxRow then its replaced
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+                myOutputTable.addRow(maxRow, hashString);
+            return myProject(myOutputTable.getRow(hashString));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,20 +1,15 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-// 	public class MaxStream : ViewStream
-// 	{
-//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
-//       {
-//       }
-// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
-// 		{
-//          mySetRealSource (new GroupByStream (stream, 
-//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
-// 		}
+namespace BCG {
+    public class MaxStream : ViewStream {
+        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
+        }
+        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
+            mySetRealSource(new GroupByStream(stream,
+                                                new MaxGroupByEvaluator(outputCol, inputCols)));
+        }
 
-//       public MaxStream (ViewStream stream, string outputCol)
-//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-// 	}
-// }
+        public MaxStream(ViewStream stream, string outputCol)
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,40 +1,36 @@
-// using System;
-// using BCG.Utilities;
+using System;
 
-// namespace BCG
-// {
-//    public class MinGroupByEvaluator : MaxGroupByEvaluator
-//    {
-//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
-//          : base (outputCol, groupByCols)  {}
+using BCG.Utilities;
 
-//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
-//          base (colMapEpression, outputCol, groupByCols)  {}
+namespace BCG {
+    public class MinGroupByEvaluator : MaxGroupByEvaluator {
+        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
-//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
-//          if (minRow != null)
-//          {
-//             // BlueEventLog.logAssert (currMinRow != null);
-//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
-//             {
-//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
-//                if (minRow == null)
-//                   myOutputTable.removeRow (stringRep);
-//                else
-//                   myOutputTable.addRow (minRow, stringRep);
-//             }
-//          }
-//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
-//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
-//             myOutputTable.addRow (minRow, stringRep);
-//          return myOutputTable.getRow (stringRep);
-//       }
+        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
+           base(colMapEpression, outputCol, groupByCols) { }
 
-//    }
-// }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
+            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
+            if (minRow != null) {
+                // BlueEventLog.logAssert (currMinRow != null);
+                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
+                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
+                    if (minRow == null)
+                        myOutputTable.removeRow(stringRep);
+                    else
+                        myOutputTable.addRow(minRow, stringRep);
+                }
+            }
+            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
+                myOutputTable.addRow(minRow, stringRep);
+            return myOutputTable.getRow(stringRep);
+        }
+
+    }
+}