refactoring onix options

This reverts commit 4f05b95bd0.

.gitignore

@@ -55,3 +55,5 @@ trace_data/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 **/devcontainer.json
+*.Generated.cs
+*.lst

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "externals"]
 	path = externals
 	url = git@bitbucket.org:bluetradesys/externals.git
+[submodule "vcpkg"]
+	path = vcpkg
+	url = https://github.com/microsoft/vcpkg.git

CMakeLists.txt

@@ -36,29 +36,6 @@ if(BUILD_API_SERVER)
 endif()
 
 if(WIN32)
-  # flags
-  set(
-    CMAKE_CXX_FLAGS
-    "${CMAKE_CXX_FLAGS} \
-    /MP \
-    /WX /wd4251 /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 \
-    /bigobj \
-    /DCURL_STATICLIB \
-    /D_WINSOCK_DEPRECATED_NO_WARNINGS \
-    /D__TBB_NO_IMPLICIT_LINKAGE \
-    /DBOOST_ALL_NO_LIB \
-    /std:c++20 \
-    /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} /MT")
-  set(
-    CMAKE_SHARED_LINKER_FLAGS_RELEASE
-    "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} \
-    /DEBUG \
-    /OPT:REF \
-    /OPT:ICF \
-    /LTCG")
-
   add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
   add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
   # subdirs
@@ -142,42 +119,6 @@ else()
   if (BUILD_ADMIN_WEB_APP)
     add_subdirectory(src/Spark.Admin.WebApp)
   endif()
-  if (BUILD_API_SERVER)
-    set(_cs_source_dirs
-      spark_api
-      spark_api_server
-      spark_core
-      spark_websocket)
-    foreach(cs_source_dir IN LISTS _cs_source_dirs)
-      file(GLOB cs_source_dir "${CMAKE_SOURCE_DIR}/src/client/${cs_source_dir}/*.cs")
-      list(APPEND _api_server_sources ${cs_source_dir})
-    endforeach()
-
-    list(APPEND _api_server_sources
-      ${CMAKE_SOURCE_DIR}/src/client/spark_api/spark_api.csproj
-      ${CMAKE_SOURCE_DIR}/src/client/spark_api_server/spark_api_server.csproj
-      ${CMAKE_SOURCE_DIR}/src/client/spark_core/spark_core.csproj
-      ${CMAKE_SOURCE_DIR}/src/client/spark_websocket/spark_websocket.csproj)
-
-    set(_out_dir ${CMAKE_BINARY_DIR}/$<CONFIG>)
-    set(_api_server_dlls
-      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/spark_api.dll
-      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/spark_core.dll
-      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/spark_websocket.dll
-      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/spark_api_server)
-    add_custom_command(
-      OUTPUT ${_api_server_dlls}
-      COMMAND dotnet build ${CMAKE_SOURCE_DIR}/Spark.Api.Server.sln --verbosity minimal --configuration $<CONFIG>
-      COMMENT "Building Spark.Api.Server.sln"
-      DEPENDS ${_api_server_sources})
-    add_custom_command(
-      OUTPUT
-      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/libspark.so
-      COMMAND ln -sf ARGS ${CMAKE_BINARY_DIR}/src/lib/$<CONFIG>/libspark.so ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0
-      COMMENT "Symlink libspark to api server bins"
-      DEPENDS ${_api_server_dlls})
-    add_custom_target(websocket_api_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/libspark.so)
-  endif()
   if(BUILD_DOCS)
     add_subdirectory(doc/user)
   endif()
@@ -210,7 +151,6 @@ else()
       FILES
       "${SPARK_LIB}/libACE-5.7.5.so"
       "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${CMAKE_SOURCE_DIR}/externals/src/emumba/sparkd.lic"
       DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
   endif()
 endif()

CMakePresets.json

@@ -26,8 +26,11 @@
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
       }
     },
 
@@ -147,7 +150,6 @@
 
     {
       "name": "spark-api-server-base",
-      "inherits": "spark",
       "hidden": true,
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
@@ -155,38 +157,47 @@
           "type": "bool",
           "value": "ON"
         },
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
       }
     },
 
     {
       "name": "spark-api-server-windows",
-      "inherits": "spark-api-server-base",
+      "inherits": "spark",
       "displayName": "Spark Api Server",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
-      "generator": "Visual Studio 17 2022"
+      "cacheVariables": {
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        }
+      }
     },
 
     {
       "name": "spark-api-server-linux",
-      "inherits": "spark-api-server-base",
+      "inherits": "sparkd",
       "displayName": "Spark Api Server",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
-        }
+        },
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
       }
     },
 
@@ -206,6 +217,17 @@
       "cacheVariables": {
         "CCAPI_LOG_LEVEL": "Trace"
       }
+    },
+
+    {
+      "name": "data-stream-suite",
+      "inherits": "spark",
+      "displayName": "Data Stream Suite",
+      "description": "Data Stream utils ported over from Edge",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
+      }
     }
   ],
 
@@ -346,6 +368,20 @@
       "configurePreset": "sparkd-ccapi-diag",
       "displayName": "Debug (ccapi diag)",
       "inherits": "sparkd"
+    },
+
+    {
+      "name": "data-stream-suite-debug",
+      "displayName": "DataStreamSuite (Debug)",
+      "configurePreset": "data-stream-suite",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-release",
+      "displayName": "DataStreamSuite (Release)",
+      "configurePreset": "data-stream-suite",
+      "configuration": "Release"
     }
   ]
 }

dev/README.md

@@ -5,6 +5,7 @@
 [clangd ext]: https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.vscode-clangd
 [dotnet]: https://dotnet.microsoft.com/en-us/download/dotnet
 [ninja build]: https://ninja-build.org/
+[ninja generator]: https://cmake.org/cmake/help/latest/generator/Ninja.html
 [ninja multi-config generator]: https://cmake.org/cmake/help/latest/generator/Ninja%20Multi-Config.html
 [python]: https://www.python.org/
 [py venv]: https://docs.python.org/3/library/venv.html
@@ -86,7 +87,6 @@
 * **[CMake][cmake]** (version >= 3.23)
 * **[Ninja Build][ninja build]**
 * **[Python][python]** (version >= 3.9)
-* **[Docker][docker]**
 
 ### **References**
 * **[CMakePresets][cmake presets]**
@@ -178,6 +178,7 @@
 # Build Spark Api Server on Linux
 ### **Dependencies**
 * **[CMake][cmake]** (version >= 3.23)
+* **[Ninja Build][ninja build]**
 * **[Python][python]** (version >= 3.9)
 * **[Dotnet][dotnet]** (sdk version >= 7.0.0)
 

docker/sparkd/dev/Dockerfile

@@ -1,66 +1,98 @@
-FROM ubuntu:22.10
+FROM ubuntu:latest
 
-RUN \
-    apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/New_York
+
+# light deps (should be quick to download/install)
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    curl \
+    gnupg \
+    liblttng-ust-dev \
+    libpcap0.8-dev \
+    locales \
+    ninja-build \
+    software-properties-common \
+    sqlite3 \
+    tzdata
+
+# clang
+RUN apt-get update && apt-get install -y \
     clang-13 \
     clang-format-13 \
     clangd-13 \
     clang-15 \
     clang-format-15 \
-    clangd-15 \
-    cmake \
-    curl \
-    dotnet7 \
-    fish \
+    clangd-15
+
+# g++
+RUN apt-get update && apt-get install -y \
     g++-11 \
-    g++-12 \
-    gnupg \
-    liblttng-ust-dev \
-    libssh2-1-dev \
-    libpcap0.8-dev \
-    locales \
-    ninja-build \
-    nodejs \
-    npm \
-    openssh-server \
+    g++-12
+
+# python
+RUN apt-get update && apt-get install -y \
     python3-pip \
     python3-venv \
-    rapidjson-dev \
-    software-properties-common \
-    sqlite3 \
-    sudo \
-    tar \
-    vim \
-    wget
+    && python3 -m pip install --upgrade pip
 
-# this is done here so that ccache sees clang++ installed, not clang-10
-RUN \
-    apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y ccache 
+# dotnet
+RUN apt-get update && apt-get install -y \
+    dotnet7
 
-# emumba deps
+# ssh
+RUN apt-get update && apt-get install -y \
+    libssh2-1-dev \
+    openssh-server
+
+# cmake
 RUN \
-    apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    tzdata \
+    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+    | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
+    && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    && apt-get update \
+    && apt-get install -y cmake
+
+# node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+    | tee /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y nodejs
+
+# ccache is done here so that ccache sees clang++ installed, not clang-10
+RUN apt-get update && apt-get install -y \
+    ccache 
+
+# emumba (light deps)
+RUN apt-get update && apt-get install -y \
     build-essential \
-    cmake-curses-gui \
-    git-all \
     doxygen \
     python3-sphinx \
     graphviz \
     libgtest-dev \
     libssl-dev \
+    rapidjson-dev \
+    zlib1g-dev \
+    pkg-config
+
+# emumba (boost)
+RUN apt-get update && apt-get install -y \
     libboost-filesystem-dev \
     libboost-program-options-dev \
     libboost-test-dev \
-    libboost-date-time-dev \
-    zlib1g-dev \
-    netcat-traditional \
-    net-tools \
-    gdb gdbserver \
-    pkg-config
+    libboost-date-time-dev
+
+# dev utils
+RUN apt-get update && apt-get install -y \ 
+    fish \
+    git-all \
+    sudo \
+    vim \
+    wget
 
 # set up default executables (update-alternatives)
 RUN \

docker/sparkd/dev/README.md

@@ -55,7 +55,7 @@ then select your container in the dropdown
 3. Push the new image to Docker Hub
 
    ```
-   docker push johncolvin/bluetradingsystems:sparkd-dev
+   docker push bluetradingsystems/sparkd-dev:latest
    ```
    ***Note:** The image doesn't have to be on Docker Hub, we can simply build
    it from scratch on our local machines (with the Dockerfile). However, this

notes

@@ -0,0 +1,6 @@
+cd ~/spark/build/clang/debug/; cmake --build . --target install; rm -rf ~/spark/install/Debug/bin.zip; tar -zcvf  ~/spark/install/Debug/bin.zip ~/spark/install/Debug/bin; scp ~/spark/install/Debug/bin.zip bts@10.1.10.31:~/13000/; cd ~/spark/
+cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; docker container stop spark_13000; docker container start spark_13000
+cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; cp -r sparkbin ~/.sparkd-dev/mounts/sparkd_build/; 
+ES SECURITYTYPE_FUTURE at VENUE_GLBX
+./sparkd md dump --symbol ES --type future --venue glbx --platform mdp3onixsex
+BT5u53r

spark_client.sln

@@ -1,11 +1,9 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libspark", "build\src\lib\libspark.vcxproj", "{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
@@ -13,9 +11,6 @@ EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-	ProjectSection(ProjectDependencies) = postProject
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25} = {FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}
-	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
 EndProject
@@ -39,14 +34,6 @@ Global
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}.Debug|x64.ActiveCfg = Debug|x64
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}.Debug|x64.Build.0 = Debug|x64
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}.MinSizeRel|x64.Build.0 = MinSizeRel|x64
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}.Release|x64.ActiveCfg = Release|x64
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}.Release|x64.Build.0 = Release|x64
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64

src/DataStreamSuite/ArithmeticExpressions/AdditiveInverse.cs

@@ -0,0 +1,37 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Numerics;
+
+class AdditiveInverse : IArithmeticExpression<double> {
+   readonly IArithmeticExpression expression;
+
+   public AdditiveInverse(IArithmeticExpression expression) =>
+      this.expression = expression;
+
+   public static AdditiveInverse<T> Create<T>(IArithmeticExpression<T> expression)
+      where T : INumber<T> =>
+      new(expression);
+
+   public double Evaluate(IDictionary<string, object?> env) =>
+      -evaluateArg(expression, nameof(expression), env);
+
+   static double evaluateArg(
+      IArithmeticExpression expr, string name, IDictionary<string, object?> env) {
+      var obj = expr.Evaluate(env);
+      return ConvertUtils.try_convert(expr.Evaluate(env), out double dbl) ? dbl :
+         throw new Exception(
+            $"Invalid AdditiveInverse. {name} is of type {obj?.GetType()} " +
+            $"(expected {typeof(double)})");
+   }
+}
+
+class AdditiveInverse<T> : IArithmeticExpression<T>
+   where T : INumber<T> {
+   readonly IArithmeticExpression<T> expression;
+
+   public AdditiveInverse(IArithmeticExpression<T> expression) =>
+      this.expression = expression;
+
+   public T Evaluate(IDictionary<string, object?> env) =>
+      -expression.Evaluate(env);
+}

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionExtensions.cs

@@ -0,0 +1,45 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Numerics;
+
+public static class ArithmeticExpressionExtensions {
+   public static IArithmeticExpression<TResult> Add<TLeft, TRight, TResult>(
+      this IArithmeticExpression<TLeft> left, IArithmeticExpression<TRight> right)
+      where TLeft :
+         INumber<TLeft>,
+         IAdditionOperators<TLeft, TRight, TResult>
+      where TRight : INumber<TRight>
+      where TResult : INumber<TResult> =>
+      new Sum<TLeft, TRight, TResult>(left, right);
+
+   public static IArithmeticExpression<TResult> Subtract<TLeft, TRight, TResult>(
+      this IArithmeticExpression<TLeft> left, IArithmeticExpression<TRight> right)
+      where TLeft :
+         INumber<TLeft>,
+         ISubtractionOperators<TLeft, TRight, TResult>
+      where TRight : INumber<TRight>
+      where TResult : INumber<TResult> =>
+      new Difference<TLeft, TRight, TResult>(left, right);
+
+   public static IArithmeticExpression<TResult> Multiply<TLeft, TRight, TResult>(
+      this IArithmeticExpression<TLeft> left, IArithmeticExpression<TRight> right)
+      where TLeft :
+         INumber<TLeft>,
+         IMultiplyOperators<TLeft, TRight, TResult>
+      where TRight : INumber<TRight>
+      where TResult : INumber<TResult> =>
+      new Product<TLeft, TRight, TResult>(left, right);
+
+   public static IArithmeticExpression<TResult> Divide<TLeft, TRight, TResult>(
+      this IArithmeticExpression<TLeft> left, IArithmeticExpression<TRight> right)
+      where TLeft :
+         INumber<TLeft>,
+         IDivisionOperators<TLeft, TRight, TResult>
+      where TRight : INumber<TRight>
+      where TResult : INumber<TResult> =>
+      new Quotient<TLeft, TRight, TResult>(left, right);
+
+   public static IArithmeticExpression<T> Negate<T>(this IArithmeticExpression<T> expression)
+      where T : INumber<T> =>
+      new AdditiveInverse<T>(expression);
+}

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionGrammar.y

@@ -0,0 +1,99 @@
+%namespace Bts.ArithmeticExpressions
+%partial
+%visibility internal
+%tokentype Token
+%parsertype ArithmeticExpressionParserImpl
+
+%union {
+   public int Integer;
+   public double Real;
+   public string Identifier;
+   public DateTime DateTime; 
+   public ImmutableList<IArithmeticExpression> Arguments;
+   public ImmutableList<Assignment> Assignments;
+   public IArithmeticExpression Argument;
+   public Assignment Assignment;
+   public ArithmeticProgram Program;
+}
+
+/* Constants  */
+%token INTEGER_CONSTANT REAL_CONSTANT
+/* Arithmetic Operators */
+%token ADD, SUBTRACT, MULTIPLY, DIVIDE, EQUALS
+/* Syntax */
+%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, IDENTIFIER
+
+%left ADD SUBTRACT
+%left MULTIPLY DIVIDE
+%nonassoc EQUALS
+
+%type <Argument> Argument
+%type <Arguments> Arguments
+%type <Assignment> Assignment
+%type <Assignments> Assignments
+%type <Program> Program
+
+%%
+
+Program :
+   Assignments {
+      $$ = new($1);
+   }
+   ;
+
+Assignments :
+   Assignment {
+      $$ = ImmutableList<Assignment>.Empty.Add($1);
+   }
+   | Assignments Assignment {
+      $$ = $1.Add($2);
+   }
+   ;
+
+Assignment :
+   IDENTIFIER EQUALS Argument {
+      $$ = new($1.Identifier, $3);
+   }
+   ;
+
+Argument :
+   IDENTIFIER PARENTHESIS_OPEN Arguments PARENTHESIS_CLOSE {
+      $$ = new Function($1.Identifier, $3);
+   }
+   | INTEGER_CONSTANT {
+      $$ = new IntegerConstant($1.Integer);
+   }
+   | REAL_CONSTANT {
+      $$ = new RealConstant($1.Real);
+   }
+   | IDENTIFIER {
+      $$ = new Variable($1.Identifier);
+   }
+   | SUBTRACT Argument {
+      $$ = $2.Negate();
+   }
+   | PARENTHESIS_OPEN Argument PARENTHESIS_CLOSE {
+      $$ = $2;
+   }
+   | Argument MULTIPLY Argument {
+      $$ = $1.Multiply($3);
+   }
+   | Argument DIVIDE Argument {
+      $$ = $1.Divide($3);
+   }
+   | Argument ADD Argument {
+      $$ = $1.Add($3);
+   }
+   | Argument SUBTRACT Argument {
+      $$ = $1.Subtract($3);
+   }
+   ;
+
+Arguments :
+   Argument {
+      $$ = ImmutableList<IArithmeticExpression>.Empty.Add($1);
+   }
+   | Arguments COMMA Argument {
+      $$ = $1.Add($3);
+   }
+   ;

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionParserImpl.cs

@@ -0,0 +1,19 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Text;
+
+partial class ArithmeticExpressionParserImpl {
+   public ArithmeticExpressionParserImpl() : base(null) { }
+
+   public ValueType Parse(Stream stream) {
+      Scanner = new ArithmeticExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+
+   public ValueType Parse(string str) {
+      var inputBuffer = Encoding.Default.GetBytes(str);
+      using var stream = new MemoryStream(inputBuffer);
+      return Parse(stream);
+   }
+}

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionScanner.cs

@@ -0,0 +1,11 @@
+namespace Bts.ArithmeticExpressions;
+
+using System;
+
+partial class ArithmeticExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionTokens.lex

@@ -0,0 +1,70 @@
+%namespace Bts.ArithmeticExpressions
+%scannertype ArithmeticExpressionScanner
+%visibility internal
+%tokentype Token
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+Letter             [a-zA-Z]
+Identifier         {Letter}({Letter}*[0-9]*_*)*
+IntegerConstant    -?[0-9]+
+RealConstant       -?[0-9]+\.[0-9]+
+ParenthesisOpen    \(
+ParenthesisClose   \)
+Add                \+
+Subtract           \-
+Multiply           \*
+Divide             /
+Equals             =
+Comma              ,
+
+%%
+
+{IntegerConstant} {
+   yylval.Integer = int.Parse(yytext);
+   return (int)Token.INTEGER_CONSTANT;
+}
+
+{RealConstant} {
+   yylval.Real = double.Parse(yytext);
+   return (int)Token.REAL_CONSTANT;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)Token.IDENTIFIER;
+}
+
+{ParenthesisOpen} {
+   return (int)Token.PARENTHESIS_OPEN;
+}
+
+{ParenthesisClose} {
+   return (int)Token.PARENTHESIS_CLOSE;
+}
+
+{Add} {
+   return (int)Token.ADD;
+}
+
+{Subtract} {
+   return (int)Token.SUBTRACT;
+}
+
+{Multiply} {
+   return (int)Token.MULTIPLY;
+}
+
+{Divide} {
+   return (int)Token.DIVIDE;
+}
+
+{Equals} {
+   return (int)Token.EQUALS;
+}
+
+{Comma} {
+   return (int)Token.COMMA;
+}
+
+%%

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ArithmeticExpression">
+    <Scanner Include="ArithmeticExpressionTokens.lex" />
+    <Parser Include="ArithmeticExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/ArithmeticExpressions/ArithmeticProgram.cs

@@ -0,0 +1,21 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Collections.Immutable;
+
+class ArithmeticProgram : IArithmeticProgram {
+   readonly ImmutableList<Assignment>? assignments;
+
+   public ArithmeticProgram(Assignment assignment) =>
+      assignments = ImmutableList<Assignment>.Empty.Add(assignment);
+
+   public ArithmeticProgram(ImmutableList<Assignment> assignments) =>
+      this.assignments = assignments;
+
+   IEnumerable<string> IArithmeticProgram.OutputIdentifiers =>
+      assignments.empty_if_null()
+         .Select(a => a.Identifier ?? throw new Exception("Identifier string is null"))
+         .Distinct();
+
+   IDictionary<string, object?> IArithmeticProgram.Evaluate(IDictionary<string, object?> env) =>
+      assignments.empty_if_null().Aggregate(env, (e, a) => a.Evaluate(e));
+}

src/DataStreamSuite/ArithmeticExpressions/ArithmeticProgramParser.cs

@@ -0,0 +1,16 @@
+namespace Bts.ArithmeticExpressions;
+
+using Bts.ExpressionCore;
+
+public static class ArithmeticProgramParser {
+   public static IArithmeticProgram Parse(Stream expression) =>
+      new Parser().Parse(expression);
+
+   public static IArithmeticProgram Parse(string expression) =>
+      new Parser().Parse(expression);
+
+   class Parser : IParser<ArithmeticProgram> {
+      public ArithmeticProgram Parse(Stream expression) =>
+         new ArithmeticExpressionParserImpl().Parse(expression).Program;
+   }
+}

src/DataStreamSuite/ArithmeticExpressions/Assignment.cs

@@ -0,0 +1,24 @@
+namespace Bts.ArithmeticExpressions;
+
+public class Assignment {
+   public Assignment(string? identifier, IArithmeticExpression? value) {
+      Identifier = identifier ?? throw new ArgumentNullException(nameof(identifier));
+      Value = value ?? throw new ArgumentNullException(nameof(identifier));
+   }
+
+   public string? Identifier { get; }
+   public IArithmeticExpression? Value { get; }
+
+   public IDictionary<string, object?> Evaluate(IDictionary<string, object?> env) {
+      if (Identifier is null) {
+         throw new InvalidOperationException("Identifier is null");
+      }
+      if (Value is null) {
+         throw new InvalidOperationException($"Value is null (Identifier '{Identifier}')");
+      }
+      env[Identifier] = Value.Evaluate(env) ?? throw new Exception(
+         $"{nameof(Assignment)}.{nameof(Value)} (of type '{Value.GetType()}') " +
+         $"evaluated to null (Identifier '{Identifier}')");
+      return env;
+   }
+}

src/DataStreamSuite/ArithmeticExpressions/Difference.cs

@@ -0,0 +1,45 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Numerics;
+
+public class Difference : IArithmeticExpression<double> {
+   readonly IArithmeticExpression lhs;
+   readonly IArithmeticExpression rhs;
+
+   public Difference(IArithmeticExpression lhs, IArithmeticExpression rhs) {
+      this.lhs = lhs;
+      this.rhs = rhs;
+   }
+
+   public static Difference<T, T, T> Create<T>(
+      IArithmeticExpression<T> left, IArithmeticExpression<T> right)
+      where T : INumber<T>, ISubtractionOperators<T, T, T> =>
+      new(left, right);
+
+   public double Evaluate(IDictionary<string, object?> env) =>
+      evaluateArg(lhs, nameof(lhs), env) - evaluateArg(rhs, nameof(rhs), env);
+
+   static double evaluateArg(
+      IArithmeticExpression expr, string name, IDictionary<string, object?> env) {
+      var obj = expr.Evaluate(env);
+      return ConvertUtils.try_convert(expr.Evaluate(env), out double dbl) ? dbl :
+         throw new Exception(
+            $"Invalid Difference. {name} is of type {obj?.GetType()} (expected {typeof(double)})");
+   }
+}
+
+public class Difference<TLeft, TRight, TResult> : IArithmeticExpression<TResult>
+   where TLeft : INumber<TLeft>, ISubtractionOperators<TLeft, TRight, TResult>
+   where TRight : INumber<TRight>
+   where TResult : INumber<TResult> {
+   readonly IArithmeticExpression<TLeft> left;
+   readonly IArithmeticExpression<TRight> right;
+
+   public Difference(IArithmeticExpression<TLeft> left, IArithmeticExpression<TRight> right) {
+      this.left = left;
+      this.right = right;
+   }
+
+   public TResult Evaluate(IDictionary<string, object?> env) =>
+      left.Evaluate(env) - right.Evaluate(env);
+}

src/DataStreamSuite/ArithmeticExpressions/EvaluationEnvironment.cs

@@ -0,0 +1,208 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Collections;
+using System.Diagnostics.CodeAnalysis;
+
+class EvaluationEnvironment : IDictionary<string, object?>, IReadOnlyDictionary<string, object?> {
+   IDictionary<string, object?> outerEnv;
+   IDictionary<string, object?> innerEnv = new Dictionary<string, object?>();
+   int? count;
+
+   public EvaluationEnvironment(IDictionary<string, object?> initial) =>
+      outerEnv = initial;
+
+   public object? this[string key] {
+      get {
+         if (innerEnv.TryGetValue(key, out var rv)) {
+            return rv;
+         }
+         if (outerEnv.TryGetValue(key, out rv)) {
+            return rv;
+         }
+         throw new Exception($"Undefined identifier '{key}'");
+      }
+      set {
+         if (count.HasValue && !ContainsKey(key)) {
+            count = count.Value + 1;
+            innerEnv.Add(key, value);
+            return;
+         }
+         innerEnv[key] = value;
+      }
+   }
+
+   public IDictionary<string, object?> Result =>
+      innerEnv;
+
+   public ICollection<string> Keys =>
+      new KeyCollection(outerEnv, innerEnv);
+
+   public ICollection<object?> Values =>
+      new ValueCollection(outerEnv, innerEnv);
+
+   public int Count =>
+      count ??= countKeys(outerEnv, innerEnv);
+
+   public bool IsReadOnly => throw new NotImplementedException();
+
+   public bool ContainsKey(string key) =>
+      innerEnv.ContainsKey(key) || outerEnv.ContainsKey(key);
+
+   public bool TryGetValue(string key, [MaybeNullWhen(false)] out object? value) =>
+      innerEnv.TryGetValue(key, out value) || outerEnv.TryGetValue(key, out value);
+
+   public IEnumerator<KeyValuePair<string, object?>> GetEnumerator() {
+      foreach (var kv in innerEnv) {
+         yield return kv;
+      }
+      foreach (var kv in outerEnv.Except()) {
+         if (!innerEnv.ContainsKey(kv.Key)) {
+            yield return kv;
+         }
+      }
+   }
+
+   public void Add(string key, object? value) {
+      if (ContainsKey(key)) {
+         throw new ArgumentException(
+            $"An element with the same key ({key}) already exists", nameof(key));
+      }
+      innerEnv.Add(key, value);
+   }
+
+   public void Add(KeyValuePair<string, object?> item) =>
+      Add(item.Key, item.Value);
+
+   public void Clear() =>
+      throw new NotSupportedException();
+
+   public bool Contains(KeyValuePair<string, object?> item) =>
+      TryGetValue(item.Key, out var v) && Equals(v, item.Value);
+
+   public void CopyTo(KeyValuePair<string, object?>[] array, int arrayIndex) {
+      foreach (var kv in innerEnv) {
+         array[arrayIndex++] = kv;
+      }
+      foreach (var kv in outerEnv) {
+         if (!innerEnv.ContainsKey(kv.Key)) {
+            array[arrayIndex++] = kv;
+         }
+      }
+   }
+
+   public bool Remove(string key) =>
+      throw new NotSupportedException();
+
+   public bool Remove(KeyValuePair<string, object?> item) =>
+      throw new NotSupportedException();
+
+   static int countKeys(IDictionary<string, object?> d1, IDictionary<string, object?> d2) =>
+      d1.Count > d2.Count ?
+         d1.Count + d2.Keys.WhereNot(d1.ContainsKey).Count() :
+         d2.Count + d1.Keys.WhereNot(d2.ContainsKey).Count();
+
+   IEnumerable<string> IReadOnlyDictionary<string, object?>.Keys => Keys;
+   IEnumerable<object?> IReadOnlyDictionary<string, object?>.Values => Values;
+   IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+
+   class KeyCollection : ICollection<string> {
+      readonly IDictionary<string, object?> outerEnv;
+      readonly IDictionary<string, object?> innerEnv;
+
+      public KeyCollection(
+         IDictionary<string, object?> outerEnv, IDictionary<string, object?> innerEnv) {
+         this.outerEnv = outerEnv;
+         this.innerEnv = innerEnv;
+      }
+
+      public int Count =>
+         countKeys(outerEnv, innerEnv);
+
+      public bool IsReadOnly => true;
+
+      public void Add(string item) =>
+         throw new NotSupportedException();
+
+      public void Clear() =>
+         throw new NotSupportedException();
+
+      public bool Contains(string item) =>
+         innerEnv.ContainsKey(item) || outerEnv.ContainsKey(item);
+
+      public void CopyTo(string[] array, int arrayIndex) {
+         foreach (var k in this) {
+            array[arrayIndex++] = k;
+         }
+      }
+
+      public IEnumerator<string> GetEnumerator() {
+         foreach (var k in innerEnv.Keys) {
+            yield return k;
+         }
+         foreach (var k in outerEnv.Keys) {
+            if (!innerEnv.ContainsKey(k)) {
+               yield return k;
+            }
+         }
+      }
+
+      public bool Remove(string item) =>
+         throw new NotSupportedException();
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+
+   class ValueCollection : ICollection<object?> {
+      readonly IDictionary<string, object?> outerEnv;
+      readonly IDictionary<string, object?> innerEnv;
+
+      public ValueCollection(
+         IDictionary<string, object?> outerEnv, IDictionary<string, object?> innerEnv) {
+         this.outerEnv = outerEnv;
+         this.innerEnv = innerEnv;
+      }
+
+      public int Count =>
+         // Not a typo (obv, numValues == numKeys)
+         countKeys(outerEnv, innerEnv);
+
+      public bool IsReadOnly => true;
+
+      public void Add(object? item) =>
+         throw new NotSupportedException();
+
+      public void Clear() =>
+         throw new NotSupportedException();
+
+      public bool Contains(object? item) =>
+         innerEnv.Values.Contains(item) ||
+         outerEnv
+            .Where(kv => !innerEnv.ContainsKey(kv.Key))
+            .Select(kv => kv.Value)
+            .Contains(item);
+
+      public void CopyTo(object?[] array, int arrayIndex) {
+         foreach (var k in this) {
+            array[arrayIndex++] = k;
+         }
+      }
+
+      public IEnumerator<object?> GetEnumerator() {
+         foreach (var k in innerEnv.Values) {
+            yield return k;
+         }
+         foreach (var kv in outerEnv) {
+            if (!innerEnv.ContainsKey(kv.Key)) {
+               yield return kv.Value;
+            }
+         }
+      }
+
+      public bool Remove(object? item) =>
+         throw new NotSupportedException();
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+}

src/DataStreamSuite/ArithmeticExpressions/Function.cs

@@ -0,0 +1,34 @@
+namespace Bts.ArithmeticExpressions;
+
+using Spark.Core;
+
+using System.Collections.Immutable;
+using System.Data;
+
+public class Function : IArithmeticExpression {
+   public Function(string? identifier, ImmutableList<IArithmeticExpression>? arguments) {
+      Identifier = identifier;
+      Arguments = arguments;
+   }
+
+   public string? Identifier { get; }
+   public ImmutableList<IArithmeticExpression>? Arguments { get; }
+
+   public object? Evaluate(IDictionary<string, object?> env) {
+      if (Identifier is null) {
+         throw new InvalidOperationException("Identifier is null");
+      }
+      if (Arguments is null) {
+         throw new InvalidOperationException("Arguments is null");
+      }
+      if (Identifier == "ifNull") {
+         throw new NotImplementedException("'ifNull' function not yet implemented");
+      }
+      var argVals = Arguments.Select(e => e.Evaluate(env)).ToArray();
+      var argTypes = Array.ConvertAll(argVals, v => v?.GetType() ?? typeof(object));
+      var method = typeof(Math).GetMethod(Identifier, argTypes) ??
+         throw new EvaluateException(
+            $"Failed to find method '{Identifier}({", ".join(argTypes)})' in {typeof(Math)}");
+      return method.Invoke(null, argVals);
+   }
+}

src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/ArithmeticExpressions/IArithmeticExpression.cs

@@ -0,0 +1,19 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Numerics;
+
+public interface IArithmeticExpression {
+   IArithmeticExpression Add(IArithmeticExpression factor) => new Sum(this, factor);
+   IArithmeticExpression Divide(IArithmeticExpression other) => new Quotient(this, other);
+   IArithmeticExpression Multiply(IArithmeticExpression other) => new Product(this, other);
+   IArithmeticExpression Subtract(IArithmeticExpression other) => new Difference(this, other);
+   IArithmeticExpression Negate() => new AdditiveInverse(this);
+   object? Evaluate(IDictionary<string, object?> env);
+}
+
+public interface IArithmeticExpression<T> : IArithmeticExpression where T : INumber<T> {
+   new T Evaluate(IDictionary<string, object?> env);
+
+   object? IArithmeticExpression.Evaluate(IDictionary<string, object?> env) =>
+      Evaluate(env);
+}

src/DataStreamSuite/ArithmeticExpressions/IArithmeticProgram.cs

@@ -0,0 +1,7 @@
+namespace Bts.ArithmeticExpressions;
+
+public interface IArithmeticProgram {
+   IEnumerable<string> OutputIdentifiers { get; }
+
+   IDictionary<string, object?> Evaluate(IDictionary<string, object?> env);
+}

src/DataStreamSuite/ArithmeticExpressions/IntegerConstant.cs

@@ -0,0 +1,11 @@
+namespace Bts.ArithmeticExpressions;
+
+class IntegerConstant : IArithmeticExpression<int> {
+   readonly int value;
+
+   public IntegerConstant(int value) =>
+      this.value = value;
+
+   public int Evaluate(IDictionary<string, object?> env) =>
+      value;
+}

src/DataStreamSuite/ArithmeticExpressions/Product.cs

@@ -0,0 +1,45 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Numerics;
+
+class Product : IArithmeticExpression<double> {
+   readonly IArithmeticExpression left;
+   readonly IArithmeticExpression right;
+
+   public Product(IArithmeticExpression left, IArithmeticExpression right) {
+      this.left = left;
+      this.right = right;
+   }
+
+   public static Product<T, T, T> Create<T>(
+      IArithmeticExpression<T> left, IArithmeticExpression<T> right)
+      where T : INumber<T>, IMultiplyOperators<T, T, T> =>
+      new(left, right);
+
+   public double Evaluate(IDictionary<string, object?> env) =>
+      evaluateArg(left, nameof(left), env) * evaluateArg(right, nameof(right), env);
+
+   static double evaluateArg(
+      IArithmeticExpression expr, string name, IDictionary<string, object?> env) {
+      var obj = expr.Evaluate(env);
+      return ConvertUtils.try_convert(expr.Evaluate(env), out double dbl) ? dbl :
+         throw new Exception(
+            $"Invalid Product. {name} is of type {obj?.GetType()} (expected {typeof(double)})");
+   }
+}
+
+class Product<TLeft, TRight, TResult> : IArithmeticExpression<TResult>
+   where TLeft : INumber<TLeft>, IMultiplyOperators<TLeft, TRight, TResult>
+   where TRight : INumber<TRight>
+   where TResult : INumber<TResult> {
+   readonly IArithmeticExpression<TLeft> left;
+   readonly IArithmeticExpression<TRight> right;
+
+   public Product(IArithmeticExpression<TLeft> left, IArithmeticExpression<TRight> right) {
+      this.left = left;
+      this.right = right;
+   }
+
+   public TResult Evaluate(IDictionary<string, object?> env) =>
+      left.Evaluate(env) * right.Evaluate(env);
+}

src/DataStreamSuite/ArithmeticExpressions/Quotient.cs

@@ -0,0 +1,45 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Numerics;
+
+class Quotient : IArithmeticExpression<double> {
+   readonly IArithmeticExpression left;
+   readonly IArithmeticExpression right;
+
+   public Quotient(IArithmeticExpression left, IArithmeticExpression right) {
+      this.left = left;
+      this.right = right;
+   }
+
+   public static Quotient<T, T, T> Create<T>(
+      IArithmeticExpression<T> left, IArithmeticExpression<T> right)
+      where T : INumber<T>, IDivisionOperators<T, T, T> =>
+      new(left, right);
+
+   public double Evaluate(IDictionary<string, object?> env) =>
+      evaluateArg(left, nameof(left), env) / evaluateArg(right, nameof(right), env);
+
+   static double evaluateArg(
+      IArithmeticExpression expr, string name, IDictionary<string, object?> env) {
+      var obj = expr.Evaluate(env);
+      return ConvertUtils.try_convert(expr.Evaluate(env), out double dbl) ? dbl :
+         throw new Exception(
+            $"Invalid Quotient. {name} is of type {obj?.GetType()} (expected {typeof(double)})");
+   }
+}
+
+class Quotient<TLeft, TRight, TResult> : IArithmeticExpression<TResult>
+   where TLeft : INumber<TLeft>, IDivisionOperators<TLeft, TRight, TResult>
+   where TRight : INumber<TRight>
+   where TResult : INumber<TResult> {
+   readonly IArithmeticExpression<TLeft> left;
+   readonly IArithmeticExpression<TRight> right;
+
+   public Quotient(IArithmeticExpression<TLeft> left, IArithmeticExpression<TRight> right) {
+      this.left = left;
+      this.right = right;
+   }
+
+   public TResult Evaluate(IDictionary<string, object?> env) =>
+      left.Evaluate(env) / right.Evaluate(env);
+}

src/DataStreamSuite/ArithmeticExpressions/RealConstant.cs

@@ -0,0 +1,11 @@
+namespace Bts.ArithmeticExpressions;
+
+class RealConstant : IArithmeticExpression<double> {
+   readonly double value;
+
+   public RealConstant(double value) =>
+      this.value = value;
+
+   public double Evaluate(IDictionary<string, object?> env) =>
+      value;
+}

src/DataStreamSuite/ArithmeticExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/ArithmeticExpressions/Sum.cs

@@ -0,0 +1,45 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Numerics;
+
+class Sum : IArithmeticExpression<double> {
+   readonly IArithmeticExpression left;
+   readonly IArithmeticExpression right;
+
+   public Sum(IArithmeticExpression left, IArithmeticExpression right) {
+      this.left = left;
+      this.right = right;
+   }
+
+   public static Sum<T, T, T> Create<T>(
+      IArithmeticExpression<T> left, IArithmeticExpression<T> right)
+      where T : INumber<T>, IAdditionOperators<T, T, T> =>
+      new(left, right);
+
+   public double Evaluate(IDictionary<string, object?> env) =>
+      evaluateArg(left, nameof(left), env) + evaluateArg(right, nameof(right), env);
+
+   static double evaluateArg(
+      IArithmeticExpression expr, string name, IDictionary<string, object?> env) {
+      var obj = expr.Evaluate(env);
+      return ConvertUtils.try_convert(expr.Evaluate(env), out double dbl) ? dbl :
+         throw new Exception(
+            $"Invalid Sum. {name} is of type {obj?.GetType()} (expected {typeof(double)})");
+   }
+}
+
+class Sum<TLeft, TRight, TResult> : IArithmeticExpression<TResult>
+   where TLeft : INumber<TLeft>, IAdditionOperators<TLeft, TRight, TResult>
+   where TRight : INumber<TRight>
+   where TResult : INumber<TResult> {
+   readonly IArithmeticExpression<TLeft> left;
+   readonly IArithmeticExpression<TRight> right;
+
+   public Sum(IArithmeticExpression<TLeft> left, IArithmeticExpression<TRight> right) {
+      this.left = left;
+      this.right = right;
+   }
+
+   public TResult Evaluate(IDictionary<string, object?> env) =>
+      left.Evaluate(env) + right.Evaluate(env);
+}

src/DataStreamSuite/ArithmeticExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/ArithmeticExpressions/Variable.cs

@@ -0,0 +1,15 @@
+namespace Bts.ArithmeticExpressions;
+
+using System.Collections.Generic;
+
+class Variable : IArithmeticExpression {
+   readonly string identifier;
+
+   public Variable(string identifier) =>
+      this.identifier = identifier;
+
+   public object? Evaluate(IDictionary<string, object?> env) =>
+      env.TryGetValue(identifier, out var val) ? val :
+         throw new KeyNotFoundException(
+            $"Failed to find {nameof(Variable)} '{identifier}'");
+}

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/BaseUtilities/Crono.cs

@@ -0,0 +1,41 @@
+using mbts.spark;
+
+using System;
+using System.Runtime;
+
+namespace BCG.Utilities.General
+{
+	/// <summary>
+	/// Summary description for Crono.
+	/// </summary>
+	public class Crono
+	{
+      public Crono()
+      {
+         myTimer = new HighPerformanceTimer();
+      }
+
+      public void start() 
+      {myTimer.start();}
+
+      public void stop()
+      {myTimer.stop();}
+
+      public TimeSpan time
+      {
+         get 
+         {
+            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
+         }
+      }
+
+      public double miliseconds
+      {
+         get
+         {return myTimer.totalMilliseconds;}
+      }
+
+      private HighPerformanceTimer myTimer;
+	}
+
+}

src/DataStreamSuite/BlueRegistry/ArrayRegistryHandler.cs

@@ -0,0 +1,60 @@
+using System;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   internal static class ArrayRegistryHandler
+   {
+      public static void setValue (RegistryKey rk, string name, Array array)
+      {
+         rk.SetValue (name, array.Length);
+         Type t = array.GetType ().GetElementType ();
+
+         for (int i = 0; i < array.Length; i++)
+         {
+            if (t == typeof (int))
+               rk.SetValue (string.Concat (name, i.ToString ()), (int) array.GetValue (i));
+            else if (t == typeof (bool))
+               rk.SetValue (string.Concat (name, i.ToString ()), (bool) array.GetValue (i));
+            else
+               rk.SetValue (string.Concat (name, i.ToString ()), array.GetValue (i));
+         }
+      }
+
+      public static Array getValue (RegistryKey rk, string name, Type elementType)
+      {
+         object c = rk.GetValue (name, 0);
+         int count = (int) c;
+
+         Array result;
+
+         if (elementType == typeof (int) || elementType == typeof (System.Int32))
+            result = new int[count];
+         else if (elementType == typeof (bool))
+            result = new bool[count];
+         else if (elementType == typeof (double))
+            result = new double[count];
+         else if (elementType == typeof (DateTime))
+            result = new DateTime[count];
+         else
+            result = new string[count];
+
+         for (int i = 0; i < count; i++)
+         {
+            object val = rk.GetValue (string.Concat (name, i.ToString ()));
+            if (elementType == typeof (int) || elementType == typeof (System.Int32))
+               result.SetValue (int.Parse (val.ToString ()), i);
+            else if (elementType == typeof (bool))
+               result.SetValue (bool.Parse (val.ToString ()), i);
+            else if (elementType == typeof (double))
+               result.SetValue (double.Parse (val.ToString ()), i);
+            else if (elementType == typeof (DateTime))
+               result.SetValue (DateTime.Parse (val.ToString ()), i);
+            else
+               result.SetValue (val.ToString (), i);
+         }
+
+         return result;
+      }
+   }
+}

src/DataStreamSuite/BlueRegistry/BlueRegistry.cs

@@ -0,0 +1,192 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      internal virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0-windows</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <UseWindowsForms>true</UseWindowsForms>
+  </PropertyGroup>
+
+</Project>

src/DataStreamSuite/BlueRegistry/DoubleRegistryHandler.cs

@@ -0,0 +1,22 @@
+using System;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   internal static class DoubleRegistryHandler
+   {
+      public static void setValue (RegistryKey rk, string name, double value)
+      {
+         rk.SetValue (name, value.ToString ());
+      }
+
+      public static double getValue (RegistryKey rk, string name)
+      {
+         object v = rk.GetValue (name);
+         if (v != null) 
+            return double.Parse (v.ToString ());
+         else
+            return double.NaN;
+      }
+   }
+}

src/DataStreamSuite/BlueRegistry/EnumRegistryHandler.cs

@@ -0,0 +1,22 @@
+using System;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   internal static class EnumRegistryHandler
+   {
+      public static void setValue (RegistryKey rk, string name, Enum value)
+      {
+         rk.SetValue (name, value.ToString ());
+      }
+
+      public static Enum getValue (RegistryKey rk, string name, Type enumType)
+      {
+         object v = rk.GetValue (name);
+         if (v != null)
+            return (Enum) Enum.Parse (enumType, v.ToString ());
+         else
+            return null;
+      }
+   }
+}

src/DataStreamSuite/BlueRegistry/PersistWindowState.cs

@@ -0,0 +1,166 @@
+using System;
+using System.Reflection;
+using System.Windows.Forms;
+using System.Drawing;
+using System.Diagnostics;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public class PersistWindowState
+   {
+      // event info that allows form to persist extra window state data
+      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
+      public event WindowStateHandler LoadState;
+      public event WindowStateHandler SaveState;
+
+      private Form myParent;
+      private BlueRegistry myRegistry;
+
+      private int myNormalLeft;
+      private int myNormalTop;
+      private int myNormalWidth;
+      private int myNormalHeight;
+      private bool myResizable = true;
+      private FormWindowState myWindowState;
+      private bool myAllowSaveMinimized = false;
+      private bool mySavePosition = true;
+      private bool mySaveState = true;
+
+      public PersistWindowState (bool savePosition)
+      {
+         mySavePosition = savePosition;
+      }
+
+      public PersistWindowState ()
+      {
+      }
+
+      public bool saveState
+      {
+         [DebuggerStepThrough]
+         set { mySaveState = value; }
+      }
+
+      public Form parent
+      {
+         [DebuggerStepThrough]
+         get { return myParent; }
+         set
+         {
+            myParent = value;
+            myRegistry = new BlueRegistry (myParent.Text);
+
+            // subscribe to parent form's events
+            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
+            myParent.Resize += new System.EventHandler(myOnResize);
+            myParent.Move += new System.EventHandler(myOnMove);
+            myParent.Load += new System.EventHandler(myOnLoad);
+
+            // get initial width and height in case form is never resized
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      public BlueRegistry blueRegistry
+      {
+         [DebuggerStepThrough]
+         get { return myRegistry; }
+         [DebuggerStepThrough]
+         set { myRegistry = value; }
+      }
+
+      public bool resizable
+      {
+         [DebuggerStepThrough]
+         set { myResizable = value; }
+      }
+
+      public bool allowSaveMinimized
+      {
+         [DebuggerStepThrough]
+         set { myAllowSaveMinimized = value; }
+      }
+
+      private void myOnResize(object sender, System.EventArgs e)
+      {
+         // save width and height
+         if (myParent.WindowState == FormWindowState.Normal && myResizable)
+         {
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      private void myOnMove(object sender, System.EventArgs e)
+      {
+         // save position
+         if(myParent.WindowState == FormWindowState.Normal)
+         {
+            myNormalLeft = myParent.Left;
+            myNormalTop = myParent.Top;
+         }
+         // save state
+         myWindowState = myParent.WindowState;
+      }
+
+      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
+      {
+         // save position, size and state
+         if (mySavePosition)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
+            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
+         }
+
+         if (myResizable)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
+            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
+         }
+
+         // check if we are allowed to save the state as minimized (not normally)
+         if (!myAllowSaveMinimized)
+         {
+            if(myWindowState == FormWindowState.Minimized)
+               myWindowState = FormWindowState.Normal;
+         }
+
+         if (mySaveState)
+            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
+        
+         // fire SaveState event
+         if (SaveState != null)
+            SaveState (this, myRegistry);
+      }
+
+      private void myOnLoad (object sender, System.EventArgs e)
+      {
+         if (mySavePosition)
+         {
+            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
+            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
+            myParent.Location = new Point (left, top);
+         }
+
+         if (myResizable)
+         {
+            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
+            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
+            myParent.Size = new Size (width, height);
+         }
+            
+         if (mySaveState)
+         {
+            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
+               (int) myParent.WindowState);
+            myParent.WindowState = windowState;
+         }
+
+         // fire LoadState event
+         if(LoadState != null)
+            LoadState (this, myRegistry);
+      }
+   }
+}

src/DataStreamSuite/BlueRegistry/SortedListRegistryHandler.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Collections;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   internal static class SortedListRegistryHandler
+   {
+      public static void setValue (RegistryKey rk, string name, SortedList slist)
+      {
+         rk.SetValue (name, slist.Count);
+               
+         for (int i = 0; i < slist.Count; i++)
+         {
+            object key = slist.GetKey (i);
+            object val = slist.GetByIndex (i);
+            rk.SetValue (string.Concat (name, "_key", i.ToString ()), key);
+            rk.SetValue (string.Concat (name, "_val", i.ToString ()), val);
+         }
+      }
+
+      public static SortedList getValue (RegistryKey rk, string name)
+      {
+         object c = rk.GetValue (name, 0);
+
+         SortedList result = new SortedList ();
+         try
+         {
+            int count = (int) c;
+            for (int i = 0; i < count; i++)
+            {
+               object key = rk.GetValue (string.Concat (name, "_key", i.ToString ()));
+               object val = rk.GetValue (string.Concat (name, "_val", i.ToString ()));
+               result.Add (key, val);
+            }
+         }
+         catch (Exception)
+         {
+            result.Clear ();
+         }
+         return result;
+      }
+
+      public static void deleteValue (RegistryKey rk, string name)
+      {
+         object c = rk.GetValue (name, 0);
+         int count = (int) c;
+
+         for (int i = 0; i < count; i++)
+         {
+            rk.DeleteValue (string.Concat (name, "_key", i.ToString ()), false);
+            rk.DeleteValue (string.Concat (name, "_val", i.ToString ()), false);
+         }
+         
+         rk.DeleteValue (name, false);
+      }
+   }
+}

src/DataStreamSuite/BooleanExpressions/Arguments/Argument.cs

@@ -0,0 +1,21 @@
+namespace Bts.BooleanExpressions.Arguments;
+
+class Argument : IArgument {
+   readonly Func<IDictionary<string, object?>, object?> getValue;
+
+   public Argument(Func<IDictionary<string, object?>, object?> getValue) =>
+      this.getValue = getValue;
+
+   public object? GetValue(IDictionary<string, object?> env) =>
+      getValue(env);
+}
+
+class Argument<T> : IArgument<T> {
+   readonly Func<IDictionary<string, object?>, T> getValue;
+
+   public Argument(Func<IDictionary<string, object?>, T> getValue) =>
+      this.getValue = getValue;
+
+   public T GetValue(IDictionary<string, object?> env) =>
+      getValue(env);
+}

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -0,0 +1,28 @@
+namespace Bts.BooleanExpressions.Arguments;
+
+using bts;
+
+using Spark.Core.Collections;
+
+static class ArgumentComparer {
+   public static int Compare(
+      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
+      if (value1 is null) {
+         throw new ArgumentNullException(nameof(value1));
+      }
+      if (value2 is null) {
+         throw new ArgumentNullException(nameof(value2));
+      }
+      var cmp1 = value1.Cast<IComparable>();
+      var cmp2 = value2.Cast<IComparable>();
+      try {
+         var v1 = cmp1.GetValue(env);
+         var v2 = cmp2.GetValue(env);
+         return v1.GetType().IsNumeric() ?
+            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
+      } catch (Exception ex) {
+         Log.Ui.error($"Comparison failed: {ex}");
+         throw;
+      }
+   }
+}

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentExtensions.cs

@@ -0,0 +1,11 @@
+namespace Bts.BooleanExpressions.Arguments;
+
+static class ArgumentExtensions {
+   public static IArgument<T> Cast<T>(this IArgument argument) =>
+      new Argument<T>(env => {
+         var obj = argument.GetValue(env);
+         return obj is T t ? t : throw new Exception(
+            $"Expected argument.GetValue to return instance of type {typeof(T)} " +
+            $"(actually {obj?.GetType()})");
+      });
+}

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentMatcher.cs

@@ -0,0 +1,19 @@
+namespace Bts.BooleanExpressions.Arguments;
+
+using System.Text.RegularExpressions;
+
+static class ArgumentMatcher {
+   public static bool Match(
+      IArgument arg1, IArgument arg2, IDictionary<string, object?> env) {
+      var lobj = arg1.GetValue(env);
+      var robj = arg2.GetValue(env);
+      if (DBNull.Value.Equals(lobj) && robj is string) {
+         return false;
+      }
+      if (lobj is string lstr && robj is string rstr) {
+         return Regex.IsMatch(lstr, rstr);
+      }
+      throw new Exception(
+         $"Type mismatch, cannot string match {lobj?.GetType()} with {robj?.GetType()}");
+   }
+}

src/DataStreamSuite/BooleanExpressions/Arguments/Constant.cs

@@ -0,0 +1,16 @@
+namespace Bts.BooleanExpressions.Arguments;
+
+static class Constant {
+   public static Constant<T> Create<T>(T value) =>
+      new(value);
+}
+
+class Constant<T> : IArgument<T> {
+   readonly T value;
+
+   public Constant(T value) =>
+      this.value = value;
+
+   public T GetValue(IDictionary<string, object?> env) =>
+      value;
+}

src/DataStreamSuite/BooleanExpressions/Arguments/Function.cs

@@ -0,0 +1,29 @@
+namespace Bts.BooleanExpressions.Arguments;
+
+class Function : IArgument {
+   public const string FunctionPrefix = "function:";
+
+   readonly string name;
+   readonly IEnumerable<IArgument> args;
+
+   public Function(string name, IEnumerable<IArgument>? args = null) {
+      this.name = name;
+      this.args = args ?? Enumerable.Empty<IArgument>();
+   }
+
+   public delegate object? Delegate(List<object?> args);
+
+   public object? GetValue(IDictionary<string, object?> env) {
+      var argValues = args?.Select(a => a.GetValue(env))?.ToList() ?? new(0);
+
+      var k = FunctionPrefix + name;
+      if (!env.TryGetValue(k, out var fnObj)) {
+         throw new BooleanExpressionException($"Did not find function named '{name}'");
+      }
+      if (fnObj is not Delegate fn) {
+         throw new BooleanExpressionException(
+            $"Function {name} is not of type {typeof(Delegate)}");
+      }
+      return fn(argValues);
+   }
+}

src/DataStreamSuite/BooleanExpressions/Arguments/IArgument.cs

@@ -0,0 +1,12 @@
+namespace Bts.BooleanExpressions;
+
+interface IArgument {
+   object? GetValue(IDictionary<string, object?> env);
+}
+
+interface IArgument<T> : IArgument {
+   new T GetValue(IDictionary<string, object?> env);
+   
+   object? IArgument.GetValue(IDictionary<string, object?> env) =>
+      GetValue(env);
+}

src/DataStreamSuite/BooleanExpressions/Arguments/Variable.cs

@@ -0,0 +1,12 @@
+namespace Bts.BooleanExpressions.Arguments;
+
+class Variable : IArgument {
+   readonly string name;
+
+   public Variable(string name) =>
+      this.name = name;
+
+   public object? GetValue(IDictionary<string, object?> env) =>
+      env.TryGetValue(name, out var res) ?
+         res : throw new BooleanExpressionException($"Did not find variable named '{name}'");
+}

src/DataStreamSuite/BooleanExpressions/BinaryOperator.cs

@@ -0,0 +1,13 @@
+namespace Bts.BooleanExpressions;
+
+class BinaryOperator : IBinaryOperator {
+   readonly Func<IArgument?, IArgument?, IDictionary<string, object?>, bool> op;
+
+   public BinaryOperator(
+      Func<IArgument?, IArgument?, IDictionary<string, object?>, bool> op) =>
+      this.op = op;
+
+   public bool Evaluate(
+      IArgument? arg1, IArgument? arg2, IDictionary<string, object?> bindings) =>
+      op(arg1, arg2, bindings);
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpression.cs

@@ -0,0 +1,19 @@
+namespace Bts.BooleanExpressions;
+
+using bts;
+
+class BooleanExpression : IBooleanExpression {
+   readonly Func<IDictionary<string, object?>, bool> evaluate;
+
+   public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
+      this.evaluate = evaluate;
+
+   public bool Evaluate(IDictionary<string, object?> env) {
+      try {
+         return evaluate(env);
+      } catch (Exception ex) {
+         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
+         throw;
+      }
+   }
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionException.cs

@@ -0,0 +1,6 @@
+namespace Bts.BooleanExpressions;
+
+public class BooleanExpressionException : Exception {
+   public BooleanExpressionException(string format, params object[] args)
+      : base(string.Format(format, args)) { }
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionGrammar.y

@@ -0,0 +1,176 @@
+%namespace Bts.BooleanExpressions
+%partial
+%visibility internal
+%tokentype Token
+%parsertype BooleanExpressionParserImpl
+
+%union {
+   public int Integer;
+   public double Real;
+   public bool Boolean;
+   public string String;
+   public string Identifier;
+   public DateTime DateTime;
+   public ImmutableList<Definition> Definitions;
+   public Definition Definition;
+   public ImmutableList<string> Parameters;
+   public ImmutableList<IArgument> Arguments;
+   public IArgument Argument;
+   public BooleanExpression Expression;
+   public BoundBooleanExpression BoundExpression;
+}
+
+/* Constants */
+%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, STRING_CONSTANT, NULL_CONSTANT, DATE_TIME_CONSTANT, REAL_CONSTANT
+/* Logical Operators */
+%token OR, AND, XOR, NOT
+/* Syntax */
+%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, IDENTIFIER, DEFINE, BEGIN_DEF, END_DEF
+/* Relational */
+%token EQUAL, LESS_EQUAL, GREATER_EQUAL, LESS, GREATER, DIFFERENT, DIFFERENT2, MATCHES
+
+%left OR AND XOR
+%right NOT
+%nonassoc EQUAL
+
+%type <BoundExpression> BoundExpression
+%type <Definitions> Definitions
+%type <Definition> Definition
+%type <Expression> Expression
+%type <Parameters> Parameters
+%type <Arguments> Arguments
+%type <Argument> Argument
+
+%%
+
+BoundExpression : Definitions Expression { $$ = new($2, $1); }
+   | Expression { $$ = new($1, ImmutableList<Definition>.Empty); };
+
+Definitions :
+   Definition { $$ = ImmutableList<Definition>.Empty.Add($1); }
+   | Definitions Definition { $$ = $1.Add($2); };
+
+Definition :
+   DEFINE IDENTIFIER PARENTHESIS_OPEN Parameters PARENTHESIS_CLOSE BEGIN_DEF Expression END_DEF {
+      $$ = new($2.Identifier, $4, $7);
+   };
+
+Expression :
+   Expression OR Expression {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) || right.Evaluate(env));
+   }
+   | Expression AND Expression {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) && right.Evaluate(env));
+   }
+   | Expression XOR Expression {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) != right.Evaluate(env));
+   }
+   | NOT Expression {
+      var exp = $2;
+      $$ = new(env => !exp.Evaluate(env));
+   }
+   | PARENTHESIS_OPEN Expression PARENTHESIS_CLOSE {
+      $$ = $2; 
+   }
+   | Argument GREATER Argument {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) > 0);
+   }
+   | Argument GREATER_EQUAL Argument {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) >= 0);
+   }
+   | Argument EQUAL Argument {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) == 0);
+   }
+   | Argument LESS_EQUAL Argument {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) <= 0);
+   }
+   | Argument LESS Argument {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) < 0);
+   }
+   | Argument DIFFERENT Argument {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) != 0);
+   }
+   | Argument DIFFERENT2 Argument {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) != 0);
+   }
+   | Argument MATCHES Argument {
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentMatcher.Match(left, right, env));
+   }
+   | Argument {
+      var arg = $1;
+      $$ = new(env => (bool)arg.GetValue(env));
+   }
+   ;
+
+Parameters :
+   {
+      $$ = ImmutableList<string>.Empty; 
+   }
+   | IDENTIFIER {
+      $$ = ImmutableList<string>.Empty.Add($1.Identifier); 
+   }
+   | IDENTIFIER COMMA Parameters {
+      $$ = $3.Insert(0, $1.Identifier); 
+   }
+   ;
+
+Arguments :
+   {
+      $$ = ImmutableList<IArgument>.Empty; 
+   }
+   | Argument {
+      $$ = ImmutableList<IArgument>.Empty.Add($1); 
+   }
+   | Arguments COMMA Argument {
+      $$ = $1.Add($3); 
+   }
+   ;
+
+Argument : IDENTIFIER 
+   {
+      $$ = new Arguments.Variable($1.Identifier);
+   }
+   | BOOLEAN_CONSTANT {
+      $$ = Arguments.Constant.Create($1.Boolean); 
+   }
+   | NULL_CONSTANT {
+      $$ = Arguments.Constant.Create(DBNull.Value); 
+   }
+   | REAL_CONSTANT {
+      $$ = Arguments.Constant.Create($1.Real); 
+   }
+   | STRING_CONSTANT {
+      $$ = Arguments.Constant.Create($1.String); 
+   }
+   | INTEGER_CONSTANT {
+      $$ = Arguments.Constant.Create($1.Integer); 
+   }
+   | DATE_TIME_CONSTANT {
+      $$ = Arguments.Constant.Create($1.DateTime); 
+   }
+   | IDENTIFIER PARENTHESIS_OPEN Arguments PARENTHESIS_CLOSE {
+      $$ = new Arguments.Function($1.Identifier, $3); 
+   }
+   ;

src/DataStreamSuite/BooleanExpressions/BooleanExpressionInterpreterException.cs

@@ -0,0 +1,6 @@
+namespace Bts.BooleanExpressions;
+
+public class BooleanExpressionInterpreterException : System.Exception {
+   public BooleanExpressionInterpreterException(string message)
+      : base(message) { }
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionParser.cs

@@ -0,0 +1,16 @@
+namespace Bts.BooleanExpressions;
+
+using Bts.ExpressionCore;
+
+public static class BooleanExpressionParser {
+   public static IBooleanExpression Parse(Stream expression) =>
+      new Parser().Parse(expression);
+
+   public static IBooleanExpression Parse(string expression) =>
+      new Parser().Parse(expression);
+
+   class Parser : IParser<IBooleanExpression> {
+      public IBooleanExpression Parse(Stream expression) =>
+         new BooleanExpressionParserImpl().Parse(expression).BoundExpression;
+   }
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionParserImpl.cs

@@ -0,0 +1,22 @@
+namespace Bts.BooleanExpressions;
+
+using QUT.Gppg;
+
+using System.IO;
+using System.Text;
+
+partial class BooleanExpressionParserImpl {
+   public BooleanExpressionParserImpl(AbstractScanner<ValueType, LexLocation>? scanner = null)
+      : base(scanner) { }
+
+   public ValueType Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType Parse(Stream stream) {
+      Scanner = new BooleanExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionScanner.cs

@@ -0,0 +1,13 @@
+namespace Bts.BooleanExpressions;
+
+using System;
+
+partial class BooleanExpressionScanner {
+   bts.Logger Logger => bts.Log.Gen;
+
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -0,0 +1,174 @@
+%namespace Bts.BooleanExpressions
+%scannertype BooleanExpressionScanner
+%visibility internal
+%tokentype Token
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+BooleanConstant          'true'|'false'
+CSStringConstant         \"([^\"\\]*|\\.)*\"
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+IntegerConstant          -?[0-9]+
+NullConstant             'null'
+RealConstant             -?[0-9]+\.[0-9]+
+StringConstant           \'([^\'\\]*|\\.)*\'
+
+/* Relational Operators */
+Equal          =
+LessEqual       <=
+GreaterEqual    >=
+Less            <
+Greater         >
+Different       !=
+Different2      <>
+Matches         ~=
+
+/* Logical Operators */
+Or              or
+And             and
+Not             not
+Xor             xor
+
+Define define
+BeginDef \{
+EndDef \}
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+
+/* Utility Tokens */
+OpenParenthesis          \(
+CloseParenthesis         \)
+Comma                    ,
+Negate                   '!'
+MapsTo                   ':'
+IsOfType                 '/'
+
+%%
+
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
+{BooleanConstant} {
+   yylval.Boolean = yytext.ToLower() switch {
+      "true" => true,
+      "false" => false,
+      _ => throw new Exception(
+         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
+   };
+   return (int)Token.BOOLEAN_CONSTANT;
+}
+
+{NullConstant} {
+   yylval.Null = DBNull.Value;
+   return (int)Token.NULL_CONSTANT;
+}
+
+{DateTimeConstant} {
+   yylval.DateTime = DateTime.Parse(yytext);
+   return (int)Token.DATE_TIME_CONSTANT;
+}
+
+{IntegerConstant} {
+   yylval.Integer = int.Parse(yytext);
+   return (int)Token.INTEGER_CONSTANT;
+}
+
+{RealConstant} {
+   yylval.Real = double.Parse(yytext);
+   return (int)Token.REAL_CONSTANT;
+}
+
+{OpenParenthesis} {
+   return (int)Token.PARENTHESIS_OPEN;
+}
+
+{CloseParenthesis} {
+   return (int)Token.PARENTHESIS_CLOSE;
+}
+
+{Comma} {
+   return (int)Token.COMMA;
+}
+
+{Negate} {
+   return (int)Token.NEGATE;
+}
+
+{MapsTo} {
+   return (int)Token.MAPS_TO;
+}
+
+{Equal} {
+   return (int)Token.EQUAL;
+}
+
+{LessEqual} {
+   return (int)Token.LESS_EQUAL;
+}
+
+{GreaterEqual} {
+   return (int)Token.GREATER_EQUAL;
+}
+
+{Less} {
+   return (int)Token.LESS;
+}
+
+{Greater} {
+   return (int)Token.GREATER;
+}
+
+{Different} {
+   return (int)Token.DIFFERENT;
+}
+
+{Different2} {
+   return (int)Token.DIFFERENT2;
+}
+
+{Matches} {
+   return (int)Token.MATCHES;
+}
+
+{Or} {
+   return (int)Token.OR;
+}
+
+{And} {
+   return (int)Token.AND;
+}
+
+{Not} {
+   return (int)Token.NOT;
+}
+
+{Xor} {
+   return (int)Token.XOR;
+}
+
+{IsOfType} {
+   return (int)Token.IS_OF_TYPE;
+}
+
+{Define} {
+   return (int)Token.DEFINE;
+}
+
+{BeginDef} {
+   return (int)Token.BEGIN_DEF;
+}
+
+{EndDef} {
+   return (int)Token.END_DEF;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)Token.IDENTIFIER;
+}
+
+%%

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+  
+  <ItemGroup>
+    <Scanner Include="BooleanExpressionTokens.lex" />
+    <Parser Include="BooleanExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/BooleanExpressions/BoundBooleanExpression.cs

@@ -0,0 +1,32 @@
+namespace Bts.BooleanExpressions;
+
+using Bts.BooleanExpressions.Arguments;
+
+class BoundBooleanExpression : IBooleanExpression {
+   readonly IBooleanExpression expression;
+   readonly IDictionary<string, FunctionDefinition> definitions;
+   readonly Dictionary<string, object?> identifiers;
+
+   public BoundBooleanExpression(
+      IBooleanExpression expression, IEnumerable<Definition>? definitions = null) {
+      this.expression = expression;
+      var defs = definitions ?? Enumerable.Empty<Definition>();
+      identifiers = defs.ToDictionary(
+         kv => kv.id, kv => (object?)kv.exp);
+      this.definitions = extractFunctions(defs).ToDictionary(kv => kv.Key, kv => kv.Value);
+   }
+
+   public bool Evaluate(IDictionary<string, object?> env) {
+      foreach (var kv in env) {
+         identifiers[kv.Key] = kv.Value;
+      }
+      env = new EvaluationEnvironment(identifiers, definitions);
+      return expression.Evaluate(env);
+   }
+
+   static IEnumerable<KeyValuePair<string, FunctionDefinition>> extractFunctions(
+      IEnumerable<Definition> definitions) =>
+      definitions.Select(d =>
+         new KeyValuePair<string, FunctionDefinition>(
+            Function.FunctionPrefix + d.id, new(d.id, d.args.ToList(), d.exp)));
+}

src/DataStreamSuite/BooleanExpressions/Definition.cs

@@ -0,0 +1,18 @@
+namespace Bts.BooleanExpressions;
+
+using System.Collections.Immutable;
+
+public class Definition {
+   public Definition(string id, ImmutableList<string> args, IBooleanExpression exp) {
+      this.id = id;
+      this.args = args;
+      this.exp = exp;
+   }
+
+   public string id { get; }
+   public ImmutableList<string> args { get; }
+   public IBooleanExpression exp { get; }
+
+   public override string ToString() =>
+      $"{id}({string.Join(", ", args)} | {exp?.GetType()})";
+}

src/DataStreamSuite/BooleanExpressions/EvaluationEnvironment.cs

@@ -0,0 +1,123 @@
+namespace Bts.BooleanExpressions;
+
+using Bts.BooleanExpressions.Arguments;
+
+using Spark.Core.Collections;
+
+using System.Collections;
+
+class EvaluationEnvironment : IDictionary<string, object?> {
+   IDictionary<string, object?> userBindings;
+   IDictionary<string, FunctionDefinition> definitions;
+
+   public EvaluationEnvironment(
+      IDictionary<string, object?> userBindings,
+      IDictionary<string, FunctionDefinition> definitions) {
+      this.userBindings = userBindings;
+      this.definitions = definitions;
+   }
+
+   public int Count =>
+      definitions.Count + userBindings.Count;
+
+   public bool IsReadOnly => true;
+
+   public ICollection<string> Keys =>
+      AggregateCollection.Create(userBindings.Keys, definitions.Keys);
+
+   public ICollection<object?> Values =>
+      AggregateCollection.Create(
+         userBindings.Values, SelectorCollection.Create(definitions.Values, v => (object?)v));
+
+   public object? this[string key] {
+      get {
+         var keystr = key.ToString();
+         if (keystr is null || !definitions.TryGetValue(keystr, out var fd)) {
+            return userBindings.TryGetValue(key, out var ubnd) ? ubnd :
+               throw new KeyNotFoundException(
+                  $"EvaluationEnvironment does not contain key '{key}'");
+         }
+         return new Function.Delegate(Evaluate);
+
+         object? Evaluate(List<object?> args) {
+            if (args.Count != fd.args.Count)
+               throw new BooleanExpressionInterpreterException(
+                  $"Parameter count mismatch for {fd.name}. Expected: {fd.args} Got: {args.Count}");
+            else {
+               var newBindings = fd.args.Zip(args).ToDictionary();
+               return fd.expression.Evaluate(new EvaluationEnvironment(
+                  AggregateDictionary.Create(newBindings, userBindings), definitions));
+            }
+         }
+      }
+      set => throw new NotSupportedException();
+   }
+
+   public bool ContainsKey(string key) {
+      var keystr = key.ToString();
+      return
+         (keystr is not null && definitions.ContainsKey(keystr)) ||
+         userBindings.ContainsKey(key);
+   }
+
+   public IEnumerator<KeyValuePair<string, object?>> GetEnumerator() {
+      foreach (var kv in userBindings) {
+         yield return kv;
+      }
+      foreach (var kv in definitions) {
+         if (!userBindings.ContainsKey(kv.Key)) {
+            yield return new(kv.Key, kv.Value);
+         }
+      }
+   }
+
+   public bool TryGetValue(string key, out object? value) {
+      var keystr = key.ToString();
+      if (keystr is null || !definitions.TryGetValue(keystr, out var fd)) {
+         return userBindings.TryGetValue(key, out value);
+      }
+      value = new Function.Delegate(Evaluate);
+      return true;
+
+      object? Evaluate(List<object?> args) {
+         if (args.Count != fd.args.Count)
+            throw new BooleanExpressionInterpreterException(
+               $"Parameter count mismatch for {fd.name}. Expected: {fd.args} Got: {args.Count}");
+         else {
+            var newBindings = fd.args.Zip(args).ToDictionary();
+            return fd.expression.Evaluate(new EvaluationEnvironment(
+               AggregateDictionary.Create(newBindings, userBindings), definitions));
+         }
+      }
+   }
+
+   public void CopyTo(KeyValuePair<string, object?>[] array, int arrayIndex) {
+      foreach (var kv in userBindings) {
+         array[arrayIndex++] = kv;
+      }
+      foreach (var kv in definitions) {
+         array[arrayIndex++] = new(kv.Key, kv.Value);
+      }
+   }
+
+   void IDictionary<string, object?>.Add(string key, object? value) =>
+      throw new NotSupportedException();
+
+   void ICollection<KeyValuePair<string, object?>>.Add(KeyValuePair<string, object?> item) =>
+      throw new NotSupportedException();
+
+   void ICollection<KeyValuePair<string, object?>>.Clear() =>
+      throw new NotSupportedException();
+
+   bool ICollection<KeyValuePair<string, object?>>.Contains(KeyValuePair<string, object?> item) =>
+      TryGetValue(item.Key, out var v) && Equals(v, item.Value);
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+
+   bool IDictionary<string, object?>.Remove(string key) =>
+      throw new NotSupportedException();
+
+   bool ICollection<KeyValuePair<string, object?>>.Remove(KeyValuePair<string, object?> item) =>
+      throw new NotSupportedException();
+}

src/DataStreamSuite/BooleanExpressions/FunctionDefinition.cs

@@ -0,0 +1,14 @@
+namespace Bts.BooleanExpressions;
+
+class FunctionDefinition {
+   public FunctionDefinition(
+      string name, IList<string> args, IBooleanExpression expression) {
+      this.name = name;
+      this.args = args;
+      this.expression = expression;
+   }
+
+   public string name { get; }
+   public IList<string> args { get; }
+   public IBooleanExpression expression { get; }
+}

src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/BooleanExpressions/IBinaryOperator.cs

@@ -0,0 +1,5 @@
+namespace Bts.BooleanExpressions;
+
+interface IBinaryOperator {
+   bool Evaluate(IArgument? left, IArgument? right, IDictionary<string, object?> env);
+}

src/DataStreamSuite/BooleanExpressions/IBooleanExpression.cs

@@ -0,0 +1,5 @@
+namespace Bts.BooleanExpressions;
+
+public interface IBooleanExpression {
+   bool Evaluate(IDictionary<string, object?> env);
+}

src/DataStreamSuite/BooleanExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/BooleanExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpression.cs

@@ -0,0 +1,26 @@
+namespace Bts.ColumnMapExpressions;
+
+using Bts.ExpressionCore;
+
+using System.Collections.Immutable;
+
+public class ColumnMapExpression {
+   public ColumnMapExpression(ImmutableDictionary<string, string> columns, bool isInclusive) {
+      Columns = columns;
+      IsInclusive = isInclusive;
+   }
+
+   public ImmutableDictionary<string, string> Columns { get; }
+   public bool IsInclusive { get; }
+
+   public static ColumnMapExpression? Parse(string expression) =>
+      new Parser().Parse(expression);
+
+   public static ColumnMapExpression? Parse(Stream expression) =>
+      new Parser().Parse(expression);
+
+   class Parser : IParser<ColumnMapExpression?> {
+      public ColumnMapExpression? Parse(Stream expression) =>
+         new ColumnMapExpressionParser().Parse(expression)?.Expression;
+   }
+}

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionGrammar.y

@@ -0,0 +1,40 @@
+%namespace Bts.ColumnMapExpressions
+%partial
+%visibility internal
+%tokentype ColumnMapExpressionToken
+%parsertype ColumnMapExpressionParser
+
+%union {
+   public string Identifier;
+   public KeyValuePair<string, string> MappedColumn;
+   public System.Collections.Immutable.ImmutableDictionary<string, string> MappedColumns;
+   public ColumnMapExpression Expression;
+}
+
+%token IDENTIFIER, MAP, NEGATE 
+
+%type <MappedColumns> MappedColumns
+%type <Expression> Expression
+%type <MappedColumn> MappedColumn
+
+%%
+
+Expression :
+   NEGATE MappedColumns { $$ = new($2, false); }
+   | NEGATE { $$ = new(System.Collections.Immutable.ImmutableDictionary<string, string>.Empty, false); }
+   | MappedColumns { $$ = new($1, true); }
+   ;
+
+MappedColumn :
+   IDENTIFIER {
+      $$ = new($1.Identifier, $1.Identifier);
+   }
+   | IDENTIFIER MAP IDENTIFIER {
+      $$ = new($1.Identifier, $3.Identifier);
+   }
+   ;
+
+MappedColumns :
+   MappedColumn { $$ = System.Collections.Immutable.ImmutableDictionary<string, string>.Empty.Add($1.Key, $1.Value); }
+   | MappedColumn MappedColumns { $$ = $2.Add($1.Key, $1.Value); }
+   ;

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnMapExpressions;
+
+using System.Text;
+
+partial class ColumnMapExpressionParser {
+   public ColumnMapExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnMapExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionTokens.lex

@@ -0,0 +1,30 @@
+%namespace Bts.ColumnMapExpressions
+%scannertype ColumnMapExpressionScanner
+%visibility internal
+%tokentype ColumnMapExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+Map                      :
+Negate                   !
+
+%%
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnMapExpressionToken.IDENTIFIER;
+}
+
+{Map} {
+   return (int)ColumnMapExpressionToken.MAP;
+}
+
+{Negate} {
+   return (int)ColumnMapExpressionToken.NEGATE;
+}
+
+%%

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnMapExpression">
+    <Scanner Include="ColumnMapExpressionTokens.lex" />
+    <Parser Include="ColumnMapExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/ColumnMapExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
+    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -0,0 +1,32 @@
+using BCG;
+
+namespace BCG
+{
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("add_to_cloud")]
+   public class AddToCloudStream : ViewStream
+   {
+      private readonly string _name, _stream_exp;
+      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
+      public AddToCloudStream(string name, params string[] lines)
+      : this(name, 0, lines)
+      {
+      }
+
+      public AddToCloudStream(string name, int persist, params string[] lines)
+      {
+         _name = name;
+         _stream_exp = getExpression(lines);
+         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
+         mySetRealSource(new TableStream("cloud_streams"));
+      }
+
+      static string getExpression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -0,0 +1,100 @@
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
+using Bts.DataStreamExpressions;
+using System.Collections.Generic;
+using BCG.Utilities.General;
+
+namespace BCG
+{
+   // listens to the requests posted on "cloud_streams" request table, and 
+   // if unfulfilled starts a ViewPubsliher
+   public class CloudStreamManager : IWorker
+   {
+      public class StreamPubUnit
+      {
+         public string name;
+         public string stream_expression;
+         public ViewStream stream;
+         public bool persist;
+         public ViewPublisher view_publisher;
+
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
+         {
+            name = name_;
+            stream_expression = stream_expression_;
+            stream = stream_;
+            persist = persist_;
+            view_publisher = view_publisher_;
+         }
+      }
+      static ViewStream build_stream(string stream_expression)
+      {
+         try
+         {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         }
+         catch (Exception ex)
+         {
+            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+      private IDictionary<string, StreamPubUnit>
+         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
+      private ViewStream _stream = new TableStream("cloud_streams");
+      private PubTableBroker _broker = new PubTableBroker();
+
+      public CloudStreamManager()
+      {
+         _stream.onChange += _on_str;
+
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         foreach (IBlueRow r in dp.newRows)
+         {
+            string name = r.get<string>("name");
+            bool persist = r.get<bool>("persist");
+            string stream_exp = r.get<string>("stream_exp");
+            if (!_stream_pub_map.ContainsKey(name))
+            {
+               _assign(name, stream_exp, persist);
+            }
+         }
+      }
+
+      private void _assign(string name, string stream_expression, bool persist)
+      {
+         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
+         ViewStream stream = build_stream(stream_expression);
+         _expose_to_ws(name, stream);
+         if (persist)
+         {
+            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
+         }
+         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
+         StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
+         _stream_pub_map.Add(name, u);
+      }
+
+      private void _expose_to_ws(string name, ViewStream stream)
+      {
+         // TODO: john
+      }
+
+      public void doWork()
+      {
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -0,0 +1,59 @@
+using BCG;
+
+using System.Linq;
+
+namespace BCG
+{
+   public class StreamOffloadRequestor
+   {
+      public class OffloadOp
+      {
+         string name;
+         string stream_exp;
+      }
+      private IBlueTableSchema _schema;
+      private IBlueRowCollection _rows;
+      private PubTableBroker _broker = new PubTableBroker();
+      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+      public StreamOffloadRequestor()
+      {
+         _schema = new BlueTableSchema("cloud_streams",
+                                   new ColumnInfo("id", 0, typeof(int)),
+                                   new ColumnInfo("name", 1, typeof(string)),
+                                   new ColumnInfo("persist", 2, typeof(bool)),
+                                   new ColumnInfo("stream_exp", 3, typeof(string)));
+         _existing_offloads_vs.onChange += _on_str;
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         _rows = dp.newRows;
+         // this is needed so the broker knows the state of the table
+         _broker.createTable(_rows);
+      }
+      public void add_to_cloud(string name, string stream_exp, bool persist)
+      {
+         // check if it exists
+         if (_rows == null)
+         {
+            BlueTable table = new BlueTable(_schema);
+            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+            table.addRow(nr);
+            _broker.createTable(table);
+            // create table  
+         }
+         else
+         {
+            if (exists(name))
+               return;
+            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+            _broker.addRow(nr);
+         }
+      }
+
+      bool exists(string name)
+      {
+         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs

@@ -0,0 +1,105 @@
+using System;
+using System.Collections;
+
+namespace BCG
+{
+   internal class AddColumnsRow : IBlueRow
+   {
+      private IBlueTableSchema mySchema;
+      private IBlueRow myRow;
+      //private object[] myCachedValues;
+
+      #region Constructors
+      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
+      {
+         mySchema = schema;
+         myRow = row;
+         //myCachedValues = new object[schema.columnNames.Length];
+      }
+      #endregion
+
+      #region IBlueRow members
+      public IBlueTableSchema schema{get{return mySchema;}}
+      #endregion
+
+      #region ICollection members
+      public int Count {get{return mySchema.columnNames.Length;}}
+
+      public bool IsSynchronized {get{return false;}}
+
+      public object SyncRoot {get{return this;}}
+
+      public void CopyTo (Array array, int index)  
+      {
+         int i = index;
+         foreach (object o in this as IEnumerable)
+            array.SetValue (o, i++);
+      }
+      #endregion
+
+      #region IDictionary members
+      public bool IsFixedSize {get{return true;}}
+
+      public bool IsReadOnly {get{return true;}}
+
+      /// <summary>
+      /// The key can either be the string corrsponding to the column name or
+      /// the order of the column
+      /// </summary>
+      public object this [object key]
+      {
+         get
+         {
+            if (key is string)
+               return this[(string)key];
+            return this[(int)key];
+         }
+         set {throw new NotSupportedException ("The IBlueRow is readonly");}
+      }
+
+      public object this[int index]
+      {
+         get
+         {
+            //if (myCachedValues[index] == null)
+            //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+            //return myCachedValues[index];
+            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+         }
+      }
+
+      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}
+
+      public ICollection Keys{get{return mySchema.columnNames;}}
+
+      public ICollection Values
+      {
+         get
+         {
+            object[] retVal = new object[mySchema.columnNames.Length];
+            CopyTo (retVal, 0);
+            return retVal;
+         }
+      }
+
+      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+      public bool Contains (object key){return mySchema[(string)key] != null;}
+
+      /// <summary>
+      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+      /// iterate through the values, typecast this into an IEnumerable
+      /// </summary>
+      /// <returns></returns>
+      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
+      
+      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
+
+      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+      #endregion
+
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using BCG.Utilities.General;
+namespace BCG
+{
+
+   public class AddColumnsStream : MapStream
+   {
+      public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef))  {}    
+   }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddRegexColumnsStream.cs

@@ -0,0 +1,13 @@
+
+// using System;
+
+// namespace BCG {
+
+//    public class AddRegexColumnsStream : ViewStream {
+
+//       public AddRegexColumnsStream (ViewStream vs, string srcCol, string regex, string newColDefs) {
+//          IStreamMapper mapper = new RegexMatchAdder (srcCol, newColDefs, regex);
+//          mySetRealSource (new MapStream (vs, mapper));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -0,0 +1,73 @@
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.AddColumnsExpression;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Provides methods to add columns to the schema
+//    /// </summary>
+//    public class ColumnAdder : IStreamMapper
+//    {
+//       protected IDictionary myNewColumns;
+//       protected IBlueTableSchema mySchema;
+
+//       static syntax myParser =
+//          new syntax();
+
+//       /// <summary>
+//       /// string represting the columns to be added to the output table. 
+//       /// </summary>
+//       /// <param name="colAddExpression"></param>
+//       public ColumnAdder (string columnDef)
+//       {
+//          AddColumnsExpression ace;
+
+//          if (DataStreamLoadFileConfig.instance.useStaticParser)
+//          {
+//             lock (myParser)
+//             {
+//                ace =
+//                (AddColumnsExpression)myParser.Parse(columnDef);
+//             }
+//          }
+//          else
+//          {
+//             syntax parser = new syntax();
+//             ace =
+//             (AddColumnsExpression)parser.Parse(columnDef);
+//          }
+
+// 	 if (ace == null)
+// 	   throw new Exception ("Invalid columnn expression: " + columnDef);
+//          myNewColumns = ace.columns;
+//       }
+         
+//       public virtual IBlueRow map (IBlueRow row)
+//       {
+//          if (mySchema == null)
+//             myConstructSchema (row.schema);
+//          return new AddColumnsRow (mySchema, row);
+//       }
+
+//       public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
+//       {
+//          myConstructSchema (inputSchema);
+//          return mySchema;
+//       }
+
+//       private void myConstructSchema (IBlueTableSchema schema)
+//       {
+//          ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
+//          int ciIndex = 0;
+//          foreach (ColumnInfo ci in schema.columnInfo)
+//             cis[ciIndex++] = ci;
+//          foreach (DictionaryEntry de in myNewColumns)
+//             cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++, 
+//                ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
+//          mySchema = new BlueTableSchema (cis, schema.tableName);
+//       }
+
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/MathStream.cs

@@ -0,0 +1,102 @@
+using Bts.ArithmeticExpressions;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+
+   public class BooleanStringReplaceStream : ViewStream, IStreamMapper
+   {
+      private BooleanExpressionFilter _bool;
+      private string _column, _value;
+      private IBlueTableSchema _schema;
+      public BooleanStringReplaceStream(ViewStream vs, string boolean_expression, string column, string value)
+      {
+         _column = column;
+         _value = value;
+         if(!string.IsNullOrEmpty(boolean_expression))
+            _bool = new BooleanExpressionFilter(boolean_expression);
+         mySetRealSource(new MapStream(vs, this));
+      }
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         _schema = inputSchema;
+         return inputSchema;
+      }
+      public IBlueRow map(IBlueRow row)
+      {
+         MutableBlueRow result = new MutableBlueRow(_schema, row);
+         if(_bool == null || _bool.filter(row))
+            result.setValue(_column, _value);
+         return result;
+      }
+
+   }
+
+   public class MathStream : ViewStream, IStreamMapper
+   {
+      private IArithmeticProgram myExp;
+      private BooleanExpressionFilter _bool;
+      List<string> myExtraColumns = new List<string>();
+      BlueTableSchema mySchema;
+       
+
+      public MathStream(ViewStream vs, string mathExpression)
+         : this(vs, mathExpression, string.Empty)
+      {
+      }
+
+      public MathStream(ViewStream vs, string mathExpression, string boolean_expression)
+      {
+         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
+            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
+         myExtraColumns = myExp.OutputIdentifiers.ToList();
+         if(!string.IsNullOrEmpty(boolean_expression))
+            _bool = new BooleanExpressionFilter(boolean_expression);
+         mySetRealSource(new MapStream(vs, this));
+      }
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         if (mySchema == null)
+         {
+            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
+            foreach (ColumnInfo ci in inputSchema.columnInfo)
+               colIdx[ci.colName] = ci;
+
+            foreach (string ncn in myExtraColumns)
+            {
+               ColumnInfo newC;
+               if (colIdx.ContainsKey(ncn))
+                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
+               else
+                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
+               colIdx[ncn] = newC;
+            }
+            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
+         }
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         MutableBlueRow result = new MutableBlueRow(mySchema, row);
+         if(_bool == null || _bool.filter(row))
+         {
+            myExp.Evaluate(result.AsDictionary());
+         }
+         return result;
+      }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/RegexMatchAdder.cs

@@ -0,0 +1,48 @@
+
+// using System;
+// using System.Collections;
+// using System.Collections.Generic;
+// using System.Text.RegularExpressions;
+// using System.ComponentModel;
+
+// using BCG.Utilities;
+
+// namespace BCG {
+
+//    public class RegexMatchAdder : ColumnAdder {
+
+//       private int[] myAddedColNos;
+//       private string mySrcCol;
+//       private Regex myRegex;
+
+//       public RegexMatchAdder (string srcCol, string columnDef, string regex) : base (columnDef) {
+//          mySrcCol = srcCol;
+//          myRegex = new Regex (regex);
+//       }
+
+//       public override IBlueRow map (IBlueRow row) {
+//          MutableBlueRow result = new MutableBlueRow (mySchema, row);
+
+//          if (myAddedColNos == null) {
+//             myAddedColNos = new int[myNewColumns.Count];
+//             for (int i = 0; i < myAddedColNos.Length; ++i)
+//                myAddedColNos[i] = i + row.Count;
+//          }
+
+//          object[] values = new object[myNewColumns.Count];
+//          string src = (string)row[mySrcCol];
+//          // extract the requested regex substrings
+//          Match m = myRegex.Match (src);
+//          for (int icol = 0; icol < myAddedColNos.Length; ++icol) {
+//             int i = icol + row.Count;
+//             Group g = m.Groups[mySchema.columnNames[i]];
+//             object val = TypeDescriptor.GetConverter (mySchema[i].colType)
+//                .ConvertFromString (g.Success ? g.Value: string.Empty);
+//             result.setValue (i, val);
+//          }
+
+//          return result;
+//       }
+
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/XYTranspositionStream.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BCG
+{
+   public class XYTranspositionStream : ViewStream
+   {
+      private ViewStream _stream;
+      private IBlueTableSchema _schema;
+      private string[] _transposed_cols;
+
+      public XYTranspositionStream(ViewStream stream, string transposed_cols)
+      {
+         _transposed_cols = transposed_cols.Split(' ');
+         _stream = stream;
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _stream.onChange += _stream_onChange;
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         if (_stream != null)
+            _stream.onChange -= _stream_onChange;
+      }
+
+      private void _stream_onChange(object newValue)
+      {
+         DataPacket dp = newValue as DataPacket;
+         if (dp != null)
+         {
+            if (_schema == null)
+               _make_schema(dp.schema);
+            BlueTable result = new BlueTable(_schema);
+            int id = 1;
+            foreach (IBlueRow row in dp.newRows)
+            {
+               foreach (string transposed_col in _transposed_cols)
+               {
+                  MutableBlueRow mbr = new MutableBlueRow(_schema);
+                  for (int i = 0; i < _schema.columnNames.Length; i++)
+                  {
+                     if (_schema.columnNames[i] == "id")
+                        mbr.setValue(i, id++);
+                     else if (_schema.columnNames[i] == "param_name")
+                        mbr.setValue(i, transposed_col);
+                     else if (_schema.columnNames[i] == "param_value")
+                        mbr.setValue(i, row[transposed_col]);
+                     else
+                        mbr.setValue(i, row[_schema.columnNames[i]]);
+                  }
+                  result.addRow(mbr);
+               }
+            }
+            myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(_schema) : (myLastValue as DataPacket).newRows));
+         }
+      }
+
+      private void _make_schema(IBlueTableSchema input_schema)
+      {
+         List<ColumnInfo> info = new List<ColumnInfo>();
+         int col_counter = 0;
+         info.Add(new ColumnInfo("id", col_counter++, typeof(int)));
+         info.Add(new ColumnInfo("param_name", col_counter++, typeof(string)));
+         info.Add(new ColumnInfo("param_value", col_counter++, typeof(object)));
+         foreach (ColumnInfo ci in input_schema.columnInfo)
+            if (!_is_transposing_column(ci.colName) && ci.colName != "id")
+               info.Add(new ColumnInfo(ci.colName, col_counter++, ci.colType));
+         _schema = new BlueTableSchema(info.ToArray(), String.Format("xy_transposed_{0}", input_schema.tableName));
+      }
+
+      private bool _is_transposing_column(string col_name)
+      {
+         foreach (string tc in _transposed_cols)
+            if (col_name == tc)
+               return true;
+         return false;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs

@@ -0,0 +1,244 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
+
+namespace BCG
+{
+   public class YXTranspositionStream : ViewStream, INonIncGroupByEvaluator
+   {
+      private class OnlyFireOnceStream : ViewStream
+      {
+         private ViewStream _stream;
+         private bool _hooked = false;
+
+         public OnlyFireOnceStream(ViewStream stream)
+         {
+            _stream = stream;
+         }
+
+         protected override void myOnFirstSubscriber()
+         {
+            _stream.onChange += _stream_onChange;
+            _hooked = true;
+         }
+
+         protected override void myOnLastSubscriber()
+         {
+            if (_hooked && _stream != null)
+               _stream.onChange -= _stream_onChange;
+         }
+
+         private void _stream_onChange(object newValue)
+         {
+            DataPacket dp = newValue as DataPacket;
+            if (dp != null && dp.newRows.Count > 0)
+            {
+               _stream.onChange -= _stream_onChange;
+               myFireEvent(new DataPacket(dp.newRows, dp.newRows, new BlueTable(dp.newRows.schema)));
+            }
+         }
+      }
+
+      public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
+      public delegate string GetColumnPrefix(object[] transposing_col_values);
+
+      public delegate string GetColumnSuffix(string col_name);
+      private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
+
+      private IncrementValues _increment_values;
+      private GetColumnPrefix _get_column_prefix;
+
+      private IBlueTableSchema _schema;
+
+      private string[] _group_by_cols;
+      private Pair<string, Type>[] _cols_per_transposed;
+      private string[] _transposing_cols;
+
+      private int[] _transposing_col_indices;
+
+      private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
+
+      public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
+         : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
+                get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
+      { }
+
+      public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
+      {
+         if (get_column_suffix_del != null)
+            _get_column_suffix = get_column_suffix_del;
+         if (transposed_col_value_comparer != null)
+         {
+            _transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
+               {
+                  return transposed_col_value_comparer(p1.second, p2.second);
+               });
+         }
+         _group_by_cols = group_by_cols.Split(' ');
+         _transposing_cols = transposed_cols_group.Split(' ');
+         string[] cols = cols_per_transposed.Split(' ');
+         _cols_per_transposed = new Pair<string, Type>[cols.Length];
+         for (int i = 0; i < _cols_per_transposed.Length; i++)
+         {
+            string[] name_type = cols[i].Split('\\');
+            // mbts.Logger.assert(name_type.Length == 2, "Expected cols_per_transposed to be in format name\\type");
+            _cols_per_transposed[i] = new Pair<string, Type>(name_type[0], Type.GetType(name_type[1]));
+         }
+         _increment_values = increment_values_del;
+         _get_column_prefix = get_column_prefix_del;
+         if(only_fire_once)
+            mySetRealSource(new NonIncGroupByStream(new OnlyFireOnceStream(stream), this));
+         else
+            mySetRealSource(new NonIncGroupByStream(stream, this));
+      }
+
+      private void _set_schema(List<string> tc_prefixes, Pair<string, Type>[] group_by_col_info, string table_name)
+      {
+         int col_index = 0;
+         List<ColumnInfo> col_info = new List<ColumnInfo>();
+         col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
+         foreach (Pair<string, Type> p in group_by_col_info)
+            col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
+         foreach (string tc_prefix in tc_prefixes)
+            foreach (Pair<string, Type> p in _cols_per_transposed)
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
+         IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
+         if (_schema == null)
+            _schema = schema;
+      }
+
+      private string _get_full_column_name(string prefix, string col_name)
+      {
+         string suffix = _get_column_suffix(col_name);
+         if (suffix == string.Empty)
+            return prefix;
+         else
+            return prefix + "_" + suffix;
+      }
+
+      #region INonIncGroupByEvaluator Members
+
+      public string[] get_group_by_cols()
+      {
+         return _group_by_cols;
+      }
+
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
+      {
+         Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
+         Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
+         BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
+         Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
+         string table_name = string.Empty;
+         if (grouped_rows.Count > 0)
+         {
+            IBlueTableSchema schema = null;
+            foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
+            {
+               schema = kvp.Value[0].schema;
+               break;
+            }
+            table_name = schema.tableName;
+            _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
+            int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
+            for (int i = 0; i < group_by_indices.Length; i++)
+            {
+               ColumnInfo ci = schema.columnInfo[group_by_indices[i]];
+               group_by_col_info[i] = new Pair<string, Type>(ci.colName, ci.colType);
+            }
+         }
+         else
+            return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); 
+         Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
+         List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
+         foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
+         {
+            //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values 
+            Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
+            foreach (IBlueRow r in kvp.Value)
+            {
+               int hash = hasher.get_hash(r, _transposing_col_indices);
+               object[] values;
+               if (!incrementing_values.TryGetValue(hash, out values))
+               {
+                  incrementing_values.Add(hash, _get_per_transposing_col_values(r));
+                  string prefix = _get_column_prefix(_get_transposing_col_values(r));
+                  if (!tc_prefixes.ContainsKey(hash))
+                  {
+                     tc_prefixes.Add(hash, prefix);
+                     tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
+                  }
+               }
+               else
+                  _increment_values(values, _get_per_transposing_col_values(r));
+            }
+            all_tc_infos_and_values.Add(kvp.Key, incrementing_values);
+            //An ugly way to extract group information needed to insert values later
+            foreach (IBlueRow r in kvp.Value)
+            {
+               group_by_cols_value_dict.Add(kvp.Key, _get_group_by_cols_values(r));
+               break;
+            }
+         }
+         //Sort the transposed schema
+         if (_transpsed_col_value_comparer_wrapper != null)
+            tc_values.Sort(_transpsed_col_value_comparer_wrapper);
+         List<string> prefixes = new List<string>();
+         foreach (Pair<int, object[]> p in tc_values)
+            prefixes.Add(tc_prefixes[p.first]);
+         _set_schema(prefixes, group_by_col_info, table_name);
+         BlueTable result = new BlueTable(_schema);
+         int id = 1;
+         foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
+         {
+            MutableBlueRow mbr = new MutableBlueRow(_schema);
+            mbr.setValue("id", id++);
+            for (int i = 0; i < _group_by_cols.Length; i++)
+               mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
+            foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
+            {
+               string prefix = tc_prefixes[tc_cols.Key];
+               for (int i = 0; i < _cols_per_transposed.Length; i++)
+               {
+                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
+                  if(mbr.schema.columnNames.Contains(column))
+                     mbr.setValue(column, tc_cols.Value[i]);
+               }
+            }
+            result.addRow(mbr);
+         }
+         return result;
+      }
+
+      #endregion
+
+      private object[] _get_transposing_col_values(IBlueRow row)
+      {
+         object[] values = new object[_transposing_cols.Length];
+         for (int i = 0; i < _transposing_cols.Length; i++)
+            values[i] = row[_transposing_cols[i]];
+         return values;
+      }
+
+      private object[] _get_per_transposing_col_values(IBlueRow row)
+      {
+         object[] values = new object[_cols_per_transposed.Length];
+         for (int i = 0; i < _cols_per_transposed.Length; i++)
+            values[i] = row[_cols_per_transposed[i].first];
+         return values;
+      }
+
+      private object[] _get_group_by_cols_values(IBlueRow row)
+      {
+         object[] values = new object[_group_by_cols.Length];
+         for (int i = 0; i < values.Length; i++)
+            values[i] = row[_group_by_cols[i]];
+         return values;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs

@@ -0,0 +1,22 @@
+// using System;
+// using BCG;
+// using BCG.Utilities;
+
+// namespace BCG
+// {
+//    public class CartesianProductStream : ViewStream
+//    {
+//       public CartesianProductStream(ViewStream v1, ViewStream v2)
+//       {
+//          mySetRealSource(
+//          new JoinStream (new AddColumnsStream (v1,
+//             "hiddenCartesianColumn=1"),
+//             "hiddenCartesianColumn",
+//             "!id hiddenCartesianColumn dateTime:dateBoundary",
+//             new AddColumnsStream (v2,
+//             "hiddenCartesianColumn=1"),
+//             "hiddenCartesianColumn",
+//             "!id hiddenCartesianColumn"));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Case Changing/CaseChangeStream.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+   public class CaseChangeStream : ViewStream, IStreamMapper
+   {
+
+      private string[] myCaseChangingColumns;
+      bool myIsUpperCase;
+
+      public CaseChangeStream(ViewStream input, bool toUpper, params string[] columnsToChangeCase)
+      {
+         myCaseChangingColumns = columnsToChangeCase;         
+         mySetRealSource(new SuperMapStream(input, this));
+         myIsUpperCase = toUpper;
+      }
+
+
+
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         return inputSchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         object[] outValues = new object[row.Count];
+
+         row.CopyTo(outValues,0);
+
+         foreach (string colName in myCaseChangingColumns)
+         {
+            ColumnInfo thisCol = row.schema[colName];
+            object rowVal = outValues[thisCol.colNo];
+
+            if (rowVal is string && !string.IsNullOrEmpty((string)rowVal))
+               outValues[thisCol.colNo] = myIsUpperCase ? ((string)rowVal).ToUpper() : ((string)rowVal).ToLower();
+         }
+
+         return new BlueRow(row.schema, outValues);
+      }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Case Changing/ToLowerStream.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+   public class ToLowerStream : CaseChangeStream
+   {
+      public ToLowerStream(ViewStream input, params string[] colNames)
+         : base(input, false, colNames) { }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Case Changing/ToUpperStream.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+   public class ToUpperStream : CaseChangeStream
+   {
+      public ToUpperStream(ViewStream input, params string[] colNames)
+         : base(input, true, colNames) { }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsMapper.cs

@@ -0,0 +1,135 @@
+using System;
+using System.Collections;
+using System.Text;
+
+using BCG;
+using BCG.Utilities;
+
+namespace BCG
+{
+   public class ConcatenateColumnsMapper : IStreamMapper
+   {
+      protected IDictionary myTempColMap = new SortedList ();
+      protected ArrayList myColMap;
+      protected BlueTableSchema mySchema;
+
+      public ConcatenateColumnsMapper(params string [] columnDefs)
+      {
+         foreach (string cd in columnDefs)
+         {
+            string outColumn = 
+               cd.Substring(0,cd.IndexOf(":"));
+
+            string [] inColumns =
+               cd.Substring(cd.IndexOf(":") + 1).Split(' ');
+
+            string [] trimedIn = new string[inColumns.Length];
+            int n = 0;
+            foreach (string s in inColumns)
+               trimedIn[n++] = s.Trim();
+
+            myTempColMap[outColumn.Trim()] = trimedIn;
+         }
+      }
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
+      {
+         if (mySchema == null) 
+         {
+            /*
+             * The new columns form the first columns, and the existing columns in the input
+             * schema are appened.
+             * */
+            ArrayList cis = new ArrayList ();
+            foreach (string cn in myTempColMap.Keys) 
+               cis.Add (new ColumnInfo (cn, cis.Count, typeof(string)));
+            foreach (ColumnInfo ci in inputSchema.columnInfo)
+               cis.Add (new ColumnInfo (ci.colName, cis.Count, ci.colType));
+            mySchema = new BlueTableSchema("ConcatenateColumns " + inputSchema.tableName,
+               cis.ToArray (typeof(ColumnInfo)) as ColumnInfo[]);
+            /*
+             * An arraylist is formed which indexes the new columns to the existing ones that form it. Hashtables
+             * are avoided wherever possible (which is natural as well, as the keys are ordered integers)
+             * */
+            myColMap = new ArrayList (myTempColMap.Count);
+            foreach (DictionaryEntry de in myTempColMap)
+            {
+               string[] cols = de.Value as string[];
+               ArrayList temp = new ArrayList ();
+               foreach (string col in cols)
+                  temp.Add (inputSchema[col].colNo);
+               myColMap.Add (temp.ToArray(typeof(int)) as int[]);
+            }
+            myTempColMap.Clear ();
+         }
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public virtual IBlueRow map (IBlueRow row)
+      {
+         object[] retVal = new object[mySchema.columnInfo.Length];
+         row.CopyTo (retVal, myColMap.Count);
+         for (int colNo = 0; colNo < myColMap.Count; colNo ++)
+            retVal[colNo] = myGetConcatValue (row, colNo);
+         return new BlueRow (mySchema, retVal);
+      }
+
+      #endregion
+
+      /// <summary>
+      /// Given a row and a column number, the method extracts the constituent columns indices
+      /// (from the colMap array) and form the resultant value corresponding to colNo and the row passed
+      /// </summary>
+      /// <param name="row"></param>
+      /// <param name="colNo"></param>
+      /// <returns></returns>
+      protected virtual object myGetConcatValue (IBlueRow row, int colNo)
+      {
+         StringBuilder b = new StringBuilder();
+         foreach (int cn in myColMap[colNo] as int[])
+            b.Append(row[cn].ToString());
+         return b.ToString ();
+      }
+   }
+
+   public class ConcatenateColumnsMapperForSameDataSet : ConcatenateColumnsMapper 
+   {
+      private HashTableND[] myDicts;
+
+      public ConcatenateColumnsMapperForSameDataSet (params string [] columnDefs)
+         : base (columnDefs)
+      {
+         myDicts = new HashTableND [columnDefs.Length];
+         int i = 0;
+         foreach (DictionaryEntry de in myTempColMap)
+            myDicts[i++] = new HashTableND ((de.Value as string[]).Length);
+      }
+
+      protected override object myGetConcatValue (IBlueRow row, int colNo)
+      {
+         int[] cols = myColMap[colNo] as int[];
+         object[] vals = new object[cols.Length];
+         for (int i = 0; i < vals.Length; i++)
+            vals[i] = row[cols[i]];
+         HashTableND ht = myDicts[colNo];
+         object rv = ht.getValue (vals);
+         if (rv != null)
+            return rv;
+         else
+         {
+            rv = base.myGetConcatValue (row, colNo);
+            object[] args = new object[vals.Length + 1];
+            vals.CopyTo (args, 0);
+            args[args.Length-1] = rv;
+            ht.add (args);
+            return rv;
+         }
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsStream.cs

@@ -0,0 +1,42 @@
+using System;
+using BCG;
+using BCG.Utilities;
+
+namespace BCG
+{
+   public class ConcatenateColumnsStream : ViewStream
+   {
+      // Concatenates the string representation of some of the fields
+      // in the input stream
+      // use the following syntax for the columnDef
+      // a:b c d
+      // where a is the name of the output column and b, c, and d are
+      // names of input colums to be concatenated
+      public ConcatenateColumnsStream(
+         ViewStream sourceStream,
+         params string [] columnDefs)
+         : this (sourceStream, true, columnDefs)
+      {}
+
+     
+
+      /// <summary>
+      /// 
+      /// </summary>
+      /// <param name="sourceStream"></param>
+      /// <param name="isConcatenatedDataConfined">true when the data being appended does not change often</param>
+      /// <param name="columnDefs"></param>
+      public ConcatenateColumnsStream (ViewStream sourceStream,
+                                       bool isConcatenatedDataConfined,
+                                       params string[] columnDefs)
+         
+      {
+         mySetRealSource (
+            new MapStream (sourceStream, 
+                           isConcatenatedDataConfined ? new ConcatenateColumnsMapperForSameDataSet (columnDefs) :
+                                                         new ConcatenateColumnsMapper (columnDefs)
+                        )
+         );
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -0,0 +1,55 @@
+using System;
+
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
+      private DataPacket myLastFiredDataPacket;
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
+      private int myId = 0;
+      private ViewStream myVS;
+
+      public CountStream (ViewStream vs)
+      {
+         myVS = vs;
+      }
+
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}
+
+      protected override void myOnFirstSubscriber () {
+         myVS.onChange += new ChangeDelegate(myHandleChanges);
+      }
+      protected override void myOnLastSubscriber () {
+         myVS.onChange -= new ChangeDelegate(myHandleChanges);
+      }
+
+
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
+            myLastFiredDataPacket =
+               myLastFiredDataPacket == null ?
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
+                  :
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  myLastFiredDataPacket.newRows)
+            );
+      }
+
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
+         retVal.addRow
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
+         return retVal;
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Crushing/AverageCollectorStream.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+    public class AverageCollectorStream : ViewStream, ITableCrusher
+    {
+        public AverageCollectorStream(ViewStream stream, params string[] columnsToAverage)
+        {        
+            myAveragingColumns = columnsToAverage;
+            mySetRealSource(new CrushingCollectorStream(stream, this));
+        }
+        
+        private string[] myAveragingColumns;
+
+        #region ITableCrusher Members
+        public IBlueRow crush(IBlueRowCollection rows, int idToUse)
+        {
+            IDictionary<string, double> columnSums = new Dictionary<string, double>();
+
+            IBlueRow resultRow = null;
+            foreach (IBlueRow row in rows)
+            {
+                resultRow = row;
+
+                foreach (string colName in myAveragingColumns)
+                {
+                    double sum = 0.0f;
+                    columnSums.TryGetValue(colName, out sum);
+                    sum += (double) row[colName];
+                    columnSums[colName] = sum;
+                }
+            }
+
+            object[] resultValues = new object[rows.schema.columnNames.Length];
+            
+            if (resultRow != null)
+                resultRow.CopyTo(resultValues, 0);
+
+            resultValues[rows.schema["id"].colNo] = idToUse;
+            resultRow = new BlueRow(rows.schema, resultValues);            
+            return resultRow;
+        }
+        #endregion
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingCollectorStream.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+    public class CrushingCollectorStream : ViewStream
+    {
+        
+        /*      Arguments: a single
+         * Every time one of input the streams updates, it is crushed down to a single row,
+         * and added to the collection of output rows.  
+         * 
+         * The column 'tableIdentifierColumnName' is added to the schema, and each row from the viewstream
+         * is put into the 
+         * 
+         * */
+        public CrushingCollectorStream(ViewStream toCrush, ITableCrusher crusher)
+        {
+            myToCrush = toCrush;
+            myRows = new List<IBlueRow>();
+            myCrusher = crusher;
+        }
+
+        private ViewStream myToCrush;
+        private List<IBlueRow> myRows;
+        private ITableCrusher myCrusher;
+        private int myNextId = 0;
+
+        protected override void myOnFirstSubscriber()
+        {
+            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
+        }
+
+        protected override void myOnLastSubscriber()
+        {
+            myToCrush.onChange -= myToCrush_onChange;
+        }
+
+        void myToCrush_onChange(object newValue)
+        {
+            DataPacket dp = (DataPacket)newValue;
+
+            IBlueRow crushed = myCrusher.crush(dp.newRows, myNextId++);
+
+            BlueTable newRows = new BlueTable(dp.schema);
+            newRows.addRows(myRows);
+            newRows.addRow(crushed);
+            myRows.Add(crushed);
+
+            BlueTable added = new BlueTable(dp.schema);
+            added.addRow(crushed);
+
+            myFireEvent(new DataPacket(newRows, added, new BlueTable(dp.schema)));
+
+        }
+
+        
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingStream.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+    public class CrushingStream : ViewStream
+    {
+        public CrushingStream(ITableCrusher crusher, params ViewStream[] streamsToCrush)
+        {
+            myToCrush = streamsToCrush;
+            myCrusher = crusher;
+            myOldRows = new IBlueRow[myToCrush.Length];
+            
+            //set up an anonymous delegate for each of the streams
+            //we do this so that we can unsubscribe
+            myDelegates = new ChangeDelegate[myToCrush.Length];
+            for (int i = 0; i < myToCrush.Length; ++i)
+            {
+                int tempI = i;
+                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
+            }
+        }
+
+        private IBlueRow[] myOldRows;
+        private ViewStream[] myToCrush;
+        private ITableCrusher myCrusher;
+
+        private ChangeDelegate[] myDelegates;
+
+        protected override void myOnFirstSubscriber()
+        {
+            for (int i = 0; i < myToCrush.Length; ++ i)
+                myToCrush[i].onChange += myDelegates[i];
+        }
+        protected override void myOnLastSubscriber()
+        {
+            for (int i = 0; i < myToCrush.Length; ++i)
+                myToCrush[i].onChange -= myDelegates[i];
+        }
+
+        protected virtual void myHandleChange(int streamIndex, IBlueRowCollection newRows)
+        {
+            IBlueRow oldRow = myOldRows[streamIndex];
+
+            IBlueRow newRow = myCrusher.crush(newRows, streamIndex);
+            myOldRows[streamIndex] = newRow;
+
+            BlueTable newState = new BlueTable(newRows.schema);
+            newState.addRows(myOldRows);
+            BlueTable addedRows = new BlueTable(newRows.schema);
+            addedRows.addRow(newRow);
+            BlueTable removedRows = new BlueTable(newRows.schema);
+            removedRows.addRow(oldRow);
+
+            myFireEvent(new DataPacket(newState, addedRows, removedRows));
+            
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Crushing/ITableCrusher.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BCG
+{
+    public interface ITableCrusher
+    {
+        IBlueRow crush(IBlueRowCollection rows, int idToUse);
+    }
+}