check pt: Before going to Chi (11/07/23, random scripts, etc.)

It should always be a number between 1-65535

.editorconfig

@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles
 
-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower
 
-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion

.gitignore

@@ -47,8 +47,8 @@ trace_data/
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
 /src/server/sparkd/cmds/spartan/analysis/data
-.tmp/
-.venv*/
+*.tmp/*
+/.venv*/*
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://x-token-auth:ATCTT3xFfGN0h_ENnUSfEAIzbfVzkQPLG26cIli50MtcTwZ7VEc8lin5tJYqibjlBJ9PM9R642Ra5GjiL52TBfbO8KeUBT0UoMT7gLQHL2Y1TkW9yZr6NkvsoDkSl49DeD4Ijiu4_0u4qEm4oMVDDg2_fB059-uOWOo7ZL94tGZET8CSVpFIGvs=B0E03660@bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.gitpod.yml

@@ -1,49 +0,0 @@
-# This configuration file was automatically generated by Gitpod.
-# Please adjust to your needs (see https://www.gitpod.io/docs/introduction/learn-gitpod/gitpod-yaml)
-# and commit this file to your remote git repository to share the goodness with others.
-
-# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
-
-image: gitpod/workspace-dotnet-8
-tasks:
-  - name: Install Deps
-    init: |
-      (
-        set -e
-        sudo install-packages rapidjson-dev
-        sudo install-packages libssh2-1-dev
-        sudo install-packages openssh-server
-        sudo install-packages liblttng-ust-dev
-        sudo install-packages libpcap0.8-dev
-        sudo install-packages ccache
-        sudo ln -s $(which ccache) /usr/local/bin/clang
-        sudo ln -s $(which ccache) /usr/local/bin/clang++
-        sudo ln -s $(which ccache) /usr/local/bin/clang-17
-        sudo ln -s $(which ccache) /usr/local/bin/clang++-17
-        sudo ln -s $(which ccache) /usr/local/bin/gcc
-        sudo ln -s $(which ccache) /usr/local/bin/g++
-        sudo ln -s $(which ccache) /usr/local/bin/gcc-12
-        sudo ln -s $(which ccache) /usr/local/bin/g++-12
-        if [ -e externals/src/emumba/install_emumba-deribit-connector.sh ]; then
-          externals/src/emumba/install_emumba-deribit-connector.sh
-        fi
-      )
-  - name: Setup Python venv
-    init: |
-      python -m pip install --upgrade pip
-      python -m venv .venv
-      . .venv/bin/activate
-      python -m pip install --upgrade pip
-      pip install -r requirements.txt
-
-ports:
-  - name: sparkd config
-    port: 8000
-    onOpen: notify
-  - name: sparkd bus
-    port: 8001
-
-gitConfig:
-  alias.st: "status"
-  status.short: "true"
-  core.pager: "less -FXRS"

CMakePresets.json

@@ -29,8 +29,8 @@
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
       }
     },
 
@@ -197,7 +197,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
       }
     },
 

docker/sparkd/dev/Dockerfile

@@ -1,112 +1,128 @@
 FROM ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/New_York
 
-# Light Deps - should be quick to download/install.
-# Longer running procs should be executed in individual RUNs.
+# light deps (should be quick to download/install)
 RUN apt-get update && apt-get install -y \
-   build-essential \
-   ca-certificates \
-   curl \
-   gnupg \
-   liblttng-ust-dev \
-   libpcap0.8-dev \
-   locales \
-   ninja-build \
-   software-properties-common \
-   sqlite3 \
-   tzdata
+    ca-certificates \
+    curl \
+    gnupg \
+    liblttng-ust-dev \
+    libpcap0.8-dev \
+    locales \
+    ninja-build \
+    software-properties-common \
+    sqlite3 \
+    tzdata
 
-# Clang 17
+# clang
+RUN apt-get update && apt-get install -y \
+    clang-13 \
+    clang-format-13 \
+    clangd-13 \
+    clang-15 \
+    clang-format-15 \
+    clangd-15
+
+# g++
+RUN apt-get update && apt-get install -y \
+    g++-11 \
+    g++-12
+
+# python
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    && python3 -m pip install --upgrade pip
+
+# dotnet
+RUN apt-get update && apt-get install -y \
+    dotnet7
+
+# ssh
+RUN apt-get update && apt-get install -y \
+    libssh2-1-dev \
+    openssh-server
+
+# cmake
 RUN \
-   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
-   | tee -a "/etc/apt/sources.list.d/llvm.list" \
-   && apt-get update \
-   && apt-get install -y \
-   clang-17 \
-   clang-format-17 \
-   clangd-17
+    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+    && apt-get update \
+    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
+    && apt-get install -y kitware-archive-keyring \
+    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
+    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
+    && apt-get install -y cmake
 
-# G++
-RUN apt-get update && apt-get install -y \
-   g++
-
-# Python
-RUN apt-get update && apt-get install -y \
-   python3-pip \
-   python3-venv \
-   && python3 -m pip install --upgrade pip
-
-# Dotnet
-RUN \
-   curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" \
-   && dpkg -i packages-microsoft-prod.deb \
-   && rm packages-microsoft-prod.deb \
-   && apt-get update \
-   && apt-get install -y dotnet-sdk-8.0
-
-# SSH
-RUN apt-get update && apt-get install -y \
-   libssh2-1-dev \
-   openssh-server
-
-# CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
-
-# Node
+# node
 ARG node_major=18
 RUN mkdir -p /etc/apt/keyrings \
-   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
-   | tee /etc/apt/sources.list.d/nodesource.list \
-   && apt-get update \
-   && apt-get install -y nodejs
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+    | tee /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y nodejs
 
-# Dev Utils
+# ccache is done here so that ccache sees clang++ installed, not clang-10
+RUN apt-get update && apt-get install -y \
+    ccache 
+
+# emumba (light deps)
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    doxygen \
+    python3-sphinx \
+    graphviz \
+    libgtest-dev \
+    libssl-dev \
+    rapidjson-dev \
+    zlib1g-dev \
+    pkg-config
+
+# emumba (boost)
+RUN apt-get update && apt-get install -y \
+    libboost-filesystem-dev \
+    libboost-program-options-dev \
+    libboost-test-dev \
+    libboost-date-time-dev
+
+# dev utils
 RUN apt-get update && apt-get install -y \ 
-   ccache \
-   fish \
-   gdb \
-   git-all \
-   sudo \
-   vim
+    fish \
+    git-all \
+    sudo \
+    vim \
+    wget
 
-# Set up default executables (update-alternatives)
+# set up default executables (update-alternatives)
 RUN \
-   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
-   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
-   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
-   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
-   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
-   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
 
 # Add compiler symlinks for ccache 
 RUN \
-   ln -s /usr/bin/ccache /usr/local/bin/clang; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++-12
+    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-12
 
 RUN locale-gen en_US.UTF-8
 
@@ -115,11 +131,11 @@ ARG docker_uid="1000"
 ARG docker_pass="bts"
 
 RUN \
-   groupadd -g "$docker_uid" "$docker_user" \
-   && useradd \
-   -m \
-   -u "$docker_uid" \
-   -g "$docker_uid" \
-   -G sudo \
-   --password "$(openssl passwd -1 "$docker_pass")" \
-   "$docker_user"
+    groupadd -g "$docker_uid" "$docker_user" \
+    && useradd \
+    -m \
+    -u "$docker_uid" \
+    -g "$docker_uid" \
+    -G sudo \
+    --password "$(openssl passwd -1 "$docker_pass")" \
+    "$docker_user"

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnExpression">
-    <Scanner Include="ColumnExpressionTokens.lex" />
-    <Parser Include="ColumnExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -1,62 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-public class ColumnExpression {
-   public ColumnExpression(string name, Type? type = null, object? value = null) {
-      Name = name;
-      Type = type
-         ?? value?.GetType()
-         ?? throw new ArgumentException(
-            "Type must be specified explicitly when value is null", nameof(type));
-      Value = value
-         ?? DBNull.Value;
-   }
-
-   public string Name { get; }
-
-   public Type Type { get; }
-
-   public object Value { get; }
-
-   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
-      // Try to convert value to 'type' to verify they are compatible
-      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
-      // account for implicit conversions (e.g., int -> long), just inheritance.
-      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
-         throw new ArgumentException(
-            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
-      }
-      return new(name, type ?? typeof(T), value);
-   }
-
-   internal static Type ParseType(string typeId) {
-      return Candidates().ExceptNull().FirstOrDefault() ??
-         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
-
-      IEnumerable<Type?> Candidates() {
-         if (typeId.Length == 0) {
-            yield break;
-         }
-         yield return Type.GetType(typeId);
-         yield return Type.GetType($"System.{typeId}");
-         if (typeId.Length > 1) {
-            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
-         }
-         yield return typeId.ToLower() switch {
-            "float64" or "double" or "real64" or "number" => typeof(double),
-            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
-            "int16" or "short" or "integer16" or "signed16" => typeof(short),
-            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
-            "int64" or "long" or "integer64" or "signed64" => typeof(long),
-            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
-            "uint32" or "uint" or "unsigned" => typeof(uint),
-            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
-            "char" => typeof(char),
-            "byte" or "uint8" => typeof(byte),
-            "bool" or "boolean" => typeof(bool),
-            "string" or "text" => typeof(string),
-            "date" or "datetime" => typeof(DateTime),
-            _ => null
-         };
-      }
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -1,49 +0,0 @@
-%namespace Bts.ColumnExpressions
-%partial
-%visibility internal
-%tokentype ColumnExpressionToken
-%parsertype ColumnExpressionParser
-
-%union {
-   public int Integer;
-   public double Real;
-   public bool Boolean;
-   public string String;
-   public string Identifier;
-   public string TypeId;
-   public DateTime DateTime;
-   public Tuple<string, Type> ColumnDefinition;
-   public ColumnExpression ColumnExpression;
-   public ImmutableList<ColumnExpression> ColumnExpressions;
-}
-
-%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
-%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
-
-%nonassoc EQUALS
-
-%type <ColumnDefinition> ColumnDefinition
-%type <ColumnExpression> ColumnExpression
-%type <ColumnExpressions> ColumnExpressions
-
-%%
-
-ColumnExpressions :
-   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
-   ;
-
-ColumnExpression :
-   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
-   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
-   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
-   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
-   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
-   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
-   ;
-
-ColumnDefinition :
-   IDENTIFIER { $$ = new($1.Identifier, null); }
-   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
-   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
-   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnExpressionInterpreter {
-   public static IList<ColumnExpression> Evaluate(Stream stream) {
-      var parser = new ColumnExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
-         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.Text;
-
-partial class ColumnExpressionParser {
-   public ColumnExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-partial class ColumnExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -1,85 +0,0 @@
-%namespace Bts.ColumnExpressions
-%scannertype ColumnExpressionScanner
-%visibility internal
-%tokentype ColumnExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-BooleanConstant          true|false
-StringConstant           '([^\'\\]*|\\.)*'
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
-IntegerConstant          -?[0-9]+
-NullConstant             'null'
-RealConstant             -?[0-9]+\.[0-9]+
-
-/* Relational Operators */
-Equals                   =
-OfType                   /
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-NamespacePrefix          {Identifier}\.
-TypeId                   {NamespacePrefix}*{Identifier}
-
-/* Utility Tokens */
-OpenParenthesis          \(
-CloseParenthesis         \)
-Comma                    ,
-QuoteOpen                \[
-QuoteClose               \]
-
-/* Comments */
-LineComment \/\/[^\n\r]*(\n|\r|\r\n)
-
-%%
-
-{Equals} {
-   return (int)ColumnExpressionToken.EQUALS;
-}
-
-{OfType} {
-   return (int)ColumnExpressionToken.OF_TYPE;
-}
-
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
-   return (int)ColumnExpressionToken.STRING_CONSTANT;
-}
-
-{BooleanConstant} {
-   yylval.Boolean = yytext.ToLower() switch {
-      "true" => true,
-      "false" => false,
-      _ => throw new Exception(
-         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
-   };
-   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
-}
-
-{DateTimeConstant} {
-   yylval.DateTime = DateTime.Parse(yytext);
-   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
-}
-
-{IntegerConstant} {
-   yylval.Integer = int.Parse(yytext);
-   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
-}
-
-{RealConstant} {
-   yylval.Real = double.Parse(yytext);
-   return (int)ColumnExpressionToken.REAL_CONSTANT;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnExpressionToken.IDENTIFIER;
-}
-
-{TypeId} {
-   yylval.TypeId = yytext;   
-   return (int)ColumnExpressionToken.TYPE_ID;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnSortExpression">
-    <Scanner Include="ColumnSortExpressionTokens.lex" />
-    <Parser Include="ColumnSortExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -1,11 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-public class ColumnSortExpression {
-   public ColumnSortExpression(string name, bool isAscending = true) {
-      Name = name;
-      IsAscending = isAscending;
-   }
-
-   public string Name { get; }
-   public bool IsAscending { get; }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -1,29 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%partial
-%visibility internal
-%tokentype ColumnSortExpressionToken
-%parsertype ColumnSortExpressionParser
-
-%union {
-   public string Identifier;
-   public ColumnSortExpression Column;
-   public ImmutableList<ColumnSortExpression> Columns;
-}
-
-%token ASCENDING, DESCENDING, IDENTIFIER
-
-%type <Column> Column
-%type <Columns> Columns
-
-%%
-
-Columns :
-   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | Columns Column { $$ = $1.Add($2); }
-   ;
-
-Column :
-   IDENTIFIER { $$ = new($1.Identifier, true); }
-   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
-   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
-   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnSortExpressionInterpreter {
-   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
-      var parser = new ColumnSortExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
-         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnSortExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.Text;
-
-partial class ColumnSortExpressionParser {
-   public ColumnSortExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnSortExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-partial class ColumnSortExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -1,30 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%scannertype ColumnSortExpressionScanner
-%visibility internal
-%tokentype ColumnSortExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-Ascending                +
-Descending               -
-
-%%
-
-{Ascending} {
-   return (int)ColumnSortExpressionToken.ASCENDING;
-}
-
-{Descending} {
-   return (int)ColumnSortExpressionToken.DESCENDING;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnSortExpressionToken.IDENTIFIER;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -1,301 +1,229 @@
-namespace Bts.DataStream.Websocket;
-
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.collections;
-
-using Bts.DataStreamExpressions;
-using Bts.Websocket;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-using Spark.Core.Json;
-
-using System.Buffers;
-using System.Reactive.Subjects;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-/// <summary>
-/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
-/// requests (from clients) as input, and emits data/notifications (to clients) as output.
-/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
-/// </summary>
-public class DataSync(JsonSerializerOptions? serializerOpts = null) :
-   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
-   protected readonly object lockObj = new();
-   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
-   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
-
-   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
-
-   public void OnCompleted() =>
-      Log.Api.info("DataSync incoming request stream has completed.");
-
-   public void OnError(Exception error) =>
-      Log.Api.error($"DataSync error: {error}");
-
-   public void OnNext(WebsocketConnectionUpdate value) {
-      switch (value.Type) {
-         case WebsocketConnectionUpdateType.IncomingMessage:
-            onIncomingMessage(value.IncomingMessage);
-            break;
-         case WebsocketConnectionUpdateType.Connect:
-            onConnect(value.Connect);
-            break;
-         case WebsocketConnectionUpdateType.Disconnect:
-            onDisconnect(value.Disconnect);
-            break;
-         default:
-            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
-            break;
-      }
-   }
-
-   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
-      lock (lockObj) {
-         requestSend += observer.OnNext;
-         return Disposable.Create(Dispose);
-      }
-      void Dispose() {
-         lock (lockObj) {
-            requestSend -= observer.OnNext;
-         }
-      }
-   }
-
-   static JsonRpcId? extractRpcId(string messageBody) {
-      try {
-         return JsonNode.Parse(messageBody)
-            ?.AsObject()
-            ?.GetValueOrDefault("id")
-            ?.Deserialize<JsonRpcId>();
-      } catch {
-         return null;
-      }
-   }
-
-   static string getRpcNotification(DataPacket dp) {
-      using (var buffer = new MemoryStream())
-      using (var writer = new Utf8JsonWriter(buffer))
-      using (writer.WriteObject()) {
-         writer.WriteString("method", "subscription");
-         using (writer.WriteObject("params")) {
-            writer.Write("tableName", dp.schema.tableName);
-            writer.Write("columns", dp.schema.columnInfo);
-            var cellJsonFactories = dp.schema.columnInfo.Select(getCellJsonFactory).ToList();
-            using (writer.WriteArray("added")) {
-               foreach (IBlueRow row in (dp.addedRows?.Cast<IBlueRow>()).empty_if_null()) {
-                  using (writer.WriteObject()) {
-                     for (int i = 0; i < cellJsonFactories.Count; i++) {
-                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
-                     }
-                  }
-               }
-            }
-            using (writer.WriteArray("deleted")) {
-               foreach (IBlueRow row in (dp.deletedRows?.Cast<IBlueRow>()).empty_if_null()) {
-                  using (writer.WriteObject()) {
-                     for (int i = 0; i < cellJsonFactories.Count; i++) {
-                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
-                     }
-                  }
-               }
-            }
-         }
-         writer.WriteString("jsonrpc", "2.0");
-         writer.Flush();
-         return Encoding.UTF8.GetString(buffer.ToArray());
-      }
-   }
-
-   static Func<IBlueRow, JsonNode?> getCellJsonFactory(ColumnInfo colInfo) {
-      var t = colInfo.colType;
-      if (t == typeof(int)) {
-         return NullCheckWrapper(v => (int)v);
-      }
-      if (t == typeof(uint)) {
-         return NullCheckWrapper(v => (uint)v);
-      }
-      if (t == typeof(long)) {
-         return NullCheckWrapper(v => (long)v);
-      }
-      if (t == typeof(ulong)) {
-         return NullCheckWrapper(v => (ulong)v);
-      }
-      if (t == typeof(short)) {
-         return NullCheckWrapper(v => (short)v);
-      }
-      if (t == typeof(ushort)) {
-         return NullCheckWrapper(v => (ushort)v);
-      }
-      if (t == typeof(double)) {
-         return NullCheckWrapper(v => (double)v);
-      }
-      if (t == typeof(float)) {
-         return NullCheckWrapper(v => (float)v);
-      }
-      if (t == typeof(bool)) {
-         return NullCheckWrapper(v => (bool)v);
-      }
-      return NullCheckWrapper(v => (string)v);
-
-      Func<IBlueRow, JsonNode?> NullCheckWrapper(Func<object, JsonNode> toJson) =>
-         r => {
-            var obj = r[colInfo.colNo];
-            if (obj is null) {
-               return null;
-            }
-            return toJson(obj);
-         };
-   }
-
-   void onConnect(WebsocketConnectNotification notification) {
-      var id = notification.Id;
-      lock (lockObj) {
-         clients.Add(id, []);
-      }
-      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
-      if (notification.Message is not null) {
-         onIncomingMessage(notification.Message);
-      }
-   }
-
-   void onDisconnect(WebsocketDisconnectNotification notification) {
-      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
-      unsubscribeAll(notification.Id);
-      clients.Remove(notification.Id);
-   }
-
-   void onIncomingMessage(WebsocketIncomingMessage message) {
-      var id = message.sender_client_id;
-      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
-         onBadRequest(message, err);
-         return;
-      }
-      switch (req.Method!.ToLower()) {
-         case "subscribe":
-            subscribe(id, req);
-            break;
-         case "publish":
-            Log.Api.error("DataStreamWebsocket api 'publish' not implemented.");
-            // TODO
-            break;
-         case "unsubscribe":
-            unsubscribe(id, req);
-            break;
-         case "unsubscribe_all":
-            unsubscribeAll(id);
-            break;
-         default:
-            Log.Api.error($"Received request of unknown type: {req}");
-            break;
-      }
-   }
-
-   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
-      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send([new(id, rpcResStr)]);
-   }
-
-   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
-      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send([new(id, rpcResStr)]);
-   }
-
-   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
-      lock (lockObj) {
-         if (requestSend is null) {
-            Log.Api.warn(
-               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
-            return false;
-         }
-         requestSend(messages);
-         return true;
-      }
-   }
-
-   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
-      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
-      lock (lockObj) {
-         clients[id].Add(sub);
-         sendResponse(id, req.Id!.Value, req.Params);
-         stream.onChange += OnChange;
-      }
-
-      void OnChange(object? newValue) {
-         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
-         var notif = getRpcNotification(dp);
-         send([new(id, notif)]);
-      }
-
-      void Dispose() =>
-         stream.onChange -= OnChange;
-   }
-
-   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>();
-      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
-      // Minimize amount of code inside lock (just need it to find/remove the subscription)
-      List<ClientSubscription>? subs = null;
-      ClientSubscription? sub = null;
-      lock (lockObj) {
-         subs = clients.GetValueOrDefault(id);
-         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
-      }
-      if (subs is null) {
-         // Didn't find client (log/throw).
-         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      if (sub is null) {
-         // Found client, but didn't find subscription (log/throw).
-         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
-            $"Id: {id}. Request:\n{subParams}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      // Found client and subscription (dispose/remove subscription).
-      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
-      sub.Dispose();
-   }
-
-   void unsubscribeAll(WebsocketClientID id) {
-      lock (lockObj) {
-         if (!clients.TryGetValue(id, out var subs)) {
-            throw new Exception(
-               $"Unable to unsubscribe: did not find client with id {id}");
-         }
-         subs.DisposeAll();
-         subs.Clear();
-      }
-   }
-
-   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
-      var rpcId = extractRpcId(req.message_body);
-      var logMsg = $"Recieved invalid request:\n{req.message_body}";
-      if (errorMsg is not null) {
-         logMsg += $"\n\n**Exception**\n{errorMsg}";
-      }
-      Log.Api.error(logMsg);
-      if (rpcId is null) {
-         return;
-      }
-      // Send error back to client
-      var rpcErr = JsonRpcError.Create(
-         (long)JsonRpcErrorCode.InvalidRequest,
-         $"Received invalid request: {errorMsg ?? "unknown error"}",
-         req.message_body);
-      sendError(req.sender_client_id, rpcId.Value, rpcErr);
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.collections;
+
+using Bts.DataStreamExpressions;
+using Bts.Websocket;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+
+using System.Reactive.Subjects;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+/// <summary>
+/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
+/// requests (from clients) as input, and emits data/notifications (to clients) as output.
+/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
+/// </summary>
+public class DataSync(ServiceContext serviceCtx, JsonSerializerOptions? serializerOpts = null) :
+   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
+   readonly ServiceContext serviceCtx = serviceCtx;
+   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+   readonly object lockObj = new();
+
+   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
+
+   public void OnCompleted() =>
+      Log.Api.info("DataSync incoming request stream has completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"DataSync error: {error}");
+
+   public void OnNext(WebsocketConnectionUpdate value) {
+      switch (value.Type) {
+         case WebsocketConnectionUpdateType.IncomingMessage:
+            onIncomingMessage(value.IncomingMessage);
+            break;
+         case WebsocketConnectionUpdateType.Connect:
+            onConnect(value.Connect);
+            break;
+         case WebsocketConnectionUpdateType.Disconnect:
+            onDisconnect(value.Disconnect);
+            break;
+         default:
+            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
+            break;
+      }
+   }
+
+   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
+      lock (lockObj) {
+         requestSend += observer.OnNext;
+         return Disposable.Create(dispose);
+      }
+      void dispose() {
+         lock (lockObj) {
+            requestSend -= observer.OnNext;
+         }
+      }
+   }
+
+   static JsonRpcId? extractRpcId(string messageBody) {
+      try {
+         return JsonNode.Parse(messageBody)
+            ?.AsObject()
+            ?.GetValueOrDefault("id")
+            ?.Deserialize<JsonRpcId>();
+      } catch {
+         return null;
+      }
+   }
+
+   void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, new());
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   void onIncomingMessage(WebsocketIncomingMessage message) {
+      var id = message.sender_client_id;
+      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
+         onBadRequest(message, err);
+         return;
+      }
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
+      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
+      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
+      send(new WebsocketOutgoingMessage[] { new(id, rpcNotificationStr) });
+   }
+
+   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
+      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send(new WebsocketOutgoingMessage[] { new(id, rpcResStr )});
+   }
+
+   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
+      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send(new WebsocketOutgoingMessage[] { new(id, rpcResStr) });
+   }
+
+   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
+      lock (lockObj) {
+         if (requestSend is null) {
+            Log.Api.warn(
+               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
+            return false;
+         }
+         requestSend(messages);
+         return true;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += onChange;
+      }
+
+      void onChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var rpcParams = new {
+            addedRows = dp.addedRows,
+            deletedRows = dp.deletedRows
+         };
+         sendNotification(id, "subscription", rpcParams);
+      }
+
+      void dispose() =>
+         stream.onChange -= onChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
+      var rpcId = extractRpcId(req.message_body);
+      var logMsg = $"Recieved invalid request:\n{req.message_body}";
+      if (errorMsg is not null) {
+         logMsg += $"\n\n**Exception**\n{errorMsg}";
+      }
+      Log.Api.error(logMsg);
+      if (rpcId is null) {
+         return;
+      }
+      // Send error back to client
+      var rpcErr = JsonRpcError.Create(
+         (long)JsonRpcErrorCode.InvalidRequest,
+         $"Received invalid request: {errorMsg ?? "unknown error"}",
+         req.message_body);
+      sendError(req.sender_client_id, rpcId.Value, rpcErr);
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs

@@ -1,48 +1,48 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class MultilineStringJsonConverter : JsonConverter<string> {
-   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (typeToConvert != typeof(string)) {
-         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-      }
-      return reader.TokenType switch {
-         JsonTokenType.String => StringImpl(ref reader),
-         JsonTokenType.StartArray => ArrayImpl(ref reader),
-         _ => throw new ArgumentException(
-            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-      };
-
-      static string? StringImpl(ref Utf8JsonReader reader) =>
-         new(reader.GetString()!);
-
-      static string? ArrayImpl(ref Utf8JsonReader reader) {
-         var lines = new List<string>(2);
-         while (reader.Read()) {
-            switch (reader.TokenType) {
-               case JsonTokenType.String:
-                  lines.Add(reader.GetString()!);
-                  break;
-               case JsonTokenType.EndArray:
-                  return string.Join("\n", lines);
-            }
-         }
-         throw new JsonException(
-            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-      }
-   }
-
-   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
-      var lines = value.Split("\n");
-      if (lines.Length < 2) {
-         writer.WriteRawValue(value);
-         return;
-      }
-      writer.WriteRawValue($"[{", ".join(lines)}]");
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class MultilineStringJsonConverter : JsonConverter<string> {
+   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (typeToConvert != typeof(string)) {
+         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+      }
+      return reader.TokenType switch {
+         JsonTokenType.String => stringImpl(ref reader),
+         JsonTokenType.StartArray => arrayImpl(ref reader),
+         _ => throw new ArgumentException(
+            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+      };
+
+      static string? stringImpl(ref Utf8JsonReader reader) =>
+         new(reader.GetString()!);
+
+      static string? arrayImpl(ref Utf8JsonReader reader) {
+         var lines = new List<string>(2);
+         while (reader.Read()) {
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  lines.Add(reader.GetString()!);
+                  break;
+               case JsonTokenType.EndArray:
+                  return string.Join("\n", lines);
+            }
+         }
+         throw new JsonException(
+            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
+      var lines = value.Split("\n");
+      if (lines.Length < 2) {
+         writer.WriteRawValue(value);
+         return;
+      }
+      writer.WriteRawValue($"[{", ".join(lines)}]");
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -1,27 +1,27 @@
-namespace Bts.DataStream.Websocket;
-
-using bts;
-
-using Bts.Websocket;
-
-using Google.Protobuf;
-
-using System.Text.Json;
-
-public static class Session {
-   static int sessionIdCounter;
-
-   public static async Task Start(
-      WebsocketServerConfig config, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      var sessionId = Interlocked.Increment(ref sessionIdCounter);
-      var gateway = new WebsocketGatewayObservable(config);
-      var sync = new DataSync(opts);
-      using (gateway.Subscribe(sync))
-      using (sync.Subscribe(gateway)) {
-         Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
-         Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
-         await gateway.ConnectAsync();
-      }
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using bts;
+using bts.api.core;
+
+using Bts.Websocket;
+
+using Google.Protobuf;
+
+using System.Text.Json;
+
+public static class Session {
+   static int sessionIdCounter;
+
+   public static async Task Start(
+      ServiceContext context, WebsocketServerConfig config, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      var sessionId = Interlocked.Increment(ref sessionIdCounter);
+      var gateway = new WebsocketGatewayObservable(config);
+      var sync = new DataSync(context, opts);
+      using var syncSub = gateway.Subscribe(sync);
+      using var gwSub = sync.Subscribe(gateway);
+      Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
+      Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
+      await gateway.ConnectAsync();
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -1,110 +1,110 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-public class SubscriptionParams : IEquatable<SubscriptionParams> {
-   static class PropNames {
-      public const string QUERY = "query";
-   }
-
-   public SubscriptionParams(string? query) {
-      Query = query;
-   }
-
-   // Serializer supports plain string, or an array of lines for convenience
-   [MaybeNull]
-   [JsonPropertyName(PropNames.QUERY)]
-   [JsonConverter(typeof(MultilineStringJsonConverter))]
-   public string? Query { get; set; }
-
-   public bool Equals(SubscriptionParams? other) =>
-      other is not null &&
-      Query == other.Query;
-
-   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
-      public override bool CanConvert(Type typeToConvert) =>
-         typeToConvert == typeof(SubscriptionParams);
-
-      public override SubscriptionParams Read(
-         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-         if (typeToConvert != typeof(SubscriptionParams)) {
-            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-         }
-         return read(ref reader);
-      }
-
-      SubscriptionParams read(ref Utf8JsonReader reader) {
-         return reader.TokenType switch {
-            JsonTokenType.String => stringImpl(ref reader),
-            JsonTokenType.StartArray => arrayImpl(ref reader),
-            JsonTokenType.StartObject => objectImpl(ref reader),
-            _ => throw new ArgumentException(
-               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-         };
-
-         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
-            new(reader.GetString()!);
-
-         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-
-         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.PropertyName:
-                     var propName = reader.GetString()!;
-                     if (propName == PropNames.QUERY) {
-                        switch (reader.TokenType) {
-                           case JsonTokenType.StartArray:
-                              return arrayImpl(ref reader);
-                           case JsonTokenType.String:
-                              return stringImpl(ref reader);
-                        }
-                     }
-                     break;
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-      }
-
-      public override void Write(
-         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
-         var q = value.Query;
-         var lines = q?.Split("\n");
-         if (lines is null || lines.Length < 2) {
-            writer.WriteString(PropNames.QUERY, value.Query);
-            return;
-         }
-         writer.WriteStartArray(PropNames.QUERY);
-         foreach (var l in lines) {
-            writer.WriteStringValue(l);
-         }
-         writer.WriteEndArray();
-      }
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SubscriptionParams : IEquatable<SubscriptionParams> {
+   static class PropNames {
+      public const string QUERY = "query";
+   }
+
+   public SubscriptionParams(string? query) {
+      Query = query;
+   }
+
+   // Serializer supports plain string, or an array of lines for convenience
+   [MaybeNull]
+   [JsonPropertyName(PropNames.QUERY)]
+   [JsonConverter(typeof(MultilineStringJsonConverter))]
+   public string? Query { get; set; }
+
+   public bool Equals(SubscriptionParams? other) =>
+      other is not null &&
+      Query == other.Query;
+
+   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
+      public override bool CanConvert(Type typeToConvert) =>
+         typeToConvert == typeof(SubscriptionParams);
+
+      public override SubscriptionParams Read(
+         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+         if (typeToConvert != typeof(SubscriptionParams)) {
+            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+         }
+         return read(ref reader);
+      }
+
+      SubscriptionParams read(ref Utf8JsonReader reader) {
+         return reader.TokenType switch {
+            JsonTokenType.String => stringImpl(ref reader),
+            JsonTokenType.StartArray => arrayImpl(ref reader),
+            JsonTokenType.StartObject => objectImpl(ref reader),
+            _ => throw new ArgumentException(
+               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+         };
+
+         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
+            new(reader.GetString()!);
+
+         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+
+         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.PropertyName:
+                     var propName = reader.GetString()!;
+                     if (propName == PropNames.QUERY) {
+                        switch (reader.TokenType) {
+                           case JsonTokenType.StartArray:
+                              return arrayImpl(ref reader);
+                           case JsonTokenType.String:
+                              return stringImpl(ref reader);
+                        }
+                     }
+                     break;
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+      }
+
+      public override void Write(
+         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
+         var q = value.Query;
+         var lines = q?.Split("\n");
+         if (lines is null || lines.Length < 2) {
+            writer.WriteString(PropNames.QUERY, value.Query);
+            return;
+         }
+         writer.WriteStartArray(PropNames.QUERY);
+         foreach (var l in lines) {
+            writer.WriteStringValue(l);
+         }
+         writer.WriteEndArray();
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Bts.DataStream.WebsocketClient.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <EnableDefaultContentItems>True</EnableDefaultContentItems>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs

@@ -39,7 +39,7 @@ static async Task run(
    Console.WriteLine("Starting DataStreamWebsocketServer...");
    var serviceCtx = ServiceContext.create(sparkdConfigSource);
    await serviceCtx.login(user, password);
-   await Session.Start(wssConfig, opts);
+   await Session.Start(serviceCtx, wssConfig, opts);
    // if (sparkdConfigSource is not null && sparkdConfigSource.Length > 9999) {
    // } else {
    //    await ChatGptSession.Start();

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -1,20 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.2' } },
-  plugins: ['react-refresh'],
-  rules: {
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -1,11 +0,0 @@
-<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
-  <PropertyGroup>
-    <StartupCommand>npm run dev</StartupCommand>
-    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
-    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
-    <!-- Allows the build (or compile) script located on package.json to run on Build -->
-    <ShouldRunBuildScript>false</ShouldRunBuildScript>
-    <!-- Folder where production build objects will be placed -->
-    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\dist</PublishAssetsDirectory>
-  </PropertyGroup>
-</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -1,8 +0,0 @@
-# React + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
-- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
-- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -1,13 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite + React</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.jsx"></script>
-  </body>
-</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-  </packageSources>
-  <disabledPackageSources>
-    <clear />
-  </disabledPackageSources>
-</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -1,27 +0,0 @@
-{
-  "name": "bts.datastreamwebapp.client",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
-    "preview": "vite preview"
-  },
-  "dependencies": {
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "react-use-websocket": "^4.5.0"
-  },
-  "devDependencies": {
-    "@types/react": "^18.2.43",
-    "@types/react-dom": "^18.2.17",
-    "@vitejs/plugin-react": "^4.2.1",
-    "eslint": "^8.55.0",
-    "eslint-plugin-react": "^7.33.2",
-    "eslint-plugin-react-hooks": "^4.6.0",
-    "eslint-plugin-react-refresh": "^0.4.5",
-    "vite": "^5.0.8"
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -1,44 +0,0 @@
-#root {
-    max-width: 1280px;
-    margin: 0 auto;
-    padding: 2rem;
-    text-align: center;
-    --cell-background-color: #293136;
-    --cell-color: #cbe3ef;
-}
-
-tr:nth-child(even) {
-    background: #F2F2F2;
-}
-
-tr:nth-child(odd) {
-    background: #FFF;
-}
-
-th, td {
-    padding-left: 1rem;
-    padding-right: 1rem;
-    background-color: var(--cell-background-color);
-    color: var(--cell-color);
-}
-
-textarea#query-input {
-  height: 5em;
-  width: 50em;
-}
-
-button#query-button {
-  height: 5em;
-  vertical-align: top;
-}
-
-table#query-output {
-  float: none;
-  clear: both;
-}
-
-p#footer {
-  float: none;
-  clear: both;
-  vertical-align: bottom;
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -1,93 +0,0 @@
-import { useEffect, useState, Component } from "react";
-import { useWebSocket } from "react-use-websocket";
-import "./App.css";
-
-export class Home extends Component {
-  static displayName = Home.name;
-  // static sparkdHost = '192.168.50.94';
-  static sparkdHost = "localhost";
-
-  // static websocket_url = 'wss://markets.api.uat.cmegroup.com/marketdatastream/v1';
-  // static access_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjF0al9MTVFrT2pGVzJWQTZvdkZzOEZHUWViMCIsInBpLmF0bSI6IjEifQ.eyJzY29wZSI6W10sImNsaWVudF9pZCI6ImFwaV90aGlua2FscGhhMzIxIiwiYXVkIjoiTnIiLCJqdGkiOiIweERtb3NnU2NHcE5GUEFaIiwic3ViIjoiYXBpX3RoaW5rYWxwaGEzMjEiLCJleHAiOjE2OTgzNTY5OTJ9.uJJEhWpujGiyJ5_lypiIPB8xR0GkuXqydtCugp5xD0hyTuX6VkQdNoY_oaRXXGRCUPCvfsXpU-HfL7pRkcXCjMLTrVIqf3RjD4_gar_ExbIb180nbYMG87E4rOILi5bIin51KOcMNBdHW7pTKwsS221XGyIAKIj65WxD-K8QSVr1HN6s7KDfYdez5dstWWC6xGg_3IZkrJjx8Df1HNT73K9PwUaAl7gcK4_2RSr8L636QbF3_y4Tv26g5IEv90Mc1mnAFPvUGI-nDR4s5Vso0398DlYl_tw-zbQQmn-QuQ6rS_ZIcZKgyoHKwD-3lgV-J1_--ndHdh8qwJC7Hk8ZKg'
-
-  render() {
-    let socket = null;
-    const wsUrl = `ws://localhost:5268/ws`;
-    function connectToSparkd() {
-      // const wsUrl = Home.websocket_url;
-      if (socket == null) {
-        socket = new WebSocket(wsUrl);
-      }
-      if (socket && socket.readyState === WebSocket.OPEN) {
-        return;
-      }
-
-      const output = document.getElementById("query-output");
-      socket.onopen = (event) => {
-        console.log(`Connected to ${wsUrl}`);
-      };
-
-      socket.onmessage = (event) => {
-        console.log(`Received message: ${event.data}`);
-        output.innerHTML = event.data;
-      };
-
-      socket.onerror = (error) => {
-        console.log(`Error on websocket ${wsUrl}: ${error}`);
-      };
-
-      socket.onclose = (event) => {
-        if (event.wasClean) {
-          const msg = `Connection to ${wsUrl} closed cleanly, code=${event.code}, reason=${event.reason}`;
-          console.log(msg);
-        } else {
-          const msg = `Connection to ${wsUrl} abruptly closed`;
-          console.log(msg);
-        }
-      };
-    }
-
-    function executeQuery() {
-      if (socket == null || !socket) {
-        connectToSparkd();
-      }
-      if (socket == null || !socket) {
-        console.log("Failed to connect to websocket server");
-        return;
-      }
-      if (socket.readyState === WebSocket.OPEN) {
-        const queryInput = document.getElementById("query-input");
-        const rpcReq = {
-          jsonrpc: "2.0",
-          method: "subscribe",
-          params: {
-            query: queryInput.value,
-          },
-          id: 1,
-        };
-        socket.send(JSON.stringify(rpcReq));
-      }
-    }
-
-    return (
-      <div>
-        <h1>Data Stream Query Analyzer</h1>
-        <p>Run queries against the BTS Cloud</p>
-        <div>
-          <textarea id="query-input" placeholder="Enter Query"></textarea>
-          <button id="query-button" onClick={executeQuery}>
-            Execute
-          </button>
-        </div>
-        <table id="query-output"></table>
-        <p id="footer">
-          The <code>ClientApp</code> subdirectory is a standard React
-          application based on the <code>create-react-app</code> template. If
-          you open a command prompt in that directory, you can run{" "}
-          <code>npm</code> commands such as <code>npm test</code> or{" "}
-          <code>npm install</code>.
-        </p>
-      </div>
-    );
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -1,68 +0,0 @@
-:root {
-  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
-  line-height: 1.5;
-  font-weight: 400;
-
-  color-scheme: light dark;
-  color: rgba(255, 255, 255, 0.87);
-  background-color: #242424;
-
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-a {
-  font-weight: 500;
-  color: #646cff;
-  text-decoration: inherit;
-}
-a:hover {
-  color: #535bf2;
-}
-
-body {
-  margin: 0;
-  display: flex;
-  place-items: center;
-  min-width: 320px;
-  min-height: 100vh;
-}
-
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-button {
-  border-radius: 8px;
-  border: 1px solid transparent;
-  padding: 0.6em 1.2em;
-  font-size: 1em;
-  font-weight: 500;
-  font-family: inherit;
-  background-color: #1a1a1a;
-  cursor: pointer;
-  transition: border-color 0.25s;
-}
-button:hover {
-  border-color: #646cff;
-}
-button:focus,
-button:focus-visible {
-  outline: 4px auto -webkit-focus-ring-color;
-}
-
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #ffffff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -1,10 +0,0 @@
-import React from 'react'
-import ReactDOM from 'react-dom/client'
-import {Home} from './App.jsx'
-import './index.css'
-
-ReactDOM.createRoot(document.getElementById('root')).render(
-  <React.StrictMode>
-    <Home />
-  </React.StrictMode>,
-)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -1,60 +0,0 @@
-import { fileURLToPath, URL } from 'node:url';
-
-import { defineConfig } from 'vite';
-import plugin from '@vitejs/plugin-react';
-import fs from 'fs';
-import path from 'path';
-import child_process from 'child_process';
-
-const baseFolder =
-    process.env.APPDATA !== undefined && process.env.APPDATA !== ''
-        ? `${process.env.APPDATA}/ASP.NET/https`
-        : `${process.env.HOME}/.aspnet/https`;
-
-const certificateArg = process.argv.map(arg => arg.match(/--name=(?<value>.+)/i)).filter(Boolean)[0];
-const certificateName = certificateArg ? certificateArg.groups.value : "bts.datastreamwebapp.client";
-
-if (!certificateName) {
-    console.error('Invalid certificate name. Run this script in the context of an npm/yarn script or pass --name=<<app>> explicitly.')
-    process.exit(-1);
-}
-
-const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
-const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
-
-if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
-    if (0 !== child_process.spawnSync('dotnet', [
-        'dev-certs',
-        'https',
-        '--export-path',
-        certFilePath,
-        '--format',
-        'Pem',
-        '--no-password',
-    ], { stdio: 'inherit', }).status) {
-        throw new Error("Could not create certificate.");
-    }
-}
-
-// https://vitejs.dev/config/
-export default defineConfig({
-    plugins: [plugin()],
-    resolve: {
-        alias: {
-            '@': fileURLToPath(new URL('./src', import.meta.url))
-        }
-    },
-    server: {
-        proxy: {
-            '^/weatherforecast': {
-                target: 'https://localhost:7128/',
-                secure: false
-            }
-        },
-        port: 5173,
-        https: {
-            key: fs.readFileSync(keyFilePath),
-            cert: fs.readFileSync(certFilePath),
-        }
-    }
-})

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -1,35 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <InvariantGlobalization>true</InvariantGlobalization>
-    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
-    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
-    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy">
-      <Version>8.*-*</Version>
-    </PackageReference>
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
-  </ItemGroup>
-
-  <!-- <ItemGroup>
-    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
-  </ItemGroup> -->
-</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.http

@@ -1,6 +0,0 @@
-@Bts.DataStreamWebApp.Server_HostAddress = http://localhost:5268
-
-GET {{Bts.DataStreamWebApp.Server_HostAddress}}/weatherforecast/
-Accept: application/json
-
-###

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,117 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.api.core;
-
-using Bts.DataStreamExpressions;
-
-using Microsoft.AspNetCore.Mvc;
-
-using Spark.Core.Disposables;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-
-// [Route("[controller]")]
-[Route("/ws")]
-public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
-   readonly ILogger<DataStreamSubscriptionController> _logger = logger;
-   IDisposable? streamSubscription;
-
-   [Route("/ws")]
-   public async Task Get() {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         _logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      Console.WriteLine("Establishing websocket connection...");
-      using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-      Console.WriteLine($"Websocket connection established.");
-      await runWebSocketServer(null, "bts", string.Empty, webSocket);
-   }
-
-   public static async Task Echo(WebSocket webSocket) {
-      var buffer = new byte[1024 * 4];
-      var receiveResult = await webSocket.ReceiveAsync(
-          new ArraySegment<byte>(buffer), CancellationToken.None);
-
-      while (!receiveResult.CloseStatus.HasValue) {
-         await webSocket.SendAsync(
-             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-
-         receiveResult = await webSocket.ReceiveAsync(
-             new ArraySegment<byte>(buffer), CancellationToken.None);
-      }
-
-      await webSocket.CloseAsync(
-          receiveResult.CloseStatus.Value,
-          receiveResult.CloseStatusDescription,
-          CancellationToken.None);
-   }
-
-   async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
-      var context = ServiceContext.create(sparkd);
-      await context.login(user, password);
-      var buffer = new byte[1024 * 4];
-      var receiveResult = await webSocket.ReceiveAsync(
-          new ArraySegment<byte>(buffer), CancellationToken.None);
-
-      try {
-         while (!receiveResult.CloseStatus.HasValue) {
-            receiveResult = await webSocket.ReceiveAsync(
-                new ArraySegment<byte>(buffer), CancellationToken.None);
-            var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
-            // var queryArgs = new QueryArguments { Query = msg };
-            var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
-            var queryArgs = rpcReq!.Params;
-            // if (queryArgs is null) {
-            //    Console.WriteLine("Invalid queryArgs.");
-            //    continue;
-            // }
-            var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
-            if (exp is null) {
-               Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
-               continue;
-            }
-            var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
-            Disposable.dispose(ref streamSubscription);
-            vs.onChange += onStreamChanged;
-            streamSubscription = Disposable.Create(() => vs.onChange -= onStreamChanged);
-            // await webSocket.SendAsync(
-            //     new ArraySegment<byte>(buffer, 0, receiveResult.Count),
-            //     receiveResult.MessageType,
-            //     receiveResult.EndOfMessage,
-            //     CancellationToken.None);
-         }
-      } catch (Exception ex) {
-         Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
-         return 1;
-      }
-
-      return 0;
-
-      async void onStreamChanged(object newValue) {
-         if (newValue is not DataPacket dp) {
-            Console.WriteLine(
-               $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
-            return;
-         }
-         Console.WriteLine("Received stream update.");
-         var added_rows_str = dp.addedRows.get_printed_rows(val => val.Length);
-         var msg = $"Received new rows:\n{added_rows_str}";
-         // var payload = new ReadOnlyMemory<byte>([.. msg]);
-         var encoded = Encoding.UTF8.GetBytes(msg);
-         var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
-         await webSocket.SendAsync(
-             payload,
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-      }
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/WeatherForecastController.cs

@@ -1,22 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-using Microsoft.AspNetCore.Mvc;
-
-[ApiController]
-[Route("[controller]")]
-public class WeatherForecastController(ILogger<WeatherForecastController> logger) : ControllerBase {
-   static readonly string[] Summaries = [
-      "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
-   ];
-
-   readonly ILogger<WeatherForecastController> _logger = logger;
-
-   [HttpGet(Name = "GetWeatherForecast")]
-   public IEnumerable<WeatherForecast> Get() {
-      return Enumerable.Range(1, 5).Select(index => new WeatherForecast {
-         Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)),
-         TemperatureC = Random.Shared.Next(-20, 55),
-         Summary = Summaries[Random.Shared.Next(Summaries.Length)]
-      })
-      .ToArray();
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using System.Text.Json.Serialization;
-
-public class DataStreamSubscription {
-   [JsonPropertyName("query")]
-   public string? Query { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamUpdate.cs

@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using BCG;
-
-class DataStreamUpdate {
-   public List<IBlueRow> addedRows { get; set; } = new(0);
-   public List<IBlueRow> deletedRows { get; set; } = new(0);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerOptions.cs

@@ -1,14 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using CommandLine;
-
-class DataStreamWebAppServerOptions {
-   [Option("sparkd")]
-   public string? sparkd { get; set; }
-
-   [Option("user", Default = "bts")]
-   public string? user { get; set; }
-
-   [Option("password", Default = "")]
-   public string? password { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebsocketHandler.cs

@@ -1,58 +0,0 @@
-using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.Logging;
-
-using System;
-using System.Net.WebSockets;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace Bts.DataStreamWebApp.Server {
-   // public class DataStreamWebSocketHandler {
-   //    readonly WebSocketConnectionManager _connectionManager = new();
-   //    readonly ILogger<DataStreamWebSocketHandler> _logger;
-
-   //    public DataStreamWebSocketHandler(
-   //       WebSocketConnectionManager connectionManager,
-   //       ILogger<DataStreamWebSocketHandler> logger
-   //    ) {
-   //       _connectionManager = connectionManager;
-   //       _logger = logger;
-   //    }
-
-   //    public async Task HandleWebSocket(HttpContext context, WebSocket webSocket) {
-   //       var socketId = _connectionManager.AddSocket(webSocket);
-
-   //       _logger.LogInformation($"WebSocket connection established with ID {socketId}");
-
-   //       while (webSocket.State == WebSocketState.Open) {
-   //          var message = await ReceiveMessageAsync(webSocket);
-   //          if (message != null) {
-   //             _logger.LogInformation($"Received message from ID {socketId}: {message}");
-   //             await BroadcastMessageAsync(message);
-   //          }
-   //       }
-
-   //       _connectionManager.RemoveSocket(socketId);
-   //       _logger.LogInformation($"WebSocket connection closed with ID {socketId}");
-   //    }
-
-   //    async Task<string?> ReceiveMessageAsync(WebSocket webSocket) {
-   //       var buffer = new byte[1024];
-   //       var result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
-
-   //       if (result.CloseStatus.HasValue) {
-   //          return null;
-   //       }
-
-   //       return System.Text.Encoding.UTF8.GetString(buffer, 0, result.Count);
-   //    }
-
-   //    async Task BroadcastMessageAsync(string message) {
-   //       foreach (var socket in _connectionManager.GetAllSockets()) {
-   //          if (socket.Value.State == WebSocketState.Open) {
-   //             await socket.Value.SendAsync(System.Text.Encoding.UTF8.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
-   //          }
-   //       }
-   //    }
-   // }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,166 +0,0 @@
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.utils;
-
-using Bts.DataStreamExpressions;
-using Bts.DataStreamWebApp.Server;
-using Bts.DataStreamWebApp.Server.Controllers;
-
-using CommandLine;
-
-using mbts.config;
-
-using System.Net.WebSockets;
-using System.Reactive.Disposables;
-using System.Reactive.Linq;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-static string fmtReq(HttpRequest req) =>
-   $"{req.Protocol}://{req.Host}/{req.Path}?{req.QueryString}";
-
-static string fmtDataPacket(DataPacket dp) {
-   var sb = new StringBuilder();
-   var rows = dp.addedRows.Cast<IBlueRow>().ToList();
-   var colWidths = dp.schema.columnInfo.Select(c => rows.Max(r => r[c.colNo]?.ToString()?.Length ?? 0)).ToArray();
-   foreach (IBlueRow r in dp.addedRows) {
-      for (int i = 0; i < colWidths.Length; i++) {
-         var str = r[i]?.ToString() ?? string.Empty;
-         if (i > 0) {
-            sb.Append(" | ");
-         }
-         sb.Append(str);
-         var width = colWidths[i];
-         if (str.Length < width) {
-            sb.Append(new string(' ', width - str.Length));
-         }
-      }
-      sb.AppendLine();
-   }
-   return sb.ToString();
-   // var upd = new DataStreamUpdate();
-   // upd.addedRows.AddRange(dp.addedRows.Cast<IBlueRow>());
-   // upd.deletedRows.AddRange(dp.deletedRows.Cast<IBlueRow>());
-   // return fmtDataStreamUpdate(upd);
-}
-
-static string fmtDataStreamUpdate(DataStreamUpdate rows) {
-   return JsonSerializer.Serialize(rows);
-}
-
-var parser = new Parser(ps => {
-   ps.CaseInsensitiveEnumValues = true;
-   ps.HelpWriter = Console.Out;
-});
-
-await parser.ParseArguments<DataStreamWebAppServerOptions>(args).
-   MapResult(run, errors => Task.FromResult(1));
-
-async Task<int> run(DataStreamWebAppServerOptions opts) {
-   BTSNativeMethods.provide_command_line(args, args.Length);
-   Configuration.initialize();
-   var context = ServiceContext.create(opts.sparkd);
-   await context.login(opts.user!, opts.password!);
-   var vsWorkLoop = Observable.Interval(TimeSpan.FromMilliseconds(200)).Subscribe(_ => {
-      ViewStreamDatabase.instance.doWork();
-      MasterWorker.getInstance(false).doWork();
-   });
-   var builder = WebApplication.CreateBuilder(args);
-   builder.Services.AddDirectoryBrowser();
-   var app = builder.Build();
-   var wsOpts = new WebSocketOptions {
-      KeepAliveInterval = TimeSpan.FromMinutes(2)
-   };
-   app.UseWebSockets(wsOpts);
-   app.UseDefaultFiles();
-   app.UseStaticFiles();
-   app.MapFallbackToFile("/index.html");
-
-   app.Use(async (context, next) => {
-      var reqDesc = fmtReq(context.Request);
-      Console.WriteLine($"Webserver received: {reqDesc}");
-      if (context.Request.Path == "/ws") {
-         if (context.WebSockets.IsWebSocketRequest) {
-            Console.WriteLine("Accepting websocket connection...");
-            using var webSocket = await context.WebSockets.AcceptWebSocketAsync();
-            Console.WriteLine("Websocket connection accepted.");
-            await runWebSocketServer(
-               opts.sparkd, opts.user!, opts.password!, webSocket);
-         } else {
-            context.Response.StatusCode = StatusCodes.Status400BadRequest;
-         }
-      } else {
-         Console.WriteLine($"Webserver ignoring req: {reqDesc}");
-         await next(context);
-      }
-   });
-
-   try {
-      await app.RunAsync();
-   } catch (Exception ex) {
-      Console.WriteLine($"Exception on DataStreamWebsocketServer thread: {ex}");
-      return 1;
-   }
-   Spark.Core.Disposables.Disposable.dispose(ref vsWorkLoop);
-   return 0;
-}
-
-async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
-   var buffer = new byte[1024 * 4];
-   var receiveResult = await webSocket.ReceiveAsync(
-       new ArraySegment<byte>(buffer), CancellationToken.None);
-
-   try {
-      while (!receiveResult.CloseStatus.HasValue) {
-         Console.WriteLine("Awaiting websocket message from client");
-         receiveResult = await webSocket.ReceiveAsync(
-             new ArraySegment<byte>(buffer), CancellationToken.None);
-         var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
-         var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
-         var queryArgs = rpcReq!.Params!;
-         // var queryArgs = new DataStreamSubscription { Query = msg };
-         var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
-         if (exp is null) {
-            Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
-            continue;
-         }
-         var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
-         Console.WriteLine("Subscribing to stream...");
-         vs.onChange += onStreamChanged;
-
-         await webSocket.SendAsync(
-             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-      }
-   } catch (Exception ex) {
-      Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
-      return 1;
-   }
-
-   return 0;
-
-   async void onStreamChanged(object newValue) {
-      if (newValue is not DataPacket dp) {
-         Console.WriteLine(
-            $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
-         return;
-      }
-      Console.WriteLine("Received stream update.");
-      var sb = new StringBuilder();
-      var updateDesc = fmtDataPacket(dp);
-      var msg = $"Received new rows:\n{updateDesc}";
-      // var payload = new ReadOnlyMemory<byte>([.. msg]);
-      var encoded = Encoding.UTF8.GetBytes(msg);
-      var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
-      await webSocket.SendAsync(
-          payload,
-          receiveResult.MessageType,
-          receiveResult.EndOfMessage,
-          CancellationToken.None);
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -1,48 +0,0 @@
-{
-  "profiles": {
-    "default": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "applicationUrl": "http://localhost:5400"
-    },
-    "http": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      },
-      "dotnetRunMessages": true,
-      "applicationUrl": "http://localhost:5268"
-    },
-    "https": {
-      "commandName": "Project",
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      },
-      "dotnetRunMessages": true,
-      "applicationUrl": "https://localhost:7128;http://localhost:5268"
-    },
-    "IIS Express": {
-      "commandName": "IISExpress",
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      }
-    }
-  },
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "iisSettings": {
-    "windowsAuthentication": false,
-    "anonymousAuthentication": true,
-    "iisExpress": {
-      "applicationUrl": "http://localhost:2083",
-      "sslPort": 44383
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WeatherForecast.cs

@@ -1,12 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class WeatherForecast
-{
-   public DateOnly Date { get; set; }
-
-   public int TemperatureC { get; set; }
-
-   public int TemperatureF => 32 + (int)(TemperatureC / 0.5556);
-
-   public string? Summary { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WebsocketConnectionManager.cs

@@ -1,37 +0,0 @@
-using System;
-using System.Collections.Concurrent;
-using System.Net.WebSockets;
-
-namespace Bts.DataStreamWebApp.Server {
-   // public class WebSocketConnectionManager {
-   //    private readonly ConcurrentDictionary<Guid, WebSocket> _sockets = new ConcurrentDictionary<Guid, WebSocket>();
-
-   //    public Guid AddSocket(WebSocket socket) {
-   //       var socketId = Guid.NewGuid();
-   //       _sockets.TryAdd(socketId, socket);
-   //       return socketId;
-   //    }
-
-   //    public WebSocket? GetSocket(Guid socketId) {
-   //       _sockets.TryGetValue(socketId, out var socket);
-   //       return socket;
-   //    }
-
-   //    public ConcurrentDictionary<Guid, WebSocket>.ValueCollection GetAllSockets() {
-   //       return _sockets.Values;
-   //    }
-
-   //    public Guid? GetSocketId(WebSocket socket) {
-   //       foreach (var (key, value) in _sockets) {
-   //          if (value == socket) {
-   //             return key;
-   //          }
-   //       }
-   //       return null;
-   //    }
-
-   //    public void RemoveSocket(Guid socketId) {
-   //       _sockets.TryRemove(socketId, out _);
-   //    }
-   // }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -1,8 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -1,9 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*"
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -1,168 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>ALT WebSocket Sample Application</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Message to send:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            var data = sendMessage.value;
-            socket.send(data);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -1,176 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>DataStreams via WebSocket</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Query:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            const rpcReq = {
-              jsonrpc: "2.0",
-              method: "subscribe",
-              params: {
-                query: sendMessage.value,
-              },
-              id: 1,
-            };
-            const rpcReqStr = JSON.stringify(rpcReq);
-            socket.send(rpcReqStr);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream.Console/DataStream.Console.csproj

@@ -7,7 +7,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream.Console/Program.cs

@@ -1,6 +1,4 @@
 // See https://aka.ms/new-console-template for more information
-using BCG;
-
 using DataStream.Console;
 
 using ExpressionTests;
@@ -9,43 +7,26 @@ using Spark.Core.Collections;
 
 using System.Reactive.Disposables;
 
-var schema = new BlueTableSchema([
-      new("id", 0, typeof(int)),
-      new("name", 1, typeof(string)),
-      new("value", 2, typeof(int)),
-   ],
-   "MyTable"
-);
-var rows = new BlueTable(schema);
-var mbr = new MutableBlueRow(schema);
-mbr.setValue("id", 1);
-mbr.setValue("name", "Jerry");
-mbr.setValue("value", 13);
-rows.addRow(mbr);
-var serializer = new UniversalSerializer(schema);
-serializer.serialize(StreamOpCode.create, new[] { rows }, out var bytes);
-var bytesDesc = BitConverter.ToString(bytes).Replace("-", "\\x");
-Console.WriteLine(bytesDesc);
 
-// try {
-//    foreach (var arg in args) {
-//       if (File.Exists(arg)) {
-//          using var fs = File.OpenRead(arg) ??
-//             throw new Exception("File.OpenRead returned null. LIES!");
-//          var expressionLines = new StringStreamEnumerable(new(fs));
-//          var test = ExpressionTestParser.Parse(expressionLines);
-//          TempExpressionTests.Run(test);
-//       }
-//    }
-//    if (args.Length == 0) {
-//       TempExpressionTests.Run("rv = 3 + 4", new Dictionary<string, string> { { "rv", "7" } });
-//    }
-// } catch (Exception ex) {
-//    using (ConsoleHelper.PushBg(ConsoleColor.Red)) {
-//       Console.WriteLine($"Failed! {ex}");
-//    }
-//    return;
-// }
+try {
+   foreach (var arg in args) {
+      if (File.Exists(arg)) {
+         using var fs = File.OpenRead(arg) ??
+            throw new Exception("File.OpenRead returned null. LIES!");
+         var expressionLines = new StringStreamEnumerable(new(fs));
+         var test = ExpressionTestParser.Parse(expressionLines);
+         TempExpressionTests.Run(test);
+      }
+   }
+   if (args.Length == 0) {
+      TempExpressionTests.Run("rv = 3 + 4", new Dictionary<string, string> { { "rv", "7" } });
+   }
+} catch (Exception ex) {
+   using (ConsoleHelper.PushBg(ConsoleColor.Red)) {
+      Console.WriteLine($"Failed! {ex}");
+   }
+   return;
+}
 
-// using var _ = ConsoleHelper.PushFg(ConsoleColor.Green);
-// Console.WriteLine($"Passed!");
+using var _ = ConsoleHelper.PushFg(ConsoleColor.Green);
+Console.WriteLine($"Passed!");

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
@@ -9,15 +9,12 @@
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
   </ItemGroup>

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -1,79 +0,0 @@
-using BCG.Utilities.General;
-
-using System;
-
-namespace BCG {
-    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
-        class State {
-            public int n_sends = 0;
-            public int n_recvs = 0;
-        }
-        private readonly string _ip;
-        private readonly int _port;
-        private readonly ViewStreamDatabase _vsd;
-        State _state = new State();
-        private readonly int _msg_size;
-
-        private IBlueTableSchema _schema;
-        private TimeSpan _send_period;
-        private ITCPConnection _connection = null;
-        private byte[] _buff;
-
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
-        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
-            _ip = ip;
-            _port = port;
-            _msg_size = msg_size;
-            _vsd = vsd;
-            _schema = BlueDataUtility.form_schema("TCPClient",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("n_sends", TypeCode.UInt32),
-            Tuple.Create("n_recvs", TypeCode.UInt32));
-            _send_period = TimeSpan.FromSeconds(send_period_seconds);
-            _buff = new byte[msg_size];
-            for (int i = 0; i < _buff.Length; i++) {
-                _buff[i] = (byte)i;
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_address = new BlutoSubAddress(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            if (_connection != null) {
-                _connection.send(_buff);
-                _state.n_sends++;
-            }
-            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
-            BlueTable rows = new BlueTable(r.schema);
-            rows.addRow(r);
-            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        public void onConnection(ITCPConnection connection) {
-            _connection = connection;
-            _connection.register(this);
-        }
-        public bool isDisposed { get; }
-
-        public void onData(RemoteMessage msg) {
-            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
-            _state.n_recvs++;
-        }
-
-        public void Dispose() {
-
-        }
-
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,6 +1,11 @@
-namespace BCG;
-
-public class AddColumnsStream : MapStream {
-   public AddColumnsStream(ViewStream vs, string columnDef)
-      : base(vs, new ColumnAdder(columnDef)) { }
-}
+// using System;
+// using System.Data;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Utilities.General;
+// namespace BCG {
+
+//     public class AddColumnsStream : MapStream {
+//         public AddColumnsStream(ViewStream vs, string columnDef) : base(vs, new ColumnAdder(columnDef)) { }
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,52 +1,63 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-
-using Bts.ColumnExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Provides methods to add columns to the schema
-/// </summary>
-/// <param name="columnsExpression">
-/// String represting the columns to add to the output table
-/// </param>
-public class ColumnAdder : IStreamMapper {
-   // Store tuples of (column, index) to preserve order
-   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
-   protected IBlueTableSchema? mySchema;
-
-   public ColumnAdder(string columnsExpression) =>
-      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
-         throw new Exception("Invalid columnn expression: " + columnsExpression))
-         .Select((c, i) => (c, i))
-         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
-
-   [MemberNotNull(nameof(mySchema))]
-   public virtual IBlueRow map(IBlueRow row) {
-      // Only rebuild schema if necessary (i.e., if it's null)
-      mySchema ??= buildSchema(row.schema, myNewColumns);
-      return new AddColumnsRow(mySchema, row);
-   }
-
-   [MemberNotNull(nameof(mySchema))]
-   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
-      // Always rebuild schema (even if it's already non-null)
-      mySchema = buildSchema(inputSchema, myNewColumns);
-
-   [MemberNotNull(nameof(mySchema))]
-   static BlueTableSchema buildSchema(
-      IBlueTableSchema schema,
-      IDictionary<string, (ColumnExpression column, int index)> newColumns
-   ) {
-      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
-      schema.columnInfo.CopyTo(cis, 0);
-      foreach (var (col, index) in newColumns.Values) {
-         var schemaIdx = schema.columnInfo.Length + index;
-         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
-      }
-      return new BlueTableSchema(cis, schema.tableName);
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.AddColumnsExpression;
+
+// namespace BCG {
+//     /// <summary>
+//     /// Provides methods to add columns to the schema
+//     /// </summary>
+//     public class ColumnAdder : IStreamMapper {
+//         protected IDictionary myNewColumns;
+//         protected IBlueTableSchema mySchema;
+
+//         static syntax myParser =
+//            new syntax();
+
+//         /// <summary>
+//         /// string represting the columns to be added to the output table. 
+//         /// </summary>
+//         /// <param name="colAddExpression"></param>
+//         public ColumnAdder(string columnDef) {
+//             AddColumnsExpression ace;
+
+//             if (DataStreamLoadFileConfig.instance.useStaticParser) {
+//                 lock (myParser) {
+//                     ace =
+//                     (AddColumnsExpression)myParser.Parse(columnDef);
+//                 }
+//             } else {
+//                 syntax parser = new syntax();
+//                 ace =
+//                 (AddColumnsExpression)parser.Parse(columnDef);
+//             }
+
+//             if (ace == null)
+//                 throw new Exception("Invalid columnn expression: " + columnDef);
+//             myNewColumns = ace.columns;
+//         }
+
+//         public virtual IBlueRow map(IBlueRow row) {
+//             if (mySchema == null)
+//                 myConstructSchema(row.schema);
+//             return new AddColumnsRow(mySchema, row);
+//         }
+
+//         public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+//             myConstructSchema(inputSchema);
+//             return mySchema;
+//         }
+
+//         private void myConstructSchema(IBlueTableSchema schema) {
+//             ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
+//             int ciIndex = 0;
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 cis[ciIndex++] = ci;
+//             foreach (DictionaryEntry de in myNewColumns)
+//                 cis[ciIndex] = new ColumnInfo((string)de.Key, ciIndex++,
+//                    ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
+//             mySchema = new BlueTableSchema(cis, schema.tableName);
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -30,8 +30,8 @@ namespace BCG
            {
               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
               {
-#pragma warning disable SYSLIB0011
                   BinaryFormatter bf = new BinaryFormatter();
+#pragma warning disable SYSLIB0011
                   BlueTable bt = (BlueTable)bf.Deserialize(fs);
 #pragma warning restore SYSLIB0011
                   mySetRealSource(new BlueRowCollectionStaticStream(bt));

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -1,31 +0,0 @@
-namespace BCG {
-    using System.Collections.Generic;
-
-    public class KeyValueToRelationalStream : ViewStream {
-        private readonly ViewStream _kv_str;
-        private readonly string _doc_id_col, _key_col, _val_col;
-        public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
-            _kv_str = kv_str;
-            _doc_id_col = doc_id_col;
-            _key_col = key_col;
-            _val_col = val_col;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _kv_str.onChange += _on_str_update;
-        }
-
-        protected override void myOnLastSubscriber() {
-            _kv_str.onChange -= _on_str_update;
-        }
-
-        private void _on_str_update(object dp_obj) {
-            DataPacket dp = dp_obj as DataPacket;
-            var rows = dp.newRows;
-            var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
-            var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-        }
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs

@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Security.Cryptography.X509Certificates;
-
-using BCG;
-using BCG.Utilities.General;
-
-using HtmlAgilityPack;
-
-namespace BCG {
-    class Program {
-        static void Main() {
-            string url = "http://quotes.toscrape.com";
-            List<string> quotes = ScrapeQuotes(url);
-
-            Console.WriteLine("Quotes from http://quotes.toscrape.com:");
-            foreach (var quote in quotes) {
-                Console.WriteLine(quote);
-            }
-        }
-
-        static List<string> ScrapeQuotes(string url) {
-            List<string> quotes = new List<string>();
-
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-
-                // Extract quotes from the HTML document
-                var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
-                if (quoteNodes != null) {
-                    foreach (var quoteNode in quoteNodes) {
-                        string quote = quoteNode.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
-
-            return quotes;
-        }
-    }
-
-    public class WebScapperStream : ViewStream, IWorker {
-        private readonly string _url;
-        private readonly string _xpath;
-        private readonly ViewStreamDatabase _vsd;
-        private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
-        private readonly IBlueTableSchema _schema;
-
-        public WebScapperStream()
-        :
-        this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath)
-        : this(url, xpath, ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
-            _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("url", TypeCode.UInt32),
-                                                Tuple.Create("line", TypeCode.String));
-            _url = url;
-            _xpath = xpath;
-            _vsd = vsd;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            var lines = scrape_lines(_url, _xpath);
-            Func<string, int> get_id = (line) => {
-                int rv = 0;
-                if (!_line_id_map.TryGetValue(line, out rv))
-                    _line_id_map.Add(line, _line_id_map.Count);
-                return rv;
-            };
-            BlueTable nrows = new BlueTable(_schema);
-            foreach (var l in lines) {
-                bts.Log.Gen.info($"line: {l}");
-                var id = get_id(l);
-                BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
-                nrows.addRow(nr);
-            }
-            var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        static List<string> scrape_lines(string url, string xpath) {
-            List<string> quotes = new List<string>();
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-                // Extract quotes from the HTML document
-                var quote_nodes = document.DocumentNode.SelectNodes(xpath);
-                if (quote_nodes != null) {
-                    foreach (var qn in quote_nodes) {
-                        string quote = qn.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
-
-            return quotes;
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,212 +1,37 @@
 using System;
-using System.Collections.Generic;
+
 using MongoDB.Driver;
 using MongoDB.Bson;
-using BCG;
-using BCG.Utilities.General;
-using ZstdSharp.Unsafe;
-using System.Runtime.CompilerServices;
-using mbts.spark;
 
 
-namespace BCG {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
+class Program {
+    static void Main() {
+        // Set the connection string and database name
+        string connectionString = "mongodb://localhost:27017";
+        string databaseName = "your_database_name";
+        string collectionName = "your_collection_name";
+
+        // Create a MongoClient
+        MongoClient client = new MongoClient(connectionString);
 
         // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+        IMongoDatabase database = client.GetDatabase(databaseName);
 
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        // Access the collection
+        IMongoCollection<BsonDocument> collection = database.GetCollection<BsonDocument>(collectionName);
 
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+        // Fetch all documents from the collection
+        var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
+        // Iterate through the documents and print values
+        foreach (var document in documents.ToEnumerable()) {
+            // Assuming your document structure has fields named "Field1", "Field2", etc.
+            Console.WriteLine("Field1: " + document["Field1"]);
+            Console.WriteLine("Field2: " + document["Field2"]);
+            // Add more fields as needed
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            _read_mogodb();
-        }
-
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
-
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
-
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
-
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
+            // Print a separator between documents
+            Console.WriteLine(new string('-', 20));
         }
     }
-
-    [DataStreamUserRespresentation("mongodb")]
-    public class MongoDBStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
-
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
-
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
-
-        }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            _read_mogodb();
-        }
-
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
-
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_doc(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
-
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("doc_id", TypeCode.String),
-            Tuple.Create("key", TypeCode.String),
-            Tuple.Create("value", TypeCode.String));
-            _table = new BlueTable(schema);
-        }
-
-        private void _add_doc(BsonDocument doc) {
-            var doc_id_bson = doc["_id"];
-            var doc_id = doc_id_bson.ToString();
-            Func<string, string, int> get_id = (string doc_id, string key) => {
-                string hash_key = $"{doc_id}_{key}";
-                int id = 0;
-                if (!_id_map.TryGetValue(hash_key, out id)) {
-                    id = _id_map.Count;
-                    _id_map.Add(hash_key, id);
-                }
-                return id;
-            };
-            foreach (var n in doc.Names) {
-                string key = n;
-                var id = get_id(doc_id, key);
-                MutableBlueRow nr = new MutableBlueRow(_table.schema);
-                var val = doc[n].ToString();
-                nr.setValue("id", id);
-                nr.setValue("doc_id", doc_id);
-                nr.setValue("key", key);
-                nr.setValue("value", val);
-                _table.addRow(nr);
-            }
-        }
-    }
-
-}
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -1,107 +1,115 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace BCG;
-
-public class SortRow : IComparable, IBlueRow {
-   readonly IBlueRow row;
-   readonly IEnumerable<ColumnSortExpression>? columns;
-   readonly IComparer? comparer;
-
-   public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) {
-      this.row = row;
-      this.columns = columns;
-   }
-
-   public SortRow(IBlueRow row, IComparer comparer) {
-      this.row = row;
-      this.comparer = comparer;
-   }
-
-   public IBlueRow internal_row { get { return row; } }
-
-   #region IComparable method
-   public int CompareTo(object? obj) {
-      if (obj is not IBlueRow otherRow) {
-         throw new ArgumentException(
-            $"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})",
-            nameof(obj));
-      }
-      if (comparer is not null) {
-         return comparer.Compare(this, otherRow);
-      }
-      return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0);
-      int Compare(ColumnSortExpression col) {
-         var otherVal = otherRow[col.Name];
-         int rv = myIsNull(this[col.Name]) ?
-            (myIsNull(otherVal) ? 0 : -1) :
-            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
-         rv *= col.IsAscending ? 1 : -1;
-         return rv;
-      }
-   }
-   #endregion
-
-   #region IBlueRow members
-   public IBlueTableSchema schema { get { return row.schema; } }
-   #endregion
-
-   #region ICollection members
-   public int Count { get { return row.Count; } }
-
-   public bool IsSynchronized { get { return false; } }
-
-   public object SyncRoot { get { return this; } }
-
-   public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
-   #endregion
-
-   #region IDictionary members
-   public bool IsFixedSize { get { return true; } }
-
-   public bool IsReadOnly { get { return true; } }
-
-   /// <summary>
-   /// The key can either be the string corrsponding to the column name or
-   /// the order of the column
-   /// </summary>
-   public object this[object key] {
-      get {
-         if (key is string)
-            return this[(string)key];
-         return this[(int)key];
-      }
-      set { throw new NotSupportedException("The IBlueRow is readonly"); }
-   }
-
-   public object this[int index] => row[index];
-
-   public object this[string colName] => row[colName];
-
-   public ICollection Keys => row.Keys;
-
-   public ICollection Values => row.Values;
-
-   public bool Contains(object key) => row.Contains(key);
-
-   void IDictionary.Add(object key, object? value) =>
-      throw new NotSupportedException("The IBlueRow is readonly");
-
-   void IDictionary.Clear() =>
-      throw new NotSupportedException("The IBlueRow is readonly");
-
-   IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); }
-
-   IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); }
-
-   public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-   #endregion
-
-   bool myIsNull(object? o) =>
-      o is null || o == null || o is DBNull;
-
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.SortExpression;
+
+
+// namespace BCG
+// {
+//    public class SortRow : IComparable, IBlueRow
+//    {
+//       private IBlueRow myRow;
+//       private IEnumerable myColumns;
+//       private IComparer myComparer;
+
+//       public SortRow (IBlueRow row, SortExpression sortExpression)
+//       {
+//          myRow = row;
+//          myColumns = sortExpression.columns;
+//       }
+
+//       public SortRow (IBlueRow row, IEnumerable columns)
+//       {
+//          myRow = row;
+//          myColumns = columns;
+//       }
+
+//       public SortRow (IBlueRow row, IComparer comparer)
+//       {
+//          myRow = row;
+//          myComparer = comparer;
+//       }
+
+//       public IBlueRow internal_row { get { return myRow; } }
+
+//       #region IComparable method
+//       public int CompareTo (object obj)
+//       {  
+//          IBlueRow otherRow = (IBlueRow)obj;
+//          if (myComparer == null)
+//          {
+//             foreach (SortExpression.ColumnCriteria cc in myColumns)
+//             {
+//                int retVal = myIsNull (this[cc.columnName]) ?
+//                   myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
+//                   myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
+//                retVal *= cc.ascending ? 1 : -1;
+//                if (retVal != 0)
+//                   return retVal;
+//             }
+//             return 0;
+//          }
+//          else
+//             return myComparer.Compare (this, otherRow);
+//       }
+//       #endregion
+
+//       #region IBlueRow members
+//       public IBlueTableSchema schema{get{return myRow.schema;}}
+//       #endregion
+
+//       #region ICollection members
+//       public int Count {get{return myRow.Count;}}
+
+//       public bool IsSynchronized {get{return false;}}
+
+//       public object SyncRoot {get{return this;}}
+
+//       public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
+//       #endregion
+
+//       #region IDictionary members
+//       public bool IsFixedSize {get{return true;}}
+
+//       public bool IsReadOnly {get{return true;}}
+
+//       /// <summary>
+//       /// The key can either be the string corrsponding to the column name or
+//       /// the order of the column
+//       /// </summary>
+//       public object this [object key]
+//       {
+//          get
+//          {
+//             if (key is string)
+//                return this[(string)key];
+//             return this[(int)key];
+//          }
+//          set {throw new NotSupportedException ("The IBlueRow is readonly");}
+//       }
+
+//       public object this [int index]   {get{return myRow[index];}}
+
+//       public object this [string colName] {get{return myRow[colName];}}
+
+//       public ICollection Keys{get{return myRow.Keys;}}
+
+//       public ICollection Values{get{return myRow.Values;}}
+
+//       public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+//       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+//       public bool Contains (object key){return myRow.Contains (key);}
+
+//       IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
+
+//       IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
+
+//       public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+//       #endregion
+
+//       private bool myIsNull (object o) {return o == null || o is DBNull;}
+
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,99 +1,129 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-namespace BCG;
-
-/// <summary>
-/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-/// to merge the additions to the accumulates table
-/// </summary>
-public class SortStream : ViewStream {
-   ViewStream stream;
-   IList<ColumnSortExpression> sortCols;
-   IComparer? comparer;
-   DataPacket? lastFiredPacket;
-   DataPacket? lastReceivedPacket;
-
-   public SortStream(ViewStream stream, string sortExpression) {
-      this.stream = stream;
-      SortExpression = sortExpression;
-   }
-
-   public SortStream(ViewStream vs, string colName, bool isAsc) {
-      stream = vs;
-      SortExpression = isAsc ? "+" + colName : "-" + colName;
-   }
-
-   public SortStream(ViewStream stream, IComparer comparer) {
-      this.stream = stream;
-      this.comparer = comparer;
-      sortCols = Array.Empty<ColumnSortExpression>();
-   }
-
-   [MemberNotNull(nameof(sortCols))]
-   public string SortExpression {
-      set {
-         comparer = null;
-         lastFiredPacket = null;
-         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public IComparer Comparer {
-      set {
-         comparer = value;
-         lastFiredPacket = null;
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public void myChangeHandlerTest(object o) =>
-      myChangeHandler(o);
-
-   [MemberNotNull(nameof(lastFiredPacket))]
-   internal void myChangeHandler(object o) {
-      lastReceivedPacket = (DataPacket)o;
-      // First callback, so we need to sort the whole table, this is the
-      // only time we will do that for all the other sorting operations would
-      // be sorting the increments and inserting them into the whole list
-      if (lastFiredPacket is null) {
-         lastFiredPacket = new() {
-            newRows = comparer is null ?
-               new SortedTable(lastReceivedPacket.newRows, sortCols) :
-               new SortedTable(lastReceivedPacket.newRows, comparer)
-         };
-         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-         myFireEvent(lastFiredPacket.Clone());
-         return;
-      }
-      // Deletes
-      lastFiredPacket.deletedRows =
-         comparer == null ?
-         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-         new SortedTable(lastReceivedPacket.deletedRows, comparer);
-      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-      // Adds
-      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-      myFireEvent(lastFiredPacket.Clone());
-   }
-
-   protected override void myOnFirstSubscriber() =>
-      stream.onChange += myChangeHandler;
-
-   protected override void myOnLastSubscriber() {
-      lastFiredPacket = null;
-      lastReceivedPacket = null;
-      stream.onChange -= myChangeHandler;
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Evaluator.SortExpression;
+
+
+// namespace BCG
+// {
+// 	/// <summary>
+// 	/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+// 	/// to merge the additions to the accumulates table
+// 	/// </summary>
+// 	public class SortStream : ViewStream
+// 	{
+//       private ViewStream myStream;
+//       private IEnumerable mySortCols;
+//       private IComparer myComparer;
+//       private DataPacket myLastFiredPacket;
+//       private DataPacket myLastRecievedPacket;
+
+//       public SortStream (ViewStream stream, string sortExpression)
+//       {
+//          myStream = stream;
+//          mySetSortExpression (sortExpression);
+//       }
+
+//       public SortStream (ViewStream vs, string colName, bool isAsc)
+//       {
+//          myStream = vs;
+//          string sortExpression = isAsc ? "+" + colName : "-" + colName;
+//          mySetSortExpression (sortExpression);
+//       }
+
+//       public SortStream (ViewStream vs, IComparer comparer)
+//       {
+//          myStream = vs;
+//          myComparer = comparer;
+//       }
+
+//       public string sortExpression
+//       {
+//          set
+//          {
+//             myComparer = null;
+//             myLastFiredPacket = null;
+//             mySetSortExpression (value);   
+//             myChangeHandler (myLastRecievedPacket);
+//          }
+//       }
+
+//       public IComparer comparer
+//       {
+//          set
+//          {
+//             myComparer = value;
+//             myLastFiredPacket = null;
+//             myChangeHandler(myLastRecievedPacket);
+//          }
+//       }
+
+//       private void mySetSortExpression (string sortExpression)
+//       {
+//          syntax parser = 
+//             new syntax();
+         
+//          mySortCols = ((SortExpression)
+//             parser.Parse (sortExpression)).columns;
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); 
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          myInit ();
+//          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); 
+//       }
+
+//       internal void myChangeHandler (object o)
+//       {
+//          myLastRecievedPacket = (DataPacket)o;
+//          //first callback, so we need to sort the whole table, this is the
+//          //only time we will do that for all the other sorting operations would
+//          //be sorting the increments and inserting them into the whole list
+//          if (myLastFiredPacket == null)
+//          {
+//             myLastFiredPacket = new DataPacket ();
+//             myLastFiredPacket.newRows = 
+//                myComparer == null ?
+//                new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
+//                new SortedTable (myLastRecievedPacket.newRows, myComparer);
+//             myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+//             myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+//             myFireEvent (myLastFiredPacket.Clone());
+//             return;
+//          }
+//          myHandleDeletes (myLastRecievedPacket);
+//          myHandleAdds (myLastRecievedPacket);
+//          myFireEvent (myLastFiredPacket.Clone());
+//       }
+
+//       public void myChangeHandlerTest(object o)
+//       {
+//           myChangeHandler(o);
+//       }
+
+//       private void myHandleDeletes (DataPacket dp)
+//       {
+//          myLastFiredPacket.deletedRows = 
+//             myComparer == null ?
+//             new SortedTable (dp.deletedRows, mySortCols) :
+//             new SortedTable (dp.deletedRows, myComparer);
+//          ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
+//       }
+
+//       private void myHandleAdds (DataPacket dp)
+//       {
+//          myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
+//          ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
+//       }
+
+//       private void myInit ()
+//       {
+//          myLastFiredPacket = null;
+//          myLastRecievedPacket = null;
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs

@@ -1,108 +1,136 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-using Bts.ColumnSortExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Represents an IBlueRowCollection with the rows sorted according to the
-/// </summary>
-public class SortedTable : IBlueRowCollection {
-   readonly List<SortRow> rows = new(0);
-   readonly Dictionary<int, IBlueRow> rowIndex = new(0);
-   readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>();
-   readonly IComparer? comparer;
-
-   /// <summary>
-   /// Sorts the rows using sortExpression as the sorting criteria
-   /// </summary>
-   /// <param name="rows">Rows to be inserted into the table</param>
-   /// <param name="sortCols">Sorting criteria</param>
-   public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) {
-      this.sortCols = sortCols;
-      init(rows);
-   }
-
-   public SortedTable(IBlueRowCollection rows, IComparer comparer) {
-      this.comparer = comparer;
-      init(rows);
-   }
-
-   public int Count => rows.Count;
-   public bool IsSynchronized => false;
-   public IBlueTableSchema schema { get; private set; }
-   public object SyncRoot => this;
-   public IBlueRow this[int id] => rowIndex[id]
-      ?? throw new BlueDataException(
-         $"ID '{id}' does not exist in table '{schema?.tableName}'");
-
-   public void addRows(IBlueRowCollection rows) {
-      // Requires merger of the addedRows to the already existing ones
-      if (rows.Count == 0) {
-         // At times there are calls with 0 additions, in that case
-         // we of course need no processing
-         return;
-      }
-      SortedTable addedRows =
-         comparer == null ? new SortedTable(rows, sortCols) :
-            new SortedTable(rows, comparer);
-      int addedRowsIndex = 0;
-      SortRow newRow = addedRows.getRowByIndex(addedRowsIndex);
-      for (int i = 0; i < this.rows.Count; i++) {
-         if (newRow.CompareTo(this.rows[i]) <= 0) {
-            this.rows.Insert(i, newRow);
-            rowIndex[(int)newRow["id"]] = newRow;
-            if (++addedRowsIndex == addedRows.Count) {
-               return;
-            }
-            newRow = addedRows.getRowByIndex(addedRowsIndex);
-         }
-      }
-      if (addedRowsIndex < addedRows.Count) {
-         for (int i = addedRowsIndex; i < addedRows.Count; i++) {
-            newRow = addedRows.getRowByIndex(i);
-            this.rows.Add(newRow);
-            rowIndex[(int)newRow["id"]] = newRow;
-         }
-      }
-   }
-
-   public void deleteRows(IBlueRowCollection deletedRows) {
-      foreach (IBlueRow deletedRow in deletedRows) {
-         var row = rowIndex[(int)deletedRow["id"]!] as SortRow
-            ?? throw new InvalidOperationException("Index did not return IBlueRow");
-         rows.Remove(row);
-         rowIndex.Remove((int)row["id"]!);
-      }
-   }
-
-   public SortRow getRowByIndex(int index) =>
-      rows[index]!;
-
-   public bool containsID(int id) =>
-      rowIndex[id] is not null;
-
-   public IEnumerator GetEnumerator() =>
-      rows.GetEnumerator();
-
-   public void CopyTo(Array array, int index) {
-      for (int i = 0; i < Count; i++) {
-         array.SetValue(rows[i], index + i);
-      }
-   }
-
-   [MemberNotNull(nameof(schema))]
-   void init(IBlueRowCollection rows) {
-      schema = rows.schema;
-      foreach (IBlueRow row in rows) {
-         SortRow newRow = comparer is null ?
-            new(row, sortCols) : new(row, comparer);
-         this.rows.Add(newRow);
-         rowIndex[(int)row["id"]!] = newRow;
-      }
-      this.rows.Sort();
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Evaluator.SortExpression;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Represents an IBlueRowCollection with the rows sorted according to the
+//    /// 
+//    /// </summary>
+//    public class SortedTable : IBlueRowCollection 
+//    {
+//       private ArrayList myRows = new ArrayList ();
+//       private Hashtable myIndex = new Hashtable ();
+//       private IBlueTableSchema mySchema;
+//       private IEnumerable mySortCols;
+//       private IComparer myComparer;
+
+//       #region SortedTable method(s)
+//       public void addRows (IBlueRowCollection rows)
+//       {
+//          //requires merger of the addedRows to the already existing ones
+//          if (rows.Count == 0)
+//             //at times there are calls with 0 additions, in that case
+//             //we ofcourse need no processing
+//             return;
+//          SortedTable addedRows = 
+//             myComparer == null ? new SortedTable (rows, mySortCols) :
+//                new SortedTable (rows, myComparer);
+//          int addedRowsIndex = 0;
+//          SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
+//          for (int i = 0; i < myRows.Count; i ++)
+//             if (newRow.CompareTo(myRows[i]) <= 0)
+//             {
+//                myRows.Insert (i, newRow);
+//                myIndex[newRow["id"]] = newRow;
+//                if (++addedRowsIndex == addedRows.Count)
+//                   return;
+//                newRow = addedRows.getRowByIndex (addedRowsIndex);
+//             }
+//          if (addedRowsIndex < addedRows.Count)
+//             for (int i = addedRowsIndex; i < addedRows.Count; i++)
+//             {
+//                newRow = addedRows.getRowByIndex (i);
+//                myRows.Add (newRow);
+//                myIndex[newRow["id"]] = newRow;
+//             }
+//       }
+
+//       public void deleteRows (IBlueRowCollection deletedRows)
+//       {
+//          foreach (IBlueRow deletedRow in deletedRows)
+//          {
+//             IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
+//             myRows.Remove (row);
+//             myIndex.Remove (row["id"]);
+//          }
+//       }
+
+//       public SortRow getRowByIndex (int index)  {return (SortRow)myRows[index];}
+//       #endregion
+
+//       #region Constructor(s)
+//       /// <summary>
+//       /// Sorts the rows using sortExpression as the sorting criteria
+//       /// </summary>
+//       /// <param name="rows">
+//       /// Rows to be inserted into the table
+//       /// </param>
+//       /// <param name="sortExpression">
+//       /// Sorting criteria
+//       /// </param>
+//       public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
+//       {
+//          mySortCols = sortExpression.columns;
+//          myConstructorHelper (rows);
+//       }
+
+//       public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
+//       {
+//          mySortCols = sortCols;
+//          myConstructorHelper (rows);
+//       }
+
+//       public SortedTable (IBlueRowCollection rows, IComparer comparer)
+//       {
+//          myComparer = comparer;
+//          myConstructorHelper (rows);
+//       }
+
+//       private void myConstructorHelper (IBlueRowCollection rows)
+//       {
+//          mySchema = rows.schema;
+//          foreach (IBlueRow row in rows)
+//          {
+//             SortRow newRow = 
+//                myComparer == null ?
+//                new SortRow (row, mySortCols) : new SortRow (row, myComparer);
+//             myRows.Add (newRow);
+//             myIndex[row["id"]] = newRow;
+//          }
+//          myRows.Sort ();
+//       }
+
+//       #endregion
+
+//       #region IBlueRowCollections members
+//       public IBlueRow this[int id] 
+//       {
+//          get
+//          {
+//             if (myIndex[id] == null)
+//                throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
+//             return (IBlueRow)myIndex[id];
+//          }
+//       }
+//       public IBlueTableSchema schema {get{return mySchema;}}
+
+//       public bool containsID (int id)  {return myIndex[id] != null;}
+
+//       #endregion
+
+//       #region ICollection members
+//       public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
+
+//       public bool IsSynchronized {get{return false;}}
+
+//       public object SyncRoot {get{return this;}}
+
+//       public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
+
+//       public int Count{get{return myRows.Count;}}
+//          #endregion
+//    }
+
+// }

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,103 +1,95 @@
-using System;
-using System.Text.Json.Serialization;
-
-namespace BCG
-{
-   /// <summary>
-   /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
-   /// </summary>
-   [Serializable]
-   public class ColumnInfo : IComparable
-   {
-      public static bool isAllowedType (Type t)
-      {
-         return 
-            t.Equals (typeof (System.Int32)) ||
-            t.Equals (typeof (System.Int64)) ||
-            t.Equals (typeof (System.Double)) ||
-            t.Equals (typeof (System.String)) ||
-            t.Equals (typeof (System.DateTime)) ? true :
-            false;
-      }
-
-      //the default column info does not cach values in a lookup table, therefore it is in group 0
-      public ColumnInfo(string colName, int colNo, Type colType)
-         : this(colName, colNo, colType, 0)
-      {
-
-
-      }
-
-      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
-      {
-         myColName = colName;
-         myColNo = colNo;
-         myColType = colType;
-         myLookupTableGroup = colGroup;
-         myColValue = null;
-      }
-
-
-      public ColumnInfo (string colName, int colNo, object colValue)
-         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
-
-      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
-         : this (colName, colNo, colType) 
-      {
-
-         myColValue = colValue == null ? DBNull.Value : colValue;
-      
-      }
-
-
-      public int CompareTo (object obj)
-      {
-         int otherColNo = ((ColumnInfo)(obj)).colNo;
-         if (myColNo < otherColNo)
-            return -1;
-         else if (myColNo > otherColNo)
-            return 1;
-         return 0;
-      }
-
-      //public bool isCalculated{get{return myExpression != null;}}
-
-      public bool isConstant {get{return myColValue != null;}}
-
-      private int myColNo;
-      private string myColName;
-      private Type myColType;
-      //private string myExpression;
-      private object myColValue;
-
-      //no table will ever have more than 65,536 columns
-      //if it does, we are in serious trouble
-      private short myLookupTableGroup;
-
-      [JsonPropertyName("index")]
-      public int colNo{get{return myColNo;}}
-
-      [JsonPropertyName("name")]
-      public string colName {get{return myColName;}}
-
-      [JsonPropertyName("type")]
-      public Type colType {get{return myColType;}}
-
-
-      //this value corresponds to how the data in the column is serialized. 
-      //if the group number is 0, the values are not stored in a table
-      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
-      //are stored together in a lookup table
-
-      
-      [JsonIgnore]
-      public short lookupTableGroup
-      {
-         get { return myLookupTableGroup; }
-         set { myLookupTableGroup = value; }
-      }
-      //public string expression {get{return myExpression;}}
-      [JsonIgnore]
-      public object colValue {get{return myColValue;}}
-   }
-}
+using System;
+
+namespace BCG
+{
+   /// <summary>
+   /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
+   /// </summary>
+   [Serializable]
+   public class ColumnInfo : IComparable
+   {
+      public static bool isAllowedType (Type t)
+      {
+         return 
+            t.Equals (typeof (System.Int32)) ||
+            t.Equals (typeof (System.Int64)) ||
+            t.Equals (typeof (System.Double)) ||
+            t.Equals (typeof (System.String)) ||
+            t.Equals (typeof (System.DateTime)) ? true :
+            false;
+      }
+
+      //the default column info does not cach values in a lookup table, therefore it is in group 0
+      public ColumnInfo(string colName, int colNo, Type colType)
+         : this(colName, colNo, colType, 0)
+      {
+
+
+      }
+
+      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
+      {
+         myColName = colName;
+         myColNo = colNo;
+         myColType = colType;
+         myLookupTableGroup = colGroup;
+         myColValue = null;
+      }
+
+
+      public ColumnInfo (string colName, int colNo, object colValue)
+         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
+
+      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
+         : this (colName, colNo, colType) 
+      {
+
+         myColValue = colValue == null ? DBNull.Value : colValue;
+      
+      }
+
+
+      public int CompareTo (object obj)
+      {
+         int otherColNo = ((ColumnInfo)(obj)).colNo;
+         if (myColNo < otherColNo)
+            return -1;
+         else if (myColNo > otherColNo)
+            return 1;
+         return 0;
+      }
+
+      //public bool isCalculated{get{return myExpression != null;}}
+
+      public bool isConstant {get{return myColValue != null;}}
+
+      private int myColNo;
+      private string myColName;
+      private Type myColType;
+      //private string myExpression;
+      private object myColValue;
+
+      //no table will ever have more than 65,536 columns
+      //if it does, we are in serious trouble
+      private short myLookupTableGroup;
+
+      public int colNo{get{return myColNo;}}
+      public string colName {get{return myColName;}}
+      public Type colType {get{return myColType;}}
+
+
+      //this value corresponds to how the data in the column is serialized. 
+      //if the group number is 0, the values are not stored in a table
+      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
+      //are stored together in a lookup table
+
+      
+      public short lookupTableGroup
+      {
+         get { return myLookupTableGroup; }
+         set { myLookupTableGroup = value; }
+      }
+      //public string expression {get{return myExpression;}}
+      public object colValue {get{return myColValue;}}
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs

@@ -3,7 +3,8 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BCG {
+namespace BCG
+{
    /*
       These fields are first going to be used by  BlueRay to help it determine what differnet tags mean, but eventually
     * the goal is that they will be used for all services in this system. 
@@ -11,12 +12,11 @@ namespace BCG {
     * not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service
     * probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated
     */
-   public enum MetaDataField {
-      Protocol,       // is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
-      Type,           // if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
-      Symbol,
-      Exchange,
-      MarketDataType, // BAM (bid, ask, mid), last, etc
-      RealName
-   }; //does this thing have a better name we could call it
+   public enum MetaDataField {   Protocol,  //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
+                                 Type,   //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
+                                 Symbol, 
+                                 Exchange,
+                                 Platform,
+                                 MarketDataType, // BAM (bid, ask, mid), last, etc
+                                 RealName }; //does this thing have a better name we could call it
 }

src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs

@@ -1,63 +0,0 @@
-// #nullable enable
-// using BCG;
-
-// using Spark.Core.Json;
-
-// using System;
-// using System.Text.Json;
-// using System.Text.Json.Nodes;
-// using System.Text.Json.Serialization;
-
-// namespace Bts.DataStreamCore;
-
-// class BlueRowJsonConverter : JsonConverter<IBlueRow> {
-//    public override IBlueRow Read(
-//       ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-//       var row = JsonSerializer.Deserialize<JsonObject>(ref reader, options)!;
-//       var schemaj = row["schema"]!.GetValue<JsonObject>() ??
-//          throw new InvalidOperationException("Failed to read schema");
-//       // var columns = schemaj["columns"]
-//       var mbr = new MutableBlueRow(schema);
-//       int colNum = 0;
-//       switch (reader.TokenType) {
-//          case JsonTokenType.StartArray:
-//             while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
-//                mbr.setValue(colNum++, readCellValue(ref reader));
-//                colNum++;
-//             }
-//             break;
-//          case JsonTokenType.StartObject:
-//             while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
-//                if (reader.TokenType != JsonTokenType.PropertyName) {
-//                   throw new InvalidOperationException(
-//                      "Expected 'PropertyName' immediately after 'StartObject' token in row json");
-//                }
-//                var colName = reader.GetString();
-//                if (!reader.Read()) {
-//                   throw new InvalidOperationException(
-//                      "Failed to read token immediately after 'PropertyName' token in row json");
-//                }
-//                mbr.setValue(colName, readCellValue(ref reader));
-//             }
-//             break;
-//       }
-//       if (mbr.Count != schema.columnInfo.Length) {
-//          throw new InvalidOperationException(
-//             $"Row json array length '{mbr.Count}' does not match the schema's " +
-//             $"column length '{schema.columnInfo.Length}'.");
-//       }
-//       return mbr;
-
-//       static object? readCellValue(ref Utf8JsonReader reader) =>
-//          reader.TokenType switch {
-//             JsonTokenType.String => reader.GetString(),
-//             JsonTokenType.Number => reader.GetDouble(),
-//             JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
-//             _ => throw new InvalidOperationException("Failed to read cell value"),
-//          };
-//    }
-
-//    public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
-//       throw new NotImplementedException();
-//    }
-// }

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -1,186 +0,0 @@
-// #nullable enable
-// using BCG;
-
-// using Spark.Core.Json;
-
-// using System;
-// using System.Collections.Generic;
-// using System.Text.Json;
-// using System.Text.Json.Nodes;
-// using System.Text.Json.Serialization;
-
-// namespace Bts.DataStreamCore;
-
-// public class DataStreamUpdateJsonConverter : JsonConverter<DataStreamUpdate> {
-//    /// <summary>
-//    /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
-//    /// default, only incremental changes are included ('added' and 'deleted').
-//    /// </summary>
-//    public bool IncludeCurrent { get; set; }
-
-//    /// <summary>
-//    /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
-//    /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
-//    /// </summary>
-//    public bool UseObjectRows { get; set; }
-
-//    public override DataStreamUpdate? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-//       var dpj = JsonSerializer.Deserialize<JsonObject>(ref reader);
-//       if (dpj is null) {
-//          return null;
-//       }
-//       var tableName = dpj["tableName"]!.AsString();
-//       var columns = dpj["columns"]!.AsArray()
-//          .Select((c, i) => {
-//             var dataType = c!["type"]!.AsEnum<DataStreamDataType>();
-//             return (info: new ColumnInfo(c["name"]!.AsString(), i, dataType.ToType()), dataType);
-//          })
-//          .ToArray();
-//       var schema = new BlueTableSchema(columns.Cut1().ToArray(), tableName);
-//       return new() {
-//          TableName = tableName,
-//          AddedRows = ToRows(dpj.GetArrayOrEmpty("added")).ToList(),
-//          DeletedRows = ToRows(dpj.GetArrayOrEmpty("deleted")).ToList(),
-//          Columns = [.. schema.columnInfo]
-//       };
-
-//       IEnumerable<IBlueRow> ToRows(JsonArray rows) {
-//          foreach (var row in rows) {
-//             var mbr = new MutableBlueRow(schema);
-//             switch (row!.GetValueKind()) {
-//                case JsonValueKind.Object:
-//                   var rowObj = row.AsObject();
-//                   for (int i = 0; i < columns.Length; i++) {
-//                      SetCellValue(mbr, i, rowObj[columns[i].info.colName]!, columns[i].dataType);
-//                   }
-//                   break;
-//                case JsonValueKind.Array:
-//                   var rowArr = row.AsArray();
-//                   for (int i = 0; i < columns.Length; i++) {
-//                      SetCellValue(mbr, i, rowArr[i]!, columns[i].dataType);
-//                   }
-//                   break;
-//             }
-//             yield return mbr;
-//          }
-//       }
-
-//       static void SetCellValue(MutableBlueRow mbr, int i, JsonNode cell, DataStreamDataType dataType) {
-//          switch (dataType) {
-//             case DataStreamDataType.String:
-//                mbr.setValue(i, cell.GetValue<string>());
-//                break;
-//             case DataStreamDataType.Int32:
-//                mbr.setValue(i, cell.GetValue<int>());
-//                break;
-//             case DataStreamDataType.Int64:
-//                mbr.setValue(i, cell.GetValue<long>());
-//                break;
-//             case DataStreamDataType.Single:
-//                mbr.setValue(i, cell.GetValue<float>());
-//                break;
-//             case DataStreamDataType.Double:
-//                mbr.setValue(i, cell.GetValue<double>());
-//                break;
-//          }
-//       }
-//    }
-
-//    public override void Write(Utf8JsonWriter writer, DataStreamUpdate value, JsonSerializerOptions options) {
-//       using var _ = writer.WriteObject();
-//       writer.WriteString("tableName", value.TableName);
-//       using (writer.WriteArray("columns")) {
-//          foreach (var col in value.Columns.empty_if_null()) {
-//             using (writer.WriteObject()) {
-//                writer.WriteString("name", col.colName);
-//                writer.WriteNumber("index", col.colNo);
-//                // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
-//                writer.WriteString("type", col.colType.ToString());
-//             }
-//          }
-//       }
-
-//       var cellWriters = value.Columns!.Select(getCellWriter).ToList();
-//       using (writer.WriteArray("added")) {
-//          foreach (IBlueRow row in value.AddedRows!) {
-//             using (writer.WriteArray()) {
-//                for (int i = 0; i < cellWriters.Count; i++) {
-//                   cellWriters[i](row[i]);
-//                }
-//             }
-//          }
-//       }
-//       using (writer.WriteArray("deleted")) {
-//          foreach (IBlueRow row in value.DeletedRows!) {
-//             using (writer.WriteArray()) {
-//                for (int i = 0; i < cellWriters.Count; i++) {
-//                   cellWriters[i](row[i]);
-//                }
-//             }
-//          }
-//       }
-//       if (IncludeCurrent) {
-//          using (writer.WriteArray("current")) {
-//             foreach (IBlueRow row in value.newRows) {
-//                using (writer.WriteArray()) {
-//                   for (int i = 0; i < cellWriters.Count; i++) {
-//                      cellWriters[i](row[i]);
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-//       Action<object?> getCellWriter(ColumnInfo colInfo) {
-//          var writeVal = getCellValueWriter(colInfo);
-//          if (!UseObjectRows) {
-//             return writeVal;
-//          }
-//          return v => {
-//             writer.WritePropertyName(colInfo.colName);
-//             writeVal(v);
-//          };
-//       }
-
-//       Action<object?> getCellValueWriter(ColumnInfo colInfo) {
-//          var t = colInfo.colType;
-//          if (t == typeof(int)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
-//          }
-//          if (t == typeof(uint)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
-//          }
-//          if (t == typeof(long)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
-//          }
-//          if (t == typeof(ulong)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
-//          }
-//          if (t == typeof(short)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
-//          }
-//          if (t == typeof(ushort)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
-//          }
-//          if (t == typeof(double)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
-//          }
-//          if (t == typeof(float)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
-//          }
-//          if (t == typeof(bool)) {
-//             return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
-//          }
-//          return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
-
-//          Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
-//             v => {
-//                if (v is null) {
-//                   writer.WriteNullValue();
-//                } else {
-//                   writeNonNull(v);
-//                }
-//             };
-//       }
-//    }
-// }

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -2,9 +2,8 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>disable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamCore/DataStreamDataType.cs

@@ -1,11 +0,0 @@
-namespace Bts.DataStreamCore;
-
-public enum DataStreamDataType {
-   Any,
-   Bytes,
-   String,
-   Int32,
-   Int64,
-   Single,
-   Double
-}

src/DataStreamSuite/DataStreamCore/DataStreamExtensions.cs

@@ -1,42 +0,0 @@
-#nullable enable
-
-using Bts.DataStreamCore;
-
-public static class DataStreamExtensions {
-   public static Type ToType(this DataStreamDataType dataType) =>
-      dataType switch {
-         DataStreamDataType.Any => typeof(object),
-         DataStreamDataType.String => typeof(string),
-         DataStreamDataType.Int32 => typeof(int),
-         DataStreamDataType.Int64 => typeof(long),
-         DataStreamDataType.Single => typeof(float),
-         DataStreamDataType.Double => typeof(double),
-         _ => throw new ArgumentException($"DataType '{dataType}' not supported.", nameof(dataType)),
-      };
-
-   public static DataStreamDataType FromType(Type type) {
-      if (type == typeof(object)) {
-         return DataStreamDataType.Any;
-      }
-      if (type == typeof(string)) {
-         return DataStreamDataType.String;
-      }
-      if (type == typeof(int)) {
-         return DataStreamDataType.Int32;
-      }
-      if (type == typeof(long)) {
-         return DataStreamDataType.Int64;
-      }
-      if (type == typeof(float)) {
-         return DataStreamDataType.Single;
-      }
-      if (type == typeof(double)) {
-         return DataStreamDataType.Double;
-      }
-      if (type == typeof(byte[])) {
-         return DataStreamDataType.Bytes;
-      }
-      throw new ArgumentException(
-         $"Unable to convert type '{type}' to {typeof(DataStreamDataType)}");
-   }
-}

src/DataStreamSuite/DataStreamCore/DataStreamUpdate.cs

@@ -1,41 +0,0 @@
-#nullable enable
-using BCG;
-
-using Spark.Core.Json;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Bts.DataStreamCore;
-
-public class DataStreamUpdate {
-   [JsonPropertyName("tableName")]
-   public string? TableName { get; set; }
-
-   [JsonPropertyName("columns")]
-   public List<ColumnInfo>? Columns { get; set; }
-
-   [JsonPropertyName("addedRows")]
-   public List<IBlueRow>? AddedRows { get; set; }
-
-   [JsonPropertyName("deletedRows")]
-   public List<IBlueRow>? DeletedRows { get; set; }
-
-   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
-      var schema = new BlueTableSchema([.. Columns!], TableName);
-      var addedRowsTable = new BlueTable(schema);
-      var deletedRowsTable = new BlueTable(schema);
-      if (applyChanges) {
-         state.deleteRows(deletedRowsTable);
-         state.addRows(addedRowsTable);
-      }
-      return new(state, addedRowsTable, deletedRowsTable);
-   }
-
-   public static string ToJson() {
-      return "";
-   }
-}

src/DataStreamSuite/DataStreamCore/Usings.cs

@@ -1 +0,0 @@
-using System.Linq;

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>