DataStreamCore Json stuff

* IsSocketStateOpen
* IsClosedProperly

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://x-token-auth:ATCTT3xFfGN0h_ENnUSfEAIzbfVzkQPLG26cIli50MtcTwZ7VEc8lin5tJYqibjlBJ9PM9R642Ra5GjiL52TBfbO8KeUBT0UoMT7gLQHL2Y1TkW9yZr6NkvsoDkSl49DeD4Ijiu4_0u4qEm4oMVDDg2_fB059-uOWOo7ZL94tGZET8CSVpFIGvs=B0E03660@bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.gitpod.yml

@@ -1,49 +0,0 @@
-# This configuration file was automatically generated by Gitpod.
-# Please adjust to your needs (see https://www.gitpod.io/docs/introduction/learn-gitpod/gitpod-yaml)
-# and commit this file to your remote git repository to share the goodness with others.
-
-# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
-
-image: gitpod/workspace-dotnet-8
-tasks:
-  - name: Install Deps
-    init: |
-      (
-        set -e
-        sudo install-packages rapidjson-dev
-        sudo install-packages libssh2-1-dev
-        sudo install-packages openssh-server
-        sudo install-packages liblttng-ust-dev
-        sudo install-packages libpcap0.8-dev
-        sudo install-packages ccache
-        sudo ln -s $(which ccache) /usr/local/bin/clang
-        sudo ln -s $(which ccache) /usr/local/bin/clang++
-        sudo ln -s $(which ccache) /usr/local/bin/clang-17
-        sudo ln -s $(which ccache) /usr/local/bin/clang++-17
-        sudo ln -s $(which ccache) /usr/local/bin/gcc
-        sudo ln -s $(which ccache) /usr/local/bin/g++
-        sudo ln -s $(which ccache) /usr/local/bin/gcc-12
-        sudo ln -s $(which ccache) /usr/local/bin/g++-12
-        if [ -e externals/src/emumba/install_emumba-deribit-connector.sh ]; then
-          externals/src/emumba/install_emumba-deribit-connector.sh
-        fi
-      )
-  - name: Setup Python venv
-    init: |
-      python -m pip install --upgrade pip
-      python -m venv .venv
-      . .venv/bin/activate
-      python -m pip install --upgrade pip
-      pip install -r requirements.txt
-
-ports:
-  - name: sparkd config
-    port: 8000
-    onOpen: notify
-  - name: sparkd bus
-    port: 8001
-
-gitConfig:
-  alias.st: "status"
-  status.short: "true"
-  core.pager: "less -FXRS"

docker/sparkd/dev/Dockerfile

@@ -1,46 +1,55 @@
 FROM ubuntu:latest
 
 ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/New_York
 
-# Light Deps - should be quick to download/install.
-# Longer running procs should be executed in individual RUNs.
+# light deps (should be quick to download/install)
 RUN apt-get update && apt-get install -y \
-   build-essential \
-   ca-certificates \
-   curl \
-   gnupg \
-   liblttng-ust-dev \
-   libpcap0.8-dev \
-   locales \
-   ninja-build \
-   software-properties-common \
-   sqlite3 \
-   tzdata
+    ca-certificates \
+    curl \
+    gnupg \
+    liblttng-ust-dev \
+    libpcap0.8-dev \
+    locales \
+    ninja-build \
+    software-properties-common \
+    sqlite3 \
+    tzdata
 
-# Clang 17
+# clang 13, 15 (so we can build old branches)
+RUN apt-get update && apt-get install -y \
+    clang-13 \
+    clang-format-13 \
+    clangd-13 \
+    clang-15 \
+    clang-format-15 \
+    clangd-15
+
+# clang 17 (so we can build new branches)
 RUN \
-   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
-   | tee -a "/etc/apt/sources.list.d/llvm.list" \
-   && apt-get update \
-   && apt-get install -y \
-   clang-17 \
-   clang-format-17 \
-   clangd-17
+    curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
+    | tee -a "/etc/apt/sources.list.d/llvm.list" \
+    && apt-get update \
+    && apt-get install -y \
+    clang-17 \
+    clang-format-17 \
+    clangd-17
 
-# G++
+# g++
 RUN apt-get update && apt-get install -y \
-   g++
+    g++-11 \
+    g++-12
 
-# Python
+# python
 RUN apt-get update && apt-get install -y \
-   python3-pip \
-   python3-venv \
-   && python3 -m pip install --upgrade pip
+    python3-pip \
+    python3-venv \
+    && python3 -m pip install --upgrade pip
 
-# Dotnet
+# dotnet
 RUN \
    curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" \
    && dpkg -i packages-microsoft-prod.deb \
@@ -48,65 +57,88 @@ RUN \
    && apt-get update \
    && apt-get install -y dotnet-sdk-8.0
 
-# SSH
+# ssh
 RUN apt-get update && apt-get install -y \
-   libssh2-1-dev \
-   openssh-server
+    libssh2-1-dev \
+    openssh-server
 
-# CMake
+# cmake
 RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
+    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+    && apt-get update \
+    && apt-get install -y cmake
 
-# Node
+# node
 ARG node_major=18
 RUN mkdir -p /etc/apt/keyrings \
-   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
-   | tee /etc/apt/sources.list.d/nodesource.list \
-   && apt-get update \
-   && apt-get install -y nodejs
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+    | tee /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y nodejs
 
-# Dev Utils
+# ccache is done here so that ccache sees clang++ installed, not clang-10
+RUN apt-get update && apt-get install -y \
+    ccache 
+
+# emumba (light deps)
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    doxygen \
+    python3-sphinx \
+    graphviz \
+    libgtest-dev \
+    libssl-dev \
+    rapidjson-dev \
+    zlib1g-dev \
+    pkg-config
+
+# emumba (boost)
+RUN apt-get update && apt-get install -y \
+    libboost-filesystem-dev \
+    libboost-program-options-dev \
+    libboost-test-dev \
+    libboost-date-time-dev
+
+# dev utils
 RUN apt-get update && apt-get install -y \ 
-   ccache \
-   fish \
-   gdb \
-   git-all \
-   sudo \
-   vim
+    fish \
+    gdb \
+    git-all \
+    sudo \
+    vim \
+    wget
 
-# Set up default executables (update-alternatives)
+# set up default executables (update-alternatives)
 RUN \
-   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
-   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
-   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
-   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
-   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
-   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
 
 # Add compiler symlinks for ccache 
 RUN \
-   ln -s /usr/bin/ccache /usr/local/bin/clang; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++-12
+    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-12
 
 RUN locale-gen en_US.UTF-8
 
@@ -115,11 +147,11 @@ ARG docker_uid="1000"
 ARG docker_pass="bts"
 
 RUN \
-   groupadd -g "$docker_uid" "$docker_user" \
-   && useradd \
-   -m \
-   -u "$docker_uid" \
-   -g "$docker_uid" \
-   -G sudo \
-   --password "$(openssl passwd -1 "$docker_pass")" \
-   "$docker_user"
+    groupadd -g "$docker_uid" "$docker_user" \
+    && useradd \
+    -m \
+    -u "$docker_uid" \
+    -g "$docker_uid" \
+    -G sudo \
+    --password "$(openssl passwd -1 "$docker_pass")" \
+    "$docker_user"

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnExpression">
-    <Scanner Include="ColumnExpressionTokens.lex" />
-    <Parser Include="ColumnExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -1,62 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-public class ColumnExpression {
-   public ColumnExpression(string name, Type? type = null, object? value = null) {
-      Name = name;
-      Type = type
-         ?? value?.GetType()
-         ?? throw new ArgumentException(
-            "Type must be specified explicitly when value is null", nameof(type));
-      Value = value
-         ?? DBNull.Value;
-   }
-
-   public string Name { get; }
-
-   public Type Type { get; }
-
-   public object Value { get; }
-
-   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
-      // Try to convert value to 'type' to verify they are compatible
-      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
-      // account for implicit conversions (e.g., int -> long), just inheritance.
-      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
-         throw new ArgumentException(
-            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
-      }
-      return new(name, type ?? typeof(T), value);
-   }
-
-   internal static Type ParseType(string typeId) {
-      return Candidates().ExceptNull().FirstOrDefault() ??
-         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
-
-      IEnumerable<Type?> Candidates() {
-         if (typeId.Length == 0) {
-            yield break;
-         }
-         yield return Type.GetType(typeId);
-         yield return Type.GetType($"System.{typeId}");
-         if (typeId.Length > 1) {
-            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
-         }
-         yield return typeId.ToLower() switch {
-            "float64" or "double" or "real64" or "number" => typeof(double),
-            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
-            "int16" or "short" or "integer16" or "signed16" => typeof(short),
-            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
-            "int64" or "long" or "integer64" or "signed64" => typeof(long),
-            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
-            "uint32" or "uint" or "unsigned" => typeof(uint),
-            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
-            "char" => typeof(char),
-            "byte" or "uint8" => typeof(byte),
-            "bool" or "boolean" => typeof(bool),
-            "string" or "text" => typeof(string),
-            "date" or "datetime" => typeof(DateTime),
-            _ => null
-         };
-      }
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -1,49 +0,0 @@
-%namespace Bts.ColumnExpressions
-%partial
-%visibility internal
-%tokentype ColumnExpressionToken
-%parsertype ColumnExpressionParser
-
-%union {
-   public int Integer;
-   public double Real;
-   public bool Boolean;
-   public string String;
-   public string Identifier;
-   public string TypeId;
-   public DateTime DateTime;
-   public Tuple<string, Type> ColumnDefinition;
-   public ColumnExpression ColumnExpression;
-   public ImmutableList<ColumnExpression> ColumnExpressions;
-}
-
-%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
-%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
-
-%nonassoc EQUALS
-
-%type <ColumnDefinition> ColumnDefinition
-%type <ColumnExpression> ColumnExpression
-%type <ColumnExpressions> ColumnExpressions
-
-%%
-
-ColumnExpressions :
-   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
-   ;
-
-ColumnExpression :
-   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
-   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
-   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
-   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
-   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
-   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
-   ;
-
-ColumnDefinition :
-   IDENTIFIER { $$ = new($1.Identifier, null); }
-   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
-   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
-   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnExpressionInterpreter {
-   public static IList<ColumnExpression> Evaluate(Stream stream) {
-      var parser = new ColumnExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
-         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-using System.Text;
-
-partial class ColumnExpressionParser {
-   public ColumnExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnExpressions;
-
-partial class ColumnExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -1,85 +0,0 @@
-%namespace Bts.ColumnExpressions
-%scannertype ColumnExpressionScanner
-%visibility internal
-%tokentype ColumnExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-BooleanConstant          true|false
-StringConstant           '([^\'\\]*|\\.)*'
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
-IntegerConstant          -?[0-9]+
-NullConstant             'null'
-RealConstant             -?[0-9]+\.[0-9]+
-
-/* Relational Operators */
-Equals                   =
-OfType                   /
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-NamespacePrefix          {Identifier}\.
-TypeId                   {NamespacePrefix}*{Identifier}
-
-/* Utility Tokens */
-OpenParenthesis          \(
-CloseParenthesis         \)
-Comma                    ,
-QuoteOpen                \[
-QuoteClose               \]
-
-/* Comments */
-LineComment \/\/[^\n\r]*(\n|\r|\r\n)
-
-%%
-
-{Equals} {
-   return (int)ColumnExpressionToken.EQUALS;
-}
-
-{OfType} {
-   return (int)ColumnExpressionToken.OF_TYPE;
-}
-
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
-   return (int)ColumnExpressionToken.STRING_CONSTANT;
-}
-
-{BooleanConstant} {
-   yylval.Boolean = yytext.ToLower() switch {
-      "true" => true,
-      "false" => false,
-      _ => throw new Exception(
-         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
-   };
-   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
-}
-
-{DateTimeConstant} {
-   yylval.DateTime = DateTime.Parse(yytext);
-   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
-}
-
-{IntegerConstant} {
-   yylval.Integer = int.Parse(yytext);
-   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
-}
-
-{RealConstant} {
-   yylval.Real = double.Parse(yytext);
-   return (int)ColumnExpressionToken.REAL_CONSTANT;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnExpressionToken.IDENTIFIER;
-}
-
-{TypeId} {
-   yylval.TypeId = yytext;   
-   return (int)ColumnExpressionToken.TYPE_ID;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,24 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Label="ColumnSortExpression">
-    <Scanner Include="ColumnSortExpressionTokens.lex" />
-    <Parser Include="ColumnSortExpressionGrammar.y" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -1,11 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-public class ColumnSortExpression {
-   public ColumnSortExpression(string name, bool isAscending = true) {
-      Name = name;
-      IsAscending = isAscending;
-   }
-
-   public string Name { get; }
-   public bool IsAscending { get; }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -1,29 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%partial
-%visibility internal
-%tokentype ColumnSortExpressionToken
-%parsertype ColumnSortExpressionParser
-
-%union {
-   public string Identifier;
-   public ColumnSortExpression Column;
-   public ImmutableList<ColumnSortExpression> Columns;
-}
-
-%token ASCENDING, DESCENDING, IDENTIFIER
-
-%type <Column> Column
-%type <Columns> Columns
-
-%%
-
-Columns :
-   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
-   | Columns Column { $$ = $1.Add($2); }
-   ;
-
-Column :
-   IDENTIFIER { $$ = new($1.Identifier, true); }
-   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
-   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
-   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.IO;
-using System.Text;
-
-public class ColumnSortExpressionInterpreter {
-   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
-      var parser = new ColumnSortExpressionParser();
-      var res = parser.Parse(stream);
-      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
-         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
-   }
-
-   public static IList<ColumnSortExpression> Evaluate(string str) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
-      return Evaluate(ms);
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -1,18 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-using System.Text;
-
-partial class ColumnSortExpressionParser {
-   public ColumnSortExpressionParser() : base(null) { }
-
-   public ValueType? Parse(string s) {
-      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
-      return Parse(ms);
-   }
-
-   public ValueType? Parse(Stream stream) {
-      Scanner = new ColumnSortExpressionScanner(stream);
-      Parse();
-      return CurrentSemanticValue;
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -1,9 +0,0 @@
-namespace Bts.ColumnSortExpressions;
-
-partial class ColumnSortExpressionScanner {
-   public override void yyerror(string format, params object[] args) {
-      base.yyerror(format, args);
-      Console.WriteLine(format, args);
-      Console.WriteLine();
-   }
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -1,30 +0,0 @@
-%namespace Bts.ColumnSortExpressions
-%scannertype ColumnSortExpressionScanner
-%visibility internal
-%tokentype ColumnSortExpressionToken
-
-%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
-
-/* Literal Constants */
-Letter                   [a-zA-Z]
-
-Identifier               {Letter}({Letter}*[0-9]*_*)*
-Ascending                +
-Descending               -
-
-%%
-
-{Ascending} {
-   return (int)ColumnSortExpressionToken.ASCENDING;
-}
-
-{Descending} {
-   return (int)ColumnSortExpressionToken.DESCENDING;
-}
-
-{Identifier} {
-   yylval.Identifier = yytext;
-   return (int)ColumnSortExpressionToken.IDENTIFIER;
-}
-
-%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -1,720 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
-    [Serializable]
-    internal class BufferException : Exception
-    {
-        public BufferException() { }
-        public BufferException(string message) : base(message) { }
-        public BufferException(string message, Exception innerException)
-            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
-    }
-
-    internal abstract class ScanBuff
-    {
-        private string fileNm;
-
-        public const int EndOfFile = -1;
-        public const int UnicodeReplacementChar = 0xFFFD;
-
-        public bool IsFile { get { return (fileNm != null); } }
-        public string FileName { get { return fileNm; } set { fileNm = value; } }
-
-        public abstract int Pos { get; set; }
-        public abstract int Read();
-        public virtual void Mark() { }
-
-        public abstract string GetString(int begin, int limit);
-
-        public static ScanBuff GetBuffer(string source)
-        {
-            return new StringBuffer(source);
-        }
-
-        public static ScanBuff GetBuffer(IList<string> source)
-        {
-            return new LineBuffer(source);
-        }
-
-#if (!NOFILES)
-        public static ScanBuff GetBuffer(Stream source)
-        {
-            return new BuildBuffer(source);
-        }
-
-#if (!BYTEMODE)
-        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
-        {
-            return new BuildBuffer(source, fallbackCodePage);
-        }
-#endif // !BYTEMODE
-#endif // !NOFILES
-    }
-
-    #region Buffer classes
-
-    // ==============================================================
-    // =====  Definitions for various ScanBuff derived classes   ====
-    // ==============================================================
-    // ===============         String input          ================
-    // ==============================================================
-
-    /// <summary>
-    /// This class reads characters from a single string as
-    /// required, for example, by Visual Studio language services
-    /// </summary>
-    sealed class StringBuffer : ScanBuff
-    {
-        string str;        // input buffer
-        int bPos;          // current position in buffer
-        int sLen;
-
-        public StringBuffer(string source)
-        {
-            this.str = source;
-            this.sLen = source.Length;
-            this.FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (bPos < sLen) return str[bPos++];
-            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
-            else { bPos++; return EndOfFile; }                // two strikes and you're out!
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            //  "limit" can be greater than sLen with the BABEL
-            //  option set.  Read returns a "virtual" EOL if
-            //  an attempt is made to read past the end of the
-            //  string buffer.  Without the guard any attempt 
-            //  to fetch yytext for a token that includes the 
-            //  EOL will throw an index exception.
-            if (limit > sLen) limit = sLen;
-            if (limit <= begin) return "";
-            else return str.Substring(begin, limit - begin);
-        }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-        public override string ToString() { return "StringBuffer"; }
-    }
-
-    // ==============================================================
-    //  The LineBuff class contributed by Nigel Horspool, 
-    //  nigelh@cs.uvic.cs
-    // ==============================================================
-
-    sealed class LineBuffer : ScanBuff
-    {
-        IList<string> line;    // list of source lines from a file
-        int numLines;          // number of strings in line list
-        string curLine;        // current line in that list
-        int cLine;             // index of current line in the list
-        int curLen;            // length of current line
-        int curLineStart;      // position of line start in whole file
-        int curLineEnd;        // position of line end in whole file
-        int maxPos;            // max position ever visited in whole file
-        int cPos;              // ordinal number of code in source
-
-        // Constructed from a list of strings, one per source line.
-        // The lines have had trailing '\n' characters removed.
-        public LineBuffer(IList<string> lineList)
-        {
-            line = lineList;
-            numLines = line.Count;
-            cPos = curLineStart = 0;
-            curLine = (numLines > 0 ? line[0] : "");
-            maxPos = curLineEnd = curLen = curLine.Length;
-            cLine = 1;
-            FileName = null;
-        }
-
-        public override int Read()
-        {
-            if (cPos < curLineEnd)
-                return curLine[cPos++ - curLineStart];
-            if (cPos++ == curLineEnd)
-                return '\n';
-            if (cLine >= numLines)
-                return EndOfFile;
-            curLine = line[cLine];
-            curLen = curLine.Length;
-            curLineStart = curLineEnd + 1;
-            curLineEnd = curLineStart + curLen;
-            if (curLineEnd > maxPos)
-                maxPos = curLineEnd;
-            cLine++;
-            return curLen > 0 ? curLine[0] : '\n';
-        }
-
-        // To speed up searches for the line containing a position
-        private int cachedPosition;
-        private int cachedIxdex;
-        private int cachedLineStart;
-
-        // Given a position pos within the entire source, the results are
-        //   ix     -- the index of the containing line
-        //   lstart -- the position of the first character on that line
-        private void findIndex(int pos, out int ix, out int lstart)
-        {
-            if (pos >= cachedPosition)
-            {
-                ix = cachedIxdex; lstart = cachedLineStart;
-            }
-            else
-            {
-                ix = lstart = 0;
-            }
-            while (ix < numLines)
-            {
-                int len = line[ix].Length + 1;
-                if (pos < lstart + len) break;
-                lstart += len;
-                ix++;
-            }
-            cachedPosition = pos;
-            cachedIxdex = ix;
-            cachedLineStart = lstart;
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            if (begin >= maxPos || limit <= begin) return "";
-            int endIx, begIx, endLineStart, begLineStart;
-            findIndex(begin, out begIx, out begLineStart);
-            int begCol = begin - begLineStart;
-            findIndex(limit, out endIx, out endLineStart);
-            int endCol = limit - endLineStart;
-            string s = line[begIx];
-            if (begIx == endIx)
-            {
-                // the usual case, substring all on one line
-                return (endCol <= s.Length) ?
-                    s.Substring(begCol, endCol - begCol)
-                    : s.Substring(begCol) + "\n";
-            }
-            // the string spans multiple lines, yuk!
-            StringBuilder sb = new StringBuilder();
-            if (begCol < s.Length)
-                sb.Append(s.Substring(begCol));
-            for (; ; )
-            {
-                sb.Append("\n");
-                s = line[++begIx];
-                if (begIx >= endIx) break;
-                sb.Append(s);
-            }
-            if (endCol <= s.Length)
-            {
-                sb.Append(s.Substring(0, endCol));
-            }
-            else
-            {
-                sb.Append(s);
-                sb.Append("\n");
-            }
-            return sb.ToString();
-        }
-
-        public override int Pos
-        {
-            get { return cPos; }
-            set
-            {
-                cPos = value;
-                findIndex(cPos, out cLine, out curLineStart);
-                // cLine should be the *next* line after curLine.
-                curLine = (cLine < numLines ? line[cLine++] : "");
-                curLineEnd = curLineStart + curLine.Length;
-            }
-        }
-
-        public override string ToString() { return "LineBuffer"; }
-    }
-
-#if (!NOFILES)
-    // ==============================================================
-    // =====     class BuildBuff : for unicode text files    ========
-    // ==============================================================
-
-    class BuildBuffer : ScanBuff
-    {
-        // Double buffer for char stream.
-        class BufferElement
-        {
-            StringBuilder bldr = new StringBuilder();
-            StringBuilder next = new StringBuilder();
-            int minIx;
-            int maxIx;
-            int brkIx;
-            bool appendToNext;
-
-            internal BufferElement() { }
-
-            internal int MaxIndex { get { return maxIx; } }
-            // internal int MinIndex { get { return minIx; } }
-
-            internal char this[int index]
-            {
-                get
-                {
-                    if (index < minIx || index >= maxIx)
-                        throw new BufferException("Index was outside data buffer");
-                    else if (index < brkIx)
-                        return bldr[index - minIx];
-                    else
-                        return next[index - brkIx];
-                }
-            }
-
-            internal void Append(char[] block, int count)
-            {
-                maxIx += count;
-                if (appendToNext)
-                    this.next.Append(block, 0, count);
-                else
-                {
-                    this.bldr.Append(block, 0, count);
-                    brkIx = maxIx;
-                    appendToNext = true;
-                }
-            }
-
-            internal string GetString(int start, int limit)
-            {
-                if (limit <= start)
-                    return "";
-                if (start >= minIx && limit <= maxIx)
-                    if (limit < brkIx) // String entirely in bldr builder
-                        return bldr.ToString(start - minIx, limit - start);
-                    else if (start >= brkIx) // String entirely in next builder
-                        return next.ToString(start - brkIx, limit - start);
-                    else // Must do a string-concatenation
-                        return
-                            bldr.ToString(start - minIx, brkIx - start) +
-                            next.ToString(0, limit - brkIx);
-                else
-                    throw new BufferException("String was outside data buffer");
-            }
-
-            internal void Mark(int limit)
-            {
-                if (limit > brkIx + 16) // Rotate blocks
-                {
-                    StringBuilder temp = bldr;
-                    bldr = next;
-                    next = temp;
-                    next.Length = 0;
-                    minIx = brkIx;
-                    brkIx = maxIx;
-                }
-            }
-        }
-
-        BufferElement data = new BufferElement();
-
-        int bPos;            // Postion index in the StringBuilder
-        BlockReader NextBlk; // Delegate that serves char-arrays;
-
-        private string EncodingName
-        {
-            get
-            {
-                StreamReader rdr = NextBlk.Target as StreamReader;
-                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
-            }
-        }
-
-        public BuildBuffer(Stream stream)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
-        }
-
-#if (!BYTEMODE)
-        public BuildBuffer(Stream stream, int fallbackCodePage)
-        {
-            FileStream fStrm = (stream as FileStream);
-            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
-        }
-#endif
-
-        /// <summary>
-        /// Marks a conservative lower bound for the buffer,
-        /// allowing space to be reclaimed.  If an application 
-        /// needs to call GetString at arbitrary past locations 
-        /// in the input stream, Mark() is not called.
-        /// </summary>
-        public override void Mark() { data.Mark(bPos - 2); }
-
-        public override int Pos
-        {
-            get { return bPos; }
-            set { bPos = value; }
-        }
-
-
-        /// <summary>
-        /// Read returns the ordinal number of the next char, or 
-        /// EOF (-1) for an end of stream.  Note that the next
-        /// code point may require *two* calls of Read().
-        /// </summary>
-        /// <returns></returns>
-        public override int Read()
-        {
-            //
-            //  Characters at positions 
-            //  [data.offset, data.offset + data.bldr.Length)
-            //  are available in data.bldr.
-            //
-            if (bPos < data.MaxIndex)
-            {
-                // ch0 cannot be EOF
-                return (int)data[bPos++];
-            }
-            else // Read from underlying stream
-            {
-                // Experimental code, blocks of page size
-                char[] chrs = new char[4096];
-                int count = NextBlk(chrs, 0, 4096);
-                if (count == 0)
-                    return EndOfFile;
-                else
-                {
-                    data.Append(chrs, count);
-                    return (int)data[bPos++];
-                }
-            }
-        }
-
-        public override string GetString(int begin, int limit)
-        {
-            return data.GetString(begin, limit);
-        }
-
-        public override string ToString()
-        {
-            return "StringBuilder buffer, encoding: " + this.EncodingName;
-        }
-    }
-
-    // =============== End ScanBuff-derived classes ==================
-
-    internal delegate int BlockReader(char[] block, int index, int number);
-
-    // A delegate factory, serving up a delegate that
-    // reads a block of characters from the underlying
-    // encoded stream, via a StreamReader object.
-    //
-    internal static class BlockReaderFactory
-    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
-
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
-        {
-            Encoding encoding;
-            int preamble = Preamble(stream);
-
-            if (preamble != 0)  // There is a valid BOM here!
-                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
-                encoding = Encoding.GetEncoding(fallbackCodePage);
-            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
-            StreamReader reader = new StreamReader(stream, encoding);
-            return reader.Read;
-        }
-
-        static int Preamble(Stream stream)
-        {
-            int b0 = stream.ReadByte();
-            int b1 = stream.ReadByte();
-
-            if (b0 == 0xfe && b1 == 0xff)
-                return 1201; // UTF16BE
-            if (b0 == 0xff && b1 == 0xfe)
-                return 1200; // UTF16LE
-
-            int b2 = stream.ReadByte();
-            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
-                return 65001; // UTF8
-            //
-            // There is no unicode preamble, so we
-            // return denoter for the machine default.
-            //
-            stream.Seek(0, SeekOrigin.Begin);
-            return 0;
-        }
-#endif // !BYTEMODE
-    }
-#endif // !NOFILES
-    #endregion Buffer classes
-
-    // ==============================================================
-    // ============      class CodePageHandling         =============
-    // ==============================================================
-#if (!NOFILES)
-    public static class CodePageHandling
-    {
-        public static int GetCodePage(string option)
-        {
-            string command = option.ToUpperInvariant();
-            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
-                command = command.Substring(9);
-            try
-            {
-                if (command.Equals("RAW"))
-                    return -1;
-                else if (command.Equals("GUESS"))
-                    return -2;
-                else if (command.Equals("DEFAULT"))
-                    return 0;
-                else if (char.IsDigit(command[0]))
-                    return int.Parse(command, CultureInfo.InvariantCulture);
-                else
-                {
-                    Encoding enc = Encoding.GetEncoding(command);
-                    return enc.CodePage;
-                }
-            }
-            catch (FormatException)
-            {
-                Console.Error.WriteLine(
-                    "Invalid format \"{0}\", using machine default", option);
-            }
-            catch (ArgumentException)
-            {
-                Console.Error.WriteLine(
-                    "Unknown code page \"{0}\", using machine default", option);
-            }
-            return 0;
-        }
-    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -1,884 +0,0 @@
-// ==============================================================
-// <auto-generated>
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with 
-// the specification from which it was generated.
-// </auto-generated>
-// ==============================================================
-
-// Gardens Point Parser Generator
-// Copyright (c) Wayne Kelly, QUT 2005-2010
-// (see accompanying GPPGcopyright.rtf)
-
-using System;
-using System.Text;
-using System.Globalization;
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Diagnostics.CodeAnalysis;
-
-namespace QUT.Gppg {
-    /// <summary>
-    /// Abstract class for GPPG shift-reduce parsers.
-    /// Parsers generated by GPPG derive from this base
-    /// class, overriding the abstract Initialize() and
-    /// DoAction() methods.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type</typeparam>
-    /// <typeparam name="TSpan">Location type</typeparam>
-#if EXPORT_GPPG
-    public abstract class ShiftReduceParser<TValue, TSpan>
-#else
-    internal abstract class ShiftReduceParser<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan>, new() {
-        private AbstractScanner<TValue, TSpan> scanner;
-        /// <summary>
-        /// The abstract scanner for this parser.
-        /// </summary>
-        protected AbstractScanner<TValue, TSpan> Scanner {
-            get { return scanner; }
-            set { scanner = value; }
-        }
-
-        /// <summary>
-        /// Constructor for base class
-        /// </summary>
-        /// <param name="scanner">Scanner instance for this parser</param>
-        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
-            this.scanner = scanner;
-        }
-
-        // ==============================================================
-        //                    TECHNICAL EXPLANATION.
-        //   Why the next two fields are not exposed via properties.
-        // ==============================================================
-        // These fields are of the generic parameter types, and are
-        // frequently instantiated as struct types in derived classes.
-        // Semantic actions are defined in the derived classes and refer
-        // to instance fields of these structs.  In such cases the code
-        // "get_CurrentSemanticValue().myField = blah;" will fail since
-        // the getter pushes the value of the field, not the reference.
-        // So, in the presence of properties, gppg would need to encode
-        // such field accesses as ... 
-        //  "tmp = get_CurrentSemanticValue(); // Fetch value
-        //   tmp.myField = blah;               // update
-        //   set_CurrentSemanticValue(tmp); "  // Write update back.
-        // There is no issue if TValue is restricted to be a ref type.
-        // The same explanation applies to scanner.yylval.
-        // ==============================================================
-        /// <summary>
-        /// The current value of the "$$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TValue CurrentSemanticValue;
-
-        /// <summary>
-        /// The current value of the "@$" symbolic variable in the parser
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        protected TSpan CurrentLocationSpan;
-        protected int NextToken;
-
-        private TSpan LastSpan;
-        private State FsaState;
-        private bool recovering;
-        private int tokensSinceLastError;
-
-        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
-        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
-        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
-
-        /// <summary>
-        /// The stack of semantic value (YYSTYPE) values.
-        /// </summary>
-        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
-
-        /// <summary>
-        /// The stack of location value (YYLTYPE) varlues.
-        /// </summary>
-        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
-
-        private int errorToken;
-        private int endOfFileToken;
-        private string[] nonTerminals;
-        private State[] states;
-        private Rule[] rules;
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the rule list into this base class.
-        /// </summary>
-        /// <param name="rules">The array of Rule objects</param>
-        protected void InitRules( Rule[] rules ) { this.rules = rules; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the states table into this base class.
-        /// </summary>
-        /// <param name="states">The pre-initialized states table</param>
-        protected void InitStates( State[] states ) { this.states = states; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// </summary>
-        /// <param name="size"></param>
-        protected void InitStateTable( int size ) { states = new State[size]; }
-
-        /// <summary>
-        /// Initialization method to allow derived classes
-        /// to insert the special value for the error and EOF tokens.
-        /// </summary>
-        /// <param name="err">The error state ordinal</param>
-        /// <param name="end">The EOF stat ordinal</param>
-        protected void InitSpecialTokens( int err, int end ) {
-            errorToken = err;
-            endOfFileToken = end;
-        }
-
-        /// <summary>
-        /// Initialization method to allow derived classes to
-        /// insert the non-terminal symbol names into this base class.
-        /// </summary>
-        /// <param name="names">Non-terminal symbol names</param>
-        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
-
-        #region YYAbort, YYAccept etcetera.
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AcceptException : Exception {
-            internal AcceptException() { }
-            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class AbortException : Exception {
-            internal AbortException() { }
-            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-        [Serializable]
-        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
-        // Reason for FxCop message suppression -
-        // This exception cannot escape from the local context
-        private class ErrorException : Exception {
-            internal ErrorException() { }
-            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
-        }
-
-        // The following methods are only called from within
-        // a semantic action. The thrown exceptions can never
-        // propagate outside the ShiftReduceParser class in 
-        // which they are nested.
-
-        /// <summary>
-        /// Force parser to terminate, returning "true"
-        /// </summary>
-        protected static void YYAccept() { throw new AcceptException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning "false"
-        /// </summary>
-        protected static void YYAbort() { throw new AbortException(); }
-
-        /// <summary>
-        /// Force parser to terminate, returning
-        /// "false" if error recovery fails.
-        /// </summary>
-        protected static void YYError() { throw new ErrorException(); }
-
-        /// <summary>
-        /// Check if parser in error recovery state.
-        /// </summary>
-        protected bool YYRecovering { get { return recovering; } }
-        #endregion
-
-        /// <summary>
-        /// Abstract base method. ShiftReduceParser calls this
-        /// to initialize the base class data structures.  Concrete
-        /// parser classes must override this method.
-        /// </summary>
-        protected abstract void Initialize();
-
-        /// <summary>
-        /// Main entry point of the Shift-Reduce Parser.
-        /// </summary>
-        /// <returns>True if parse succeeds, else false for
-        /// unrecoverable errors</returns>
-        public bool Parse() {
-            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
-
-            NextToken = 0;
-            FsaState = states[0];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-
-            while (true) {
-#if TRACE_ACTIONS
-                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-                DisplayStack();
-#endif
-                int action = FsaState.defaultAction;
-
-                if (FsaState.ParserTable != null) {
-                    if (NextToken == 0) {
-                        // We save the last token span, so that the location span
-                        // of production right hand sides that begin or end with a
-                        // nullable production will be correct.
-                        LastSpan = scanner.yylloc;
-                        NextToken = scanner.yylex();
-#if TRACE_ACTIONS
-                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
-#endif
-                    }
-#if TRACE_ACTIONS
-                    else 
-                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
-#endif
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-                }
-
-                if (action > 0)         // shift
-                {
-                    Shift( action );
-                }
-                else if (action < 0)   // reduce
-                {
-                    try {
-                        Reduce( -action );
-                        if (action == -1)	// accept
-                            return true;
-                    }
-                    catch (AbortException) {
-                            return false;
-                    }
-                    catch (AcceptException) {
-                        return true;
-                    }
-                    catch (ErrorException) {
-                        if (!ErrorRecovery())
-                            return false;
-                        else
-                            throw;  // Rethrow x, preserving information.
-                    }
-                }
-                else if (action == 0 && !ErrorRecovery()) // error
-                    return false;
-            }
-        }
-
-        private void Shift( int stateIndex ) {
-#if TRACE_ACTIONS
-				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
-#endif
-            FsaState = states[stateIndex];
-
-            valueStack.Push( scanner.yylval );
-            StateStack.Push( FsaState );
-            LocationStack.Push( scanner.yylloc );
-
-            if (recovering) {
-                if (NextToken != errorToken)
-                    tokensSinceLastError++;
-
-                if (tokensSinceLastError > 5)
-                    recovering = false;
-            }
-
-            if (NextToken != endOfFileToken)
-                NextToken = 0;
-        }
-
-        private void Reduce( int ruleNumber ) {
-#if TRACE_ACTIONS
-				DisplayRule(ruleNumber);
-#endif
-            Rule rule = rules[ruleNumber];
-            int rhLen = rule.RightHandSide.Length;
-            //
-            //  Default actions for unit productions.
-            //
-            if (rhLen == 1) {
-                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
-                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
-            }
-            else if (rhLen == 0) {
-                // Create a new blank value.
-                // Explicit semantic action may mutate this value
-                CurrentSemanticValue = default( TValue );
-                // The location span for an empty production will start with the
-                // beginning of the next lexeme, and end with the finish of the
-                // previous lexeme.  This gives the correct behaviour when this
-                // nonsense value is used in later Merge operations.
-                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
-                    scanner.yylloc.Merge( LastSpan ) :
-                    default( TSpan ));
-            }
-            else {
-                // Default action: $$ = $1;
-                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
-                //  Default action "@$ = @1.Merge(@N)" for location info.
-                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
-                TSpan atN = LocationStack[LocationStack.Depth - 1];
-                CurrentLocationSpan =
-                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
-            }
-
-            DoAction( ruleNumber );
-
-            for (int i = 0; i < rule.RightHandSide.Length; i++) {
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-            }
-            FsaState = StateStack.TopElement();
-
-            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
-                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
-
-            StateStack.Push( FsaState );
-            valueStack.Push( CurrentSemanticValue );
-            LocationStack.Push( CurrentLocationSpan );
-        }
-
-        /// <summary>
-        /// Execute the selected action from array.
-        /// Must be overriden in derived classes.
-        /// </summary>
-        /// <param name="actionNumber">Index of the action to perform</param>
-        protected abstract void DoAction( int actionNumber );
-
-        private bool ErrorRecovery() {
-            bool discard;
-
-            if (!recovering) // if not recovering from previous error
-                ReportError();
-
-            if (!FindErrorRecoveryState())
-                return false;
-            //
-            //  The interim fix for the "looping in error recovery"
-            //  artifact involved moving the setting of the recovering 
-            //  bool until after invalid tokens have been discarded.
-            //
-            ShiftErrorToken();
-            discard = DiscardInvalidTokens();
-            recovering = true;
-            tokensSinceLastError = 0;
-            return discard;
-        }
-
-        private void ReportError() {
-            StringBuilder errorMsg = new StringBuilder();
-            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
-
-            if (FsaState.ParserTable.Count < 7) {
-                bool first = true;
-                foreach (int terminal in FsaState.ParserTable.Keys) {
-                    if (first)
-                        errorMsg.Append( ", expecting " );
-                    else
-                        errorMsg.Append( ", or " );
-
-                    errorMsg.Append( TerminalToString( terminal ) );
-                    first = false;
-                }
-            }
-            scanner.yyerror( errorMsg.ToString() );
-        }
-
-        private void ShiftErrorToken() {
-            int old_next = NextToken;
-            NextToken = errorToken;
-
-            Shift( FsaState.ParserTable[NextToken] );
-
-#if TRACE_ACTIONS
-				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
-#endif
-            NextToken = old_next;
-        }
-
-        private bool FindErrorRecoveryState() {
-            while (true)    // pop states until one found that accepts error token
-            {
-                if (FsaState.ParserTable != null &&
-                    FsaState.ParserTable.ContainsKey( errorToken ) &&
-                    FsaState.ParserTable[errorToken] > 0) // shift
-                    return true;
-
-#if TRACE_ACTIONS
-					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
-#endif
-                StateStack.Pop();
-                valueStack.Pop();
-                LocationStack.Pop();
-
-#if TRACE_ACTIONS
-			    DisplayStack();
-#endif
-                if (StateStack.IsEmpty()) {
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
-#endif
-                    return false;
-                }
-                else
-                    FsaState = StateStack.TopElement();
-            }
-        }
-
-        private bool DiscardInvalidTokens() {
-
-            int action = FsaState.defaultAction;
-
-            if (FsaState.ParserTable != null) {
-                // Discard tokens until find one that works ...
-                while (true) {
-                    if (NextToken == 0) {
-#if TRACE_ACTIONS
-                            Console.Error.Write("Reading a token: ");
-#endif
-                        NextToken = scanner.yylex();
-                    }
-#if TRACE_ACTIONS
-                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
-#endif
-                    if (NextToken == endOfFileToken)
-                        return false;
-
-                    if (FsaState.ParserTable.ContainsKey( NextToken ))
-                        action = FsaState.ParserTable[NextToken];
-
-                    if (action != 0)
-                        return true;
-                    else {
-#if TRACE_ACTIONS
-                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
-#endif
-                        NextToken = 0;
-                    }
-                }
-            }
-            else if (recovering && tokensSinceLastError == 0) {
-                // 
-                //  Boolean recovering is not set until after the first
-                //  error token has been shifted.  Thus if we get back 
-                //  here with recovering set and no tokens read we are
-                //  looping on the same error recovery action.  This 
-                //  happens if current_state.ParserTable is null because
-                //  the state has an LR(0) reduction, but not all
-                //  lookahead tokens are valid.  This only occurs for
-                //  error productions that *end* on "error".
-                //
-                //  This action discards tokens one at a time until
-                //  the looping stops.  Another attack would be to always
-                //  use the LALR(1) table if a production ends on "error"
-                //
-#if TRACE_ACTIONS
-                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
-#endif
-                if (NextToken == endOfFileToken)
-                    return false;
-                NextToken = 0;
-                return true;
-            }
-            else
-                return true;
-        }
-
-        /// <summary>
-        /// Traditional YACC method.  Discards the next input token.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyclearin() { NextToken = 0; }
-
-        /// <summary>
-        /// Tradional YACC method. Clear the "recovering" flag.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        protected void yyerrok() { recovering = false; }
-
-        /// <summary>
-        /// OBSOLETE FOR VERSION 1.4.0
-        /// Method used by derived types to insert new
-        /// state instances in the "states" array.
-        /// </summary>
-        /// <param name="stateNumber">index of the state</param>
-        /// <param name="state">data for the state</param>
-        protected void AddState( int stateNumber, State state ) {
-            states[stateNumber] = state;
-            state.number = stateNumber;
-        }
-
-        private void DisplayStack() {
-            Console.Error.Write( "State stack is now:" );
-            for (int i = 0; i < StateStack.Depth; i++)
-                Console.Error.Write( " {0}", StateStack[i].number );
-            Console.Error.WriteLine();
-        }
-
-        private void DisplayRule( int ruleNumber ) {
-            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
-            DisplayProduction( rules[ruleNumber] );
-        }
-
-        private void DisplayProduction( Rule rule ) {
-            if (rule.RightHandSide.Length == 0)
-                Console.Error.Write( "/* empty */ " );
-            else
-                foreach (int symbol in rule.RightHandSide)
-                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
-
-            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
-        }
-
-        /// <summary>
-        /// Abstract state class naming terminal symbols.
-        /// This is overridden by derived classes with the
-        /// name (or alias) to be used in error messages.
-        /// </summary>
-        /// <param name="terminal">The terminal ordinal</param>
-        /// <returns></returns>
-        protected abstract string TerminalToString( int terminal );
-
-        private string SymbolToString( int symbol ) {
-            if (symbol < 0)
-                return nonTerminals[-symbol - 1];
-            else
-                return TerminalToString( symbol );
-        }
-
-        /// <summary>
-        /// Return text representation of argument character
-        /// </summary>
-        /// <param name="input">The character to convert</param>
-        /// <returns>String representation of the character</returns>
-        protected static string CharToString( char input ) {
-            switch (input) {
-                case '\a': return @"'\a'";
-                case '\b': return @"'\b'";
-                case '\f': return @"'\f'";
-                case '\n': return @"'\n'";
-                case '\r': return @"'\r'";
-                case '\t': return @"'\t'";
-                case '\v': return @"'\v'";
-                case '\0': return @"'\0'";
-                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
-            }
-        }
-    }
-
-    /// <summary>
-    /// Classes implementing this interface must supply a
-    /// method that merges two location objects to return
-    /// a new object of the same type.
-    /// GPPG-generated parsers have the default location
-    /// action equivalent to "@$ = @1.Merge(@N);" where N
-    /// is the right-hand-side length of the production.
-    /// </summary>
-    /// <typeparam name="TSpan">The Location type</typeparam>
-#if EXPORT_GPPG
-    public interface IMerge<TSpan>
-#else
-    internal interface IMerge<TSpan>
-#endif
- {
-        /// <summary>
-        /// Interface method that creates a location object from
-        /// the current and last object.  Typically used to create
-        /// a location object extending from the start of the @1
-        /// object to the end of the @N object.
-        /// </summary>
-        /// <param name="last">The lexically last object to merge</param>
-        /// <returns>The merged location object</returns>
-        TSpan Merge( TSpan last );
-    }
-
-    /// <summary>
-    /// This is the default class that carries location
-    /// information from the scanner to the parser.
-    /// If you don't declare "%YYLTYPE Foo" the parser
-    /// will expect to deal with this type.
-    /// </summary>
-#if EXPORT_GPPG
-    public class LexLocation : IMerge<LexLocation>
-#else
-    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
-    internal class LexLocation : IMerge<LexLocation>
-#endif
- {
-        private int startLine;   // start line
-        private int startColumn; // start column
-        private int endLine;     // end line
-        private int endColumn;   // end column
-
-        /// <summary>
-        /// The line at which the text span starts.
-        /// </summary>
-        public int StartLine { get { return startLine; } }
-
-        /// <summary>
-        /// The column at which the text span starts.
-        /// </summary>
-        public int StartColumn { get { return startColumn; } }
-
-        /// <summary>
-        /// The line on which the text span ends.
-        /// </summary>
-        public int EndLine { get { return endLine; } }
-
-        /// <summary>
-        /// The column of the first character
-        /// beyond the end of the text span.
-        /// </summary>
-        public int EndColumn { get { return endColumn; } }
-
-        /// <summary>
-        /// Default no-arg constructor.
-        /// </summary>
-        public LexLocation() { }
-
-        /// <summary>
-        /// Constructor for text-span with given start and end.
-        /// </summary>
-        /// <param name="sl">start line</param>
-        /// <param name="sc">start column</param>
-        /// <param name="el">end line </param>
-        /// <param name="ec">end column</param>
-        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
-
-        /// <summary>
-        /// Create a text location which spans from the 
-        /// start of "this" to the end of the argument "last"
-        /// </summary>
-        /// <param name="last">The last location in the result span</param>
-        /// <returns>The merged span</returns>
-        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
-    }
-
-    /// <summary>
-    /// Abstract scanner class that GPPG expects its scanners to 
-    /// extend.
-    /// </summary>
-    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
-    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
-#if EXPORT_GPPG
-    public abstract class AbstractScanner<TValue, TSpan>
-#else
-    internal abstract class AbstractScanner<TValue, TSpan>
-#endif
- where TSpan : IMerge<TSpan> {
-        /// <summary>
-        /// Lexical value optionally set by the scanner. The value
-        /// is of the %YYSTYPE type declared in the parser spec.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        // A field must be declared for this value of parametric type,
-        // since it may be instantiated by a value struct.  If it were 
-        // implemented as a property, machine generated code in derived
-        // types would not be able to select on the returned value.
-#pragma warning disable 649
-        public TValue yylval;                     // Lexical value: set by scanner
-#pragma warning restore 649
-
-        /// <summary>
-        /// Current scanner location property. The value is of the
-        /// type declared by %YYLTYPE in the parser specification.
-        /// </summary>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual TSpan yylloc {
-            get { return default( TSpan ); }       // Empty implementation allowing
-            set { /* skip */ }                   // yylloc to be ignored entirely.
-        }
-
-        /// <summary>
-        /// Main call point for LEX-like scanners.  Returns an int
-        /// corresponding to the token recognized by the scanner.
-        /// </summary>
-        /// <returns>An int corresponding to the token</returns>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public abstract int yylex();
-
-        /// <summary>
-        /// Traditional error reporting provided by LEX-like scanners
-        /// to their YACC-like clients.
-        /// </summary>
-        /// <param name="format">Message format string</param>
-        /// <param name="args">Optional array of args</param>
-        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
-        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
-        // Reason for FxCop message suppression -
-        // This is a traditional name for YACC-like functionality
-        public virtual void yyerror( string format, params object[] args ) { }
-    }
-
-    /// <summary>
-    /// Encapsulated state for the parser.
-    /// Opaque to users, visible to the tool-generated code.
-    /// </summary>
-#if EXPORT_GPPG
-    public class State
-    {
-        /// <summary>
-        /// The number of states in the automaton.
-        /// </summary>
-        public int number;
-#else
-    internal class State {
-        /// <summary>
-        /// The index of this state in the states array.
-        /// </summary>
-        internal int number;
-#endif
-        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
-        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
-        internal int defaultAction; // = 0;		     // ParseAction
-
-        /// <summary>
-        /// State transition data for this state. Pairs of elements of the 
-        /// goto array associate symbol ordinals with next state indices.
-        /// The actions array is passed to another constructor. 
-        /// </summary>
-        /// <param name="actions">The action list</param>c
-        /// <param name="goToList">Next state data</param>
-        public State( int[] actions, int[] goToList )
-            : this( actions ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-
-        /// <summary>
-        /// Action data for this state. Pairs of elements of the 
-        /// action array associate action ordinals with each of
-        /// those symbols that have actions in the current state.
-        /// </summary>
-        /// <param name="actions">The action array</param>
-        public State( int[] actions ) {
-            ParserTable = new Dictionary<int, int>();
-            for (int i = 0; i < actions.Length; i += 2)
-                ParserTable.Add( actions[i], actions[i + 1] );
-        }
-
-        /// <summary>
-        /// Set the default action for this state.
-        /// </summary>
-        /// <param name="defaultAction">Ordinal of the default action</param>
-        public State( int defaultAction ) {
-            this.defaultAction = defaultAction;
-        }
-
-        /// <summary>
-        /// Set the default action and the state transition table.
-        /// </summary>
-        /// <param name="defaultAction">The default action</param>
-        /// <param name="goToList">Transitions from this state</param>
-        public State( int defaultAction, int[] goToList )
-            : this( defaultAction ) {
-            Goto = new Dictionary<int, int>();
-            for (int i = 0; i < goToList.Length; i += 2)
-                Goto.Add( goToList[i], goToList[i + 1] );
-        }
-    }
-
-    /// <summary>
-    /// Rule representation at runtime.
-    /// </summary>
-#if EXPORT_GPPG
-    public class Rule
-#else
-    internal class Rule
-#endif
- {
-        internal int LeftHandSide; // symbol
-        internal int[] RightHandSide; // symbols
-
-        /// <summary>
-        /// Rule constructor.  This holds the ordinal of
-        /// the left hand side symbol, and the list of
-        /// right hand side symbols, in lexical order.
-        /// </summary>
-        /// <param name="left">The LHS non-terminal</param>
-        /// <param name="right">The RHS symbols, in lexical order</param>
-        public Rule( int left, int[] right ) {
-            this.LeftHandSide = left;
-            this.RightHandSide = right;
-        }
-    }
-
-    /// <summary>
-    /// Stack utility for the shift-reduce parser.
-    /// GPPG parsers have three instances:
-    /// (1) The parser state stack, T = QUT.Gppg.State,
-    /// (2) The semantic value stack, T = TValue,
-    /// (3) The location stack, T = TSpan.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-#if EXPORT_GPPG
-    public class PushdownPrefixState<T>
-#else
-    internal class PushdownPrefixState<T>
-#endif
- {
-        //  Note that we cannot use the BCL Stack<T> class
-        //  here as derived types need to index into stacks.
-        //
-        private T[] array = new T[8];
-        private int tos = 0;
-
-        /// <summary>
-        /// Indexer for values of the stack below the top.
-        /// </summary>
-        /// <param name="index">index of the element, starting from the bottom</param>
-        /// <returns>the selected element</returns>
-        public T this[int index] { get { return array[index]; } }
-
-        /// <summary>
-        /// The current depth of the stack.
-        /// </summary>
-        public int Depth { get { return tos; } }
-
-        internal void Push( T value ) {
-            if (tos >= array.Length) {
-                T[] newarray = new T[array.Length * 2];
-                System.Array.Copy( array, newarray, tos );
-                array = newarray;
-            }
-            array[tos++] = value;
-        }
-
-        internal T Pop() {
-            T rslt = array[--tos];
-            array[tos] = default( T );
-            return rslt;
-        }
-
-        internal T TopElement() { return array[tos - 1]; }
-
-        internal bool IsEmpty() { return tos == 0; }
-    }
-}
-

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -1,4 +0,0 @@
-global using System;
-global using System.Linq;
-global using System.Collections.Generic;
-global using System.Collections.Immutable;

src/DataStreamSuite/Bts.DataStream.Websocket/DataStreamDataSync.cs

@@ -0,0 +1,119 @@
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.collections;
+using bts.websocket_core;
+
+using Bts.DataStreamExpressions;
+using Bts.Websocket;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamDataSync(JsonSerializerOptions? serializerOpts = null) : JsonRpcDataSync(serializerOpts) {
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+
+   protected override void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, []);
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   protected override void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   protected override void onRequest(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += OnChange;
+      }
+
+      void OnChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var rpcParams = new {
+            dp.addedRows,
+            dp.deletedRows
+         };
+         sendNotification(id, "subscription", rpcParams);
+      }
+
+      void Dispose() =>
+         stream.onChange -= OnChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+}

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -12,11 +12,8 @@ using Bts.Websocket;
 using mbts.utils;
 
 using Spark.Core.Disposables;
-using Spark.Core.Json;
 
-using System.Buffers;
 using System.Reactive.Subjects;
-using System.Text;
 using System.Text.Json;
 using System.Text.Json.Nodes;
 
@@ -79,81 +76,6 @@ public class DataSync(JsonSerializerOptions? serializerOpts = null) :
       }
    }
 
-   static string getRpcNotification(DataPacket dp) {
-      using (var buffer = new MemoryStream())
-      using (var writer = new Utf8JsonWriter(buffer))
-      using (writer.WriteObject()) {
-         writer.WriteString("method", "subscription");
-         using (writer.WriteObject("params")) {
-            writer.Write("tableName", dp.schema.tableName);
-            writer.Write("columns", dp.schema.columnInfo);
-            var cellJsonFactories = dp.schema.columnInfo.Select(getCellJsonFactory).ToList();
-            using (writer.WriteArray("added")) {
-               foreach (IBlueRow row in (dp.addedRows?.Cast<IBlueRow>()).empty_if_null()) {
-                  using (writer.WriteObject()) {
-                     for (int i = 0; i < cellJsonFactories.Count; i++) {
-                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
-                     }
-                  }
-               }
-            }
-            using (writer.WriteArray("deleted")) {
-               foreach (IBlueRow row in (dp.deletedRows?.Cast<IBlueRow>()).empty_if_null()) {
-                  using (writer.WriteObject()) {
-                     for (int i = 0; i < cellJsonFactories.Count; i++) {
-                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
-                     }
-                  }
-               }
-            }
-         }
-         writer.WriteString("jsonrpc", "2.0");
-         writer.Flush();
-         return Encoding.UTF8.GetString(buffer.ToArray());
-      }
-   }
-
-   static Func<IBlueRow, JsonNode?> getCellJsonFactory(ColumnInfo colInfo) {
-      var t = colInfo.colType;
-      if (t == typeof(int)) {
-         return NullCheckWrapper(v => (int)v);
-      }
-      if (t == typeof(uint)) {
-         return NullCheckWrapper(v => (uint)v);
-      }
-      if (t == typeof(long)) {
-         return NullCheckWrapper(v => (long)v);
-      }
-      if (t == typeof(ulong)) {
-         return NullCheckWrapper(v => (ulong)v);
-      }
-      if (t == typeof(short)) {
-         return NullCheckWrapper(v => (short)v);
-      }
-      if (t == typeof(ushort)) {
-         return NullCheckWrapper(v => (ushort)v);
-      }
-      if (t == typeof(double)) {
-         return NullCheckWrapper(v => (double)v);
-      }
-      if (t == typeof(float)) {
-         return NullCheckWrapper(v => (float)v);
-      }
-      if (t == typeof(bool)) {
-         return NullCheckWrapper(v => (bool)v);
-      }
-      return NullCheckWrapper(v => (string)v);
-
-      Func<IBlueRow, JsonNode?> NullCheckWrapper(Func<object, JsonNode> toJson) =>
-         r => {
-            var obj = r[colInfo.colNo];
-            if (obj is null) {
-               return null;
-            }
-            return toJson(obj);
-         };
-   }
-
    void onConnect(WebsocketConnectNotification notification) {
       var id = notification.Id;
       lock (lockObj) {
@@ -181,10 +103,6 @@ public class DataSync(JsonSerializerOptions? serializerOpts = null) :
          case "subscribe":
             subscribe(id, req);
             break;
-         case "publish":
-            Log.Api.error("DataStreamWebsocket api 'publish' not implemented.");
-            // TODO
-            break;
          case "unsubscribe":
             unsubscribe(id, req);
             break;
@@ -197,10 +115,16 @@ public class DataSync(JsonSerializerOptions? serializerOpts = null) :
       }
    }
 
+   void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
+      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
+      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
+      send([new(id, rpcNotificationStr)]);
+   }
+
    void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
       var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
       var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send([new(id, rpcResStr)]);
+      send([new(id, rpcResStr )]);
    }
 
    void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
@@ -234,8 +158,11 @@ public class DataSync(JsonSerializerOptions? serializerOpts = null) :
 
       void OnChange(object? newValue) {
          var dp = ExceptionAssert.of_type<DataPacket>(newValue);
-         var notif = getRpcNotification(dp);
-         send([new(id, notif)]);
+         var rpcParams = new {
+            dp.addedRows,
+            dp.deletedRows
+         };
+         sendNotification(id, "subscription", rpcParams);
       }
 
       void Dispose() =>

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -1,22 +1,26 @@
 namespace Bts.DataStream.Websocket;
 
 using bts;
+using bts.api.core;
 
 using Bts.Websocket;
 
 using Google.Protobuf;
 
+using Spark.Core.Disposables;
+
+
 using System.Text.Json;
 
 public static class Session {
    static int sessionIdCounter;
 
    public static async Task Start(
-      WebsocketServerConfig config, JsonSerializerOptions? opts) {
+      ServiceContext context, WebsocketServerConfig config, JsonSerializerOptions? opts) {
       opts ??= JsonSerializerOptions.Default;
       var sessionId = Interlocked.Increment(ref sessionIdCounter);
       var gateway = new WebsocketGatewayObservable(config);
-      var sync = new DataSync(opts);
+      var sync = new DataSync(context, opts);
       using (gateway.Subscribe(sync))
       using (sync.Subscribe(gateway)) {
          Log.Api.info($"Starting DataStream Websocket Session {sessionId}");

src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs

@@ -1,41 +1,41 @@
-#nullable enable
-using BCG;
-
-using Spark.Core.Json;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Bts.DataStreamCore;
-
-public class DataStreamUpdate {
-   [JsonPropertyName("tableName")]
-   public string? TableName { get; set; }
-
-   [JsonPropertyName("columns")]
-   public List<ColumnInfo>? Columns { get; set; }
-
-   [JsonPropertyName("addedRows")]
-   public List<IBlueRow>? AddedRows { get; set; }
-
-   [JsonPropertyName("deletedRows")]
-   public List<IBlueRow>? DeletedRows { get; set; }
-
-   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
-      var schema = new BlueTableSchema([.. Columns!], TableName);
-      var addedRowsTable = new BlueTable(schema);
-      var deletedRowsTable = new BlueTable(schema);
-      if (applyChanges) {
-         state.deleteRows(deletedRowsTable);
-         state.addRows(addedRowsTable);
-      }
-      return new(state, addedRowsTable, deletedRowsTable);
-   }
-
-   public static string ToJson() {
-      return "";
-   }
-}
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class ViewStreamUpdate {
+   [JsonPropertyName("tableName")]
+   public string? TableName { get; set; }
+
+   [JsonPropertyName("columns")]
+   public List<ColumnInfo>? Columns { get; set; }
+
+   [JsonPropertyName("addedRows")]
+   public List<Dictionary<string, object?>>? AddedRows { get; set; }
+
+   [JsonPropertyName("deletedRows")]
+   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
+
+   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
+      var schema = new BlueTableSchema([.. Columns!], TableName);
+      var addedRowsTable = new BlueTable(schema);
+      var deletedRowsTable = new BlueTable(schema);
+      if (applyChanges) {
+         state.deleteRows(deletedRowsTable);
+         state.addRows(addedRowsTable);
+      }
+      return new(state, addedRowsTable, deletedRowsTable);
+   }
+
+   public static string ToJson() {
+      return "";
+   }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs

@@ -39,7 +39,7 @@ static async Task run(
    Console.WriteLine("Starting DataStreamWebsocketServer...");
    var serviceCtx = ServiceContext.create(sparkdConfigSource);
    await serviceCtx.login(user, password);
-   await Session.Start(wssConfig, opts);
+   await Session.Start(serviceCtx, wssConfig, opts);
    // if (sparkdConfigSource is not null && sparkdConfigSource.Length > 9999) {
    // } else {
    //    await ChatGptSession.Start();

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream.Console/Program.cs

@@ -1,6 +1,4 @@
 // See https://aka.ms/new-console-template for more information
-using BCG;
-
 using DataStream.Console;
 
 using ExpressionTests;
@@ -9,43 +7,26 @@ using Spark.Core.Collections;
 
 using System.Reactive.Disposables;
 
-var schema = new BlueTableSchema([
-      new("id", 0, typeof(int)),
-      new("name", 1, typeof(string)),
-      new("value", 2, typeof(int)),
-   ],
-   "MyTable"
-);
-var rows = new BlueTable(schema);
-var mbr = new MutableBlueRow(schema);
-mbr.setValue("id", 1);
-mbr.setValue("name", "Jerry");
-mbr.setValue("value", 13);
-rows.addRow(mbr);
-var serializer = new UniversalSerializer(schema);
-serializer.serialize(StreamOpCode.create, new[] { rows }, out var bytes);
-var bytesDesc = BitConverter.ToString(bytes).Replace("-", "\\x");
-Console.WriteLine(bytesDesc);
 
-// try {
-//    foreach (var arg in args) {
-//       if (File.Exists(arg)) {
-//          using var fs = File.OpenRead(arg) ??
-//             throw new Exception("File.OpenRead returned null. LIES!");
-//          var expressionLines = new StringStreamEnumerable(new(fs));
-//          var test = ExpressionTestParser.Parse(expressionLines);
-//          TempExpressionTests.Run(test);
-//       }
-//    }
-//    if (args.Length == 0) {
-//       TempExpressionTests.Run("rv = 3 + 4", new Dictionary<string, string> { { "rv", "7" } });
-//    }
-// } catch (Exception ex) {
-//    using (ConsoleHelper.PushBg(ConsoleColor.Red)) {
-//       Console.WriteLine($"Failed! {ex}");
-//    }
-//    return;
-// }
+try {
+   foreach (var arg in args) {
+      if (File.Exists(arg)) {
+         using var fs = File.OpenRead(arg) ??
+            throw new Exception("File.OpenRead returned null. LIES!");
+         var expressionLines = new StringStreamEnumerable(new(fs));
+         var test = ExpressionTestParser.Parse(expressionLines);
+         TempExpressionTests.Run(test);
+      }
+   }
+   if (args.Length == 0) {
+      TempExpressionTests.Run("rv = 3 + 4", new Dictionary<string, string> { { "rv", "7" } });
+   }
+} catch (Exception ex) {
+   using (ConsoleHelper.PushBg(ConsoleColor.Red)) {
+      Console.WriteLine($"Failed! {ex}");
+   }
+   return;
+}
 
-// using var _ = ConsoleHelper.PushFg(ConsoleColor.Green);
-// Console.WriteLine($"Passed!");
+using var _ = ConsoleHelper.PushFg(ConsoleColor.Green);
+Console.WriteLine($"Passed!");

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -9,8 +9,6 @@
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -1,79 +0,0 @@
-using BCG.Utilities.General;
-
-using System;
-
-namespace BCG {
-    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
-        class State {
-            public int n_sends = 0;
-            public int n_recvs = 0;
-        }
-        private readonly string _ip;
-        private readonly int _port;
-        private readonly ViewStreamDatabase _vsd;
-        State _state = new State();
-        private readonly int _msg_size;
-
-        private IBlueTableSchema _schema;
-        private TimeSpan _send_period;
-        private ITCPConnection _connection = null;
-        private byte[] _buff;
-
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
-        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
-            _ip = ip;
-            _port = port;
-            _msg_size = msg_size;
-            _vsd = vsd;
-            _schema = BlueDataUtility.form_schema("TCPClient",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("n_sends", TypeCode.UInt32),
-            Tuple.Create("n_recvs", TypeCode.UInt32));
-            _send_period = TimeSpan.FromSeconds(send_period_seconds);
-            _buff = new byte[msg_size];
-            for (int i = 0; i < _buff.Length; i++) {
-                _buff[i] = (byte)i;
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_address = new BlutoSubAddress(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            if (_connection != null) {
-                _connection.send(_buff);
-                _state.n_sends++;
-            }
-            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
-            BlueTable rows = new BlueTable(r.schema);
-            rows.addRow(r);
-            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        public void onConnection(ITCPConnection connection) {
-            _connection = connection;
-            _connection.register(this);
-        }
-        public bool isDisposed { get; }
-
-        public void onData(RemoteMessage msg) {
-            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
-            _state.n_recvs++;
-        }
-
-        public void Dispose() {
-
-        }
-
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,6 +1,11 @@
-namespace BCG;
-
-public class AddColumnsStream : MapStream {
-   public AddColumnsStream(ViewStream vs, string columnDef)
-      : base(vs, new ColumnAdder(columnDef)) { }
-}
+// using System;
+// using System.Data;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Utilities.General;
+// namespace BCG {
+
+//     public class AddColumnsStream : MapStream {
+//         public AddColumnsStream(ViewStream vs, string columnDef) : base(vs, new ColumnAdder(columnDef)) { }
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,52 +1,63 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-
-using Bts.ColumnExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Provides methods to add columns to the schema
-/// </summary>
-/// <param name="columnsExpression">
-/// String represting the columns to add to the output table
-/// </param>
-public class ColumnAdder : IStreamMapper {
-   // Store tuples of (column, index) to preserve order
-   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
-   protected IBlueTableSchema? mySchema;
-
-   public ColumnAdder(string columnsExpression) =>
-      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
-         throw new Exception("Invalid columnn expression: " + columnsExpression))
-         .Select((c, i) => (c, i))
-         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
-
-   [MemberNotNull(nameof(mySchema))]
-   public virtual IBlueRow map(IBlueRow row) {
-      // Only rebuild schema if necessary (i.e., if it's null)
-      mySchema ??= buildSchema(row.schema, myNewColumns);
-      return new AddColumnsRow(mySchema, row);
-   }
-
-   [MemberNotNull(nameof(mySchema))]
-   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
-      // Always rebuild schema (even if it's already non-null)
-      mySchema = buildSchema(inputSchema, myNewColumns);
-
-   [MemberNotNull(nameof(mySchema))]
-   static BlueTableSchema buildSchema(
-      IBlueTableSchema schema,
-      IDictionary<string, (ColumnExpression column, int index)> newColumns
-   ) {
-      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
-      schema.columnInfo.CopyTo(cis, 0);
-      foreach (var (col, index) in newColumns.Values) {
-         var schemaIdx = schema.columnInfo.Length + index;
-         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
-      }
-      return new BlueTableSchema(cis, schema.tableName);
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.AddColumnsExpression;
+
+// namespace BCG {
+//     /// <summary>
+//     /// Provides methods to add columns to the schema
+//     /// </summary>
+//     public class ColumnAdder : IStreamMapper {
+//         protected IDictionary myNewColumns;
+//         protected IBlueTableSchema mySchema;
+
+//         static syntax myParser =
+//            new syntax();
+
+//         /// <summary>
+//         /// string represting the columns to be added to the output table. 
+//         /// </summary>
+//         /// <param name="colAddExpression"></param>
+//         public ColumnAdder(string columnDef) {
+//             AddColumnsExpression ace;
+
+//             if (DataStreamLoadFileConfig.instance.useStaticParser) {
+//                 lock (myParser) {
+//                     ace =
+//                     (AddColumnsExpression)myParser.Parse(columnDef);
+//                 }
+//             } else {
+//                 syntax parser = new syntax();
+//                 ace =
+//                 (AddColumnsExpression)parser.Parse(columnDef);
+//             }
+
+//             if (ace == null)
+//                 throw new Exception("Invalid columnn expression: " + columnDef);
+//             myNewColumns = ace.columns;
+//         }
+
+//         public virtual IBlueRow map(IBlueRow row) {
+//             if (mySchema == null)
+//                 myConstructSchema(row.schema);
+//             return new AddColumnsRow(mySchema, row);
+//         }
+
+//         public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+//             myConstructSchema(inputSchema);
+//             return mySchema;
+//         }
+
+//         private void myConstructSchema(IBlueTableSchema schema) {
+//             ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
+//             int ciIndex = 0;
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 cis[ciIndex++] = ci;
+//             foreach (DictionaryEntry de in myNewColumns)
+//                 cis[ciIndex] = new ColumnInfo((string)de.Key, ciIndex++,
+//                    ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
+//             mySchema = new BlueTableSchema(cis, schema.tableName);
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -1,107 +1,115 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace BCG;
-
-public class SortRow : IComparable, IBlueRow {
-   readonly IBlueRow row;
-   readonly IEnumerable<ColumnSortExpression>? columns;
-   readonly IComparer? comparer;
-
-   public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) {
-      this.row = row;
-      this.columns = columns;
-   }
-
-   public SortRow(IBlueRow row, IComparer comparer) {
-      this.row = row;
-      this.comparer = comparer;
-   }
-
-   public IBlueRow internal_row { get { return row; } }
-
-   #region IComparable method
-   public int CompareTo(object? obj) {
-      if (obj is not IBlueRow otherRow) {
-         throw new ArgumentException(
-            $"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})",
-            nameof(obj));
-      }
-      if (comparer is not null) {
-         return comparer.Compare(this, otherRow);
-      }
-      return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0);
-      int Compare(ColumnSortExpression col) {
-         var otherVal = otherRow[col.Name];
-         int rv = myIsNull(this[col.Name]) ?
-            (myIsNull(otherVal) ? 0 : -1) :
-            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
-         rv *= col.IsAscending ? 1 : -1;
-         return rv;
-      }
-   }
-   #endregion
-
-   #region IBlueRow members
-   public IBlueTableSchema schema { get { return row.schema; } }
-   #endregion
-
-   #region ICollection members
-   public int Count { get { return row.Count; } }
-
-   public bool IsSynchronized { get { return false; } }
-
-   public object SyncRoot { get { return this; } }
-
-   public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
-   #endregion
-
-   #region IDictionary members
-   public bool IsFixedSize { get { return true; } }
-
-   public bool IsReadOnly { get { return true; } }
-
-   /// <summary>
-   /// The key can either be the string corrsponding to the column name or
-   /// the order of the column
-   /// </summary>
-   public object this[object key] {
-      get {
-         if (key is string)
-            return this[(string)key];
-         return this[(int)key];
-      }
-      set { throw new NotSupportedException("The IBlueRow is readonly"); }
-   }
-
-   public object this[int index] => row[index];
-
-   public object this[string colName] => row[colName];
-
-   public ICollection Keys => row.Keys;
-
-   public ICollection Values => row.Values;
-
-   public bool Contains(object key) => row.Contains(key);
-
-   void IDictionary.Add(object key, object? value) =>
-      throw new NotSupportedException("The IBlueRow is readonly");
-
-   void IDictionary.Clear() =>
-      throw new NotSupportedException("The IBlueRow is readonly");
-
-   IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); }
-
-   IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); }
-
-   public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-   #endregion
-
-   bool myIsNull(object? o) =>
-      o is null || o == null || o is DBNull;
-
-}
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
+// using BCG.Evaluator.SortExpression;
+
+
+// namespace BCG
+// {
+//    public class SortRow : IComparable, IBlueRow
+//    {
+//       private IBlueRow myRow;
+//       private IEnumerable myColumns;
+//       private IComparer myComparer;
+
+//       public SortRow (IBlueRow row, SortExpression sortExpression)
+//       {
+//          myRow = row;
+//          myColumns = sortExpression.columns;
+//       }
+
+//       public SortRow (IBlueRow row, IEnumerable columns)
+//       {
+//          myRow = row;
+//          myColumns = columns;
+//       }
+
+//       public SortRow (IBlueRow row, IComparer comparer)
+//       {
+//          myRow = row;
+//          myComparer = comparer;
+//       }
+
+//       public IBlueRow internal_row { get { return myRow; } }
+
+//       #region IComparable method
+//       public int CompareTo (object obj)
+//       {  
+//          IBlueRow otherRow = (IBlueRow)obj;
+//          if (myComparer == null)
+//          {
+//             foreach (SortExpression.ColumnCriteria cc in myColumns)
+//             {
+//                int retVal = myIsNull (this[cc.columnName]) ?
+//                   myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
+//                   myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
+//                retVal *= cc.ascending ? 1 : -1;
+//                if (retVal != 0)
+//                   return retVal;
+//             }
+//             return 0;
+//          }
+//          else
+//             return myComparer.Compare (this, otherRow);
+//       }
+//       #endregion
+
+//       #region IBlueRow members
+//       public IBlueTableSchema schema{get{return myRow.schema;}}
+//       #endregion
+
+//       #region ICollection members
+//       public int Count {get{return myRow.Count;}}
+
+//       public bool IsSynchronized {get{return false;}}
+
+//       public object SyncRoot {get{return this;}}
+
+//       public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
+//       #endregion
+
+//       #region IDictionary members
+//       public bool IsFixedSize {get{return true;}}
+
+//       public bool IsReadOnly {get{return true;}}
+
+//       /// <summary>
+//       /// The key can either be the string corrsponding to the column name or
+//       /// the order of the column
+//       /// </summary>
+//       public object this [object key]
+//       {
+//          get
+//          {
+//             if (key is string)
+//                return this[(string)key];
+//             return this[(int)key];
+//          }
+//          set {throw new NotSupportedException ("The IBlueRow is readonly");}
+//       }
+
+//       public object this [int index]   {get{return myRow[index];}}
+
+//       public object this [string colName] {get{return myRow[colName];}}
+
+//       public ICollection Keys{get{return myRow.Keys;}}
+
+//       public ICollection Values{get{return myRow.Values;}}
+
+//       public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+//       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+
+//       public bool Contains (object key){return myRow.Contains (key);}
+
+//       IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
+
+//       IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
+
+//       public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+//       #endregion
+
+//       private bool myIsNull (object o) {return o == null || o is DBNull;}
+
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,99 +1,129 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-namespace BCG;
-
-/// <summary>
-/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-/// to merge the additions to the accumulates table
-/// </summary>
-public class SortStream : ViewStream {
-   ViewStream stream;
-   IList<ColumnSortExpression> sortCols;
-   IComparer? comparer;
-   DataPacket? lastFiredPacket;
-   DataPacket? lastReceivedPacket;
-
-   public SortStream(ViewStream stream, string sortExpression) {
-      this.stream = stream;
-      SortExpression = sortExpression;
-   }
-
-   public SortStream(ViewStream vs, string colName, bool isAsc) {
-      stream = vs;
-      SortExpression = isAsc ? "+" + colName : "-" + colName;
-   }
-
-   public SortStream(ViewStream stream, IComparer comparer) {
-      this.stream = stream;
-      this.comparer = comparer;
-      sortCols = Array.Empty<ColumnSortExpression>();
-   }
-
-   [MemberNotNull(nameof(sortCols))]
-   public string SortExpression {
-      set {
-         comparer = null;
-         lastFiredPacket = null;
-         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public IComparer Comparer {
-      set {
-         comparer = value;
-         lastFiredPacket = null;
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public void myChangeHandlerTest(object o) =>
-      myChangeHandler(o);
-
-   [MemberNotNull(nameof(lastFiredPacket))]
-   internal void myChangeHandler(object o) {
-      lastReceivedPacket = (DataPacket)o;
-      // First callback, so we need to sort the whole table, this is the
-      // only time we will do that for all the other sorting operations would
-      // be sorting the increments and inserting them into the whole list
-      if (lastFiredPacket is null) {
-         lastFiredPacket = new() {
-            newRows = comparer is null ?
-               new SortedTable(lastReceivedPacket.newRows, sortCols) :
-               new SortedTable(lastReceivedPacket.newRows, comparer)
-         };
-         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-         myFireEvent(lastFiredPacket.Clone());
-         return;
-      }
-      // Deletes
-      lastFiredPacket.deletedRows =
-         comparer == null ?
-         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-         new SortedTable(lastReceivedPacket.deletedRows, comparer);
-      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-      // Adds
-      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-      myFireEvent(lastFiredPacket.Clone());
-   }
-
-   protected override void myOnFirstSubscriber() =>
-      stream.onChange += myChangeHandler;
-
-   protected override void myOnLastSubscriber() {
-      lastFiredPacket = null;
-      lastReceivedPacket = null;
-      stream.onChange -= myChangeHandler;
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Evaluator.SortExpression;
+
+
+// namespace BCG
+// {
+// 	/// <summary>
+// 	/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+// 	/// to merge the additions to the accumulates table
+// 	/// </summary>
+// 	public class SortStream : ViewStream
+// 	{
+//       private ViewStream myStream;
+//       private IEnumerable mySortCols;
+//       private IComparer myComparer;
+//       private DataPacket myLastFiredPacket;
+//       private DataPacket myLastRecievedPacket;
+
+//       public SortStream (ViewStream stream, string sortExpression)
+//       {
+//          myStream = stream;
+//          mySetSortExpression (sortExpression);
+//       }
+
+//       public SortStream (ViewStream vs, string colName, bool isAsc)
+//       {
+//          myStream = vs;
+//          string sortExpression = isAsc ? "+" + colName : "-" + colName;
+//          mySetSortExpression (sortExpression);
+//       }
+
+//       public SortStream (ViewStream vs, IComparer comparer)
+//       {
+//          myStream = vs;
+//          myComparer = comparer;
+//       }
+
+//       public string sortExpression
+//       {
+//          set
+//          {
+//             myComparer = null;
+//             myLastFiredPacket = null;
+//             mySetSortExpression (value);   
+//             myChangeHandler (myLastRecievedPacket);
+//          }
+//       }
+
+//       public IComparer comparer
+//       {
+//          set
+//          {
+//             myComparer = value;
+//             myLastFiredPacket = null;
+//             myChangeHandler(myLastRecievedPacket);
+//          }
+//       }
+
+//       private void mySetSortExpression (string sortExpression)
+//       {
+//          syntax parser = 
+//             new syntax();
+         
+//          mySortCols = ((SortExpression)
+//             parser.Parse (sortExpression)).columns;
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); 
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          myInit ();
+//          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); 
+//       }
+
+//       internal void myChangeHandler (object o)
+//       {
+//          myLastRecievedPacket = (DataPacket)o;
+//          //first callback, so we need to sort the whole table, this is the
+//          //only time we will do that for all the other sorting operations would
+//          //be sorting the increments and inserting them into the whole list
+//          if (myLastFiredPacket == null)
+//          {
+//             myLastFiredPacket = new DataPacket ();
+//             myLastFiredPacket.newRows = 
+//                myComparer == null ?
+//                new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
+//                new SortedTable (myLastRecievedPacket.newRows, myComparer);
+//             myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+//             myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+//             myFireEvent (myLastFiredPacket.Clone());
+//             return;
+//          }
+//          myHandleDeletes (myLastRecievedPacket);
+//          myHandleAdds (myLastRecievedPacket);
+//          myFireEvent (myLastFiredPacket.Clone());
+//       }
+
+//       public void myChangeHandlerTest(object o)
+//       {
+//           myChangeHandler(o);
+//       }
+
+//       private void myHandleDeletes (DataPacket dp)
+//       {
+//          myLastFiredPacket.deletedRows = 
+//             myComparer == null ?
+//             new SortedTable (dp.deletedRows, mySortCols) :
+//             new SortedTable (dp.deletedRows, myComparer);
+//          ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
+//       }
+
+//       private void myHandleAdds (DataPacket dp)
+//       {
+//          myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
+//          ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
+//       }
+
+//       private void myInit ()
+//       {
+//          myLastFiredPacket = null;
+//          myLastRecievedPacket = null;
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs

@@ -1,108 +1,136 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-using Bts.ColumnSortExpressions;
-
-namespace BCG;
-
-/// <summary>
-/// Represents an IBlueRowCollection with the rows sorted according to the
-/// </summary>
-public class SortedTable : IBlueRowCollection {
-   readonly List<SortRow> rows = new(0);
-   readonly Dictionary<int, IBlueRow> rowIndex = new(0);
-   readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>();
-   readonly IComparer? comparer;
-
-   /// <summary>
-   /// Sorts the rows using sortExpression as the sorting criteria
-   /// </summary>
-   /// <param name="rows">Rows to be inserted into the table</param>
-   /// <param name="sortCols">Sorting criteria</param>
-   public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) {
-      this.sortCols = sortCols;
-      init(rows);
-   }
-
-   public SortedTable(IBlueRowCollection rows, IComparer comparer) {
-      this.comparer = comparer;
-      init(rows);
-   }
-
-   public int Count => rows.Count;
-   public bool IsSynchronized => false;
-   public IBlueTableSchema schema { get; private set; }
-   public object SyncRoot => this;
-   public IBlueRow this[int id] => rowIndex[id]
-      ?? throw new BlueDataException(
-         $"ID '{id}' does not exist in table '{schema?.tableName}'");
-
-   public void addRows(IBlueRowCollection rows) {
-      // Requires merger of the addedRows to the already existing ones
-      if (rows.Count == 0) {
-         // At times there are calls with 0 additions, in that case
-         // we of course need no processing
-         return;
-      }
-      SortedTable addedRows =
-         comparer == null ? new SortedTable(rows, sortCols) :
-            new SortedTable(rows, comparer);
-      int addedRowsIndex = 0;
-      SortRow newRow = addedRows.getRowByIndex(addedRowsIndex);
-      for (int i = 0; i < this.rows.Count; i++) {
-         if (newRow.CompareTo(this.rows[i]) <= 0) {
-            this.rows.Insert(i, newRow);
-            rowIndex[(int)newRow["id"]] = newRow;
-            if (++addedRowsIndex == addedRows.Count) {
-               return;
-            }
-            newRow = addedRows.getRowByIndex(addedRowsIndex);
-         }
-      }
-      if (addedRowsIndex < addedRows.Count) {
-         for (int i = addedRowsIndex; i < addedRows.Count; i++) {
-            newRow = addedRows.getRowByIndex(i);
-            this.rows.Add(newRow);
-            rowIndex[(int)newRow["id"]] = newRow;
-         }
-      }
-   }
-
-   public void deleteRows(IBlueRowCollection deletedRows) {
-      foreach (IBlueRow deletedRow in deletedRows) {
-         var row = rowIndex[(int)deletedRow["id"]!] as SortRow
-            ?? throw new InvalidOperationException("Index did not return IBlueRow");
-         rows.Remove(row);
-         rowIndex.Remove((int)row["id"]!);
-      }
-   }
-
-   public SortRow getRowByIndex(int index) =>
-      rows[index]!;
-
-   public bool containsID(int id) =>
-      rowIndex[id] is not null;
-
-   public IEnumerator GetEnumerator() =>
-      rows.GetEnumerator();
-
-   public void CopyTo(Array array, int index) {
-      for (int i = 0; i < Count; i++) {
-         array.SetValue(rows[i], index + i);
-      }
-   }
-
-   [MemberNotNull(nameof(schema))]
-   void init(IBlueRowCollection rows) {
-      schema = rows.schema;
-      foreach (IBlueRow row in rows) {
-         SortRow newRow = comparer is null ?
-            new(row, sortCols) : new(row, comparer);
-         this.rows.Add(newRow);
-         rowIndex[(int)row["id"]!] = newRow;
-      }
-      this.rows.Sort();
-   }
-}
+// using System;
+// using System.Collections;
+// using BCG.Evaluator.SortExpression;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Represents an IBlueRowCollection with the rows sorted according to the
+//    /// 
+//    /// </summary>
+//    public class SortedTable : IBlueRowCollection 
+//    {
+//       private ArrayList myRows = new ArrayList ();
+//       private Hashtable myIndex = new Hashtable ();
+//       private IBlueTableSchema mySchema;
+//       private IEnumerable mySortCols;
+//       private IComparer myComparer;
+
+//       #region SortedTable method(s)
+//       public void addRows (IBlueRowCollection rows)
+//       {
+//          //requires merger of the addedRows to the already existing ones
+//          if (rows.Count == 0)
+//             //at times there are calls with 0 additions, in that case
+//             //we ofcourse need no processing
+//             return;
+//          SortedTable addedRows = 
+//             myComparer == null ? new SortedTable (rows, mySortCols) :
+//                new SortedTable (rows, myComparer);
+//          int addedRowsIndex = 0;
+//          SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
+//          for (int i = 0; i < myRows.Count; i ++)
+//             if (newRow.CompareTo(myRows[i]) <= 0)
+//             {
+//                myRows.Insert (i, newRow);
+//                myIndex[newRow["id"]] = newRow;
+//                if (++addedRowsIndex == addedRows.Count)
+//                   return;
+//                newRow = addedRows.getRowByIndex (addedRowsIndex);
+//             }
+//          if (addedRowsIndex < addedRows.Count)
+//             for (int i = addedRowsIndex; i < addedRows.Count; i++)
+//             {
+//                newRow = addedRows.getRowByIndex (i);
+//                myRows.Add (newRow);
+//                myIndex[newRow["id"]] = newRow;
+//             }
+//       }
+
+//       public void deleteRows (IBlueRowCollection deletedRows)
+//       {
+//          foreach (IBlueRow deletedRow in deletedRows)
+//          {
+//             IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
+//             myRows.Remove (row);
+//             myIndex.Remove (row["id"]);
+//          }
+//       }
+
+//       public SortRow getRowByIndex (int index)  {return (SortRow)myRows[index];}
+//       #endregion
+
+//       #region Constructor(s)
+//       /// <summary>
+//       /// Sorts the rows using sortExpression as the sorting criteria
+//       /// </summary>
+//       /// <param name="rows">
+//       /// Rows to be inserted into the table
+//       /// </param>
+//       /// <param name="sortExpression">
+//       /// Sorting criteria
+//       /// </param>
+//       public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
+//       {
+//          mySortCols = sortExpression.columns;
+//          myConstructorHelper (rows);
+//       }
+
+//       public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
+//       {
+//          mySortCols = sortCols;
+//          myConstructorHelper (rows);
+//       }
+
+//       public SortedTable (IBlueRowCollection rows, IComparer comparer)
+//       {
+//          myComparer = comparer;
+//          myConstructorHelper (rows);
+//       }
+
+//       private void myConstructorHelper (IBlueRowCollection rows)
+//       {
+//          mySchema = rows.schema;
+//          foreach (IBlueRow row in rows)
+//          {
+//             SortRow newRow = 
+//                myComparer == null ?
+//                new SortRow (row, mySortCols) : new SortRow (row, myComparer);
+//             myRows.Add (newRow);
+//             myIndex[row["id"]] = newRow;
+//          }
+//          myRows.Sort ();
+//       }
+
+//       #endregion
+
+//       #region IBlueRowCollections members
+//       public IBlueRow this[int id] 
+//       {
+//          get
+//          {
+//             if (myIndex[id] == null)
+//                throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
+//             return (IBlueRow)myIndex[id];
+//          }
+//       }
+//       public IBlueTableSchema schema {get{return mySchema;}}
+
+//       public bool containsID (int id)  {return myIndex[id] != null;}
+
+//       #endregion
+
+//       #region ICollection members
+//       public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
+
+//       public bool IsSynchronized {get{return false;}}
+
+//       public object SyncRoot {get{return this;}}
+
+//       public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
+
+//       public int Count{get{return myRows.Count;}}
+//          #endregion
+//    }
+
+// }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -48,12 +48,6 @@ namespace BCG {
             rv.Add(r.get<T>(col));
          return rv;
       }
-      public static IEnumerable<T> get_colvalues<T>(IBlueRowCollection rows, string col) {
-         List<T> rv = new List<T>();
-         foreach (IBlueRow r in rows)
-            rv.Add(r.get<T>(col));
-         return rv;
-      }
 
       public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
          Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -247,68 +241,6 @@ namespace BCG {
          return new BlueTableSchema(table_name, cis.ToArray());
       }
 
-      public static TypeCode convert_column_type(IEnumerable<string> col_vals) {
-         // bool
-         bool success = true;
-         foreach (var val in col_vals) {
-            bool parsed_val;
-            success = success && Boolean.TryParse(val, out parsed_val);
-            if (!success)
-               break;
-         }
-         if (success)
-            return TypeCode.Boolean;
-         // int
-         success = true;
-         foreach (var val in col_vals) {
-            int parsed_val;
-            success = success && Int32.TryParse(val, out parsed_val);
-            if (!success)
-               break;
-         }
-         if (success)
-            return TypeCode.Int32;
-         // try double
-         success = true;
-         foreach (var val in col_vals) {
-            double parsed_val;
-            success = success && Double.TryParse(val, out parsed_val);
-            if (!success)
-               break;
-         }
-         if (success)
-            return TypeCode.Double;
-         // try DateTime
-         success = true;
-         foreach (var val in col_vals) {
-            DateTime parsed_val;
-            success = success && DateTime.TryParse(val, out parsed_val);
-            if (!success)
-               break;
-         }
-         if (success)
-            return TypeCode.DateTime;
-         bts.Log.assert(false);
-         return TypeCode.Empty;
-      }
-
-      public static IBlueTableSchema to_typed_schema(IBlueRowCollection rows) {
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         foreach (var col in rows.schema.columnInfo) {
-            var text_vals = BlueDataUtility.get_colvalues<string>(rows, col.colName);
-            TypeCode type_code = convert_column_type(text_vals);
-            cis.Add(new ColumnInfo(col.colName, cis.Count, _typecode_to_type[type_code]));
-         }
-         return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
-      }
-
-      // public static IBlueRow convert_string_row(IBlueRow r, IBlueTableSchema out_schema) {
-      //    foreach (var ci in out_schema.columnInfo){
-
-      //    }
-      // }
-
-
       /// <summary>
       /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
       /// be alphabetic or not

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -74,13 +74,13 @@ namespace BCG
       //if it does, we are in serious trouble
       private short myLookupTableGroup;
 
-      [JsonPropertyName("index")]
+      [JsonPropertyName("colNo")]
       public int colNo{get{return myColNo;}}
 
-      [JsonPropertyName("name")]
+      [JsonPropertyName("colName")]
       public string colName {get{return myColName;}}
 
-      [JsonPropertyName("type")]
+      [JsonPropertyName("colType")]
       public Type colType {get{return myColType;}}
 
 

src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs

@@ -1,63 +1,61 @@
-// #nullable enable
-// using BCG;
+#nullable enable
+using BCG;
 
-// using Spark.Core.Json;
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
-// using System;
-// using System.Text.Json;
-// using System.Text.Json.Nodes;
-// using System.Text.Json.Serialization;
+namespace Bts.DataStreamCore;
 
-// namespace Bts.DataStreamCore;
+class BlueRowJsonConverter : JsonConverter<IBlueRow> {
+   IBlueTableSchema? schema = null;
 
-// class BlueRowJsonConverter : JsonConverter<IBlueRow> {
-//    public override IBlueRow Read(
-//       ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-//       var row = JsonSerializer.Deserialize<JsonObject>(ref reader, options)!;
-//       var schemaj = row["schema"]!.GetValue<JsonObject>() ??
-//          throw new InvalidOperationException("Failed to read schema");
-//       // var columns = schemaj["columns"]
-//       var mbr = new MutableBlueRow(schema);
-//       int colNum = 0;
-//       switch (reader.TokenType) {
-//          case JsonTokenType.StartArray:
-//             while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
-//                mbr.setValue(colNum++, readCellValue(ref reader));
-//                colNum++;
-//             }
-//             break;
-//          case JsonTokenType.StartObject:
-//             while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
-//                if (reader.TokenType != JsonTokenType.PropertyName) {
-//                   throw new InvalidOperationException(
-//                      "Expected 'PropertyName' immediately after 'StartObject' token in row json");
-//                }
-//                var colName = reader.GetString();
-//                if (!reader.Read()) {
-//                   throw new InvalidOperationException(
-//                      "Failed to read token immediately after 'PropertyName' token in row json");
-//                }
-//                mbr.setValue(colName, readCellValue(ref reader));
-//             }
-//             break;
-//       }
-//       if (mbr.Count != schema.columnInfo.Length) {
-//          throw new InvalidOperationException(
-//             $"Row json array length '{mbr.Count}' does not match the schema's " +
-//             $"column length '{schema.columnInfo.Length}'.");
-//       }
-//       return mbr;
+   public override IBlueRow Read(
+      ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (schema is null) {
+         throw new InvalidOperationException("Cannot read row from json because schema is null");
+      }
+      var mbr = new MutableBlueRow(schema);
+      int colNum = 0;
+      switch (reader.TokenType) {
+         case JsonTokenType.StartArray:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
+               mbr.setValue(colNum++, readCellValue(ref reader));
+               colNum++;
+            }
+            break;
+         case JsonTokenType.StartObject:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
+               if (reader.TokenType != JsonTokenType.PropertyName) {
+                  throw new InvalidOperationException(
+                     "Expected 'PropertyName' immediately after 'StartObject' token in row json");
+               }
+               var colName = reader.GetString();
+               if (!reader.Read()) {
+                  throw new InvalidOperationException(
+                     "Failed to read token immediately after 'PropertyName' token in row json");
+               }
+               mbr.setValue(colName, readCellValue(ref reader));
+            }
+            break;
+      }
+      if (mbr.Count != schema.columnInfo.Length) {
+         throw new InvalidOperationException(
+            $"Row json array length '{mbr.Count}' does not match the schema's " +
+            $"column length '{schema.columnInfo.Length}'.");
+      }
+      return mbr;
 
-//       static object? readCellValue(ref Utf8JsonReader reader) =>
-//          reader.TokenType switch {
-//             JsonTokenType.String => reader.GetString(),
-//             JsonTokenType.Number => reader.GetDouble(),
-//             JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
-//             _ => throw new InvalidOperationException("Failed to read cell value"),
-//          };
-//    }
+      static object? readCellValue(ref Utf8JsonReader reader) =>
+         reader.TokenType switch {
+            JsonTokenType.String => reader.GetString(),
+            JsonTokenType.Number => reader.GetDouble(),
+            JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
+            _ => throw new InvalidOperationException("Failed to read cell value"),
+         };
+   }
 
-//    public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
-//       throw new NotImplementedException();
-//    }
-// }
+   public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+}

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -1,186 +1,154 @@
-// #nullable enable
-// using BCG;
+#nullable enable
+using BCG;
 
-// using Spark.Core.Json;
+using Spark.Core.Json;
 
-// using System;
-// using System.Collections.Generic;
-// using System.Text.Json;
-// using System.Text.Json.Nodes;
-// using System.Text.Json.Serialization;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
-// namespace Bts.DataStreamCore;
+namespace Bts.DataStreamCore;
 
-// public class DataStreamUpdateJsonConverter : JsonConverter<DataStreamUpdate> {
-//    /// <summary>
-//    /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
-//    /// default, only incremental changes are included ('added' and 'deleted').
-//    /// </summary>
-//    public bool IncludeCurrent { get; set; }
+class DataPacketJsonConverter : JsonConverter<DataPacket> {
+   /// <summary>
+   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
+   /// default, only incremental changes are included ('added' and 'deleted').
+   /// </summary>
+   public bool IncludeCurrent { get; set; }
 
-//    /// <summary>
-//    /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
-//    /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
-//    /// </summary>
-//    public bool UseObjectRows { get; set; }
+   /// <summary>
+   /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
+   /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
+   /// </summary>
+   public bool UseObjectRows { get; set; }
 
-//    public override DataStreamUpdate? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-//       var dpj = JsonSerializer.Deserialize<JsonObject>(ref reader);
-//       if (dpj is null) {
-//          return null;
-//       }
-//       var tableName = dpj["tableName"]!.AsString();
-//       var columns = dpj["columns"]!.AsArray()
-//          .Select((c, i) => {
-//             var dataType = c!["type"]!.AsEnum<DataStreamDataType>();
-//             return (info: new ColumnInfo(c["name"]!.AsString(), i, dataType.ToType()), dataType);
-//          })
-//          .ToArray();
-//       var schema = new BlueTableSchema(columns.Cut1().ToArray(), tableName);
-//       return new() {
-//          TableName = tableName,
-//          AddedRows = ToRows(dpj.GetArrayOrEmpty("added")).ToList(),
-//          DeletedRows = ToRows(dpj.GetArrayOrEmpty("deleted")).ToList(),
-//          Columns = [.. schema.columnInfo]
-//       };
+   public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (!reader.ReadObject()) {
+         return null;
+      }
+      if (!reader.ReadPropertyName(out var prop)) {
+         return null;
+      }
+      if (!reader.Read()) {
+         return null;
+      }
+      if (prop == "added") {
+         var added = new List<MutableBlueRow>();
+         if (reader.TokenType != JsonTokenType.StartArray) {
+            return null;
+         }
+         while (true) {
+            if (!reader.Read()) {
+               break;
+            }
+            if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
+               return null;
+            }
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  // DAMNIT! Need the schema.
+                  break;
+            }
+         }
+      }
+   }
 
-//       IEnumerable<IBlueRow> ToRows(JsonArray rows) {
-//          foreach (var row in rows) {
-//             var mbr = new MutableBlueRow(schema);
-//             switch (row!.GetValueKind()) {
-//                case JsonValueKind.Object:
-//                   var rowObj = row.AsObject();
-//                   for (int i = 0; i < columns.Length; i++) {
-//                      SetCellValue(mbr, i, rowObj[columns[i].info.colName]!, columns[i].dataType);
-//                   }
-//                   break;
-//                case JsonValueKind.Array:
-//                   var rowArr = row.AsArray();
-//                   for (int i = 0; i < columns.Length; i++) {
-//                      SetCellValue(mbr, i, rowArr[i]!, columns[i].dataType);
-//                   }
-//                   break;
-//             }
-//             yield return mbr;
-//          }
-//       }
+   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      using (writer.WriteArray("schema")) {
+         foreach (var col in value.schema.columnInfo) {
+            using (writer.WriteObject()) {
+               writer.WriteString("name", col.colName);
+               writer.WriteNumber("index", col.colNo);
+               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
+               writer.WriteString("type", col.colType.ToString());
+            }
+         }
+      }
 
-//       static void SetCellValue(MutableBlueRow mbr, int i, JsonNode cell, DataStreamDataType dataType) {
-//          switch (dataType) {
-//             case DataStreamDataType.String:
-//                mbr.setValue(i, cell.GetValue<string>());
-//                break;
-//             case DataStreamDataType.Int32:
-//                mbr.setValue(i, cell.GetValue<int>());
-//                break;
-//             case DataStreamDataType.Int64:
-//                mbr.setValue(i, cell.GetValue<long>());
-//                break;
-//             case DataStreamDataType.Single:
-//                mbr.setValue(i, cell.GetValue<float>());
-//                break;
-//             case DataStreamDataType.Double:
-//                mbr.setValue(i, cell.GetValue<double>());
-//                break;
-//          }
-//       }
-//    }
+      var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
+      using (writer.WriteArray("added")) {
+         foreach (IBlueRow row in value.addedRows) {
+            using (writer.WriteArray()) {
+               for (int i = 0; i < cellWriters.Count; i++) {
+                  cellWriters[i](row[i]);
+               }
+            }
+         }
+      }
+      using (writer.WriteArray("deleted")) {
+         foreach (IBlueRow row in value.deletedRows) {
+            using (writer.WriteArray()) {
+               for (int i = 0; i < cellWriters.Count; i++) {
+                  cellWriters[i](row[i]);
+               }
+            }
+         }
+      }
+      if (IncludeCurrent) {
+         using (writer.WriteArray("current")) {
+            foreach (IBlueRow row in value.newRows) {
+               using (writer.WriteArray()) {
+                  for (int i = 0; i < cellWriters.Count; i++) {
+                     cellWriters[i](row[i]);
+                  }
+               }
+            }
+         }
+      }
 
-//    public override void Write(Utf8JsonWriter writer, DataStreamUpdate value, JsonSerializerOptions options) {
-//       using var _ = writer.WriteObject();
-//       writer.WriteString("tableName", value.TableName);
-//       using (writer.WriteArray("columns")) {
-//          foreach (var col in value.Columns.empty_if_null()) {
-//             using (writer.WriteObject()) {
-//                writer.WriteString("name", col.colName);
-//                writer.WriteNumber("index", col.colNo);
-//                // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
-//                writer.WriteString("type", col.colType.ToString());
-//             }
-//          }
-//       }
+      Action<object?> getCellWriter(ColumnInfo colInfo) {
+         var writeVal = getCellValueWriter(colInfo);
+         if (!UseObjectRows) {
+            return writeVal;
+         }
+         return v => {
+            writer.WritePropertyName(colInfo.colName);
+            writeVal(v);
+         };
+      }
 
-//       var cellWriters = value.Columns!.Select(getCellWriter).ToList();
-//       using (writer.WriteArray("added")) {
-//          foreach (IBlueRow row in value.AddedRows!) {
-//             using (writer.WriteArray()) {
-//                for (int i = 0; i < cellWriters.Count; i++) {
-//                   cellWriters[i](row[i]);
-//                }
-//             }
-//          }
-//       }
-//       using (writer.WriteArray("deleted")) {
-//          foreach (IBlueRow row in value.DeletedRows!) {
-//             using (writer.WriteArray()) {
-//                for (int i = 0; i < cellWriters.Count; i++) {
-//                   cellWriters[i](row[i]);
-//                }
-//             }
-//          }
-//       }
-//       if (IncludeCurrent) {
-//          using (writer.WriteArray("current")) {
-//             foreach (IBlueRow row in value.newRows) {
-//                using (writer.WriteArray()) {
-//                   for (int i = 0; i < cellWriters.Count; i++) {
-//                      cellWriters[i](row[i]);
-//                   }
-//                }
-//             }
-//          }
-//       }
+      Action<object?> getCellValueWriter(ColumnInfo colInfo) {
+         var t = colInfo.colType;
+         if (t == typeof(int)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
+         }
+         if (t == typeof(uint)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
+         }
+         if (t == typeof(long)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
+         }
+         if (t == typeof(ulong)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
+         }
+         if (t == typeof(short)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
+         }
+         if (t == typeof(ushort)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
+         }
+         if (t == typeof(double)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
+         }
+         if (t == typeof(float)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
+         }
+         if (t == typeof(bool)) {
+            return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
+         }
+         return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
 
-//       Action<object?> getCellWriter(ColumnInfo colInfo) {
-//          var writeVal = getCellValueWriter(colInfo);
-//          if (!UseObjectRows) {
-//             return writeVal;
-//          }
-//          return v => {
-//             writer.WritePropertyName(colInfo.colName);
-//             writeVal(v);
-//          };
-//       }
-
-//       Action<object?> getCellValueWriter(ColumnInfo colInfo) {
-//          var t = colInfo.colType;
-//          if (t == typeof(int)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
-//          }
-//          if (t == typeof(uint)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
-//          }
-//          if (t == typeof(long)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
-//          }
-//          if (t == typeof(ulong)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
-//          }
-//          if (t == typeof(short)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
-//          }
-//          if (t == typeof(ushort)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
-//          }
-//          if (t == typeof(double)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
-//          }
-//          if (t == typeof(float)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
-//          }
-//          if (t == typeof(bool)) {
-//             return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
-//          }
-//          return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
-
-//          Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
-//             v => {
-//                if (v is null) {
-//                   writer.WriteNullValue();
-//                } else {
-//                   writeNonNull(v);
-//                }
-//             };
-//       }
-//    }
-// }
+         Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
+            v => {
+               if (v is null) {
+                  writer.WriteNullValue();
+               } else {
+                  writeNonNull(v);
+               }
+            };
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -4,7 +4,6 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <TargetFramework>net8.0</TargetFramework>
     <Nullable>disable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamCore/DataStreamDataType.cs

@@ -1,11 +0,0 @@
-namespace Bts.DataStreamCore;
-
-public enum DataStreamDataType {
-   Any,
-   Bytes,
-   String,
-   Int32,
-   Int64,
-   Single,
-   Double
-}

src/DataStreamSuite/DataStreamCore/DataStreamExtensions.cs

@@ -1,42 +0,0 @@
-#nullable enable
-
-using Bts.DataStreamCore;
-
-public static class DataStreamExtensions {
-   public static Type ToType(this DataStreamDataType dataType) =>
-      dataType switch {
-         DataStreamDataType.Any => typeof(object),
-         DataStreamDataType.String => typeof(string),
-         DataStreamDataType.Int32 => typeof(int),
-         DataStreamDataType.Int64 => typeof(long),
-         DataStreamDataType.Single => typeof(float),
-         DataStreamDataType.Double => typeof(double),
-         _ => throw new ArgumentException($"DataType '{dataType}' not supported.", nameof(dataType)),
-      };
-
-   public static DataStreamDataType FromType(Type type) {
-      if (type == typeof(object)) {
-         return DataStreamDataType.Any;
-      }
-      if (type == typeof(string)) {
-         return DataStreamDataType.String;
-      }
-      if (type == typeof(int)) {
-         return DataStreamDataType.Int32;
-      }
-      if (type == typeof(long)) {
-         return DataStreamDataType.Int64;
-      }
-      if (type == typeof(float)) {
-         return DataStreamDataType.Single;
-      }
-      if (type == typeof(double)) {
-         return DataStreamDataType.Double;
-      }
-      if (type == typeof(byte[])) {
-         return DataStreamDataType.Bytes;
-      }
-      throw new ArgumentException(
-         $"Unable to convert type '{type}' to {typeof(DataStreamDataType)}");
-   }
-}

src/DataStreamSuite/DataStreamCore/Usings.cs

@@ -1 +0,0 @@
-using System.Linq;

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json

@@ -1,12 +0,0 @@
-{
-  "user": "bts",
-  "prepublish_tables": {
-    "bbo": "bbo()",
-    "instruments": "instruments()",
-    "timesales": "timesales(10000)",
-    "books": "books()",
-    "seconds_1_10": "second(1,10)",
-    "literal_420": "literal(\"x\", \"420.0\")",
-    "mongodb_testcollection": "mongodb(\"mongodb+srv://johnc-dev:DMhDhP!g-TKeK.5@atlas-online-archive-5e18ab3c77c74d1cffa64c03-m7jkr.a.query.mongodb.net/?ssl=true&authSource=admin\",\"admin\",\"test_collection\")"
-  }
-}

src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleEchoClient.json

@@ -1,4 +0,0 @@
-{
-  "user": "bts",
-  "simple_echo_client": true
-}

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -1,16 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-  </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
@@ -20,4 +9,11 @@
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
 </Project>

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs

@@ -1,76 +0,0 @@
-namespace Bts.DataStreamPublisher;
-
-using CommandLine;
-
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-public class DataStreamPublisherConfig {
-   /// <summary>
-   /// The login user for sparkd
-   /// </summary>
-   [JsonPropertyName("user")]
-   [Option("user", HelpText = "The sparkd login user")]
-   public string? User { get; set; }
-
-   /// <summary>
-   /// The login password for sparkd
-   /// </summary>
-   [JsonPropertyName("password")]
-   [Option("pass", HelpText = "The sparkd login password")]
-   public string? Password { get; set; }
-
-   /// <summary>
-   /// The config server url for sparkd
-   /// </summary>
-   [JsonPropertyName("sparkd")]
-   [Option("sparkd", HelpText = "The sparkd config server")]
-   public string? Sparkd { get; set; }
-
-   /// <summary>
-   /// The tables to publish on startup. Each key/value represents a table name/query to publish.
-   /// </summary>
-   [JsonPropertyName("prepublish_tables")]
-   public Dictionary<string, string>? PrepublishTables { get; set; }
-
-   [JsonPropertyName("simple_echo_client")]
-   [Option("simple-echo-client", HelpText = "Run a simple echo client (won't publish tables or connect to sparkd)")]
-   public bool? SimpleEchoClient { get; set; }
-
-   [Option(
-      "config",
-      MetaValue = "PATH",
-      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
-   [JsonIgnore]
-   public string? ConfigFile { get; set; }
-
-   public void MergeWith(DataStreamPublisherConfig other) {
-      User ??= other.User;
-      Password ??= other.Password;
-      Sparkd ??= other.Sparkd;
-      PrepublishTables ??= other.PrepublishTables;
-      SimpleEchoClient ??= other.SimpleEchoClient;
-      ConfigFile ??= other.ConfigFile;
-   }
-
-   /// <summary>
-   /// Merges options with those defined in `ConfigFile` (does not overwrite
-   /// non-null existing values).
-   /// </summary>
-   /// <exception cref="InvalidOperationException">
-   /// JsonSerializer.Deserialize returns null
-   /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
-         return;
-      }
-      var cfgString = File.ReadAllText(ConfigFile);
-      var parsedCfg = JsonSerializer.Deserialize<DataStreamPublisherConfig>(cfgString) ??
-         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
-      MergeWith(parsedCfg);
-      ConfigFile = null;
-   }
-}

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -1,91 +1,91 @@
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.utils;
-
-using Bts.DataStreamExpressions;
-using Bts.DataStreamPublisher;
-
-using CommandLine;
-
-using mbts.config;
-
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-
-await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
-   .MapResult(RunAsync, errs => Task.FromResult(1));
-
-async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
-   try {
-      cfg.MergeWithConfigFile();
-      Configuration.initialize(cfg.Sparkd);
-      Log.Gen.info("Starting DataStreamPublisher");
-      var service_ctx = ServiceContext.create(cfg.Sparkd);
-      await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
-      if (cfg.SimpleEchoClient ?? false) {
-         start_echo_client();
-      } else {
-         _publish_streams(cfg.PrepublishTables ?? []);
-      }
-   } catch (Exception ex) {
-      // Write the error to console first, in case the logging system is compromised.
-      var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
-      Console.Error.WriteLine(error_msg);
-      Console.Error.Flush();
-      bts.Log.Gen.error(error_msg);
-      return 1;
-   }
-   return 0;
-}
-
-static ViewStream build_stream(string stream_expression) {
-   try {
-      bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
-      var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-      var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-      return rv;
-
-   } catch (Exception ex) {
-      bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
-      return null;
-   }
-}
-
-static void start_echo_client() {
-   Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
-   Console.ReadLine();
-   SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345);
-   client.start();
-   while (true) {
-      System.Threading.Thread.Sleep(1000);
-   }
-   // await SimpleTCPClient.start("127.0.0.1", 12345);
-   // Task.Delay(1000000000).Wait();
-}
-
-void _publish_streams(IEnumerable<KeyValuePair<string, string>> prepublish_streams) {
-   PubTableBroker pb_broker = new PubTableBroker();
-   CloudStreamManager csm = new CloudStreamManager();
-   foreach (var kv in prepublish_streams) {
-      var name = kv.Key;
-      var exp = kv.Value;
-      BCG.ViewStream vs = build_stream(exp);
-      ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
-      vs.onChange += on_change;
-   }
-   while (true) {
-      System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
-      ViewStreamDatabase.instance.doWork();
-      csm.doWork();
-      MasterWorker.getInstance(false).doWork();
-   }
-}
-
-
-
-void on_change(object obj) {
-   bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
-}
+using System;
+using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
+using Bts.DataStreamExpressions;
+using System.Collections.Generic;
+
+using BCG;
+using System.Threading;
+
+
+bts.Log.Gen.info($"Starting DSPublisher");
+Console.WriteLine("Starting DSPublisher...");
+bool run_echo_tcp_client_only = false;
+static void init(string[] args) {
+   Func<string, string, Task> login = async (l, p) => {
+      var login = l;
+      var passwd = p;
+      var service_ctx = bts.api.core.ServiceContext.create();
+      await service_ctx.login(login, passwd);
+   };
+   BTSNativeMethods.provide_command_line(args, args.Length);
+   Configuration.initialize();
+   Task.Run(async () => await login("bts", String.Empty));
+}
+init(args);
+if (run_echo_tcp_client_only)
+   start_echo_client();
+else
+   _publish_streams();
+
+static ViewStream build_stream(string stream_expression) {
+   try {
+      bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
+      var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+      var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+      return rv;
+
+   } catch (Exception ex) {
+      bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+      return null;
+   }
+}
+
+static void start_echo_client() {
+   Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
+   Console.ReadLine();
+   SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345);
+   client.start();
+   while (true) {
+      System.Threading.Thread.Sleep(1000);
+   }
+   // await SimpleTCPClient.start("127.0.0.1", 12345);
+   // Task.Delay(1000000000).Wait();
+}
+
+void _publish_streams() {
+   PubTableBroker pb_broker = new PubTableBroker();
+   CloudStreamManager csm = new CloudStreamManager();
+   // get it from config
+   Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
+        //{"bbo","bbo()"},
+        //{"instruments","instruments()"},
+        //{"timesales","timesales(10000)"},
+        //{"books","books()"},
+       {"seconds_1_10","second(1,10)"},
+        {"literal_420","literal(\"x\", \"420.0\")"},
+        {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
+        };
+   //  {"bbo_ts","timeSeries(bbo())"}
+
+   foreach (var kv in streams_to_publish) {
+      var name = kv.Key;
+      var exp = kv.Value;
+      BCG.ViewStream vs = build_stream(exp);
+      ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
+      vs.onChange += on_change;
+   }
+   while (true) {
+      System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
+      ViewStreamDatabase.instance.doWork();
+      csm.doWork();
+      MasterWorker.getInstance(false).doWork();
+   }
+}
+
+
+
+void on_change(object obj) {
+   bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
+}

src/DataStreamSuite/DataStreamSuite.sln

@@ -38,10 +38,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStre
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{134C3915-CB2B-494C-BD62-E0B18250CBF5}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStreamWebApp.Server", "Bts.DataStreamWebApp.Server\Bts.DataStreamWebApp.Server.csproj", "{6DE5E4CE-423E-4571-81F7-539D268D6059}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.Websocket", "Bts.DataStream.Websocket\Bts.DataStream.Websocket.csproj", "{088655DA-3126-4F8A-9AC7-9061FDA891FF}"
@@ -56,8 +52,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionTests", "Expressi
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionTester", "ExpressionTester\ExpressionTester.csproj", "{0908C8F3-5E8A-41C7-979C-1C7C38D431C8}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream.Console", "DataStream.Console\DataStream.Console.csproj", "{B2968D7D-A41A-41E0-940A-37C7C2CA9513}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -139,14 +133,6 @@ Global
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU
-		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
 		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -175,9 +161,5 @@ Global
 		{0908C8F3-5E8A-41C7-979C-1C7C38D431C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0908C8F3-5E8A-41C7-979C-1C7C38D431C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0908C8F3-5E8A-41C7-979C-1C7C38D431C8}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B2968D7D-A41A-41E0-940A-37C7C2CA9513}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B2968D7D-A41A-41E0-940A-37C7C2CA9513}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B2968D7D-A41A-41E0-940A-37C7C2CA9513}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B2968D7D-A41A-41E0-940A-37C7C2CA9513}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs

@@ -64,7 +64,7 @@ namespace bts {
             newConn.getPeerAddress(out ip, out port);
             string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, _port);
             //         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
-            Console.WriteLine(connMsg);       
+            //Console.WriteLine(connMsg);       
         }
 
 
@@ -72,4 +72,4 @@ namespace bts {
 
     }
 
-}
+}

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -480,6 +480,7 @@ namespace bts {
                         _on_read_data_completely(rcv_message);
                         _read_next_msg();
                         return;
+                        return;
                      } else if (msg_type == HB_MSG_SIGNAL) {
                         _on_read_hb_completely(rcv_message);
                         _read_next_msg();
@@ -580,4 +581,4 @@ namespace bts {
          }
       }
    }
-}
+}

src/DataStreamSuite/PubSub/Testing/SimpleTCPServer2.cs

@@ -1,42 +0,0 @@
-using System;
-using System.Net;
-using System.Net.Sockets;
-using System.Text;
-
-namespace Bts.PubSub;
-
-public class SimpleTCPServer2 {
-   private static Socket? _acceptSocket;
-   private static int _port = 12345;
-
-   public static void StartServer() {
-      _acceptSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-      IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port);
-      _acceptSocket.Bind(endpoint);
-      _acceptSocket.Listen(250);
-
-      Console.WriteLine($"Server listening on port {_port}");
-
-      while (true) {
-         Socket clientSocket = _acceptSocket.Accept();
-         HandleClient(clientSocket);
-      }
-   }
-
-   static void HandleClient(Socket clientSocket) {
-      // Example: Echo back received messages
-      byte[] buffer = new byte[1024];
-      int bytesRead;
-
-      while ((bytesRead = clientSocket.Receive(buffer)) > 0) {
-         string receivedData = Encoding.UTF8.GetString(buffer, 0, bytesRead);
-         Console.WriteLine($"Received data: {receivedData}");
-
-         // Echo back to the client
-         clientSocket.Send(Encoding.UTF8.GetBytes(receivedData));
-      }
-
-      Console.WriteLine("Client disconnected");
-      clientSocket.Close();
-   }
-}

src/DataStreamSuite/PubSubServer/Program.cs

@@ -1,43 +1,57 @@
-using BCG;
-
-using bts.utils;
-
-using mbts.config;
-
-using System;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-
-bool run_echo_tcp_server_only = true;
-Configuration.initialize();
-if (run_echo_tcp_server_only) {
-   Console.WriteLine("Running as TCP Echo server, Hit enter to continue");
-   Console.ReadLine();
-   var port = 12345;
-   Console.WriteLine($"ready to listen to port: {port}");
-   SimpleTCPServer sts = new SimpleTCPServer(port);
-   sts.init();
-   Task.Delay(1000000000).Wait();
-}
-
-PubSub3Config config = PubSub3Config.instance;
-
-long currentMemory = 0;
-int defaultWaitTime = 100;
-bool gced = false;
-new Thread(() => {
-   while (true) {
-      gced = false;
-      currentMemory = GC.GetTotalMemory(false);
-      if (config.PS3_GCActivationThreshold > 0 && currentMemory > config.PS3_GCActivationThreshold) {
-         Console.WriteLine($"GCing memory, mem before GC = {currentMemory}");
-         GC.Collect();
-         gced = true;
-      }
-      Thread.Sleep(gced ? config.PS3_WaitPeriodAfterGC : defaultWaitTime);
-   }
-}).Start();
-
-PubSubMessageReactor pubSubServer = new PubSubMessageReactor();
-pubSubServer.start();
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Linq;
+
+//using BCG.Utilities;
+//using BCG.Utilities.General;
+
+
+namespace BCG {
+   class PubSub2Main {
+
+      /// <summary>
+      /// The main entry point for the application.
+      /// </summary>
+      [STAThread]
+      public static void Main(string[] args) {
+         bool run_echo_tcp_server_only = false;
+         bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0);
+         mbts.config.Configuration.initialize();
+         if (run_echo_tcp_server_only) {
+            Console.WriteLine("Running as TCP Echo server, Hit enter to continue");
+            Console.ReadLine();
+            SimpleTCPServer sts = new SimpleTCPServer(12345);
+            sts.init();
+            Task.Delay(1000000000).Wait();
+         }
+         if (args.Length > 0 && args[0] == "daemon")
+            args = args.Skip(1).ToArray();
+
+         //CoreServicesInit.init();
+         PubSub3Config config = PubSub3Config.instance;
+
+         long currentMemory = 0;
+         int defaultWaitTime = 100;
+         bool gced = false;
+         new Thread(() => {
+            while (true) {
+               gced = false;
+               currentMemory = GC.GetTotalMemory(false);
+               if (config.PS3_GCActivationThreshold > 0 && currentMemory > config.PS3_GCActivationThreshold) {
+                  string gcMsg = String.Format("GCing memory, mem before GC = {0}", currentMemory);
+                  // BlueEventLog.logEvent(gcMsg, PS3LogLevels.info);
+                  Console.WriteLine(gcMsg);
+                  GC.Collect();
+                  gced = true;
+               }
+               Thread.Sleep(gced ? config.PS3_WaitPeriodAfterGC : defaultWaitTime);
+            }
+         }).Start();
+
+         PubSubMessageReactor pubSubServer = new PubSubMessageReactor();
+         pubSubServer.start();
+
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs

@@ -1,127 +1,125 @@
-namespace BCG {
-   using BCG.Utilities.General;
-
-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
-   using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
-   using Spark.Core.Reactive;
-
-   using System;
-   using System.Collections.Concurrent;
-   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
-   using System.Linq;
-   using System.Reactive.Linq;
-   using bts.api.core;
-   using Spark.Api.MarketData;
-
-   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
-
-   [DataStreamUserRespresentation("price_stats")]
-   public class PriceStatsStream : ViewStream, IWorker {
-      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
-      private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase _vsd;
-      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
-      MarketDataClient _md_client;
-      private bool _dirty = false;
-      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("sec_key", TypeCode.Int32),
-                                                Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("open", TypeCode.UInt32),
-                                                Tuple.Create("close", TypeCode.Double),
-                                                Tuple.Create("settlement", TypeCode.Single));
-      IDisposable? subscription;
-
-      // TODO: add cons for symbol, exchange etc 
-      public PriceStatsStream()
-      : this(ViewStreamDatabase.instance) {
-      }
-
-      public PriceStatsStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("PriceStatsStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
-      }
-
-      private void _log_updates(TableUpdate<PriceStats, uint> update) {
-         int i = 0;
-         // foreach (BookTuple bt in update.added) {
-         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-         // foreach (Change<BookTuple> c in update.changes) {
-         //    var bt = c.New;
-         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-      }
-
-      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
-         _log_updates(update);
-         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
-         _cached_state = update.state;
-         _dirty = true;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-         Disposable.dispose(ref subscription);
-      }
-
-      private void _scan() {
-         // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
-         if (_cached_state == null)
-            return;
-         BlueTable new_state = new BlueTable(_pstats_scheam);
-         foreach (var kv in _cached_state) {
-            SecurityKey skey = kv.Key;
-            int id = Convert.ToInt32(skey);
-            PriceStats pstats = kv.Value;
-            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement);
-            new_state.addRow(r);
-         }
-         DataPacket dp = null;
-         if (myLastValue == null) {
-            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
-         } else {
-            DataPacket old_dp = myLastValue as DataPacket;
-            IBlueRowCollection added_rows, del_rows;
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
-            dp = new DataPacket(new_state, added_rows, del_rows);
-         }
-         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-            return;
-         myFireEvent(dp);
-      }
-
-      public void doWork() {
-         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
-            _last_poll_time = DateTime.Now;
-            _dirty = false;
-            _scan();
-         }
-      }
-   }
-}
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
+   using bts.md;
+   using bts.spark;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
+   using Spark.Core.Disposables;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+   using System.Collections.ObjectModel;
+   using System.Linq;
+   using System.Reactive.Linq;
+   using bts.api.core;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+   using System.Data;
+   using System.Reflection.Metadata;
+
+   [DataStreamUserRespresentation("price_stats")]
+   public class PriceStatsStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
+      MarketDataClient _md_client;
+      private bool _dirty = false;
+      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.Int32),
+                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("open", TypeCode.UInt32),
+                                                Tuple.Create("close", TypeCode.Double),
+                                                Tuple.Create("settlement", TypeCode.Single));
+
+      // TODO: add cons for symbol, exchange etc 
+      public PriceStatsStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public PriceStatsStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("PriceStatsStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         _md_client.price_stats().Subscribe(_on_price_stats);
+      }
+
+      private void _log_updates(TableUpdate<PriceStats, uint> update) {
+         int i = 0;
+         // foreach (BookTuple bt in update.added) {
+         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+         // foreach (Change<BookTuple> c in update.changes) {
+         //    var bt = c.New;
+         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+      }
+
+      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
+         _log_updates(update);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         _cached_state = update.state;
+         _dirty = false;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private void _scan() {
+         // this shouldn be made incremental using the TableUpdate
+         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_pstats_scheam);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            int id = Convert.ToInt32(skey);
+            PriceStats pstats = kv.Value;
+            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement);
+            new_state.addRow(r);
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _dirty = false;
+            _scan();
+         }
+      }
+   }
+}

src/client/Spark.Console/src/Program.cs

@@ -1,45 +1,45 @@
-namespace Spark.Console {
-   using bts;
-   using bts.utils;
-
-   using System;
-
-   using CommandLine;
-
-   class Program {
-      static int Main(string[] args) {
-         // Initializing config is up to each individual command
-         // We call provide_command_line here in case config is first
-         // initialized from c++ instead of c#
-         BTSNativeMethods.provide_command_line(args, args.Length);
-
-         var parser = new Parser(ps => {
-            ps.CaseInsensitiveEnumValues = true;
-            ps.HelpWriter = Console.Out;
-         });
-
-         // Note: this can be made to infer the options type list from the list of delegates
-         return parser.parse_and_run<
-            ConfigCommandOptions,
-            ScriptCommandOptions,
-            TailServiceOptions,
-            TestServiceOptions,
-            SettingsCommandOptions,
-            TestCommandOptions,
-            OrderStressOptions,
-            MBOCommandOptions,
-            int
-         >(
-            args,
-            ConfigCommand.run,
-            ScriptCommand.run,
-            TailService.run,
-            TestService.run,
-            SettingsCommand.run,
-            TestCommand.run,
-            OrderStressCommand.run,
-            MBOCommand.run,
-            errors => 1);
-      }
-   }
-}
+namespace Spark.Console {
+   using bts;
+   using bts.utils;
+
+   using System;
+
+   using CommandLine;
+
+   class Program {
+      static int Main(string[] args) {
+         // Initializing config is up to each individual command
+         // We call provide_command_line here in case config is first
+         // initialized from c++ instead of c#
+         BTSNativeMethods.provide_command_line(args, args.Length);
+
+         var parser = new Parser(ps => {
+            ps.CaseInsensitiveEnumValues = true;
+            ps.HelpWriter = Console.Out;
+         });
+
+         // Note: this can be made to infer the options type list from the list of delegates
+         return parser.parse_and_run<
+            ConfigCommandOptions,
+            ScriptCommandOptions,
+            TailServiceOptions,
+            TestServiceOptions,
+            SettingsCommandOptions,
+            TestCommandOptions,
+            OrderStressOptions,
+            MBOCommandOptions,
+            int
+         >(
+            BTS.remove_config_arg(args),
+            ConfigCommand.run,
+            ScriptCommand.run,
+            TailService.run,
+            TestService.run,
+            SettingsCommand.run,
+            TestCommand.run,
+            OrderStressCommand.run,
+            MBOCommand.run,
+            errors => 1);
+      }
+   }
+}

src/client/spark/view/Controls/DIYGrid/CellRender/CellRender.cs

@@ -70,7 +70,8 @@ namespace mbts.ui.controls {
             public double border_thickness => _border_thickness.Value;
 
             void _apply_animation() {
-               if (_animations.Select(a => a.background_animation).ExceptNull().TryGetFirst(out var bga)) {
+               var bga = _animations.OfType<BackgroundAnimation>().FirstOrDefault();
+               if (bga != null) {
                   _background.set(bga.brush);
                   if (bga.pen != null) {
                      _border.set(bga.pen.Brush);

src/client/spark_api/MarketData/MarketDataClient.cs

@@ -1,280 +1,287 @@
-#nullable enable
-namespace bts.md {
-   using bts.api;
-   using bts.api.core;
-   using bts.collections;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.spark;
-
-   using Spark.Api.MarketData;
-   using Spark.Core;
-   using Spark.Core.Reactive;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Reactive.Linq;
-
-   public class MarketDataClient {
-      readonly MDSource _source;
-      readonly TimeSpan _poll_interval;
-      readonly TimeSpan _slow_poll_interval;
-      readonly uint _num_depth_levels;
-
-      public MarketDataClient(MDSource source)
-         : this(source, 5, TimeSpan.FromSeconds(0.1), TimeSpan.FromSeconds(1)) { }
-
-      public MarketDataClient(
-         MDSource source,
-         uint num_depth_levels,
-         TimeSpan poll_interval,
-         TimeSpan slow_poll_interval) {
-         _source = source;
-         _poll_interval = poll_interval;
-         if (slow_poll_interval < poll_interval) {
-            Log.Api.warn("MarketDataClient 'slow_poll_interval' smaller than 'poll_interval'");
-         }
-         _slow_poll_interval = slow_poll_interval;
-         _num_depth_levels = num_depth_levels;
-      }
-
-      public IObservable<IReadOnlyDictionary<uint, SecurityImmutable>> securities(
-         SecurityCriteria? criteria = null) =>
-         criteria.subscribe(_source.subscribe_securities())
-            .Select(secs => secs.ToDictionary(s => s.Id));
-
-      public IObservable<TableUpdate<SecurityImmutable, uint>> securities_incremental(
-         SecurityCriteria? criteria = null) =>
-         criteria is null ? securities_incremental() : securities_incremental(criteria.includes);
-
-      public IObservable<TableUpdate<SecurityImmutable, uint>> securities_incremental(
-         Func<SecurityImmutable, bool> filter) {
-         var secs_src = _source.subscribe_securities().ToSelector(
-            sec_mds => MappedDictionary.create(sec_mds, s => s.security));
-         if (filter is not null) {
-            secs_src = secs_src.extend(
-               secs => new FilteredDictionary<uint, SecurityImmutable>(secs, filter));
-         }
-         return secs_src.Table(s => s.Id);
-      }
-
-      public IObservable<TableUpdate<BookTuple, uint>> books(SecurityCriteria? criteria = null) =>
-         books(criteria is null ? null :
-            new Func<SecurityImmutable, bool>(s => criteria.includes(s)));
-
-      public IObservable<TableUpdate<BookTuple, uint>> books(Func<SecurityImmutable, bool>? filter) {
-         // This MDSource's IObservable provides callbacks when a security is added or removed, but
-         // doesn't detect book changes. To avoid maintaining complicated state, create a brand new
-         // IObservable of book updates whenever the securities change.
-         var secs = _source.subscribe_securities();
-         var filtered_secs = filter is null ?
-            secs.Select(mds => mds.Values) :
-            secs.Select(mds => mds.Values.Where(md => filter(md.security)));
-         return filtered_secs.Select(fixed_sec_market_data).FlattenTableSeries();
-
-         IObservable<TableUpdate<BookTuple, uint>> fixed_sec_market_data(
-            IEnumerable<SecurityMD> security_mds) {
-            // Convert 'security_mds' to a list of PollableBooks.
-            var cached_books = security_mds
-               .Select(s => new PollableBook(s, _num_depth_levels))
-               .ToList();
-
-            return Observable.Interval(_poll_interval).Scan(
-               TableUpdate<BookTuple, uint>.mk_empty(get_sec_id),
-               poll_books);
-
-            uint get_sec_id(BookTuple book) =>
-               book.security.Id;
-
-            TableUpdate<BookTuple, uint> poll_books(TableUpdate<BookTuple, uint> table, long _) {
-               var bd = table.to_builder();
-               foreach (var cbook in cached_books) {
-                  if (cbook.poll_changes()) {
-                     bd.add_or_update(cbook.to_book_tuple());
-                  }
-               }
-               return bd.to_table();
-            }
-         }
-      }
-
-      public IObservable<IList<TradeInfo>> trades(TradesSubscriptionFilter filter) {
-         var sec_mds = filter.security_criteria == null ?
-            _source.subscribe_securities() :
-            filter.security_criteria.subscribe_md_map(_source.subscribe_securities());
-         return TradesSource.create(_source.consumer, sec_mds, _poll_interval);
-      }
-
-      public IObservable<TableUpdate<PriceTicker, uint>> price_ticker(
-         SecurityCriteria? criteria = null) {
-         // This MDSource's IObservable provides callbacks when a security is added or removed, but
-         // doesn't detect ticker changes. To avoid maintaining complicated state, create a brand new
-         // IObservable of ticker updates whenever the securities change.
-         return criteria.subscribe_md(_source.subscribe_securities())
-            .Select(fixed_sec_price_tickers)
-            .FlattenTableSeries();
-
-         IObservable<TableUpdate<PriceTicker, uint>> fixed_sec_price_tickers(
-            IEnumerable<SecurityMD> security_mds) {
-            // Convert 'security_mds' to a list of PollablePriceTickers.
-            var cached_tickers = security_mds
-               .Select(s => new PollablePriceTicker(s))
-               .ToList();
-
-            return Observable.Interval(_poll_interval).Scan(
-               TableUpdate<PriceTicker, uint>.mk_empty(get_sec_id),
-               poll_tickers);
-
-            uint get_sec_id(PriceTicker ticker) =>
-               ticker.security.Id;
-
-            TableUpdate<PriceTicker, uint> poll_tickers(TableUpdate<PriceTicker, uint> table, long _) {
-               bool is_first_update = table.state.Count == 0;
-               var bd = table.to_builder();
-               foreach (var cticker in cached_tickers) {
-                  if (cticker.poll_changes() || is_first_update) {
-                     bd.add_or_update(cticker.to_ticker());
-                  }
-               }
-               return bd.to_table();
-            }
-         }
-      }
-
-      public IObservable<TableUpdate<PriceStats, uint>> price_stats(
-         SecurityCriteria? criteria = null) {
-         // This MDSource's IObservable provides callbacks when a security is added or removed, but
-         // doesn't detect stats changes. To avoid maintaining complicated state, create a brand new
-         // IObservable of stats updates whenever the securities change.
-         return criteria.subscribe_md(_source.subscribe_securities())
-            .Select(fixed_sec_price_stats)
-            .FlattenTableSeries();
-
-         IObservable<TableUpdate<PriceStats, uint>> fixed_sec_price_stats(
-            IEnumerable<SecurityMD> security_mds) {
-            // Convert 'security_mds' to a list of PollablePriceStats.
-            var cached_statss = security_mds
-               .Select(s => new PollablePriceStats(s))
-               .ToList();
-
-            return Observable.Interval(_slow_poll_interval).Scan(
-               TableUpdate<PriceStats, uint>.mk_empty(get_sec_id),
-               poll_stats);
-
-            uint get_sec_id(PriceStats stats) =>
-               stats.security.Id;
-
-            TableUpdate<PriceStats, uint> poll_stats(TableUpdate<PriceStats, uint> table, long _) {
-               bool is_first_update = table.state.Count == 0;
-               var bd = table.to_builder();
-               foreach (var cstats in cached_statss) {
-                  if (cstats.poll_changes() || is_first_update) {
-                     bd.add_or_update(cstats.to_stats());
-                  }
-               }
-               return bd.to_table();
-            }
-         }
-      }
-
-      class PollableBook {
-         readonly SecurityMD _security_md;
-         readonly uint _num_depth_levels;
-         ulong _last_update_time = 0;
-         // Reuse the same arrays as an optimization to avoid reallocating memory on each poll.
-         // Note: this will break if the code isn't called in a thread-safe manner.
-         NativePriceSize[] _bids;
-         NativePriceSize[] _asks;
-
-         public PollableBook(SecurityMD security_md, uint num_depth_levels) {
-            _security_md = security_md;
-            _num_depth_levels = num_depth_levels;
-            _bids = new NativePriceSize[_num_depth_levels];
-            _asks = new NativePriceSize[_num_depth_levels];
-         }
-
-         public BookTuple to_book_tuple() {
-            var book = new Book(
-               _security_md.security.Id, _to_book_side(_bids), _to_book_side(_asks));
-            return new(_security_md.security, book, _security_md.quantity_display);
-         }
-
-         public bool poll_changes() =>
-            _security_md.md.get_depth(
-               ref _last_update_time, ref _bids, ref _asks, _num_depth_levels);
-
-         static List<PriceSize> _to_book_side(NativePriceSize[] native_book_side) =>
-            native_book_side
-               .Select(nps => new PriceSize(nps.size, nps.direct_size, nps.implied_size, nps.price))
-               .ToList();
-      }
-
-      class PollablePriceTicker {
-         readonly SecurityMD _security_md;
-         ulong _bbo_poll_time = 0;
-         ulong _last_poll_time = 0;
-         NativePriceSize _bid;
-         NativePriceSize _ask;
-         NativePriceSize _last;
-         PriceTicker _ticker;
-         ulong _volume;
-
-         public PollablePriceTicker(SecurityMD security_md) =>
-            _security_md = security_md;
-
-         public PriceTicker to_ticker() =>
-            _ticker;
-
-         public bool poll_changes() {
-            var md = _security_md.md;
-            md.get_bbo(ref _bbo_poll_time, out _bid, out _ask);
-            md.get_last_trade(ref _last_poll_time, out _last, out _, out _volume);
-            return Swap.toggle_to(ref _ticker, _mk_ticker());
-         }
-
-         PriceTicker _mk_ticker() {
-            var md = _security_md.md;
-            return new(
-               _security_md.security,
-               _security_md.quantity_display,
-               md.session_high,
-               md.session_low,
-               _volume,
-               _bid.price,
-               _bid.size,
-               _ask.price,
-               _ask.size,
-               md.fixing_price,
-               _last.price,
-               _last.size);
-         }
-      }
-
-      class PollablePriceStats {
-         readonly SecurityMD _security_md;
-         PriceStats _stats;
-
-         public PollablePriceStats(SecurityMD security_md) {
-            _security_md = security_md;
-            _stats = _mk_stats();
-         }
-
-         public PriceStats to_stats() =>
-            _stats;
-
-         public bool poll_changes() =>
-            Swap.toggle_to(ref _stats, _mk_stats());
-         PriceStats _mk_stats() {
-            var md = _security_md.md;
-            return new(
-               _security_md.security,
-               md.open_price,
-               md.close_price,
-               md.settlement_price);
-         }
-      }
-   }
-}
+#nullable enable
+namespace bts.md {
+   using bts.api;
+   using bts.api.core;
+   using bts.collections;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.spark;
+
+   using Spark.Api.MarketData;
+   using Spark.Core;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   public class MarketDataClient {
+      readonly MDSource _source;
+      readonly TimeSpan _poll_interval;
+      readonly TimeSpan _slow_poll_interval;
+      readonly uint _num_depth_levels;
+
+      public MarketDataClient(MDSource source)
+         : this(source, 5, TimeSpan.FromSeconds(0.1), TimeSpan.FromSeconds(1)) { }
+
+      public MarketDataClient(
+         MDSource source,
+         uint num_depth_levels,
+         TimeSpan poll_interval,
+         TimeSpan slow_poll_interval) {
+         _source = source;
+         _poll_interval = poll_interval;
+         if (slow_poll_interval < poll_interval) {
+            Log.Api.warn("MarketDataClient 'slow_poll_interval' smaller than 'poll_interval'");
+         }
+         _slow_poll_interval = slow_poll_interval;
+         _num_depth_levels = num_depth_levels;
+      }
+
+      public IObservable<IReadOnlyDictionary<uint, SecurityImmutable>> securities(
+         SecurityCriteria? criteria = null) =>
+         criteria.subscribe(_source.subscribe_securities())
+            .Select(secs => secs.ToDictionary(s => s.Id));
+
+      public IObservable<TableUpdate<SecurityImmutable, uint>> securities_incremental(
+         SecurityCriteria? criteria = null) =>
+         criteria is null ? securities_incremental() : securities_incremental(criteria.includes);
+
+      public IObservable<TableUpdate<SecurityImmutable, uint>> securities_incremental(
+         Func<SecurityImmutable, bool> filter) {
+         var secs_src = _source.subscribe_securities().ToSelector(
+            sec_mds => MappedDictionary.create(sec_mds, s => s.security));
+         if (filter is not null) {
+            secs_src = secs_src.extend(
+               secs => new FilteredDictionary<uint, SecurityImmutable>(secs, filter));
+         }
+         return secs_src.Table(s => s.Id);
+      }
+
+      public IObservable<TableUpdate<BookTuple, uint>> books(SecurityCriteria? criteria = null) =>
+         books(criteria is null ? null :
+            new Func<SecurityImmutable, bool>(s => criteria.includes(s)));
+
+      public IObservable<TableUpdate<BookTuple, uint>> books(Func<SecurityImmutable, bool>? filter) {
+         // This MDSource's IObservable provides callbacks when a security is added or removed, but
+         // doesn't detect book changes. To avoid maintaining complicated state, create a brand new
+         // IObservable of book updates whenever the securities change.
+         var secs = _source.subscribe_securities();
+         var filtered_secs = filter is null ?
+            secs.Select(mds => mds.Values) :
+            secs.Select(mds => mds.Values.Where(md => filter(md.security)));
+         return filtered_secs.Select(fixed_sec_market_data).FlattenTableSeries();
+
+         IObservable<TableUpdate<BookTuple, uint>> fixed_sec_market_data(
+            IEnumerable<SecurityMD> security_mds) {
+            // Convert 'security_mds' to a list of PollableBooks.
+            var cached_books = security_mds
+               .Select(s => new PollableBook(s, _num_depth_levels))
+               .ToList();
+
+            return Observable.Interval(_poll_interval).Scan(
+               TableUpdate<BookTuple, uint>.mk_empty(get_sec_id),
+               poll_books);
+
+            uint get_sec_id(BookTuple book) =>
+               book.security.Id;
+
+            TableUpdate<BookTuple, uint> poll_books(TableUpdate<BookTuple, uint> table, long _) {
+               var bd = table.to_builder();
+               foreach (var cbook in cached_books) {
+                  if (cbook.poll_changes()) {
+                     bd.add_or_update(cbook.to_book_tuple());
+                  }
+               }
+               return bd.to_table();
+            }
+         }
+      }
+
+      public IObservable<IList<TradeInfo>> trades(TradesSubscriptionFilter filter) {
+         var sec_mds = filter.security_criteria == null ?
+            _source.subscribe_securities() :
+            filter.security_criteria.subscribe_md_map(_source.subscribe_securities());
+         return TradesSource.create(_source.consumer, sec_mds, _poll_interval);
+      }
+
+      public IObservable<TableUpdate<PriceTicker, uint>> price_ticker(
+         SecurityCriteria? criteria = null) {
+         // This MDSource's IObservable provides callbacks when a security is added or removed, but
+         // doesn't detect ticker changes. To avoid maintaining complicated state, create a brand new
+         // IObservable of ticker updates whenever the securities change.
+         return criteria.subscribe_md(_source.subscribe_securities())
+            .Select(fixed_sec_price_tickers)
+            .FlattenTableSeries();
+
+         IObservable<TableUpdate<PriceTicker, uint>> fixed_sec_price_tickers(
+            IEnumerable<SecurityMD> security_mds) {
+            // Convert 'security_mds' to a list of PollablePriceTickers.
+            var cached_tickers = security_mds
+               .Select(s => new PollablePriceTicker(s))
+               .ToList();
+
+            return Observable.Interval(_poll_interval).Scan(
+               TableUpdate<PriceTicker, uint>.mk_empty(get_sec_id),
+               poll_tickers);
+
+            uint get_sec_id(PriceTicker ticker) =>
+               ticker.security.Id;
+
+            TableUpdate<PriceTicker, uint> poll_tickers(TableUpdate<PriceTicker, uint> table, long _) {
+               var bd = table.to_builder();
+               foreach (var cticker in cached_tickers) {
+                  if (cticker.poll_changes()) {
+                     bd.add_or_update(cticker.to_ticker());
+                  }
+               }
+               return bd.to_table();
+            }
+         }
+      }
+
+      public IObservable<TableUpdate<PriceStats, uint>> price_stats(
+         SecurityCriteria? criteria = null) {
+         // This MDSource's IObservable provides callbacks when a security is added or removed, but
+         // doesn't detect stats changes. To avoid maintaining complicated state, create a brand new
+         // IObservable of stats updates whenever the securities change.
+         return criteria.subscribe_md(_source.subscribe_securities())
+            .Select(fixed_sec_price_stats)
+            .FlattenTableSeries();
+
+         IObservable<TableUpdate<PriceStats, uint>> fixed_sec_price_stats(
+            IEnumerable<SecurityMD> security_mds) {
+            // Convert 'security_mds' to a list of PollablePriceStats.
+            List<PollablePriceStats>? cached_statss = security_mds
+               .Select(s => new PollablePriceStats(s))
+               .ToList();
+            var seed = TableUpdate<PriceStats, uint>.mk_empty(get_sec_id);
+            seed = get_seed(seed);
+            return Observable.Interval(_slow_poll_interval).Scan(
+                  seed,
+               poll_stats);
+
+            uint get_sec_id(PriceStats stats) =>
+               stats.security.Id;
+            TableUpdate<PriceStats, uint> get_seed(TableUpdate<PriceStats, uint> table) {
+               if (cached_statss == null) {
+                  return table;
+               }
+               var bd = table.to_builder();
+               foreach (var cstats in cached_statss) {
+                  bd.add_or_update(cstats.to_stats());
+               }
+               return bd.to_table();
+            }
+            TableUpdate<PriceStats, uint> poll_stats(TableUpdate<PriceStats, uint> table, long _) {
+               var bd = table.to_builder();
+               foreach (var cstats in cached_statss) {
+                  if (cstats.poll_changes()) {
+                     bd.add_or_update(cstats.to_stats());
+                  }
+               }
+               return bd.to_table();
+            }
+         }
+      }
+
+      class PollableBook {
+         readonly SecurityMD _security_md;
+         readonly uint _num_depth_levels;
+         ulong _last_update_time = 0;
+         // Reuse the same arrays as an optimization to avoid reallocating memory on each poll.
+         // Note: this will break if the code isn't called in a thread-safe manner.
+         NativePriceSize[] _bids;
+         NativePriceSize[] _asks;
+
+         public PollableBook(SecurityMD security_md, uint num_depth_levels) {
+            _security_md = security_md;
+            _num_depth_levels = num_depth_levels;
+            _bids = new NativePriceSize[_num_depth_levels];
+            _asks = new NativePriceSize[_num_depth_levels];
+         }
+
+         public BookTuple to_book_tuple() {
+            var book = new Book(
+               _security_md.security.Id, _to_book_side(_bids), _to_book_side(_asks));
+            return new(_security_md.security, book, _security_md.quantity_display);
+         }
+
+         public bool poll_changes() =>
+            _security_md.md.get_depth(
+               ref _last_update_time, ref _bids, ref _asks, _num_depth_levels);
+
+         static List<PriceSize> _to_book_side(NativePriceSize[] native_book_side) =>
+            native_book_side
+               .Select(nps => new PriceSize(nps.size, nps.direct_size, nps.implied_size, nps.price))
+               .ToList();
+      }
+
+      class PollablePriceTicker {
+         readonly SecurityMD _security_md;
+         ulong _bbo_poll_time = 0;
+         ulong _last_poll_time = 0;
+         NativePriceSize _bid;
+         NativePriceSize _ask;
+         NativePriceSize _last;
+         PriceTicker _ticker;
+         ulong _volume;
+
+         public PollablePriceTicker(SecurityMD security_md) =>
+            _security_md = security_md;
+
+         public PriceTicker to_ticker() =>
+            _ticker;
+
+         public bool poll_changes() {
+            var md = _security_md.md;
+            md.get_bbo(ref _bbo_poll_time, out _bid, out _ask);
+            md.get_last_trade(ref _last_poll_time, out _last, out _, out _volume);
+            return Swap.toggle_to(ref _ticker, _mk_ticker());
+         }
+
+         PriceTicker _mk_ticker() {
+            var md = _security_md.md;
+            return new(
+               _security_md.security,
+               _security_md.quantity_display,
+               md.session_high,
+               md.session_low,
+               _volume,
+               _bid.price,
+               _bid.size,
+               _ask.price,
+               _ask.size,
+               md.fixing_price,
+               _last.price,
+               _last.size);
+         }
+      }
+
+      class PollablePriceStats {
+         readonly SecurityMD _security_md;
+         PriceStats _stats;
+
+         public PollablePriceStats(SecurityMD security_md) {
+            _security_md = security_md;
+            _stats = _mk_stats();
+         }
+
+         public PriceStats to_stats() =>
+            _stats;
+
+         public bool poll_changes() =>
+            Swap.toggle_to(ref _stats, _mk_stats());
+         PriceStats _mk_stats() {
+            var md = _security_md.md;
+            return new(
+               _security_md.security,
+               md.open_price,
+               md.close_price,
+               md.settlement_price);
+         }
+      }
+   }
+}

src/client/spark_api/api/ServiceContext.cs

@@ -1,109 +1,110 @@
-#nullable enable
-namespace bts.api.core {
-   using bts.api.user;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Threading.Tasks;
-   using System.Reflection;
-
-   public sealed class ServiceContext : Spark.Api.IServiceProvider, IDisposable {
-      IntPtr _qbus;
-      AuthToken? _token;
-
-      public ServiceContext(string? config_source = null) {
-         BTS.init(config_source);
-         _qbus = ApiNativeMethods.api_create_quickbus(null, 0);
-      }
-
-      internal ServiceContext(IntPtr qbus) =>
-         _qbus = qbus;
-
-      public AuthTokenImmutable? token => _token?.immutable();
-      public UserTuple? user { get; private set; }
-      public bool is_authenticated => _token is not null;
-
-      internal IntPtr native_context => _qbus;
-
-      public static ServiceContext create(string? config_server = null) {
-         BTS.init(config_server);
-         var qbus = ApiNativeMethods.api_create_quickbus(null, 0);
-         return new ServiceContext(qbus);
-      }
-
-      public T locate<T>(string qualifier = "") where T : class, IApiClient {
-         var ctr = typeof(T).GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance)
-            .FirstOrDefault(ci => ci.GetParameters()
-               .SingleOrDefault()?.ParameterType == typeof(ClientChannel));
-         if (ctr is null) {
-            throw new InvalidOperationException(
-               $"Unable to find appropriate constructor for type '{typeof(T)}'");
-         }
-         ClientChannel channel = new QuickbusClientChannel(this, typeof(T).Name, qualifier);
-         return (T)ctr.Invoke(new object[] { channel });
-      }
-
-      public T GetService<T>(string? qualifier) where T : class, IApiClient =>
-         locate<T>(qualifier ?? string.Empty);
-
-      public async Task<bool> try_login(string username, string password) {
-         var res = await login(username, password);
-         return res.success;
-      }
-
-      public async Task<LoginResult> login(string username, string password) {
-         var res = await impl();
-         if (!res.success) {
-            Log.Api.error($"Login failed: for username: {username} password: {password} reason: {res.reason}");
-         }
-         return res;
-
-         async Task<LoginResult> impl() {
-            var auth_service = locate<AuthService>();
-            var result = await auth_service.authenticate_login_async(
-               new AuthenticateLoginRequest() {
-                  Login = username ?? string.Empty,
-                  Password = password ?? string.Empty
-               });
-            if (result.success) {
-               if (result.value.IsValid) {
-                  _token = result.value.Token;
-                  user = new(result.value.UserId, username);
-                  return new LoginResult(result.value.Reason, true);
-               } else {
-                  return new LoginResult(result.value.Reason);
-               }
-            } else {
-               return new LoginResult(result.error_message);
-            }
-         }
-      }
-
-      public void login(AuthTokenImmutable token) {
-         user = UserTuple.empty;
-         _token = token.CloneToMutable();
-      }
-
-      public void Dispose() {
-         if (_qbus != IntPtr.Zero) {
-            ApiNativeMethods.api_destroy_quickbus(_qbus);
-            _qbus = IntPtr.Zero;
-         }
-      }
-
-      internal ServerChannel publish<T>(T service, string service_type, string qualifier) where T : IApiServer {
-         var ch = new ServerChannel(this, service_type, qualifier ?? "");
-         ch.on_request += service.dispatch;
-         return ch;
-      }
-   }
-
-   public class LoginResult {
-      public LoginResult() { success = true; }
-      public LoginResult(string reason) { success = false; this.reason = reason; }
-      public LoginResult(string reason, bool success) { this.success = success; this.reason = reason; }
-      public bool success { get; }
-      public string? reason { get; }
-   }
-}
+#nullable enable
+namespace bts.api.core {
+   using bts.api.user;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Threading.Tasks;
+   using System.Reflection;
+
+   public sealed class ServiceContext : Spark.Api.IServiceProvider, IDisposable {
+      IntPtr _qbus;
+      AuthToken? _token;
+
+      public ServiceContext(string? config_source = null) {
+         BTS.init(config_source);
+         _qbus = ApiNativeMethods.api_create_quickbus(null, 0);
+      }
+
+      internal ServiceContext(IntPtr qbus) =>
+         _qbus = qbus;
+
+      public AuthTokenImmutable? token => _token?.immutable();
+      public UserTuple? user { get; private set; }
+      public bool is_authenticated => _token is not null;
+
+      internal IntPtr native_context => _qbus;
+
+      public static ServiceContext create(string? config_server = null) {
+         var args = Environment.GetCommandLineArgs();
+         BTS.init(config_server, args);
+         var qbus = ApiNativeMethods.api_create_quickbus(null, 0);
+         return new ServiceContext(qbus);
+      }
+
+      public T locate<T>(string qualifier = "") where T : class, IApiClient {
+         var ctr = typeof(T).GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance)
+            .FirstOrDefault(ci => ci.GetParameters()
+               .SingleOrDefault()?.ParameterType == typeof(ClientChannel));
+         if (ctr is null) {
+            throw new InvalidOperationException(
+               $"Unable to find appropriate constructor for type '{typeof(T)}'");
+         }
+         ClientChannel channel = new QuickbusClientChannel(this, typeof(T).Name, qualifier);
+         return (T)ctr.Invoke(new object[] { channel });
+      }
+
+      public T GetService<T>(string? qualifier) where T : class, IApiClient =>
+         locate<T>(qualifier ?? string.Empty);
+
+      public async Task<bool> try_login(string username, string password) {
+         var res = await login(username, password);
+         return res.success;
+      }
+
+      public async Task<LoginResult> login(string username, string password) {
+         var res = await impl();
+         if (!res.success) {
+            Log.Api.error($"Login failed: for username: {username} password: {password} reason: {res.reason}");
+         }
+         return res;
+
+         async Task<LoginResult> impl() {
+            var auth_service = locate<AuthService>();
+            var result = await auth_service.authenticate_login_async(
+               new AuthenticateLoginRequest() {
+                  Login = username ?? string.Empty,
+                  Password = password ?? string.Empty
+               });
+            if (result.success) {
+               if (result.value.IsValid) {
+                  _token = result.value.Token;
+                  user = new(result.value.UserId, username);
+                  return new LoginResult(result.value.Reason, true);
+               } else {
+                  return new LoginResult(result.value.Reason);
+               }
+            } else {
+               return new LoginResult(result.error_message);
+            }
+         }
+      }
+
+      public void login(AuthTokenImmutable token) {
+         user = UserTuple.empty;
+         _token = token.CloneToMutable();
+      }
+
+      public void Dispose() {
+         if (_qbus != IntPtr.Zero) {
+            ApiNativeMethods.api_destroy_quickbus(_qbus);
+            _qbus = IntPtr.Zero;
+         }
+      }
+
+      internal ServerChannel publish<T>(T service, string service_type, string qualifier) where T : IApiServer {
+         var ch = new ServerChannel(this, service_type, qualifier ?? "");
+         ch.on_request += service.dispatch;
+         return ch;
+      }
+   }
+
+   public class LoginResult {
+      public LoginResult() { success = true; }
+      public LoginResult(string reason) { success = false; this.reason = reason; }
+      public LoginResult(string reason, bool success) { this.success = success; this.reason = reason; }
+      public bool success { get; }
+      public string? reason { get; }
+   }
+}

src/client/spark_api/utils/Locator.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.core;
+
+namespace bts.api
+{
+   public static class Locator
+   {
+      readonly static Lazy<ServiceContext> _context = new Lazy<ServiceContext>(() => new ServiceContext(), true);      
+      public static T locate<T>(string qualifier="")  where T : class, IApiClient
+      {
+         return _context.Value.locate<T>(qualifier);
+      }
+      static public async Task<LoginResult> login(string username, string password)
+      {
+         return await _context.Value.login(username, password);
+      }
+      static public void login_with_token(AuthTokenImmutable token)
+      {
+         _context.Value.login(token);
+      }      
+      static public AuthTokenImmutable? token => _context.Value.token;
+   }
+}

src/client/spark_api_server/Program.cs

@@ -40,7 +40,7 @@
             DefaultEndPoint = "/",
             Host = "titan",
             Scheme = "http",
-            Port = 8080,
+            Port = "8080",
             KeepAliveIntervalSecs = 0.1,
          };
          sync.init(gwy);

src/client/spark_core/Json/JsonExtensions.cs

@@ -6,59 +6,14 @@ using Spark.Core.Disposables;
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Text.Json;
-using System.Text.Json.Nodes;
 
 public static class JsonExtensions {
-   public static T As<T>(this JsonNode node) =>
-      node.GetValue<T>();
-
-   public static JsonArray AsArrayOrEmpty(this JsonNode node) =>
-      node.GetValueKind() == JsonValueKind.Array ? node.AsArray() : [];
-
-   public static TEnum AsEnum<TEnum>(this JsonNode node) where TEnum : struct, Enum {
-      var str = node.AsString();
-      return Enum.Parse<TEnum>(str);
-   }
-
-   public static string AsString(this JsonNode node) {
-      if (!node.AsValue().TryGetValue(out string? str)) {
-         throw new Exception("Failed to get string from JsonNode");
-      }
-      return str;
-   }
-
-   public static string AsStringOrEmpty(this JsonNode node) =>
-      node.GetValueKind() == JsonValueKind.String ? node.AsString() : string.Empty;
-
-   public static JsonArray GetArrayOrEmpty(this JsonObject obj, string propertyName) =>
-      obj.TryGetArray(propertyName, out var res) ? res : [];
-
-   public static T GetValue<T>(this JsonObject obj, string propertyName) =>
-      obj[propertyName]!.GetValue<T>();
-
-   public static JsonObject GetObjectOrEmpty(this JsonObject obj, string propertyName) =>
-      obj.TryGetObject(propertyName, out var res) ? res : [];
-
-   public static string GetStringOrEmpty(this JsonObject obj, string propertyName) =>
-      obj.TryGetString(propertyName, out var res) ? res : string.Empty;
-
-   public static bool ReadArray(
-      this Utf8JsonReader reader, [NotNullWhen(true)] out JsonArray? array) {
-      array = JsonSerializer.Deserialize<JsonArray>(ref reader);
-      return array is not null;
-   }
-
    public static bool ReadObject(this Utf8JsonReader reader) {
       return reader.Read() && reader.TokenType == JsonTokenType.StartObject;
    }
 
-   public static bool ReadObject(this Utf8JsonReader reader, [NotNullWhen(true)] out JsonObject? obj) {
-      obj = JsonSerializer.Deserialize<JsonObject>(ref reader);
-      return obj is not null;
-   }
-
    public static bool ReadPropertyName(
-      this Utf8JsonReader reader, [NotNullWhen(true)] out string? propertyName) {
+      this Utf8JsonReader reader, [NotNullWhen(true)]out string? propertyName) {
       if (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) {
          propertyName = reader.GetString()!;
          return true;
@@ -67,41 +22,6 @@ public static class JsonExtensions {
       return false;
    }
 
-   public static bool ReadString(
-      this Utf8JsonReader reader, [NotNullWhen(true)] out string? str) {
-      str = JsonSerializer.Deserialize<string>(ref reader);
-      return str is not null;
-   }
-
-   public static bool TryGetArray(this JsonObject obj, string propertyName, [NotNullWhen(true)] out JsonArray? result) {
-      if (obj.TryGetPropertyValue(propertyName, out var v) && v?.GetValueKind() == JsonValueKind.Array) {
-         result = v.AsArray();
-         return true;
-      }
-      result = null;
-      return false;
-   }
-
-   public static bool TryGetObject(
-      this JsonObject obj, string propertyName, [NotNullWhen(true)] out JsonObject? result) {
-      if (obj.TryGetPropertyValue(propertyName, out var v) && v?.GetValueKind() == JsonValueKind.Object) {
-         result = v.AsObject();
-         return true;
-      }
-      result = null;
-      return false;
-   }
-
-   public static bool TryGetString(
-      this JsonObject obj, string propertyName, [NotNullWhen(true)] out string? result) {
-      if (obj.TryGetPropertyValue(propertyName, out var v) && v?.GetValueKind() == JsonValueKind.String) {
-         result = v.AsString();
-         return true;
-      }
-      result = null;
-      return false;
-   }
-
    public static IDisposable WriteArray(this Utf8JsonWriter writer) {
       writer.WriteStartArray();
       return Disposable.Create(writer.WriteEndArray);
@@ -112,11 +32,6 @@ public static class JsonExtensions {
       return Disposable.Create(writer.WriteEndArray);
    }
 
-   public static void Write<T>(this Utf8JsonWriter writer, string propertyName, T node) {
-      writer.WritePropertyName(propertyName);
-      JsonSerializer.Serialize(writer, node);
-   }
-
    public static IDisposable WriteObject(this Utf8JsonWriter writer) {
       writer.WriteStartObject();
       return Disposable.Create(writer.WriteEndObject);
@@ -124,11 +39,6 @@ public static class JsonExtensions {
 
    public static IDisposable WriteObject(this Utf8JsonWriter writer, string propertyName) {
       writer.WritePropertyName(propertyName);
-      return writer.WriteObject();
-   }
-
-   public static void WriteObject(this Utf8JsonWriter writer, string propertyName, JsonObject obj) {
-      writer.WritePropertyName(propertyName);
-      JsonSerializer.Serialize(writer, obj);
+      return Disposable.Create(writer.WriteEndObject);
    }
 }

src/client/spark_core/utils_api/BTS.cs

@@ -1,33 +1,76 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Threading;
-
-namespace bts {
-   public static class BTS {
-      static int _needs_init_spark_lib = 0;
-
-      public static void init(string config_source = null) {
-         Console.WriteLine(
-            $"Configuring BTS with {(config_source is null ? "defaults" : $"'{config_source}'")}");
-         var args = Environment.GetCommandLineArgs();
-         utils.BTSNativeMethods.provide_command_line(args, args.Length);
-
-         var buffer = new StringBuilder(4096);
-         bool config_ok = utils.BTSNativeMethods.initialize_nothrow(
-            config_source ?? string.Empty, buffer, buffer.Capacity);
-         if (!config_ok) {
-            throw new ApplicationException(buffer.ToString());
-         }
-
-         if (Interlocked.CompareExchange(ref _needs_init_spark_lib, 1, 0) == 0) {
-            utils.BTSNativeMethods.init_spark_lib();
-         }
-
-         // This seems like a good idea but it crashes due to thread shutdown order
-         // AppDomain.CurrentDomain.ProcessExit += (o, e) => {
-         //    utils.BTSNativeMethods.finit_spark_lib();
-         // };
-      }
-   }
-}
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+
+namespace bts
+{
+   public static class BTS
+   {
+      static int _needs_init_spark_lib = 0;
+
+      public static void init(string config_source = null) {
+         var args = Environment.GetCommandLineArgs();
+         init(config_source, args);
+      }
+
+      public static void init(string config_source = null, params string[] args) {
+         utils.BTSNativeMethods.provide_command_line(args, args.Length);
+
+         var buffer = new StringBuilder(4096);
+         bool config_ok = utils.BTSNativeMethods.initialize_nothrow(
+            config_source ?? string.Empty, buffer, buffer.Capacity);
+         if (!config_ok) {
+            throw new ApplicationException(buffer.ToString());
+         }
+
+         if (Interlocked.CompareExchange(ref _needs_init_spark_lib,1,0) == 0) {
+            utils.BTSNativeMethods.init_spark_lib();
+         }
+
+         // This seems like a good idea but it crashes due to thread shutdown order
+         // AppDomain.CurrentDomain.ProcessExit += (o, e) => {
+         //    utils.BTSNativeMethods.finit_spark_lib();
+         // };
+      }
+
+      /// <summary>
+      /// Adds or replaces the '-configserver' argument with <paramref name="config_server"/>
+      /// </summary>
+      /// <remarks>
+      /// TODO: Clean up the cli, or consider getting rid of it and simply using config files as
+      /// the exclusive means to pass arguments. The configserver override is particularly awkward
+      /// because it is processed independently in c++ (see ConfigurationState::settings_path).
+      /// </remarks>
+      public static IEnumerable<string> set_config_arg(
+         string[] args, string config_server) {
+         const string CONFIG_SERVER_OPTION = "-configserver";
+         bool has_emitted_cfg_server = false;
+         for (int i = 0; i < args.Length;) {
+            yield return args[i];
+            if (args[i] == CONFIG_SERVER_OPTION) {
+               has_emitted_cfg_server = true;
+               yield return config_server;
+               i += 2;
+               continue;
+            }
+            i++;
+         }
+         if (!has_emitted_cfg_server) {
+            yield return CONFIG_SERVER_OPTION;
+            yield return config_server;
+         }
+      }
+
+      internal static IEnumerable<string> remove_config_arg(string[] args) {
+         int skip = 0;
+         foreach (var arg in args) {
+            if (arg.ToLower() == "-configserver") {
+               skip = 2;
+            }
+            if (skip > 0) { skip -= 1; }
+            else { yield return arg; }
+         }
+      }
+   }
+}

src/client/spark_sdk/samples/OrderSample.cs

@@ -25,22 +25,21 @@ namespace bts.sdk
       readonly List<IDisposable> _disps = new List<IDisposable>();
       IReadOnlyDictionary<uint, SecurityImmutable> _securities =
          new Dictionary<uint, SecurityImmutable>(0);
-      readonly Lazy<OrderService> _orders;
-      readonly Lazy<AdminService> _admin;
 
       public OrderSample(
-         ServiceContext serviceContext,
          OrdersClient orders_client,
          MarketDataClient md_client,
          IObservable<IReadOnlyDictionary<uint, SecurityImmutable>> securities_src
       ) {
          _orders_client = orders_client;
          _md = md_client;
-         _orders = new(() => serviceContext.locate<OrderService>(), true);
-         _admin = new(() => serviceContext.locate<AdminService>(), true);
          _disps.Add(securities_src.Subscribe(secs => _securities = secs));
       }
 
+      readonly static Lazy<OrderService> _orders = new Lazy<OrderService>(() => Locator.locate<OrderService>(), true);
+      readonly static Lazy<AuthService> _users = new Lazy<AuthService>(() => Locator.locate<AuthService>(), true);
+      readonly static Lazy<AdminService> _admin = new Lazy<AdminService>(() => Locator.locate<AdminService>(), true);
+
       void print_update<T>(T update) 
       {
          Console.WriteLine(update.ToString());
@@ -95,10 +94,7 @@ namespace bts.sdk
          return resp;
       }
 
-      void order_setup(Create cr, string username, string accountname) =>
-         order_setup(cr, username, accountname, _admin.Value.get_admin_data(new VoidMessage()));
-
-      static void order_setup(Create cr, string username, string accountname, AdminData admin)
+      static void order_setup(Create cr, string username, string accountname)
       {
          cr.NewOrder = new OrderDefinition();
          cr.NewOrder.ManualOrderIndicator = ManualOrderIndicator.Yes;
@@ -106,6 +102,7 @@ namespace bts.sdk
          cr.NewOrder.CustomerOrFirm = CustomerOrFirm.Firm;
          cr.NewOrder.SubType = OrderSubType.None;
          cr.NewOrder.TimeInForce = TimeInForce.Day;
+         var admin = _admin.Value.get_admin_data(new VoidMessage());
          BTSUser user = admin.Users.FirstOrDefault( x=> username != null ? x.Name.ToLowerInvariant() == username.ToLowerInvariant() : true );
          if (user != null)
          {

src/client/spark_sdk/samples/SDKMain.cs

@@ -1,168 +1,167 @@
-namespace bts.sdk {
-   using bts.api.admin;
-   using bts.api.core;
-   using bts.api.orderman;
-   using bts.api.position;
-   using bts.co2;
-   using bts.md;
-
-   using Spark.Api.Admin;
-   using Spark.Api.Orders;
-   using Spark.Api.Reactive;
-
-   using System;
-   using System.Linq;
-   using System.Reactive.Disposables;
-   using System.Reactive.Linq;
-
-   public class SDKMain
-   {
-      private static void user_test(ServiceContext ctx)
-      {   
-         var s = ctx.locate<api.user.AuthService>();
-         var r = new api.user.GetUsersRequest();
-         var resp = s.get_users(r);
-         Console.WriteLine("{0}", resp);
-      }      
-      public static void Main(string[] args)
-      {         
-         BTS.init();
-         Log.Sdk.info("Welcome to the BTS Client Side SDK example");
-         var ctx = new ServiceContext();
-
-         string u = args.FirstOrDefault(x => x.Contains("--user="));
-         string p = args.FirstOrDefault(x => x.Contains("--pass="));
-         u = u?.Replace("--user=", string.Empty) ?? string.Empty;
-         p = p?.Replace("--pass=", string.Empty) ?? string.Empty;
-
-         var lr = ctx.login(u,p).Result;
-         if (!lr.success)
-         {
-            Console.WriteLine("Login failure: " + lr.reason);
-            help();
-         }
-
-         if (args.FirstOrDefault() == "users")
-         {
-            user_test(ctx);
-         }
-         else
-         {
-            uint ms = 100;
-            string pu = args.FirstOrDefault(x => x.Contains("--pump="));
-            if (pu != null)
-               ms = uint.Parse(p.Replace("--pump=", string.Empty));
-
-            var order_service = ctx.locate<OrderService>();
-            var position_service = ctx.locate<PositionService>();
-            var admin_service = ctx.locate<AdminService>();
-            var md_relay = ctx.locate<MarketDataRelay>();
-            var md_source = new MDSource(TimeSpan.FromSeconds(0.1));
-            var md_client = new MarketDataClient(md_source);
-            var admin_client = new AdminClient(admin_service, UserTuple.admin);
-            var sec_resolver = new SecurityResolver(md_relay, md_source.subscribe_securities());
-            var orders_client = new OrdersClient(
-               order_service, position_service, admin_client, sec_resolver);
-
-            using(var disp = new CompositeDisposable(md_source)) {
-               if (args.Length > 0 && args.FirstOrDefault().Contains("md"))
-               {
-                  var s = new MarketDataSample(md_client);
-                  if (args.FirstOrDefault() == "md")
-                  {
-                     if (args.Length >= 5)
-                     {
-                        s.print_security(args[1], args[2], args[3], args[4]);
-                     }
-                     else if (args.Length >= 4)
-                     {
-                        s.print_product(args[1], args[2], args[3]);
-                     }
-                     else
-                        md_help();
-                  }
-                  else
-                     md_help();
-               }
-               else if (args.FirstOrDefault() == "order")
-               {
-                  var os = new OrderSample(
-                     ctx,
-                     orders_client,
-                     md_client,
-                     md_source.subscribe_securities().Select(sec_map =>
-                        sec_map.ToDictionary(kv => kv.Key, kv => kv.Value.security)));
-                  disp.Add(os);
-                  order_sample(os, u, args);
-               }
-               else
-                  help();
-
-               wait_for_stop();
-            }
-         }
-      }
-
-      static void order_sample(OrderSample os, string user, string[] args)
-      {
-         if (args.Length >= 10) //--user and --pass are in args[]
-         {
-            os.send_order(user, args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
-         }         
-         else if (args.Length >= 9)
-         {
-            os.send_order(user, args[1], args[2], args[3], args[4], args[5], args[6]);
-         }
-         else
-            order_help();         
-      }
-      
-      static void wait_for_stop()
-      {
-         Console.WriteLine("Hit q to stop");
-         ConsoleKeyInfo c;
-         do
-         {
-            c = Console.ReadKey();
-            Console.WriteLine();
-         } while (c.KeyChar != 'q');
-      }
-
-      static void help()
-      {
-         Console.WriteLine("BTS Spark SDK");
-         Console.WriteLine("Usages: spark_sdk [command] [arguments]");
-         Console.WriteLine();
-         Console.WriteLine("Commands:");
-         Console.WriteLine("   md <exchange> <security_type> <symbol> [<security_desc>]");
-         Console.WriteLine("   order <account> <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>]");
-         Console.WriteLine("Arguments:");
-         Console.WriteLine("   --user=<bts user name>");
-         Console.WriteLine("   --pass=<password>");
-         Console.WriteLine("   --pump=<pump_interval_in_ms=100 default>");
-         Environment.Exit(1);
-      }
-
-      static void order_help()
-      {
-         Console.WriteLine("order <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>] [<user>] [<account>]");
-         Console.WriteLine("This command sends a limit or market DAY order for the specified security_desc.  Leave off the price for a market order");
-         Console.WriteLine("   example: order \"ACC A1\" GLBX Future ES ESM8 10 2650.0");
-         Console.WriteLine("            order \"ACC A1\" SIM Future ES SIM:ESM8 -10 2650.0");
-         Console.WriteLine("            order \"ACC A1\" GLBX Future ES ESM8 10 (this is a market order)");
-         Environment.Exit(1);
-      }      
-
-      static void md_help()
-      {
-         Console.WriteLine("md <exchange> <security_type> <symbol>");
-         Console.WriteLine("This command subscribes to market data for an exchange-type-symbol channel and writes to console.");
-         Console.WriteLine("Marketdata is read off the wire on the same thread as is used to read and print");
-         Console.WriteLine("   example: md GLBX Future ES --user=bts --pass=password");
-         Console.WriteLine("            md SIM Future ES SIM:ESZ8 --user=bts --pass=password");
-         Console.WriteLine("            md GLBX Spread ES --user=bts --pass=password");
-         Environment.Exit(1);
-      }
-
-   }
-}
+namespace bts.sdk {
+   using bts.api.admin;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.api.position;
+   using bts.co2;
+   using bts.md;
+
+   using Spark.Api.Admin;
+   using Spark.Api.Orders;
+   using Spark.Api.Reactive;
+
+   using System;
+   using System.Linq;
+   using System.Reactive.Disposables;
+   using System.Reactive.Linq;
+
+   public class SDKMain
+   {
+      private static void user_test(ServiceContext ctx)
+      {   
+         var s = ctx.locate<api.user.AuthService>();
+         var r = new api.user.GetUsersRequest();
+         var resp = s.get_users(r);
+         Console.WriteLine("{0}", resp);
+      }      
+      public static void Main(string[] args)
+      {         
+         BTS.init();
+         Log.Sdk.info("Welcome to the BTS Client Side SDK example");
+         var ctx = new ServiceContext();
+
+         string u = args.FirstOrDefault(x => x.Contains("--user="));
+         string p = args.FirstOrDefault(x => x.Contains("--pass="));
+         u = u?.Replace("--user=", string.Empty) ?? string.Empty;
+         p = p?.Replace("--pass=", string.Empty) ?? string.Empty;
+
+         var lr = ctx.login(u,p).Result;
+         if (!lr.success)
+         {
+            Console.WriteLine("Login failure: " + lr.reason);
+            help();
+         }
+
+         if (args.FirstOrDefault() == "users")
+         {
+            user_test(ctx);
+         }
+         else
+         {
+            uint ms = 100;
+            string pu = args.FirstOrDefault(x => x.Contains("--pump="));
+            if (pu != null)
+               ms = uint.Parse(p.Replace("--pump=", string.Empty));
+
+            var order_service = ctx.locate<OrderService>();
+            var position_service = ctx.locate<PositionService>();
+            var admin_service = ctx.locate<AdminService>();
+            var md_relay = ctx.locate<MarketDataRelay>();
+            var md_source = new MDSource(TimeSpan.FromSeconds(0.1));
+            var md_client = new MarketDataClient(md_source);
+            var admin_client = new AdminClient(admin_service, UserTuple.admin);
+            var sec_resolver = new SecurityResolver(md_relay, md_source.subscribe_securities());
+            var orders_client = new OrdersClient(
+               order_service, position_service, admin_client, sec_resolver);
+
+            using(var disp = new CompositeDisposable(md_source)) {
+               if (args.Length > 0 && args.FirstOrDefault().Contains("md"))
+               {
+                  var s = new MarketDataSample(md_client);
+                  if (args.FirstOrDefault() == "md")
+                  {
+                     if (args.Length >= 5)
+                     {
+                        s.print_security(args[1], args[2], args[3], args[4]);
+                     }
+                     else if (args.Length >= 4)
+                     {
+                        s.print_product(args[1], args[2], args[3]);
+                     }
+                     else
+                        md_help();
+                  }
+                  else
+                     md_help();
+               }
+               else if (args.FirstOrDefault() == "order")
+               {
+                  var os = new OrderSample(
+                     orders_client,
+                     md_client,
+                     md_source.subscribe_securities().Select(sec_map =>
+                        sec_map.ToDictionary(kv => kv.Key, kv => kv.Value.security)));
+                  disp.Add(os);
+                  order_sample(os, u, args);
+               }
+               else
+                  help();
+
+               wait_for_stop();
+            }
+         }
+      }
+
+      static void order_sample(OrderSample os, string user, string[] args)
+      {
+         if (args.Length >= 10) //--user and --pass are in args[]
+         {
+            os.send_order(user, args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
+         }         
+         else if (args.Length >= 9)
+         {
+            os.send_order(user, args[1], args[2], args[3], args[4], args[5], args[6]);
+         }
+         else
+            order_help();         
+      }
+      
+      static void wait_for_stop()
+      {
+         Console.WriteLine("Hit q to stop");
+         ConsoleKeyInfo c;
+         do
+         {
+            c = Console.ReadKey();
+            Console.WriteLine();
+         } while (c.KeyChar != 'q');
+      }
+
+      static void help()
+      {
+         Console.WriteLine("BTS Spark SDK");
+         Console.WriteLine("Usages: spark_sdk [command] [arguments]");
+         Console.WriteLine();
+         Console.WriteLine("Commands:");
+         Console.WriteLine("   md <exchange> <security_type> <symbol> [<security_desc>]");
+         Console.WriteLine("   order <account> <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>]");
+         Console.WriteLine("Arguments:");
+         Console.WriteLine("   --user=<bts user name>");
+         Console.WriteLine("   --pass=<password>");
+         Console.WriteLine("   --pump=<pump_interval_in_ms=100 default>");
+         Environment.Exit(1);
+      }
+
+      static void order_help()
+      {
+         Console.WriteLine("order <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>] [<user>] [<account>]");
+         Console.WriteLine("This command sends a limit or market DAY order for the specified security_desc.  Leave off the price for a market order");
+         Console.WriteLine("   example: order \"ACC A1\" GLBX Future ES ESM8 10 2650.0");
+         Console.WriteLine("            order \"ACC A1\" SIM Future ES SIM:ESM8 -10 2650.0");
+         Console.WriteLine("            order \"ACC A1\" GLBX Future ES ESM8 10 (this is a market order)");
+         Environment.Exit(1);
+      }      
+
+      static void md_help()
+      {
+         Console.WriteLine("md <exchange> <security_type> <symbol>");
+         Console.WriteLine("This command subscribes to market data for an exchange-type-symbol channel and writes to console.");
+         Console.WriteLine("Marketdata is read off the wire on the same thread as is used to read and print");
+         Console.WriteLine("   example: md GLBX Future ES --user=bts --pass=password");
+         Console.WriteLine("            md SIM Future ES SIM:ESZ8 --user=bts --pass=password");
+         Console.WriteLine("            md GLBX Spread ES --user=bts --pass=password");
+         Environment.Exit(1);
+      }
+
+   }
+}

src/client/spark_websocket/Core/WebsocketConnectionGateway.cs

@@ -120,7 +120,7 @@ namespace bts.websocket_core {
          /// <returns>The connected WebSocket or null if it was not found or not open.</returns>
          private WebSocket? GetConnectedWebSocket(WebsocketClientID id, string error_message) {
             var socket = Connections.GetSocket(id);
-            if (socket?.IsOpen() ?? false) {
+            if (socket?.IsSocketStateOpen() ?? false) {
                return socket;
             }
             Log.Gen.warn(
@@ -143,7 +143,7 @@ namespace bts.websocket_core {
             if (socket is null) {
                return false;
             }
-            if (socket.IsOpen()) {
+            if (socket.IsSocketStateOpen()) {
                return true;
             }
             if (!socket.IsClosedProperly()) {
@@ -213,7 +213,7 @@ namespace bts.websocket_core {
             if (socket == null || que == null) {
                return;
             }
-            while (socket.IsOpen()) {
+            while (IsSocketStateOpen(socket)) {
                // Try to dequeue a message from the queue
                if (!que.TryDequeue(out var message)) {
                   // If the queue is empty, wait for 100 milliseconds before trying again
@@ -272,7 +272,7 @@ namespace bts.websocket_core {
                ArraySegment<byte> buffer = new ArraySegment<byte>(new byte[256]);
                WebSocketReceiveResult result;
                string request = "";
-               while (socket.IsOpen()) {
+               while (IsSocketStateOpen(socket)) {
                   // Receive the message
                   result = await socket.ReceiveAsync(buffer, CancellationToken.None);
                   // If the message is a close request, handle the disconnection and return

src/client/spark_websocket/Core/WebsocketDataSync.cs

@@ -1,367 +1,363 @@
-namespace bts.websocket_core {
-   using bts.api;
-   using bts.md;
-
-   using mbts;
-   using Newtonsoft.Json;
-   using System.Collections.Concurrent;
-   using bts.api.core;
-   using bts.api.orderman;
-   using bts.api.admin;
-   using bts.co2;
-   using Spark.Api.Reactive;
-   using Spark.Api.Orders;
-   using Spark.Api.Admin;
-   using Spark.Api.MarketData;
-   using bts.api.position;
-   using bts.api.synthetic;
-
-
-   /// <summary>
-   /// This does all the bookkeeping for connections and data, and passes
-   /// it over to the business layer CURRENT VERSION IS JUST TO SHOW HOW TO USE GATEWAY
-   /// </summary>
-   public class WebsocketDataSync : IWebsocketDataSync {
-      readonly AdminService adminService;
-      readonly ServiceContext _service_ctx;
-
-      public WebsocketDataSync(ServiceContext service_ctx) {
-         _service_ctx = service_ctx;
-         adminService = service_ctx.locate<AdminService>();
-      }
-
-      /// <summary>
-      /// class to manage thread safe client store, will change this to blocking collection
-      /// </summary>
-      class ClientsStore {
-         private ConcurrentDictionary<WebsocketClientID, ClientState> Clients { get; } = new();
-         /// <summary>
-         /// adds client to dictionary is thread safe
-         /// </summary>
-         /// <param name="client_id"></param>
-         public void AddClient(WebsocketClientID client_id) {
-            if (Clients.TryAdd(client_id, new ClientState())) {
-               return;
-            }
-            var log_txt = $"ClientsStore.AddClient client_id: {client_id.ToString()} Already exits";
-            bts.Log.Gen.warn(log_txt);
-            Clients[client_id] = new ClientState();
-         }
-         /// <summary>
-         /// removes client from dictionary is thread safe
-         /// </summary>
-         /// <param name="client_id"></param>
-         public void RemoveClient(WebsocketClientID client_id) {
-            if (Clients.TryRemove(client_id, out ClientState state)) {
-               state.Dispose();
-            }
-         }
-         public bool ContainsKey(WebsocketClientID client_id) => Clients.ContainsKey(client_id);
-         public IEnumerator<KeyValuePair<WebsocketClientID, ClientState>> GetEnumerator() => Clients.GetEnumerator();
-         public ClientState this[WebsocketClientID clientId] {
-            get => Clients.TryGetValue(
-               clientId,
-               out var clientState) ?
-            clientState :
-            throw new ArgumentException($"{nameof(clientId)} not found in the Clients dictionary");
-
-            set => Clients.AddOrUpdate(
-               clientId,
-               value,
-               (key, old_value) => value);
-         }
-         public int Count() => Clients.Count;
-      }
-      /// <summary>
-      /// Immutable to store the state of the client
-      /// </summary>
-      public readonly struct ClientState : IDisposable {
-
-         private readonly Dictionary<IUniqueID, IWebsocketRequestHandler?> _requests = new();
-         public ClientState() {
-
-         }
-         public ClientState(Dictionary<IUniqueID, IWebsocketRequestHandler?> requests) {
-            _requests = new Dictionary<IUniqueID, IWebsocketRequestHandler?>(requests);
-         }
-         public ClientState Add(IUniqueID id, IWebsocketRequestHandler? dis) {
-            var newIds = new Dictionary<IUniqueID, IWebsocketRequestHandler?>(_requests);
-            newIds.TryAdd(id, dis);
-            return new ClientState(newIds);
-         }
-
-         public void Dispose() {
-            foreach (var dis in _requests.Values) {
-               if (dis != null) {
-                  dis.Dispose();
-               }
-            }
-         }
-         public bool ContainsKey(IUniqueID id) => _requests.ContainsKey(id);
-         public ClientState Remove(IUniqueID id) {
-            var newIds = new Dictionary<IUniqueID, IWebsocketRequestHandler?>(_requests);
-            newIds.TryPopValue(id, out IWebsocketRequestHandler? dis);
-            if (dis != null) {
-               dis.Dispose();
-            }
-            return new ClientState(newIds);
-         }
-         public ClientState RemoveAll() {
-            Dispose();
-            return new ClientState();
-         }
-      }
-      // ServiceContext? service_ctx;
-      // ServiceContext Service_Ctx{
-      //    get => Log.notnull<ServiceContext>(service_ctx);
-      //    set {
-      //       Log.isnull(service_ctx);
-      //       service_ctx = value;
-      //    }
-      // }
-
-      MarketDataClient? mdClient;
-      MarketDataClient Md_Client {
-         get => Log.notnull<MarketDataClient>(mdClient);
-         set {
-            Log.isnull(mdClient);
-            mdClient = value;
-         }
-      }
-      MDSource? _md_source;
-      MDSource MD_Source {
-         get => Log.notnull<MDSource>(_md_source);
-         set {
-            Log.isnull(_md_source);
-            _md_source = value;
-         }
-      }
-
-      SyntheticSecurityClient? _synth_sec_client;
-      SyntheticSecurityClient Synth_Sec_Client {
-         get => Log.notnull(_synth_sec_client);
-         set {
-            Log.isnull(_synth_sec_client);
-            _synth_sec_client = value;
-         }
-      }
-
-      OrdersClient? _orders_client;
-      OrdersClient Orders_Client {
-         get => Log.notnull<OrdersClient>(_orders_client);
-         set {
-            Log.isnull<OrdersClient>(_orders_client);
-            _orders_client = value;
-         }
-      }
-      WebsocketConnectionGateway? gateway;
-      WebsocketConnectionGateway Gateway {
-         get => Log.notnull<WebsocketConnectionGateway>(gateway);
-         set {
-            Log.isnull(gateway);
-            gateway = value;
-         }
-      }
-      /// <summary>
-      /// stores the current connected clients, can be converted the concurrent dictionary
-      /// </summary>
-      private ClientsStore? clients = null;
-      private ClientsStore Clients {
-         get => Log.notnull<ClientsStore>(clients);
-         set {
-            Log.isnull(clients);
-            clients = value;
-         }
-      }
-
-      IReadOnlyDictionary<uint, SecurityMD> _securities = new Dictionary<uint, SecurityMD>();
-      IDisposable? _securities_sub;
-
-      /// <summary>
-      /// Should be called in the start, stores pointer for gtwy and creates dictionary for clients, starts broadcast thread
-      /// </summary>
-      /// <param name="gtwy"></param>
-      public void init(WebsocketConnectionGateway gtwy) {
-         Gateway = gtwy;
-         clients = new ClientsStore();
-         TimeSpan md_throttle = new TimeSpan(10000);
-         MD_Source = new MDSource(md_throttle);
-         Md_Client = new(MD_Source);
-         _securities_sub = MD_Source.subscribe_securities().Subscribe(sec_map => _securities = sec_map);
-         var order_service = _service_ctx.locate<OrderService>();
-         var position_service = _service_ctx.locate<PositionService>();
-         var admin_service = _service_ctx.locate<AdminService>();
-         var synth_sec_service = _service_ctx.locate<SyntheticSecurityService>();
-         var admin_client = new AdminClient(admin_service, UserTuple.admin);
-         var md_relay = _service_ctx.locate<MarketDataRelay>();
-         var sec_resolver = new SecurityResolver(md_relay, MD_Source.subscribe_securities());
-         Orders_Client = new OrdersClient(
-            order_service, position_service, admin_client, sec_resolver, TimeSpan.FromSeconds(0.1));
-         var sec_table_src = MD_Source.subscribe_securities()
-            .Table(s => s.security.Id)
-            .Select(s => s.security, s => s.Id);
-         Synth_Sec_Client = new(synth_sec_service, sec_table_src);
-
-
-
-         // broadcast.Start();
-      }
-      private uint get_security(Venue v, string sec_name) {
-         var securites = _securities;
-         if (securites.Values.TryGetFirst(sc_md =>
-            (
-               sc_md.security.Exchange == v &&
-               sc_md.security.ToString() == sec_name),
-            out SecurityMD sc_md
-            )) {
-            return sc_md.id;
-         }
-         throw new ArgumentException($"{sec_name} at {v} not found in securites");
-      }
-      public void handle_connection(WebsocketClientID client_id,
-                                 WebsocketIncomingMessage? optional_message) {
-         Clients.AddClient(client_id);
-         var log_txt = $"WebsocketDataSync.handle_connection client_id: {client_id} " +
-                       $"message: {(optional_message == null ? "null" : optional_message.ToString())}";
-         bts.Log.Gen.info(log_txt);
-         System.Console.WriteLine(log_txt);
-         if (optional_message != null) {
-            handle_client_message(optional_message);
-         }
-      }
-      // Called when a client is disconnected
-      public void handle_disconnection(WebsocketClientID client_id) {
-         handle_unsubscribe_all(client_id);
-         Clients.RemoveClient(client_id);
-         var log_txt = $"WebsocketDataSync.handle_disconnection client_id: {client_id}";
-         bts.Log.Gen.info(log_txt);
-         System.Console.WriteLine(log_txt);
-      }
-      // Called when a client sends data 
-      public void handle_client_message(WebsocketIncomingMessage message) {
-         WebsocketRequest? req;
-         string error;
-         if (!WebsocketRequest.TryGetWebSocketRequest(message.sender_client_id, message.message_body, out IUniqueID id, out req, out error)) {
-            handle_incorrect_request(id, message.sender_client_id, error);
-            return;
-         }
-         handle_request(req);
-         var log_txt = $"WebsocketDataSync.handle_client_message message: {req.type}";
-         bts.Log.Gen.info(log_txt);
-         System.Console.WriteLine(log_txt);
-
-      }
-      public void handle_request(WebsocketRequest req) {
-         switch (req.type) {
-            case WebsocketRequest.REQUEST_TYPES.SUBSCRIBE: {
-                  handle_subscription(req);
-                  break;
-               }
-            case WebsocketRequest.REQUEST_TYPES.UNSUBSCRIBE_ALL: {
-                  handle_unsubscribe_all(req.client_id);
-                  break;
-               }
-            case WebsocketRequest.REQUEST_TYPES.ENTER_ORDER: {
-                  handle_enter_order(req);
-                  break;
-               }
-            case WebsocketRequest.REQUEST_TYPES.DELETE_ORDER: {
-                  handle_delete_order(req);
-                  break;
-               }
-            case WebsocketRequest.REQUEST_TYPES.UPDATE_ORDER: {
-                  handle_update_order(req);
-                  break;
-               }
-            case WebsocketRequest.REQUEST_TYPES.UNKOWN: {
-                  ///send error
-                  break;
-               }
-            default: break;
-         }
-      }
-      public void handle_unsubscribe_all(WebsocketClientID client_id) {
-         try {
-            Clients[client_id] = Clients[client_id].RemoveAll();
-         } catch (ArgumentException ex) {
-            System.Console.WriteLine("error for req", client_id, ex.ToString());
-         }
-      }
-      public async void handle_delete_order(WebsocketRequest req) {
-         (bool suc, string status) = await WebsocketRestHandler.DeleteOrderAsync(req, Orders_Client);
-         if (!suc) {
-            handle_incorrect_request(req, status);
-            return;
-         }
-         handle_correct_request(req.id, status, null, req.client_id);
-      }
-      public async void handle_enter_order(WebsocketRequest req) {
-         (bool suc, uint? id, string status) = await WebsocketRestHandler.EnterOrderAsync(
-            req, Orders_Client, _securities, adminService);
-         if (!suc) {
-            handle_incorrect_request(req, status);
-            return;
-         }
-         handle_correct_request(req.id, status, new { order_key = id }, req.client_id);
-      }
-      public async void handle_update_order(WebsocketRequest req) {
-         (bool suc, string status) = await WebsocketRestHandler.UpdateOrderAsync(req, Orders_Client);
-         if (!suc) {
-            handle_incorrect_request(req, status);
-            return;
-         }
-         handle_correct_request(req.id, status, null, req.client_id);
-      }
-      public void handle_subscription(WebsocketRequest req) {
-         WebsocketSubscriptionHandler? hdlr;
-         string status;
-         if (!WebsocketSubscriptionHandler.TryCreate(
-            req,
-            Md_Client,
-            Orders_Client,
-            obj => send_message(obj, req.client_id),
-            out hdlr,
-            out status)) {
-            handle_incorrect_request(req, status);
-            return;
-         }
-         if (Clients.ContainsKey(req.client_id) && Clients[req.client_id].ContainsKey(hdlr.ID)) {
-            handle_incorrect_request(req, "subscription id already in use");
-            return;
-         }
-         handle_correct_request(req.id, status, new { subscription_id = hdlr.ID.ID }, req.client_id);
-         hdlr.subscribe();
-         Clients[req.client_id] = Clients[req.client_id].Add(hdlr.ID, hdlr);
-      }
-      public void handle_incorrect_request(WebsocketRequest req, string message = "") => handle_incorrect_request(req.id, req.client_id, message);
-      public void handle_incorrect_request(IUniqueID rid, WebsocketClientID cid, string message = "") {
-         System.Console.WriteLine($"request {rid} from {cid} failed, message: {message}");
-         var ms = new {
-            jsonrpc = 2.0,
-            id = rid.ID,
-            error = message
-         };
-         send_message(ms, cid);
-      }
-
-      public void handle_correct_request(IUniqueID rid, string status, Object? data, WebsocketClientID cid) {
-         System.Console.WriteLine($"request {rid} from {cid} suceeded, message: {status}");
-         var ms = new {
-            jsonrpc = 2.0,
-            id = rid.ID,
-            status = status,
-            result = data
-         };
-         send_message(ms, cid);
-      }
-      public void send_message(object value, WebsocketClientID id) {
-         var requestJson = JsonConvert.SerializeObject(value);
-         Gateway.send_to_clients(
-            new List<WebsocketOutgoingMessage>() {
-                new WebsocketOutgoingMessage(id, requestJson)
-                });
-
-      }
-
-   }
-}
+namespace bts.websocket_core {
+   using bts.api;
+   using bts.md;
+
+   using mbts;
+   using Newtonsoft.Json;
+   using System.Collections.Concurrent;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.api.admin;
+   using bts.co2;
+   using Spark.Api.Reactive;
+   using Spark.Api.Orders;
+   using Spark.Api.Admin;
+   using Spark.Api.MarketData;
+   using bts.api.position;
+   using bts.api.synthetic;
+
+
+   /// <summary>
+   /// This does all the bookkeeping for connections and data, and passes
+   /// it over to the business layer CURRENT VERSION IS JUST TO SHOW HOW TO USE GATEWAY
+   /// </summary>
+   public class WebsocketDataSync : IWebsocketDataSync {
+      readonly ServiceContext _service_ctx;
+
+      public WebsocketDataSync(ServiceContext service_ctx) =>
+         _service_ctx = service_ctx;
+
+      /// <summary>
+      /// class to manage thread safe client store, will change this to blocking collection
+      /// </summary>
+      class ClientsStore {
+         private ConcurrentDictionary<WebsocketClientID, ClientState> Clients { get; } = new();
+         /// <summary>
+         /// adds client to dictionary is thread safe
+         /// </summary>
+         /// <param name="client_id"></param>
+         public void AddClient(WebsocketClientID client_id) {
+            if (Clients.TryAdd(client_id, new ClientState())) {
+               return;
+            }
+            var log_txt = $"ClientsStore.AddClient client_id: {client_id.ToString()} Already exits";
+            bts.Log.Gen.warn(log_txt);
+            Clients[client_id] = new ClientState();
+         }
+         /// <summary>
+         /// removes client from dictionary is thread safe
+         /// </summary>
+         /// <param name="client_id"></param>
+         public void RemoveClient(WebsocketClientID client_id) {
+            if (Clients.TryRemove(client_id, out ClientState state)) {
+               state.Dispose();
+            }
+         }
+         public bool ContainsKey(WebsocketClientID client_id) => Clients.ContainsKey(client_id);
+         public IEnumerator<KeyValuePair<WebsocketClientID, ClientState>> GetEnumerator() => Clients.GetEnumerator();
+         public ClientState this[WebsocketClientID clientId] {
+            get => Clients.TryGetValue(
+               clientId,
+               out var clientState) ?
+            clientState :
+            throw new ArgumentException($"{nameof(clientId)} not found in the Clients dictionary");
+
+            set => Clients.AddOrUpdate(
+               clientId,
+               value,
+               (key, old_value) => value);
+         }
+         public int Count() => Clients.Count;
+      }
+      /// <summary>
+      /// Immutable to store the state of the client
+      /// </summary>
+      public readonly struct ClientState : IDisposable {
+
+         private readonly Dictionary<IUniqueID, IWebsocketRequestHandler?> _requests = new();
+         public ClientState() {
+
+         }
+         public ClientState(Dictionary<IUniqueID, IWebsocketRequestHandler?> requests) {
+            _requests = new Dictionary<IUniqueID, IWebsocketRequestHandler?>(requests);
+         }
+         public ClientState Add(IUniqueID id, IWebsocketRequestHandler? dis) {
+            var newIds = new Dictionary<IUniqueID, IWebsocketRequestHandler?>(_requests);
+            newIds.TryAdd(id, dis);
+            return new ClientState(newIds);
+         }
+
+         public void Dispose() {
+            foreach (var dis in _requests.Values) {
+               if (dis != null) {
+                  dis.Dispose();
+               }
+            }
+         }
+         public bool ContainsKey(IUniqueID id) => _requests.ContainsKey(id);
+         public ClientState Remove(IUniqueID id) {
+            var newIds = new Dictionary<IUniqueID, IWebsocketRequestHandler?>(_requests);
+            newIds.TryPopValue(id, out IWebsocketRequestHandler? dis);
+            if (dis != null) {
+               dis.Dispose();
+            }
+            return new ClientState(newIds);
+         }
+         public ClientState RemoveAll() {
+            Dispose();
+            return new ClientState();
+         }
+      }
+      // ServiceContext? service_ctx;
+      // ServiceContext Service_Ctx{
+      //    get => Log.notnull<ServiceContext>(service_ctx);
+      //    set {
+      //       Log.isnull(service_ctx);
+      //       service_ctx = value;
+      //    }
+      // }
+
+      MarketDataClient? mdClient;
+      MarketDataClient Md_Client {
+         get => Log.notnull<MarketDataClient>(mdClient);
+         set {
+            Log.isnull(mdClient);
+            mdClient = value;
+         }
+      }
+      MDSource? _md_source;
+      MDSource MD_Source {
+         get => Log.notnull<MDSource>(_md_source);
+         set {
+            Log.isnull(_md_source);
+            _md_source = value;
+         }
+      }
+
+      SyntheticSecurityClient? _synth_sec_client;
+      SyntheticSecurityClient Synth_Sec_Client {
+         get => Log.notnull(_synth_sec_client);
+         set {
+            Log.isnull(_synth_sec_client);
+            _synth_sec_client = value;
+         }
+      }
+
+      OrdersClient? _orders_client;
+      OrdersClient Orders_Client {
+         get => Log.notnull<OrdersClient>(_orders_client);
+         set {
+            Log.isnull<OrdersClient>(_orders_client);
+            _orders_client = value;
+         }
+      }
+      WebsocketConnectionGateway? gateway;
+      WebsocketConnectionGateway Gateway {
+         get => Log.notnull<WebsocketConnectionGateway>(gateway);
+         set {
+            Log.isnull(gateway);
+            gateway = value;
+         }
+      }
+      /// <summary>
+      /// stores the current connected clients, can be converted the concurrent dictionary
+      /// </summary>
+      private ClientsStore? clients = null;
+      private ClientsStore Clients {
+         get => Log.notnull<ClientsStore>(clients);
+         set {
+            Log.isnull(clients);
+            clients = value;
+         }
+      }
+
+      IReadOnlyDictionary<uint, SecurityMD> _securities = new Dictionary<uint, SecurityMD>();
+      IDisposable? _securities_sub;
+
+      /// <summary>
+      /// Should be called in the start, stores pointer for gtwy and creates dictionary for clients, starts broadcast thread
+      /// </summary>
+      /// <param name="gtwy"></param>
+      public void init(WebsocketConnectionGateway gtwy) {
+         Gateway = gtwy;
+         clients = new ClientsStore();
+         TimeSpan md_throttle = new TimeSpan(10000);
+         MD_Source = new MDSource(md_throttle);
+         Md_Client = new(MD_Source);
+         _securities_sub = MD_Source.subscribe_securities().Subscribe(sec_map => _securities = sec_map);
+         var order_service = _service_ctx.locate<OrderService>();
+         var position_service = _service_ctx.locate<PositionService>();
+         var admin_service = _service_ctx.locate<AdminService>();
+         var synth_sec_service = _service_ctx.locate<SyntheticSecurityService>();
+         var admin_client = new AdminClient(admin_service, UserTuple.admin);
+         var md_relay = _service_ctx.locate<MarketDataRelay>();
+         var sec_resolver = new SecurityResolver(md_relay, MD_Source.subscribe_securities());
+         Orders_Client = new OrdersClient(
+            order_service, position_service, admin_client, sec_resolver, TimeSpan.FromSeconds(0.1));
+         var sec_table_src = MD_Source.subscribe_securities()
+            .Table(s => s.security.Id)
+            .Select(s => s.security, s => s.Id);
+         Synth_Sec_Client = new(synth_sec_service, sec_table_src);
+
+
+
+         // broadcast.Start();
+      }
+      private uint get_security(Venue v, string sec_name) {
+         var securites = _securities;
+         if (securites.Values.TryGetFirst(sc_md =>
+            (
+               sc_md.security.Exchange == v &&
+               sc_md.security.ToString() == sec_name),
+            out SecurityMD sc_md
+            )) {
+            return sc_md.id;
+         }
+         throw new ArgumentException($"{sec_name} at {v} not found in securites");
+      }
+      public void handle_connection(WebsocketClientID client_id,
+                                 WebsocketIncomingMessage? optional_message) {
+         Clients.AddClient(client_id);
+         var log_txt = $"WebsocketDataSync.handle_connection client_id: {client_id} " +
+                       $"message: {(optional_message == null ? "null" : optional_message.ToString())}";
+         bts.Log.Gen.info(log_txt);
+         System.Console.WriteLine(log_txt);
+         if (optional_message != null) {
+            handle_client_message(optional_message);
+         }
+      }
+      // Called when a client is disconnected
+      public void handle_disconnection(WebsocketClientID client_id) {
+         handle_unsubscribe_all(client_id);
+         Clients.RemoveClient(client_id);
+         var log_txt = $"WebsocketDataSync.handle_disconnection client_id: {client_id}";
+         bts.Log.Gen.info(log_txt);
+         System.Console.WriteLine(log_txt);
+      }
+      // Called when a client sends data 
+      public void handle_client_message(WebsocketIncomingMessage message) {
+         WebsocketRequest? req;
+         string error;
+         if (!WebsocketRequest.TryGetWebSocketRequest(message.sender_client_id, message.message_body, out IUniqueID id, out req, out error)) {
+            handle_incorrect_request(id, message.sender_client_id, error);
+            return;
+         }
+         handle_request(req);
+         var log_txt = $"WebsocketDataSync.handle_client_message message: {req.type}";
+         bts.Log.Gen.info(log_txt);
+         System.Console.WriteLine(log_txt);
+
+      }
+      public void handle_request(WebsocketRequest req) {
+         switch (req.type) {
+            case WebsocketRequest.REQUEST_TYPES.SUBSCRIBE: {
+                  handle_subscription(req);
+                  break;
+               }
+            case WebsocketRequest.REQUEST_TYPES.UNSUBSCRIBE_ALL: {
+                  handle_unsubscribe_all(req.client_id);
+                  break;
+               }
+            case WebsocketRequest.REQUEST_TYPES.ENTER_ORDER: {
+                  handle_enter_order(req);
+                  break;
+               }
+            case WebsocketRequest.REQUEST_TYPES.DELETE_ORDER: {
+                  handle_delete_order(req);
+                  break;
+               }
+            case WebsocketRequest.REQUEST_TYPES.UPDATE_ORDER: {
+                  handle_update_order(req);
+                  break;
+               }
+            case WebsocketRequest.REQUEST_TYPES.UNKOWN: {
+                  ///send error
+                  break;
+               }
+            default: break;
+         }
+      }
+      public void handle_unsubscribe_all(WebsocketClientID client_id) {
+         try {
+            Clients[client_id] = Clients[client_id].RemoveAll();
+         } catch (ArgumentException ex) {
+            System.Console.WriteLine("error for req", client_id, ex.ToString());
+         }
+      }
+      public async void handle_delete_order(WebsocketRequest req) {
+         (bool suc, string status) = await WebsocketRestHandler.DeleteOrderAsync(req, Orders_Client);
+         if (!suc) {
+            handle_incorrect_request(req, status);
+            return;
+         }
+         handle_correct_request(req.id, status, null, req.client_id);
+      }
+      public async void handle_enter_order(WebsocketRequest req) {
+         (bool suc, uint? id, string status) = await WebsocketRestHandler.EnterOrderAsync(req, Orders_Client, _securities);
+         if (!suc) {
+            handle_incorrect_request(req, status);
+            return;
+         }
+         handle_correct_request(req.id, status, new { order_key = id }, req.client_id);
+      }
+      public async void handle_update_order(WebsocketRequest req) {
+         (bool suc, string status) = await WebsocketRestHandler.UpdateOrderAsync(req, Orders_Client);
+         if (!suc) {
+            handle_incorrect_request(req, status);
+            return;
+         }
+         handle_correct_request(req.id, status, null, req.client_id);
+      }
+      public void handle_subscription(WebsocketRequest req) {
+         WebsocketSubscriptionHandler? hdlr;
+         string status;
+         if (!WebsocketSubscriptionHandler.TryCreate(
+            req,
+            Md_Client,
+            Orders_Client,
+            obj => send_message(obj, req.client_id),
+            out hdlr,
+            out status)) {
+            handle_incorrect_request(req, status);
+            return;
+         }
+         if (Clients.ContainsKey(req.client_id) && Clients[req.client_id].ContainsKey(hdlr.ID)) {
+            handle_incorrect_request(req, "subscription id already in use");
+            return;
+         }
+         handle_correct_request(req.id, status, new { subscription_id = hdlr.ID.ID }, req.client_id);
+         hdlr.subscribe();
+         Clients[req.client_id] = Clients[req.client_id].Add(hdlr.ID, hdlr);
+      }
+      public void handle_incorrect_request(WebsocketRequest req, string message = "") => handle_incorrect_request(req.id, req.client_id, message);
+      public void handle_incorrect_request(IUniqueID rid, WebsocketClientID cid, string message = "") {
+         System.Console.WriteLine($"request {rid} from {cid} failed, message: {message}");
+         var ms = new {
+            jsonrpc = 2.0,
+            id = rid.ID,
+            error = message
+         };
+         send_message(ms, cid);
+      }
+
+      public void handle_correct_request(IUniqueID rid, string status, Object? data, WebsocketClientID cid) {
+         System.Console.WriteLine($"request {rid} from {cid} suceeded, message: {status}");
+         var ms = new {
+            jsonrpc = 2.0,
+            id = rid.ID,
+            status = status,
+            result = data
+         };
+         send_message(ms, cid);
+      }
+      public void send_message(object value, WebsocketClientID id) {
+         var requestJson = JsonConvert.SerializeObject(value);
+         Gateway.send_to_clients(
+            new List<WebsocketOutgoingMessage>() {
+                new WebsocketOutgoingMessage(id, requestJson)
+                });
+
+      }
+
+   }
+}

src/client/spark_websocket/Core/WebsocketExtensions.cs

@@ -3,7 +3,7 @@ namespace Bts.Websocket;
 using System.Net.WebSockets;
 
 public static class WebsocketExtensions {
-   public static bool IsOpen(this WebSocket ws) =>
+   public static bool IsSocketStateOpen(this WebSocket ws) =>
       ws.State switch {
          WebSocketState.None or WebSocketState.Closed or WebSocketState.Aborted => false,
          _ => true

src/client/spark_websocket/Core/WebsocketRestHandler.cs

@@ -1,263 +1,257 @@
-namespace bts.websocket_core {
-   using bts.api;
-   using bts.api.admin;
-   using bts.api.core;
-   using bts.api.orderman;
-
-   using mbts;
-
-   using Newtonsoft.Json.Linq;
-
-   using Spark.Api.Orders;
-
-   using System;
-
-   public class WebsocketRestHandler {
-      private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(WebsocketRequest req) {
-         string status = "";
-         CustomOrderDefinition customdef;
-         JObject? custom = req.parameters["custom"] as JObject;
-         if (!WebsocketJsonParsers.TryParse(
-            req.parameters,
-            "order_type",
-            out CustomOrderDefinition.CustomOneofCase? order_type,
-            out status)
-            || order_type == null) {
-            return (false, null, status);
-         }
-         switch (order_type) {
-            case CustomOrderDefinition.CustomOneofCase.Limit: {
-                  customdef = new CustomOrderDefinition() {
-                     Limit = new LimitOrderDefinition()
-                  };
-                  break;
-               }
-
-            // case CustomOrderDefinition.CustomOneofCase.Spreader: {
-            //       // if (custom == null) {
-            //       //    message = "custom not present";
-            //       //    return false;
-            //       // }
-            //       //ENTER SPREADER CODE HERE
-            //       break;
-            //    }
-            case CustomOrderDefinition.CustomOneofCase.Iceberg: {
-                  if (custom == null) {
-                     return (false, null, "custom not present");
-                  }
-                  if (!WebsocketJsonParsers.TryParse(custom, "show_quantity", out double? sq, out status) || sq == null) {
-                     return (false, null, status);
-                  }
-                  customdef = new CustomOrderDefinition() {
-                     Iceberg = new IcebergOrderDefinition() {
-                        ShowQuantity = (double)sq,
-                        Child = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        }
-                     }
-                  };
-                  break;
-               }
-            case CustomOrderDefinition.CustomOneofCase.Cover: {
-                  if (custom == null) {
-                     return (false, null, "custom not present");
-                  }
-                  if (!WebsocketJsonParsers.TryParse(
-                     custom,
-                     "hedge_ticks",
-                     out uint? ht,
-                     out status)
-                     || ht == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse(
-                     custom,
-                     "cancel_on_external_hedge_cancel",
-                     out bool? coehc,
-                     out status)
-                     || coehc == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse(
-                     custom,
-                     "cancel_hedges_on_cancel",
-                     out bool? choc,
-                     out status)
-                     || choc == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse(
-                     custom,
-                     "detach_hedges",
-                     out bool? dh,
-                     out status)
-                     || dh == null) {
-                     return (false, null, status);
-                  }
-                  customdef = new CustomOrderDefinition() {
-                     Cover = new CoverOrderDefinition() {
-                        Hedge = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        },
-                        HedgeTicks = (uint)ht,
-                        CancelOnExternalHedgeCancel = (bool)coehc,
-                        CancelHedgesOnCancel = (bool)choc,
-                        DetachHedges = (bool)dh,
-                        Child = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        }
-                     }
-                  };
-                  break;
-               }
-            case CustomOrderDefinition.CustomOneofCase.Market: {
-                  customdef = new CustomOrderDefinition() {
-                     Market = new MarketOrderDefinition()
-                  };
-                  break;
-               }
-            default: {
-                  return (false, null, "order_type not correct");
-               }
-         };
-         return (true, customdef, status);
-      }
-
-      static async Task<(bool, OrderDefinition?, string)> TryGetOrderDefAsync(
-         WebsocketRequest req, IReadOnlyDictionary<uint, SecurityMD> securities, AdminService adminService) {
-         if (req.parameters == null) {
-            return (false, null, "parameters not present");
-         }
-         if (!WebsocketJsonParsers.TryParse(req.parameters, "user", out string? usr, out string status)
-            || usr == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse(req.parameters, "account", out string? act, out status)
-            || act == null) {
-            return (false, null, status);
-         }
-         Result<AdminData> adminResult = await adminService.get_admin_data_async(new VoidMessage());
-         uint user_id;
-         uint account_id;
-         if (!adminResult.success || adminResult.value == null) {
-            return (false, null, adminResult.error_message);
-         }
-         AdminData admin = adminResult.value;
-         string username = usr!;
-         string accountname = act!;
-         BTSUser? user = admin.Users.FirstOrDefault(x => username != null ? x.Name.ToLowerInvariant() == username.ToLowerInvariant() : true);
-         if (user == null) {
-            return (false, null, "user not correct");
-         }
-         user_id = (uint)user.Id;
-         BTSAccount? account = admin.Accounts.FirstOrDefault(x => x.Name.ToLowerInvariant() == accountname.ToLowerInvariant());
-         if (account == null) {
-            return (false, null, "account not corrent");
-         }
-         account_id = (uint)account.Id;
-         if (!WebsocketJsonParsers.TryParse(req.parameters, "side", out Side? side, out status)
-            || side == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse(
-            req.parameters, "exchange_securty_name", out string? exchange_securty_name, out status) || exchange_securty_name == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse(req.parameters, "venue", out Venue? exchange, out status) || exchange == null) {
-            return (false, null, status);
-         }
-         var _securites = securities;
-         if (!_securites.Values.TryGetFirst(sc_md =>
-            (
-               sc_md.security.Exchange == exchange &&
-               sc_md.security.ToString() == exchange_securty_name),
-            out SecurityMD sc_md
-            )) {
-            return (false, null, "security not found");
-         }
-
-         (var check, var customdef, status) = TryGetCustomDef(req);
-         if (check == false) {
-            return (false, null, status);
-         }
-         return (true, new OrderDefinition {
-            TimeInForce = TimeInForce.Gtc,
-            SecurityKey = sc_md.id,
-            Side = (Side)side,
-            UserId = user_id,
-            AccountId = account_id,
-            CustomDef = customdef
-         }, status);
-      }
-      public static async Task<(bool success, string status)> DeleteOrderAsync(WebsocketRequest req, OrdersClient oc) {
-         if (req.parameters == null) {
-            return (false, "parameters not present");
-         }
-         if (!WebsocketJsonParsers.TryParse(req.parameters, "order_id", out uint? key, out string status) || key == null) {
-            return (false, status);
-         }
-         Response res = await oc.TryDeleteOrderAsync((uint)key);
-         status = res.ErrorCode.ToString();
-         return (res.ErrorCode != OrderError.Ok, status);
-      }
-      public static async Task<(bool success, string status)> UpdateOrderAsync(WebsocketRequest request, OrdersClient oc) {
-         if (request.parameters == null) {
-            return (false, "parameters not present");
-         }
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "order_id", out uint? key, out string status) || key == null) {
-            return (false, status);
-         }
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null) {
-            return (false, status);
-         }
-
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null) {
-            return (false, status);
-         }
-         (var check, var customdef, status) = TryGetCustomDef(request);
-         if (check == false || customdef == null) {
-            return (false, status);
-         }
-         Response res = await oc.TryUpdateOrderAsync(customdef, (double)price, (double)quantity, (uint)key);
-         status = res.ErrorCode.ToString();
-         return (res.ErrorCode != OrderError.Ok, status);
-      }
-
-      public static async Task<(bool success, uint? orderId, string status)> EnterOrderAsync(
-         WebsocketRequest request,
-         OrdersClient ordersClient,
-         IReadOnlyDictionary<uint, SecurityMD> securities,
-         AdminService adminService
-      ) {
-         double? quantity = null;
-         double? price = null;
-         OrderDefinition? orderDefinition = null;
-         uint? orderId = null;
-         string status = string.Empty;
-
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "quantity", out quantity, out status) || quantity == null) {
-            return (false, null, status);
-         }
-
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "price", out price, out status) || price == null) {
-            return (false, null, status);
-         }
-
-         (bool success, orderDefinition, status) = await TryGetOrderDefAsync(request, securities, adminService);
-         if (!success || orderDefinition == null) {
-            return (success, null, status);
-         }
-
-         var result = await ordersClient.TryEnterOrderAsync(orderDefinition, (uint)price, (double)quantity);
-
-         if (result.ErrorCode != OrderError.Ok) {
-            return (false, null, result.ErrorCode.ToString());
-         }
-
-         orderId = result.OrderKey;
-         status = result.ErrorCode.ToString();
-
-         return (true, orderId, status);
-      }
-   }
-}
+namespace bts.websocket_core {
+   using bts.api;
+   using bts.api.admin;
+   using bts.api.core;
+   using bts.api.orderman;
+
+   using mbts;
+
+   using Newtonsoft.Json.Linq;
+
+   using Spark.Api.Orders;
+
+   using System;
+
+   public class WebsocketRestHandler {
+      readonly static Lazy<AdminService> _admin = new Lazy<AdminService>(() => Locator.locate<AdminService>(), true);
+      private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(WebsocketRequest req) {
+         string status = "";
+         CustomOrderDefinition customdef;
+         JObject? custom = req.parameters["custom"] as JObject;
+         if (!WebsocketJsonParsers.TryParse(
+            req.parameters,
+            "order_type",
+            out CustomOrderDefinition.CustomOneofCase? order_type,
+            out status)
+            || order_type == null) {
+            return (false, null, status);
+         }
+         switch (order_type) {
+            case CustomOrderDefinition.CustomOneofCase.Limit: {
+                  customdef = new CustomOrderDefinition() {
+                     Limit = new LimitOrderDefinition()
+                  };
+                  break;
+               }
+
+            // case CustomOrderDefinition.CustomOneofCase.Spreader: {
+            //       // if (custom == null) {
+            //       //    message = "custom not present";
+            //       //    return false;
+            //       // }
+            //       //ENTER SPREADER CODE HERE
+            //       break;
+            //    }
+            case CustomOrderDefinition.CustomOneofCase.Iceberg: {
+                  if (custom == null) {
+                     return (false, null, "custom not present");
+                  }
+                  if (!WebsocketJsonParsers.TryParse(custom, "show_quantity", out double? sq, out status) || sq == null) {
+                     return (false, null, status);
+                  }
+                  customdef = new CustomOrderDefinition() {
+                     Iceberg = new IcebergOrderDefinition() {
+                        ShowQuantity = (double)sq,
+                        Child = new CustomOrderDefinition() {
+                           Limit = new LimitOrderDefinition()
+                        }
+                     }
+                  };
+                  break;
+               }
+            case CustomOrderDefinition.CustomOneofCase.Cover: {
+                  if (custom == null) {
+                     return (false, null, "custom not present");
+                  }
+                  if (!WebsocketJsonParsers.TryParse(
+                     custom,
+                     "hedge_ticks",
+                     out uint? ht,
+                     out status)
+                     || ht == null) {
+                     return (false, null, status);
+                  }
+                  if (!WebsocketJsonParsers.TryParse(
+                     custom,
+                     "cancel_on_external_hedge_cancel",
+                     out bool? coehc,
+                     out status)
+                     || coehc == null) {
+                     return (false, null, status);
+                  }
+                  if (!WebsocketJsonParsers.TryParse(
+                     custom,
+                     "cancel_hedges_on_cancel",
+                     out bool? choc,
+                     out status)
+                     || choc == null) {
+                     return (false, null, status);
+                  }
+                  if (!WebsocketJsonParsers.TryParse(
+                     custom,
+                     "detach_hedges",
+                     out bool? dh,
+                     out status)
+                     || dh == null) {
+                     return (false, null, status);
+                  }
+                  customdef = new CustomOrderDefinition() {
+                     Cover = new CoverOrderDefinition() {
+                        Hedge = new CustomOrderDefinition() {
+                           Limit = new LimitOrderDefinition()
+                        },
+                        HedgeTicks = (uint)ht,
+                        CancelOnExternalHedgeCancel = (bool)coehc,
+                        CancelHedgesOnCancel = (bool)choc,
+                        DetachHedges = (bool)dh,
+                        Child = new CustomOrderDefinition() {
+                           Limit = new LimitOrderDefinition()
+                        }
+                     }
+                  };
+                  break;
+               }
+            case CustomOrderDefinition.CustomOneofCase.Market: {
+                  customdef = new CustomOrderDefinition() {
+                     Market = new MarketOrderDefinition()
+                  };
+                  break;
+               }
+            default: {
+                  return (false, null, "order_type not correct");
+               }
+         };
+         return (true, customdef, status);
+      }
+      private async static Task<(bool, OrderDefinition?, string)> TryGetOrderDefAsync(WebsocketRequest req, IReadOnlyDictionary<uint, SecurityMD> securities) {
+         if (req.parameters == null) {
+            return (false, null, "parameters not present");
+         }
+         if (!WebsocketJsonParsers.TryParse(req.parameters, "user", out string? usr, out string status)
+            || usr == null) {
+            return (false, null, status);
+         }
+         if (!WebsocketJsonParsers.TryParse(req.parameters, "account", out string? act, out status)
+            || act == null) {
+            return (false, null, status);
+         }
+         Result<AdminData> adminResult = await _admin.Value.get_admin_data_async(new VoidMessage());
+         uint user_id;
+         uint account_id;
+         if (!adminResult.success || adminResult.value == null) {
+            return (false, null, adminResult.error_message);
+         }
+         AdminData admin = adminResult.value;
+         string username = usr!;
+         string accountname = act!;
+         BTSUser? user = admin.Users.FirstOrDefault(x => username != null ? x.Name.ToLowerInvariant() == username.ToLowerInvariant() : true);
+         if (user == null) {
+            return (false, null, "user not correct");
+         }
+         user_id = (uint)user.Id;
+         BTSAccount? account = admin.Accounts.FirstOrDefault(x => x.Name.ToLowerInvariant() == accountname.ToLowerInvariant());
+         if (account == null) {
+            return (false, null, "account not corrent");
+         }
+         account_id = (uint)account.Id;
+         if (!WebsocketJsonParsers.TryParse(req.parameters, "side", out Side? side, out status)
+            || side == null) {
+            return (false, null, status);
+         }
+         if (!WebsocketJsonParsers.TryParse(
+            req.parameters, "exchange_securty_name", out string? exchange_securty_name, out status) || exchange_securty_name == null) {
+            return (false, null, status);
+         }
+         if (!WebsocketJsonParsers.TryParse(req.parameters, "venue", out Venue? exchange, out status) || exchange == null) {
+            return (false, null, status);
+         }
+         var _securites = securities;
+         if (!_securites.Values.TryGetFirst(sc_md =>
+            (
+               sc_md.security.Exchange == exchange &&
+               sc_md.security.ToString() == exchange_securty_name),
+            out SecurityMD sc_md
+            )) {
+            return (false, null, "security not found");
+         }
+
+         (var check, var customdef, status) = TryGetCustomDef(req);
+         if (check == false) {
+            return (false, null, status);
+         }
+         return (true, new OrderDefinition {
+            TimeInForce = TimeInForce.Gtc,
+            SecurityKey = sc_md.id,
+            Side = (Side)side,
+            UserId = user_id,
+            AccountId = account_id,
+            CustomDef = customdef
+         }, status);
+      }
+      public static async Task<(bool success, string status)> DeleteOrderAsync(WebsocketRequest req, OrdersClient oc) {
+         if (req.parameters == null) {
+            return (false, "parameters not present");
+         }
+         if (!WebsocketJsonParsers.TryParse(req.parameters, "order_id", out uint? key, out string status) || key == null) {
+            return (false, status);
+         }
+         Response res = await oc.TryDeleteOrderAsync((uint)key);
+         status = res.ErrorCode.ToString();
+         return (res.ErrorCode != OrderError.Ok, status);
+      }
+      public static async Task<(bool success, string status)> UpdateOrderAsync(WebsocketRequest request, OrdersClient oc) {
+         if (request.parameters == null) {
+            return (false, "parameters not present");
+         }
+         if (!WebsocketJsonParsers.TryParse(request.parameters, "order_id", out uint? key, out string status) || key == null) {
+            return (false, status);
+         }
+         if (!WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null) {
+            return (false, status);
+         }
+
+         if (!WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null) {
+            return (false, status);
+         }
+         (var check, var customdef, status) = TryGetCustomDef(request);
+         if (check == false || customdef == null) {
+            return (false, status);
+         }
+         Response res = await oc.TryUpdateOrderAsync(customdef, (double)price, (double)quantity, (uint)key);
+         status = res.ErrorCode.ToString();
+         return (res.ErrorCode != OrderError.Ok, status);
+      }
+      public static async Task<(bool success, uint? orderId, string status)> EnterOrderAsync(
+        WebsocketRequest request, OrdersClient ordersClient, IReadOnlyDictionary<uint, SecurityMD> securities) {
+         double? quantity = null;
+         double? price = null;
+         OrderDefinition? orderDefinition = null;
+         uint? orderId = null;
+         string status = string.Empty;
+
+         if (!WebsocketJsonParsers.TryParse(request.parameters, "quantity", out quantity, out status) || quantity == null) {
+            return (false, null, status);
+         }
+
+         if (!WebsocketJsonParsers.TryParse(request.parameters, "price", out price, out status) || price == null) {
+            return (false, null, status);
+         }
+
+         (bool success, orderDefinition, status) = await TryGetOrderDefAsync(request, securities);
+         if (!success || orderDefinition == null) {
+            return (success, null, status);
+         }
+
+         var result = await ordersClient.TryEnterOrderAsync(orderDefinition, (uint)price, (double)quantity);
+
+         if (result.ErrorCode != OrderError.Ok) {
+            return (false, null, result.ErrorCode.ToString());
+         }
+
+         orderId = result.OrderKey;
+         status = result.ErrorCode.ToString();
+
+         return (true, orderId, status);
+      }
+   }
+}

src/client/spark_websocket/Usings.cs

@@ -1 +0,0 @@
-using System.Text.Json;

src/lib/logging/LoggerHub.cpp

@@ -79,9 +79,6 @@ void LoggerHub::config(const LoggingConfig &cfg) {
 }
 
 void LoggerHub::start() {
-   std::cout << "Starting LoggerHub '" << me->logger_name << "' on thread "
-             << os::get_thread_name() << ":" << std::this_thread::get_id()
-             << std::endl;
    bts_assert(!me->worker);
    me->worker = std::make_unique<std::thread>(threading::make_thread(
        me->logger_name.c_str(), std::bind(&LoggerHub::do_work, this),
@@ -103,9 +100,6 @@ Logger &LoggerHub::make_logger() {
 }
 
 void LoggerHub::do_work() {
-   std::cout << "Starting LoggerHub '" << me->logger_name
-             << "' worker on thread " << os::get_thread_name() << ":"
-             << std::this_thread::get_id() << std::endl;
    while (!me->done) {
       size_t consumed = 0;
       {

src/lib/utils/init_spark_lib.cpp

@@ -9,14 +9,8 @@ namespace utils {
 static log::LoggingConfigPoller *poller = nullptr;
 
 void init_spark_lib() {
-   // Write to stdout instead of normal log, because the logging system hasn't
-   // been configured yet (that's one of the things we're about to do).
-   std::cout << "Initializing spark_lib\n";
-   std::cout << "Configuring thread 'main'" << std::endl;
    threading::configure_current_thread("main");
-   std::cout << "Starting default logging thread" << std::endl;
    log::start_default_logging_thread();
-   std::cout << "Initializing LoggingConfigPoller" << std::endl;
    poller = new log::LoggingConfigPoller();
 }
 

tools/py/data_stream_tcp_client.py

@@ -1,91 +0,0 @@
-import socket
-import struct
-import time
-
-
-def tcp_client(host, port, message):
-    # Create a socket object
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        try:
-            # Connect
-            sock.connect((host, port))
-            print(f"Connected to {host}:{port}")
-            # Send
-            sock.send(message.encode('utf-8'))
-            print(f"Sent data: {message}")
-            # Receive
-            data = sock.recv(1024)
-            print(f"Received data: {data.decode('utf-8')}")
-        except Exception as e:
-            print(f"Error: {e}")
-        finally:
-            sock.close()
-            print("Socket closed")
-
-
-def serialize_msg(msg):
-    raw_msg = bytearray(msg)
-    # while (len(raw_msg) < msg_len):
-    #     raw_msg.append(0)
-    # Use struct.pack to prepend the length as a 4-byte integer
-    # length_bytes = struct.pack('!I', len(raw_msg) + 4)
-    # b = byte(36)
-    raw_msg.insert(0, 0)
-    raw_msg.insert(0, 0)
-    raw_msg.insert(0, 0)
-    raw_msg.insert(0, 0)
-    raw_msg.insert(0, 0)
-    raw_msg.insert(0, 0)
-    raw_msg.insert(0, 0)
-    raw_msg.insert(0, len(msg) + 4)
-    return raw_msg
-
-
-def tcp_client2(host, port):
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        msg = b'Hello!'
-        raw_msg = serialize_msg(msg)
-
-        try:
-            sock.connect((host, port))
-            while True:
-                print(f'Sending message {msg}')
-                sock.sendall(raw_msg)
-                print(f'Sent message: {msg}')
-                print(f"Receiving data...")
-                data = sock.recv(1024)
-                print(f"Received data: {data.decode('utf-8')}")
-                time.sleep(2)
-        except Exception as e:
-            print(f"Error: {e}")
-        finally:
-            sock.close()
-            print("Socket closed")
-        
-
-def tcp_server(host, port):
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind((host, port))
-        sock.listen()
-        conn, addr = sock.accept()
-        with conn:
-            print(f"Connected by {addr}")
-            while True:
-                data = conn.recv(1024)
-                if not data:
-                    break
-                print(f'Received data (will echo back): {data}')
-                conn.sendall('Thanks!')
-                time.sleep(2)
-
-
-def _run():
-    # Replace 'your_server_ip' and 'your_server_port' with the actual server IP and port
-    server_ip = 'your_server_ip'
-    server_port = 12345
-    tcp_client2(server_ip, server_port)
-
-
-if __name__ == '__main__':
-    print(serialize_msg(b'Hello!'))
-    tcp_client2('127.0.0.1', 12345)